feat: retry added by default (#198)

* feat: retry added by default

Author: pashidlos

lib/__data__/index.ts

@@ -0,0 +1,44 @@
+import { TestStatus, TestRunResponse } from "../types";
+
+export const testRunUnresolvedResponse: TestRunResponse = {
+  url: "url",
+  status: TestStatus.unresolved,
+  pixelMisMatchCount: 12,
+  diffPercent: 0.12,
+  diffTollerancePercent: 0,
+  id: "some id",
+  imageName: "imageName",
+  merge: false,
+};
+
+export const testRunOkResponse: TestRunResponse = {
+  url: "url",
+  status: TestStatus.ok,
+  diffPercent: 0,
+  diffTollerancePercent: 0,
+  id: "some id",
+  imageName: "imageName",
+  merge: false,
+};
+
+export const testRunNewResponse: TestRunResponse = {
+  url: "url",
+  status: TestStatus.new,
+  pixelMisMatchCount: 0,
+  diffPercent: 0,
+  diffTollerancePercent: 0,
+  id: "some id",
+  imageName: "imageName",
+  merge: false,
+};
+
+export const testRunAutoApprovedResponse: TestRunResponse = {
+  url: "url",
+  status: TestStatus.autoApproved,
+  pixelMisMatchCount: 0,
+  diffPercent: 0,
+  diffTollerancePercent: 0,
+  id: "some id",
+  imageName: "imageName",
+  merge: false,
+};

lib/helpers/index.ts

@@ -1,3 +1,4 @@
 export * from "./config.helper";
 export * from "./type.helper";
 export * from "./dto.helper";
+export * from "./track.helper";

lib/helpers/track.helper.spec.ts

@@ -0,0 +1,53 @@
+import { TestRunResponse, TestStatus } from "../types";
+import { processTestRun, shouldStopRetry } from "./track.helper";
+import {
+  testRunOkResponse,
+  testRunUnresolvedResponse,
+  testRunAutoApprovedResponse,
+  testRunNewResponse,
+} from "../__data__";
+
+describe.each<[TestStatus.new | TestStatus.unresolved, string]>([
+  [TestStatus.new, "No baseline: "],
+  [TestStatus.unresolved, "Difference found: "],
+])("processTestRun", (status, expectedMessage) => {
+  beforeEach(() => {
+    testRunUnresolvedResponse.status = status;
+  });
+
+  it(`default soft assert should throw exception if status ${status}`, () => {
+    expect(() => processTestRun(testRunUnresolvedResponse)).toThrowError(
+      new Error(expectedMessage.concat(testRunUnresolvedResponse.url))
+    );
+  });
+
+  it(`disabled soft assert should throw exception if status ${status}`, () => {
+    const enableSoftAssert = false;
+
+    expect(() =>
+      processTestRun(testRunUnresolvedResponse, enableSoftAssert)
+    ).toThrowError(
+      new Error(expectedMessage.concat(testRunUnresolvedResponse.url))
+    );
+  });
+
+  it(`enabled soft assert should log error if status ${status}`, () => {
+    console.error = jest.fn();
+    const enableSoftAssert = true;
+
+    processTestRun(testRunUnresolvedResponse, enableSoftAssert);
+
+    expect(console.error).toHaveBeenCalledWith(
+      expectedMessage.concat(testRunUnresolvedResponse.url)
+    );
+  });
+});
+
+it.each<[TestRunResponse, boolean]>([
+  [testRunOkResponse, true],
+  [testRunUnresolvedResponse, false],
+  [testRunAutoApprovedResponse, true],
+  [testRunNewResponse, true],
+])("shouldStopRetry", (response, expectedResult) => {
+  expect(shouldStopRetry(response)).toBe(expectedResult);
+});

lib/helpers/track.helper.ts

@@ -0,0 +1,48 @@
+import { TestRunResponse, TestStatus } from "../types";
+
+const getErrorMessage = (
+  testRunResponse: TestRunResponse
+): string | undefined => {
+  switch (testRunResponse.status) {
+    case TestStatus.new: {
+      return `No baseline: ${testRunResponse.url}`;
+    }
+    case TestStatus.unresolved: {
+      return `Difference found: ${testRunResponse.url}`;
+    }
+  }
+};
+
+export const processTestRun = (
+  testRunResponse: TestRunResponse,
+  enableSoftAssert?: boolean
+) => {
+  const errorMessage = getErrorMessage(testRunResponse);
+
+  if (errorMessage) {
+    if (enableSoftAssert) {
+      // eslint-disable-next-line no-console
+      console.error(errorMessage);
+    } else {
+      throw new Error(errorMessage);
+    }
+  }
+};
+
+export const shouldStopRetry = (result: TestRunResponse) =>
+  result.status !== TestStatus.unresolved;
+
+export const trackWithRetry = async (
+  trackFn: () => Promise<TestRunResponse>,
+  retryLimit: number,
+  enableSoftAssert?: boolean
+): Promise<TestRunResponse> => {
+  const result = await trackFn();
+  if (retryLimit <= 0 || shouldStopRetry(result)) {
+    processTestRun(result, enableSoftAssert);
+    return result;
+  }
+  // eslint-disable-next-line no-console
+  console.info(`Diff found... Remaining retry attempts **${retryLimit}**`);
+  return trackWithRetry(trackFn, retryLimit - 1, enableSoftAssert);
+};

lib/visualRegressionTracker.spec.ts

@@ -1,7 +1,7 @@
 import axios, { AxiosError, AxiosResponse } from "axios";
 import { mocked } from "jest-mock";
 import FormData from "form-data";
-
+import { testRunOkResponse, testRunUnresolvedResponse } from "./__data__";
 import { VisualRegressionTracker } from "./visualRegressionTracker";
 import {
   Config,
@@ -164,17 +164,6 @@ const testRunBuffer: TestRunBuffer = {
   comment: "comment",
 };
 
-const testRunResponse: TestRunResponse = {
-  url: "url",
-  status: TestStatus.unresolved,
-  pixelMisMatchCount: 12,
-  diffPercent: 0.12,
-  diffTollerancePercent: 0,
-  id: "some id",
-  imageName: "imageName",
-  merge: false,
-};
-
 describe("VisualRegressionTracker", () => {
   let vrt: VisualRegressionTracker;
 
@@ -249,34 +238,42 @@ describe("VisualRegressionTracker", () => {
       );
     });
 
-    it("should track base64", async () => {
+    it("should track base64 without retry", async () => {
+      const responce = testRunOkResponse;
       vrt["isStarted"] = jest.fn().mockReturnValueOnce(true);
-      vrt["submitTestRunBase64"] = jest
-        .fn()
-        .mockResolvedValueOnce(testRunResponse);
-      vrt["processTestRun"] = jest.fn();
+      vrt["submitTestRunBase64"] = jest.fn().mockResolvedValue(responce);
 
       await vrt.track(testRunBase64);
 
       expect(vrt["submitTestRunBase64"]).toHaveBeenCalledWith(testRunBase64);
-      expect(vrt["processTestRun"]).toHaveBeenCalledWith(testRunResponse);
+      expect(vrt["submitTestRunBase64"]).toHaveBeenCalledTimes(1);
       expect(mockedTestRunResult).toHaveBeenCalledWith(
-        testRunResponse,
+        responce,
         "http://localhost:4200"
       );
     });
 
-    it("should track multipart", async () => {
+    it("should track base64 with retry", async () => {
+      const responce = testRunUnresolvedResponse;
+      vrt["isStarted"] = jest.fn().mockReturnValueOnce(true);
+      vrt["submitTestRunBase64"] = jest.fn().mockResolvedValue(responce);
+
+      await expect(vrt.track(testRunBase64, 3)).rejects.toThrowError(
+        "Difference found: url"
+      );
+      expect(vrt["submitTestRunBase64"]).toHaveBeenCalledTimes(4);
+      expect(vrt["submitTestRunBase64"]).toHaveBeenCalledWith(testRunBase64);
+    });
+
+    it("should track multipart without retry", async () => {
+      const responce = testRunOkResponse;
       const data = new FormData();
       const buildId = "1312";
       const projectId = "asd";
       vrt["buildId"] = buildId;
       vrt["projectId"] = projectId;
       vrt["isStarted"] = jest.fn().mockReturnValueOnce(true);
-      vrt["submitTestRunMultipart"] = jest
-        .fn()
-        .mockResolvedValueOnce(testRunResponse);
-      vrt["processTestRun"] = jest.fn();
+      vrt["submitTestRunMultipart"] = jest.fn().mockResolvedValueOnce(responce);
       mockedDtoHelper.multipartDtoToFormData.mockReturnValueOnce(data);
 
       await vrt.track(testRunMultipart);
@@ -287,25 +284,47 @@ describe("VisualRegressionTracker", () => {
         branchName: config.branchName,
         ...testRunMultipart,
       });
+      expect(vrt["submitTestRunMultipart"]).toHaveBeenCalledTimes(1);
       expect(vrt["submitTestRunMultipart"]).toHaveBeenCalledWith(data);
-      expect(vrt["processTestRun"]).toHaveBeenCalledWith(testRunResponse);
       expect(mockedTestRunResult).toHaveBeenCalledWith(
-        testRunResponse,
+        responce,
         "http://localhost:4200"
       );
     });
 
+    it("should track multipart with retry", async () => {
+      const responce = testRunUnresolvedResponse;
+      const data = new FormData();
+      const buildId = "1312";
+      const projectId = "asd";
+      vrt["buildId"] = buildId;
+      vrt["projectId"] = projectId;
+      vrt["isStarted"] = jest.fn().mockReturnValueOnce(true);
+      vrt["submitTestRunMultipart"] = jest.fn().mockResolvedValue(responce);
+      mockedDtoHelper.multipartDtoToFormData.mockReturnValueOnce(data);
+
+      await expect(vrt.track(testRunMultipart, 3)).rejects.toThrowError(
+        "Difference found: url"
+      );
+      expect(mockedDtoHelper.multipartDtoToFormData).toHaveBeenCalledWith({
+        buildId,
+        projectId,
+        branchName: config.branchName,
+        ...testRunMultipart,
+      });
+      expect(vrt["submitTestRunMultipart"]).toHaveBeenCalledTimes(4);
+      expect(vrt["submitTestRunMultipart"]).toHaveBeenCalledWith(data);
+    });
+
     it("should track buffer", async () => {
+      const responce = testRunOkResponse;
       const data = new FormData();
       const buildId = "1312";
       const projectId = "asd";
       vrt["buildId"] = buildId;
       vrt["projectId"] = projectId;
       vrt["isStarted"] = jest.fn().mockReturnValueOnce(true);
-      vrt["submitTestRunMultipart"] = jest
-        .fn()
-        .mockResolvedValueOnce(testRunResponse);
-      vrt["processTestRun"] = jest.fn();
+      vrt["submitTestRunMultipart"] = jest.fn().mockResolvedValueOnce(responce);
       mockedDtoHelper.bufferDtoToFormData.mockReturnValueOnce(data);
 
       await vrt.track(testRunBuffer);
@@ -317,9 +336,8 @@ describe("VisualRegressionTracker", () => {
         ...testRunBuffer,
       });
       expect(vrt["submitTestRunMultipart"]).toHaveBeenCalledWith(data);
-      expect(vrt["processTestRun"]).toHaveBeenCalledWith(testRunResponse);
       expect(mockedTestRunResult).toHaveBeenCalledWith(
-        testRunResponse,
+        responce,
         "http://localhost:4200"
       );
     });
@@ -415,7 +433,7 @@ describe("VisualRegressionTracker", () => {
 
   describe("submitTestRunBase64", () => {
     it("should submit test run", async () => {
-      const testRunResponse: TestRunResponse = {
+      const testRunUnresolvedResponse: TestRunResponse = {
         url: "url",
         status: TestStatus.unresolved,
         pixelMisMatchCount: 12,
@@ -429,11 +447,13 @@ describe("VisualRegressionTracker", () => {
       const projectId = "asd";
       vrt["buildId"] = buildId;
       vrt["projectId"] = projectId;
-      mockedAxios.post.mockResolvedValueOnce({ data: testRunResponse });
+      mockedAxios.post.mockResolvedValueOnce({
+        data: testRunUnresolvedResponse,
+      });
 
       const result = await vrt["submitTestRunBase64"](testRunBase64);
 
-      expect(result).toBe(testRunResponse);
+      expect(result).toBe(testRunUnresolvedResponse);
       expect(mockedAxios.post).toHaveBeenCalledWith(
         `${config.apiUrl}/test-runs`,
         {
@@ -478,11 +498,13 @@ describe("VisualRegressionTracker", () => {
   describe("submitTestRunMultipart", () => {
     it("should submit test run", async () => {
       const data = new FormData();
-      mockedAxios.post.mockResolvedValueOnce({ data: testRunResponse });
+      mockedAxios.post.mockResolvedValueOnce({
+        data: testRunUnresolvedResponse,
+      });
 
       const result = await vrt["submitTestRunMultipart"](data);
 
-      expect(result).toBe(testRunResponse);
+      expect(result).toBe(testRunUnresolvedResponse);
       expect(mockedAxios.post).toHaveBeenCalledWith(
         `${config.apiUrl}/test-runs/multipart`,
         data,
@@ -546,32 +568,4 @@ describe("VisualRegressionTracker", () => {
       );
     });
   });
-
-  describe.each<[TestStatus.new | TestStatus.unresolved, string]>([
-    [TestStatus.new, "No baseline: "],
-    [TestStatus.unresolved, "Difference found: "],
-  ])("processTestRun", (status, expectedMessage) => {
-    beforeEach(() => {
-      testRunResponse.status = status;
-    });
-
-    it(`disabled soft assert should throw exception if status ${status}`, () => {
-      vrt["config"].enableSoftAssert = false;
-
-      expect(() => vrt["processTestRun"](testRunResponse)).toThrowError(
-        new Error(expectedMessage.concat(testRunResponse.url))
-      );
-    });
-
-    it(`enabled soft assert should log error if status ${status}`, () => {
-      console.error = jest.fn();
-      vrt["config"].enableSoftAssert = true;
-
-      vrt["processTestRun"](testRunResponse);
-
-      expect(console.error).toHaveBeenCalledWith(
-        expectedMessage.concat(testRunResponse.url)
-      );
-    });
-  });
 });