Merge pull request #337 from abi/qa-testing

Add e2e tests

frontend/.gitignore

@@ -25,3 +25,6 @@ dist-ssr
 
 # Env files
 .env*
+
+# Test files
+src/tests/results/

frontend/jest.config.js

@@ -0,0 +1,9 @@
+export default {
+  preset: "ts-jest",
+  testEnvironment: "node",
+  setupFiles: ["<rootDir>/src/setupTests.ts"],
+  transform: {
+    "^.+\\.tsx?$": "ts-jest",
+  },
+  testTimeout: 30000,
+};

frontend/package.json

@@ -10,7 +10,7 @@
     "build-hosted": "tsc && vite build --mode prod",
     "lint": "eslint . --ext ts,tsx --report-unused-disable-directives --max-warnings 0",
     "preview": "vite preview",
-    "test": "vitest"
+    "test": "jest"
   },
   "dependencies": {
     "@codemirror/lang-html": "^6.4.6",
@@ -49,18 +49,24 @@
     "webm-duration-fix": "^1.0.4"
   },
   "devDependencies": {
+    "@types/jest": "^29.5.12",
     "@types/node": "^20.9.0",
+    "@types/puppeteer": "^7.0.4",
     "@types/react": "^18.2.15",
     "@types/react-dom": "^18.2.7",
     "@typescript-eslint/eslint-plugin": "^6.0.0",
     "@typescript-eslint/parser": "^6.0.0",
     "@vitejs/plugin-react": "^4.0.3",
     "autoprefixer": "^10.4.16",
+    "dotenv": "^16.4.5",
     "eslint": "^8.45.0",
     "eslint-plugin-react-hooks": "^4.6.0",
     "eslint-plugin-react-refresh": "^0.4.3",
+    "jest": "^29.7.0",
     "postcss": "^8.4.31",
+    "puppeteer": "^22.6.4",
     "tailwindcss": "^3.3.5",
+    "ts-jest": "^29.1.2",
     "typescript": "^5.0.2",
     "vite": "^4.4.5",
     "vite-plugin-html": "^3.2.0",

frontend/src/.env.jest.example

@@ -0,0 +1,2 @@
+TEST_SCREENSHOTONE_API_KEY=
+TEST_ROOT_PATH=

frontend/src/App.tsx

@@ -482,7 +482,7 @@ function App() {
                     </div>
                     <Button
                       onClick={doUpdate}
-                      className="dark:text-white dark:bg-gray-700"
+                      className="dark:text-white dark:bg-gray-700 update-btn"
                     >
                       Update
                     </Button>
@@ -490,7 +490,7 @@ function App() {
                   <div className="flex items-center justify-end gap-x-2 mt-2">
                     <Button
                       onClick={regenerate}
-                      className="flex items-center gap-x-2 dark:text-white dark:bg-gray-700"
+                      className="flex items-center gap-x-2 dark:text-white dark:bg-gray-700 regenerate-btn"
                     >
                       🔄 Regenerate
                     </Button>
@@ -599,7 +599,7 @@ function App() {
                       <Button
                         onClick={downloadCode}
                         variant="secondary"
-                        className="flex items-center gap-x-2 mr-4 dark:text-white dark:bg-gray-700"
+                        className="flex items-center gap-x-2 mr-4 dark:text-white dark:bg-gray-700 download-btn"
                       >
                         <FaDownload /> Download
                       </Button>

frontend/src/components/ImageUpload.tsx

@@ -166,7 +166,7 @@ function ImageUpload({ setReferenceImages }: Props) {
       {screenRecorderState === ScreenRecorderState.INITIAL && (
         /* eslint-disable-next-line @typescript-eslint/no-explicit-any */
         <div {...getRootProps({ style: style as any })}>
-          <input {...getInputProps()} />
+          <input {...getInputProps()} className="file-input" />
           <p className="text-slate-700 text-lg">
             Drag & drop a screenshot here, <br />
             or click to upload

frontend/src/components/ImportCodeSection.tsx

@@ -38,7 +38,9 @@ function ImportCodeSection({ importFromCode }: Props) {
   return (
     <Dialog>
       <DialogTrigger asChild>
-        <Button variant="secondary">Import from Code</Button>
+        <Button className="import-from-code-btn" variant="secondary">
+          Import from Code
+        </Button>
       </DialogTrigger>
       <DialogContent className="sm:max-w-[425px]">
         <DialogHeader>
@@ -62,7 +64,7 @@ function ImportCodeSection({ importFromCode }: Props) {
         />
 
         <DialogFooter>
-          <Button type="submit" onClick={doImport}>
+          <Button className="import-btn" type="submit" onClick={doImport}>
             Import
           </Button>
         </DialogFooter>

frontend/src/components/UrlInputSection.tsx

@@ -69,7 +69,7 @@ export function UrlInputSection({ doCreate, screenshotOneApiKey }: Props) {
       <Button
         onClick={takeScreenshot}
         disabled={isLoading}
-        className="bg-slate-400"
+        className="bg-slate-400 capture-btn"
       >
         {isLoading ? "Capturing..." : "Capture"}
       </Button>

frontend/src/components/history/utils.test.ts

@@ -1,4 +1,3 @@
-import { expect, test } from "vitest";
 import { extractHistoryTree, renderHistory } from "./utils";
 import type { History } from "./history_types";
 
@@ -84,147 +83,149 @@ const basicBadHistory: History = [
   },
 ];
 
-test("should correctly extract the history tree", () => {
+describe("History Utils", () => {
-  expect(extractHistoryTree(basicLinearHistory, 2)).toEqual([
+  test("should correctly extract the history tree", () => {
-    "<html>1. create</html>",
+    expect(extractHistoryTree(basicLinearHistory, 2)).toEqual([
-    "use better icons",
+      "<html>1. create</html>",
-    "<html>2. edit with better icons</html>",
+      "use better icons",
-    "make text red",
+      "<html>2. edit with better icons</html>",
-    "<html>3. edit with better icons and red text</html>",
+      "make text red",
-  ]);
+      "<html>3. edit with better icons and red text</html>",
+    ]);
 
-  expect(extractHistoryTree(basicLinearHistory, 0)).toEqual([
+    expect(extractHistoryTree(basicLinearHistory, 0)).toEqual([
-    "<html>1. create</html>",
+      "<html>1. create</html>",
-  ]);
+    ]);
 
-  // Test branching
+    // Test branching
-  expect(extractHistoryTree(basicBranchingHistory, 3)).toEqual([
+    expect(extractHistoryTree(basicBranchingHistory, 3)).toEqual([
-    "<html>1. create</html>",
+      "<html>1. create</html>",
-    "use better icons",
+      "use better icons",
-    "<html>2. edit with better icons</html>",
+      "<html>2. edit with better icons</html>",
-    "make text green",
+      "make text green",
-    "<html>4. edit with better icons and green text</html>",
+      "<html>4. edit with better icons and green text</html>",
-  ]);
+    ]);
 
-  expect(extractHistoryTree(longerBranchingHistory, 4)).toEqual([
+    expect(extractHistoryTree(longerBranchingHistory, 4)).toEqual([
-    "<html>1. create</html>",
+      "<html>1. create</html>",
-    "use better icons",
+      "use better icons",
-    "<html>2. edit with better icons</html>",
+      "<html>2. edit with better icons</html>",
-    "make text green",
+      "make text green",
-    "<html>4. edit with better icons and green text</html>",
+      "<html>4. edit with better icons and green text</html>",
-    "make text bold",
+      "make text bold",
-    "<html>5. edit with better icons and green, bold text</html>",
+      "<html>5. edit with better icons and green, bold text</html>",
-  ]);
+    ]);
 
-  expect(extractHistoryTree(longerBranchingHistory, 2)).toEqual([
+    expect(extractHistoryTree(longerBranchingHistory, 2)).toEqual([
-    "<html>1. create</html>",
+      "<html>1. create</html>",
-    "use better icons",
+      "use better icons",
-    "<html>2. edit with better icons</html>",
+      "<html>2. edit with better icons</html>",
-    "make text red",
+      "make text red",
-    "<html>3. edit with better icons and red text</html>",
+      "<html>3. edit with better icons and red text</html>",
-  ]);
+    ]);
 
-  // Errors
+    // Errors
 
-  // Bad index
+    // Bad index
-  expect(() => extractHistoryTree(basicLinearHistory, 100)).toThrow();
+    expect(() => extractHistoryTree(basicLinearHistory, 100)).toThrow();
-  expect(() => extractHistoryTree(basicLinearHistory, -2)).toThrow();
+    expect(() => extractHistoryTree(basicLinearHistory, -2)).toThrow();
 
-  // Bad tree
+    // Bad tree
-  expect(() => extractHistoryTree(basicBadHistory, 1)).toThrow();
+    expect(() => extractHistoryTree(basicBadHistory, 1)).toThrow();
-});
+  });
-
+
-test("should correctly render the history tree", () => {
+  test("should correctly render the history tree", () => {
-  expect(renderHistory(basicLinearHistory, 2)).toEqual([
+    expect(renderHistory(basicLinearHistory, 2)).toEqual([
-    {
+      {
-      isActive: false,
+        isActive: false,
-      parentVersion: null,
+        parentVersion: null,
-      summary: "Create",
+        summary: "Create",
-      type: "Create",
+        type: "Create",
-    },
+      },
-    {
+      {
-      isActive: false,
+        isActive: false,
-      parentVersion: null,
+        parentVersion: null,
-      summary: "use better icons",
+        summary: "use better icons",
-      type: "Edit",
+        type: "Edit",
-    },
+      },
-    {
+      {
-      isActive: true,
+        isActive: true,
-      parentVersion: null,
+        parentVersion: null,
-      summary: "make text red",
+        summary: "make text red",
-      type: "Edit",
+        type: "Edit",
-    },
+      },
-  ]);
+    ]);
-
+
-  // Current version is the first version
+    // Current version is the first version
-  expect(renderHistory(basicLinearHistory, 0)).toEqual([
+    expect(renderHistory(basicLinearHistory, 0)).toEqual([
-    {
+      {
-      isActive: true,
+        isActive: true,
-      parentVersion: null,
+        parentVersion: null,
-      summary: "Create",
+        summary: "Create",
-      type: "Create",
+        type: "Create",
-    },
+      },
-    {
+      {
-      isActive: false,
+        isActive: false,
-      parentVersion: null,
+        parentVersion: null,
-      summary: "use better icons",
+        summary: "use better icons",
-      type: "Edit",
+        type: "Edit",
-    },
+      },
-    {
+      {
-      isActive: false,
+        isActive: false,
-      parentVersion: null,
+        parentVersion: null,
-      summary: "make text red",
+        summary: "make text red",
-      type: "Edit",
+        type: "Edit",
-    },
+      },
-  ]);
+    ]);
-
+
-  // Render a history with code
+    // Render a history with code
-  expect(renderHistory(basicLinearHistoryWithCode, 0)).toEqual([
+    expect(renderHistory(basicLinearHistoryWithCode, 0)).toEqual([
-    {
+      {
-      isActive: true,
+        isActive: true,
-      parentVersion: null,
+        parentVersion: null,
-      summary: "Imported from code",
+        summary: "Imported from code",
-      type: "Imported from code",
+        type: "Imported from code",
-    },
+      },
-    {
+      {
-      isActive: false,
+        isActive: false,
-      parentVersion: null,
+        parentVersion: null,
-      summary: "use better icons",
+        summary: "use better icons",
-      type: "Edit",
+        type: "Edit",
-    },
+      },
-    {
+      {
-      isActive: false,
+        isActive: false,
-      parentVersion: null,
+        parentVersion: null,
-      summary: "make text red",
+        summary: "make text red",
-      type: "Edit",
+        type: "Edit",
-    },
+      },
-  ]);
+    ]);
-
+
-  // Render a non-linear history
+    // Render a non-linear history
-  expect(renderHistory(basicBranchingHistory, 3)).toEqual([
+    expect(renderHistory(basicBranchingHistory, 3)).toEqual([
-    {
+      {
-      isActive: false,
+        isActive: false,
-      parentVersion: null,
+        parentVersion: null,
-      summary: "Create",
+        summary: "Create",
-      type: "Create",
+        type: "Create",
-    },
+      },
-    {
+      {
-      isActive: false,
+        isActive: false,
-      parentVersion: null,
+        parentVersion: null,
-      summary: "use better icons",
+        summary: "use better icons",
-      type: "Edit",
+        type: "Edit",
-    },
+      },
-    {
+      {
-      isActive: false,
+        isActive: false,
-      parentVersion: null,
+        parentVersion: null,
-      summary: "make text red",
+        summary: "make text red",
-      type: "Edit",
+        type: "Edit",
-    },
+      },
-    {
+      {
-      isActive: true,
+        isActive: true,
-      parentVersion: "v2",
+        parentVersion: "v2",
-      summary: "make text green",
+        summary: "make text green",
-      type: "Edit",
+        type: "Edit",
-    },
+      },
-  ]);
+    ]);
+  });
 });

frontend/src/setupTests.ts

@@ -0,0 +1,3 @@
+// So jest test runner can read env vars from .env file
+import { config } from "dotenv";
+config({ path: ".env.jest" });

frontend/src/tests/qa.test.ts

@@ -0,0 +1,274 @@
+import puppeteer, { Browser, Page, ElementHandle } from "puppeteer";
+import { Stack } from "../lib/stacks";
+import { CodeGenerationModel } from "../lib/models";
+
+const TESTS_ROOT_PATH = process.env.TEST_ROOT_PATH;
+
+// Fixtures
+const FIXTURES_PATH = `${TESTS_ROOT_PATH}/fixtures`;
+const SIMPLE_SCREENSHOT = FIXTURES_PATH + "/simple_button.png";
+const SCREENSHOT_WITH_IMAGES = `${FIXTURES_PATH}/simple_ui_with_image.png`;
+
+// Results
+const RESULTS_DIR = `${TESTS_ROOT_PATH}/results`;
+
+describe("e2e tests", () => {
+  let browser: Browser;
+  let page: Page;
+
+  const DEBUG = false;
+  const IS_HEADLESS = true;
+
+  const stacks = Object.values(Stack).slice(0, DEBUG ? 1 : undefined);
+  const models = Object.values(CodeGenerationModel).slice(
+    0,
+    DEBUG ? 1 : undefined
+  );
+
+  beforeAll(async () => {
+    browser = await puppeteer.launch({ headless: IS_HEADLESS });
+    page = await browser.newPage();
+    await page.goto("http://localhost:5173/");
+
+    // Set screen size
+    await page.setViewport({ width: 1080, height: 1024 });
+
+    // TODO: Does this need to be moved?
+    // const client = await page.createCDPSession();
+    // Set download behavior path
+    // await client.send("Page.setDownloadBehavior", {
+    //   behavior: "allow",
+    //   downloadPath: DOWNLOAD_PATH,
+    // });
+  });
+
+  afterAll(async () => {
+    await browser.close();
+  });
+
+  // Create tests
+  models.forEach((model) => {
+    stacks.forEach((stack) => {
+      it(
+        `Create for : ${model} & ${stack}`,
+        async () => {
+          const app = new App(
+            page,
+            stack,
+            model,
+            `create_screenshot_${model}_${stack}`
+          );
+          await app.init();
+          // Generate from screenshot
+          await app.uploadImage(SCREENSHOT_WITH_IMAGES);
+        },
+        60 * 1000
+      );
+
+      it(
+        `Create from URL for : ${model} & ${stack}`,
+        async () => {
+          const app = new App(
+            page,
+            stack,
+            model,
+            `create_url_${model}_${stack}`
+          );
+          await app.init();
+          // Generate from screenshot
+          await app.generateFromUrl("https://a.picoapps.xyz/design-fear");
+        },
+        60 * 1000
+      );
+    });
+  });
+
+  // Update tests - for every model (doesn’t need to be repeated for each stack - fix to HTML Tailwind only)
+  models.forEach((model) => {
+    ["html_tailwind"].forEach((stack) => {
+      it(
+        `update: ${model}`,
+        async () => {
+          const app = new App(page, stack, model, `update_${model}_${stack}`);
+          await app.init();
+
+          // Generate from screenshot
+          await app.uploadImage(SIMPLE_SCREENSHOT);
+          // Regenerate works for v1
+          await app.regenerate();
+          // Make an update
+          await app.edit("make the button background blue", "v2");
+          // Make another update
+          await app.edit("make the text italic", "v3");
+          // Branch off v2 and make an update
+          await app.clickVersion("v2");
+          await app.edit("make the text yellow", "v4");
+        },
+        90 * 1000
+      );
+    });
+  });
+
+  // Start from code tests - for every model
+  models.forEach((model) => {
+    ["html_tailwind"].forEach((stack) => {
+      it.skip(
+        `Start from code: ${model}`,
+        async () => {
+          const app = new App(
+            page,
+            stack,
+            model,
+            `start_from_code_${model}_${stack}`
+          );
+          await app.init();
+
+          await app.importFromCode();
+
+          // Regenerate works for v1
+          // await app.regenerate();
+          // // Make an update
+          // await app.edit("make the header blue", "v2");
+          // // Make another update
+          // await app.edit("make all text italic", "v3");
+          // // Branch off v2 and make an update
+          // await app.clickVersion("v2");
+          // await app.edit("make all text red", "v4");
+        },
+        90 * 1000
+      );
+    });
+  });
+});
+
+class App {
+  private screenshotPathPrefix: string;
+  private page: Page;
+  private stack: string;
+  private model: string;
+
+  constructor(page: Page, stack: string, model: string, testId: string) {
+    this.page = page;
+    this.stack = stack;
+    this.model = model;
+    this.screenshotPathPrefix = `${RESULTS_DIR}/${testId}`;
+  }
+
+  async init() {
+    await this.setupLocalStorage();
+  }
+
+  async setupLocalStorage() {
+    const setting = {
+      openAiApiKey: null,
+      openAiBaseURL: null,
+      screenshotOneApiKey: process.env.TEST_SCREENSHOTONE_API_KEY,
+      isImageGenerationEnabled: true,
+      editorTheme: "cobalt",
+      generatedCodeConfig: this.stack,
+      codeGenerationModel: this.model,
+      isTermOfServiceAccepted: false,
+      accessCode: null,
+    };
+
+    await this.page.evaluate((setting) => {
+      localStorage.setItem("setting", JSON.stringify(setting));
+    }, setting);
+
+    // Reload the page to apply the local storage
+    await this.page.reload();
+  }
+
+  async _screenshot(step: string) {
+    await this.page.screenshot({
+      path: `${this.screenshotPathPrefix}_${step}.png`,
+    });
+  }
+
+  async _waitUntilVersionIsReady(version: string) {
+    await this.page.waitForNetworkIdle();
+    await this.page.waitForFunction(
+      (version) => document.body.innerText.includes(version),
+      {
+        timeout: 30000,
+      },
+      version
+    );
+    // Wait for 3s so that the HTML and JS has time to render before screenshotting
+    await new Promise((resolve) => setTimeout(resolve, 3000));
+  }
+
+  async generateFromUrl(url: string) {
+    // Type in the URL
+    await this.page.type('input[placeholder="Enter URL"]', url);
+    await this._screenshot("typed_url");
+
+    // Click the capture button and wait for the code to be generated
+    await this.page.click("button.capture-btn");
+    await this._waitUntilVersionIsReady("v1");
+    await this._screenshot("url_result");
+  }
+
+  // Uploads a screenshot and generates the image
+  async uploadImage(screenshotPath: string) {
+    // Upload file
+    const fileInput = (await this.page.$(
+      ".file-input"
+    )) as ElementHandle<HTMLInputElement>;
+    if (!fileInput) {
+      throw new Error("File input element not found");
+    }
+    await fileInput.uploadFile(screenshotPath);
+    await this._screenshot("image_uploaded");
+
+    // Click the generate button and wait for the code to be generated
+    await this._waitUntilVersionIsReady("v1");
+    await this._screenshot("image_results");
+  }
+
+  // Makes a text edit and waits for a new version
+  async edit(edit: string, version: string) {
+    // Type in the edit
+    await this.page.type(
+      'textarea[placeholder="Tell the AI what to change..."]',
+      edit
+    );
+    await this._screenshot(`typed_${version}`);
+
+    // Click the update button and wait for the code to be generated
+    await this.page.click(".update-btn");
+    await this._waitUntilVersionIsReady(version);
+    await this._screenshot(`done_${version}`);
+  }
+
+  async clickVersion(version: string) {
+    await this.page.evaluate((version) => {
+      document.querySelectorAll("div").forEach((div) => {
+        if (div.innerText.includes(version)) {
+          div.click();
+        }
+      });
+    }, version);
+  }
+
+  async regenerate() {
+    await this.page.click(".regenerate-btn");
+    await this._waitUntilVersionIsReady("v1");
+    await this._screenshot("regenerate_results");
+  }
+
+  // Work in progress
+  async importFromCode() {
+    await this.page.click(".import-from-code-btn");
+
+    await this.page.type("textarea", "<html>hello world</html>");
+
+    await this.page.select("#output-settings-js", "HTML + Tailwind");
+
+    await this._screenshot("typed_code");
+
+    await this.page.click(".import-btn");
+
+    await this._waitUntilVersionIsReady("v1");
+  }
+}