Merge branch 'main' into hosted

backend/.gitignore

@@ -150,3 +150,7 @@ cython_debug/
 #  and can be added to the global gitignore or merged into this file.  For a more nuclear
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
 #.idea/
+
+
+# Temporary eval output
+evals

backend/eval.py

@@ -0,0 +1,67 @@
+# Load environment variables first
+from typing import Any, Coroutine
+from dotenv import load_dotenv
+
+from eval_config import EVALS_DIR
+from eval_utils import image_to_data_url
+
+load_dotenv()
+
+import os
+from llm import stream_openai_response
+from prompts import assemble_prompt
+import asyncio
+
+from utils import pprint_prompt
+
+
+async def generate_code_core(image_url: str, stack: str) -> str:
+    prompt_messages = assemble_prompt(image_url, stack)
+    openai_api_key = os.environ.get("OPENAI_API_KEY")
+    openai_base_url = None
+
+    pprint_prompt(prompt_messages)
+
+    async def process_chunk(content: str):
+        pass
+
+    if not openai_api_key:
+        raise Exception("OpenAI API key not found")
+
+    completion = await stream_openai_response(
+        prompt_messages,
+        api_key=openai_api_key,
+        base_url=openai_base_url,
+        callback=lambda x: process_chunk(x),
+    )
+
+    return completion
+
+
+async def main():
+    INPUT_DIR = EVALS_DIR + "/inputs"
+    OUTPUT_DIR = EVALS_DIR + "/outputs"
+
+    # Get all the files in the directory (only grab pngs)
+    evals = [f for f in os.listdir(INPUT_DIR) if f.endswith(".png")]
+
+    tasks: list[Coroutine[Any, Any, str]] = []
+    for filename in evals:
+        filepath = os.path.join(INPUT_DIR, filename)
+        data_url = await image_to_data_url(filepath)
+        task = generate_code_core(data_url, "html_tailwind")
+        tasks.append(task)
+
+    results = await asyncio.gather(*tasks)
+
+    os.makedirs(OUTPUT_DIR, exist_ok=True)
+
+    for filename, content in zip(evals, results):
+        # File name is derived from the original filename in evals
+        output_filename = f"{os.path.splitext(filename)[0]}.html"
+        output_filepath = os.path.join(OUTPUT_DIR, output_filename)
+        with open(output_filepath, "w") as file:
+            file.write(content)
+
+
+asyncio.run(main())

backend/eval_config.py

@@ -0,0 +1 @@
+EVALS_DIR = "./evals"

backend/eval_utils.py

@@ -0,0 +1,7 @@
+import base64
+
+
+async def image_to_data_url(filepath: str):
+    with open(filepath, "rb") as image_file:
+        encoded_string = base64.b64encode(image_file.read()).decode()
+    return f"data:image/png;base64,{encoded_string}"

backend/main.py

@@ -7,7 +7,7 @@ load_dotenv()
 import os
 from fastapi import FastAPI
 from fastapi.middleware.cors import CORSMiddleware
-from routes import screenshot, generate_code, home
+from routes import screenshot, generate_code, home, evals
 from config import IS_PROD
 
 # Setup Sentry (only relevant in prod)
@@ -40,3 +40,4 @@ app.add_middleware(
 app.include_router(generate_code.router)
 app.include_router(screenshot.router)
 app.include_router(home.router)
+app.include_router(evals.router)

backend/routes/evals.py

@@ -0,0 +1,46 @@
+import os
+from fastapi import APIRouter
+from pydantic import BaseModel
+from eval_utils import image_to_data_url
+from eval_config import EVALS_DIR
+
+
+router = APIRouter()
+
+
+class Eval(BaseModel):
+    input: str
+    output: str
+
+
+@router.get("/evals")
+async def get_evals():
+    # Get all evals from EVALS_DIR
+    input_dir = EVALS_DIR + "/inputs"
+    output_dir = EVALS_DIR + "/outputs"
+
+    evals: list[Eval] = []
+    for file in os.listdir(input_dir):
+        if file.endswith(".png"):
+            input_file_path = os.path.join(input_dir, file)
+            input_file = await image_to_data_url(input_file_path)
+
+            # Construct the corresponding output file name
+            output_file_name = file.replace(".png", ".html")
+            output_file_path = os.path.join(output_dir, output_file_name)
+
+            # Check if the output file exists
+            if os.path.exists(output_file_path):
+                with open(output_file_path, "r") as f:
+                    output_file_data = f.read()
+            else:
+                output_file_data = "Output file not found."
+
+            evals.append(
+                Eval(
+                    input=input_file,
+                    output=output_file_data,
+                )
+            )
+
+    return evals

frontend/package.json

@@ -18,6 +18,7 @@
     "@radix-ui/react-accordion": "^1.1.2",
     "@radix-ui/react-alert-dialog": "^1.0.5",
     "@radix-ui/react-checkbox": "^1.0.4",
+    "@radix-ui/react-collapsible": "^1.0.3",
     "@radix-ui/react-dialog": "^1.0.5",
     "@radix-ui/react-hover-card": "^1.0.7",
     "@radix-ui/react-icons": "^1.3.0",
@@ -41,6 +42,7 @@
     "react-dropzone": "^14.2.3",
     "react-hot-toast": "^2.4.1",
     "react-icons": "^4.12.0",
+    "react-router-dom": "^6.20.1",
     "tailwind-merge": "^2.0.0",
     "tailwindcss-animate": "^1.0.7",
     "thememirror": "^2.0.1",

frontend/src/components/evals/EvalsPage.tsx

@@ -0,0 +1,70 @@
+import React, { useEffect } from "react";
+import { HTTP_BACKEND_URL } from "../../config";
+import RatingPicker from "./RatingPicker";
+
+interface Eval {
+  input: string;
+  output: string;
+}
+
+function EvalsPage() {
+  const [evals, setEvals] = React.useState<Eval[]>([]);
+  const [ratings, setRatings] = React.useState<number[]>([]);
+
+  const total = ratings.reduce((a, b) => a + b, 0);
+  const max = ratings.length * 4;
+  const score = ((total / max) * 100 || 0).toFixed(2);
+
+  useEffect(() => {
+    if (evals.length > 0) return;
+
+    fetch(`${HTTP_BACKEND_URL}/evals`)
+      .then((res) => res.json())
+      .then((data) => {
+        setEvals(data);
+        setRatings(new Array(data.length).fill(0));
+      });
+  }, [evals]);
+
+  return (
+    <div className="mx-auto">
+      {/* Display total */}
+      <div className="flex items-center justify-center w-full h-12 bg-zinc-950">
+        <span className="text-2xl font-semibold text-white">
+          Total: {total} out of {max} ({score}%)
+        </span>
+      </div>
+
+      <div className="flex flex-col gap-y-4 mt-4 mx-auto justify-center">
+        {evals.map((e, index) => (
+          <div className="flex flex-col justify-center" key={index}>
+            <div className="flex gap-x-2 justify-center">
+              <div className="w-1/2 p-1 border">
+                <img src={e.input} />
+              </div>
+              <div className="w-1/2 p-1 border">
+                {/* Put output into an iframe */}
+                <iframe
+                  srcDoc={e.output}
+                  className="w-[1200px] h-[800px] transform scale-[0.60]"
+                  style={{ transformOrigin: "top left" }}
+                ></iframe>
+              </div>
+            </div>
+            <div className="ml-8 mt-4 flex justify-center">
+              <RatingPicker
+                onSelect={(rating) => {
+                  const newRatings = [...ratings];
+                  newRatings[index] = rating;
+                  setRatings(newRatings);
+                }}
+              />
+            </div>
+          </div>
+        ))}
+      </div>
+    </div>
+  );
+}
+
+export default EvalsPage;

frontend/src/components/evals/RatingPicker.tsx

@@ -0,0 +1,38 @@
+import React from "react";
+
+interface Props {
+  onSelect: (rating: number) => void;
+}
+
+function RatingPicker({ onSelect }: Props) {
+  const [selected, setSelected] = React.useState<number | null>(null);
+
+  const renderCircle = (number: number) => {
+    const isSelected = selected === number;
+    const bgColor = isSelected ? "bg-black" : "bg-gray-300";
+    const textColor = isSelected ? "text-white" : "text-black";
+
+    return (
+      <div
+        className={`flex items-center justify-center w-8 h-8 ${bgColor} rounded-full cursor-pointer`}
+        onClick={() => {
+          setSelected(number);
+          onSelect(number);
+        }}
+      >
+        <span className={`text-lg font-semibold ${textColor}`}>{number}</span>
+      </div>
+    );
+  };
+
+  return (
+    <div className="flex space-x-4">
+      {renderCircle(1)}
+      {renderCircle(2)}
+      {renderCircle(3)}
+      {renderCircle(4)}
+    </div>
+  );
+}
+
+export default RatingPicker;

frontend/src/components/history/HistoryDisplay.tsx

@@ -1,12 +1,16 @@
-import { History, HistoryItemType } from "./history_types";
+import { History } from "./history_types";
 import toast from "react-hot-toast";
 import classNames from "classnames";
-import {
-  HoverCard,
-  HoverCardTrigger,
-  HoverCardContent,
-} from "../ui/hover-card";
+
 import { Badge } from "../ui/badge";
+import { renderHistory } from "./utils";
+import {
+  Collapsible,
+  CollapsibleContent,
+  CollapsibleTrigger,
+} from "../ui/collapsible";
+import { Button } from "../ui/button";
+import { CaretSortIcon } from "@radix-ui/react-icons";
 
 interface Props {
   history: History;
@@ -15,44 +19,33 @@ interface Props {
   shouldDisableReverts: boolean;
 }
 
-function displayHistoryItemType(itemType: HistoryItemType) {
-  switch (itemType) {
-    case "ai_create":
-      return "Create";
-    case "ai_edit":
-      return "Edit";
-    case "code_create":
-      return "Imported from code";
-    default: {
-      const exhaustiveCheck: never = itemType;
-      throw new Error(`Unhandled case: ${exhaustiveCheck}`);
-    }
-  }
-}
-
 export default function HistoryDisplay({
   history,
   currentVersion,
   revertToVersion,
   shouldDisableReverts,
 }: Props) {
-  return history.length === 0 ? null : (
+  const renderedHistory = renderHistory(history, currentVersion);
+
+  return renderedHistory.length === 0 ? null : (
     <div className="flex flex-col h-screen">
       <h1 className="font-bold mb-2">Versions</h1>
       <ul className="space-y-0 flex flex-col-reverse">
-        {history.map((item, index) => (
+        {renderedHistory.map((item, index) => (
           <li key={index}>
-            <HoverCard>
-              <HoverCardTrigger
+            <Collapsible>
+              <div
                 className={classNames(
-                  "flex items-center justify-between space-x-2 p-2",
+                  "flex items-center justify-between space-x-2 w-full pr-2",
                   "border-b cursor-pointer",
                   {
-                    " hover:bg-black hover:text-white":
-                      index !== currentVersion,
-                    "bg-slate-500 text-white": index === currentVersion,
+                    " hover:bg-black hover:text-white": !item.isActive,
+                    "bg-slate-500 text-white": item.isActive,
                   }
                 )}
+              >
+                <div
+                  className="flex justify-between truncate flex-1 p-2"
                   onClick={() =>
                     shouldDisableReverts
                       ? toast.error(
@@ -61,36 +54,30 @@ export default function HistoryDisplay({
                       : revertToVersion(index)
                   }
                 >
-                {" "}
                   <div className="flex gap-x-1 truncate">
-                  <h2 className="text-sm truncate">
-                    {item.type === "ai_edit"
-                      ? item.inputs.prompt
-                      : item.type === "ai_create"
-                      ? "Create"
-                      : "Imported from code"}
-                  </h2>
-                  {/* <h2 className="text-sm">{displayHistoryItemType(item.type)}</h2> */}
-                  {item.parentIndex !== null &&
-                  item.parentIndex !== index - 1 ? (
+                    <h2 className="text-sm truncate">{item.summary}</h2>
+                    {item.parentVersion !== null && (
                       <h2 className="text-sm">
-                      (parent: v{(item.parentIndex || 0) + 1})
+                        (parent: {item.parentVersion})
                       </h2>
-                  ) : null}
+                    )}
                   </div>
                   <h2 className="text-sm">v{index + 1}</h2>
-              </HoverCardTrigger>
-              <HoverCardContent>
-                <div>
-                  {item.type === "ai_edit"
-                    ? item.inputs.prompt
-                    : item.type === "ai_create"
-                    ? "Create"
-                    : "Imported from code"}
                 </div>
-                <Badge>{displayHistoryItemType(item.type)}</Badge>
-              </HoverCardContent>
-            </HoverCard>
+                <CollapsibleTrigger asChild>
+                  <Button variant="ghost" size="sm" className="h-6">
+                    <CaretSortIcon className="h-4 w-4" />
+                    <span className="sr-only">Toggle</span>
+                  </Button>
+                </CollapsibleTrigger>
+              </div>
+              <CollapsibleContent className="w-full bg-slate-300 p-2">
+                <div>Full prompt: {item.summary}</div>
+                <div className="flex justify-end">
+                  <Badge>{item.type}</Badge>
+                </div>
+              </CollapsibleContent>
+            </Collapsible>
           </li>
         ))}
       </ul>

frontend/src/components/history/history_types.ts

@@ -32,3 +32,10 @@ export type CodeCreateInputs = {
 };
 
 export type History = HistoryItem[];
+
+export type RenderedHistoryItem = {
+  type: string;
+  summary: string;
+  parentVersion: string | null;
+  isActive: boolean;
+};

frontend/src/components/history/utils.test.ts

@@ -1,5 +1,5 @@
 import { expect, test } from "vitest";
-import { extractHistoryTree } from "./utils";
+import { extractHistoryTree, renderHistory } from "./utils";
 import type { History } from "./history_types";
 
 const basicLinearHistory: History = [
@@ -29,6 +29,18 @@ const basicLinearHistory: History = [
   },
 ];
 
+const basicLinearHistoryWithCode: History = [
+  {
+    type: "code_create",
+    parentIndex: null,
+    code: "<html>1. create</html>",
+    inputs: {
+      code: "<html>1. create</html>",
+    },
+  },
+  ...basicLinearHistory.slice(1),
+];
+
 const basicBranchingHistory: History = [
   ...basicLinearHistory,
   {
@@ -121,3 +133,98 @@ test("should correctly extract the history tree", () => {
   // Bad tree
   expect(() => extractHistoryTree(basicBadHistory, 1)).toThrow();
 });
+
+test("should correctly render the history tree", () => {
+  expect(renderHistory(basicLinearHistory, 2)).toEqual([
+    {
+      isActive: false,
+      parentVersion: null,
+      summary: "Create",
+      type: "Create",
+    },
+    {
+      isActive: false,
+      parentVersion: null,
+      summary: "use better icons",
+      type: "Edit",
+    },
+    {
+      isActive: true,
+      parentVersion: null,
+      summary: "make text red",
+      type: "Edit",
+    },
+  ]);
+
+  // Current version is the first version
+  expect(renderHistory(basicLinearHistory, 0)).toEqual([
+    {
+      isActive: true,
+      parentVersion: null,
+      summary: "Create",
+      type: "Create",
+    },
+    {
+      isActive: false,
+      parentVersion: null,
+      summary: "use better icons",
+      type: "Edit",
+    },
+    {
+      isActive: false,
+      parentVersion: null,
+      summary: "make text red",
+      type: "Edit",
+    },
+  ]);
+
+  // Render a history with code
+  expect(renderHistory(basicLinearHistoryWithCode, 0)).toEqual([
+    {
+      isActive: true,
+      parentVersion: null,
+      summary: "Imported from code",
+      type: "Imported from code",
+    },
+    {
+      isActive: false,
+      parentVersion: null,
+      summary: "use better icons",
+      type: "Edit",
+    },
+    {
+      isActive: false,
+      parentVersion: null,
+      summary: "make text red",
+      type: "Edit",
+    },
+  ]);
+
+  // Render a non-linear history
+  expect(renderHistory(basicBranchingHistory, 3)).toEqual([
+    {
+      isActive: false,
+      parentVersion: null,
+      summary: "Create",
+      type: "Create",
+    },
+    {
+      isActive: false,
+      parentVersion: null,
+      summary: "use better icons",
+      type: "Edit",
+    },
+    {
+      isActive: false,
+      parentVersion: null,
+      summary: "make text red",
+      type: "Edit",
+    },
+    {
+      isActive: true,
+      parentVersion: "v2",
+      summary: "make text green",
+      type: "Edit",
+    },
+  ]);
+});

frontend/src/components/history/utils.ts

@@ -1,4 +1,9 @@
-import { History, HistoryItem } from "./history_types";
+import {
+  History,
+  HistoryItem,
+  HistoryItemType,
+  RenderedHistoryItem,
+} from "./history_types";
 
 export function extractHistoryTree(
   history: History,
@@ -30,3 +35,62 @@ export function extractHistoryTree(
 
   return flatHistory;
 }
+
+function displayHistoryItemType(itemType: HistoryItemType) {
+  switch (itemType) {
+    case "ai_create":
+      return "Create";
+    case "ai_edit":
+      return "Edit";
+    case "code_create":
+      return "Imported from code";
+    default: {
+      const exhaustiveCheck: never = itemType;
+      throw new Error(`Unhandled case: ${exhaustiveCheck}`);
+    }
+  }
+}
+
+function summarizeHistoryItem(item: HistoryItem) {
+  const itemType = item.type;
+  switch (itemType) {
+    case "ai_create":
+      return "Create";
+    case "ai_edit":
+      return item.inputs.prompt;
+    case "code_create":
+      return "Imported from code";
+    default: {
+      const exhaustiveCheck: never = itemType;
+      throw new Error(`Unhandled case: ${exhaustiveCheck}`);
+    }
+  }
+}
+
+export const renderHistory = (
+  history: History,
+  currentVersion: number | null
+) => {
+  const renderedHistory: RenderedHistoryItem[] = [];
+
+  for (let i = 0; i < history.length; i++) {
+    const item = history[i];
+    // Only show the parent version if it's not the previous version
+    // (i.e. it's the branching point) and if it's not the first version
+    const parentVersion =
+      item.parentIndex !== null && item.parentIndex !== i - 1
+        ? `v${(item.parentIndex || 0) + 1}`
+        : null;
+    const type = displayHistoryItemType(item.type);
+    const isActive = i === currentVersion;
+    const summary = summarizeHistoryItem(item);
+    renderedHistory.push({
+      isActive,
+      summary: summary,
+      parentVersion,
+      type,
+    });
+  }
+
+  return renderedHistory;
+};

frontend/src/components/ui/collapsible.tsx

@@ -0,0 +1,9 @@
+import * as CollapsiblePrimitive from "@radix-ui/react-collapsible"
+
+const Collapsible = CollapsiblePrimitive.Root
+
+const CollapsibleTrigger = CollapsiblePrimitive.CollapsibleTrigger
+
+const CollapsibleContent = CollapsiblePrimitive.CollapsibleContent
+
+export { Collapsible, CollapsibleTrigger, CollapsibleContent }

frontend/src/main.tsx

@@ -3,13 +3,20 @@ import ReactDOM from "react-dom/client";
 import { Toaster } from "react-hot-toast";
 import AppContainer from "./components/hosted/AppContainer.tsx";
 import { ClerkProvider } from "@clerk/clerk-react";
+import EvalsPage from "./components/evals/EvalsPage.tsx";
+import { BrowserRouter as Router, Routes, Route } from "react-router-dom";
 import { CLERK_PUBLISHABLE_KEY } from "./config.ts";
 import "./index.css";
 
 ReactDOM.createRoot(document.getElementById("root")!).render(
   <React.StrictMode>
     <ClerkProvider publishableKey={CLERK_PUBLISHABLE_KEY}>
-      <AppContainer />
+      <Router>
+        <Routes>
+          <Route path="/" element={<AppContainer />} />
+          <Route path="/evals" element={<EvalsPage />} />
+        </Routes>
+      </Router>
       <Toaster
         toastOptions={{ className: "dark:bg-zinc-950 dark:text-white" }}
       />

frontend/yarn.lock

@@ -829,7 +829,7 @@
     "@radix-ui/react-use-previous" "1.0.1"
     "@radix-ui/react-use-size" "1.0.1"
 
-"@radix-ui/react-collapsible@1.0.3":
+"@radix-ui/react-collapsible@1.0.3", "@radix-ui/react-collapsible@^1.0.3":
   version "1.0.3"
   resolved "https://registry.yarnpkg.com/@radix-ui/react-collapsible/-/react-collapsible-1.0.3.tgz#df0e22e7a025439f13f62d4e4a9e92c4a0df5b81"
   integrity sha512-UBmVDkmR6IvDsloHVN+3rtx4Mi5TFvylYXpluuv0f37dtaz3H99bp8No0LGXRigVpl3UAT4l9j6bIchh42S/Gg==
@@ -1209,6 +1209,11 @@
   dependencies:
     "@babel/runtime" "^7.13.10"
 
+"@remix-run/router@1.13.1":
+  version "1.13.1"
+  resolved "https://registry.yarnpkg.com/@remix-run/router/-/router-1.13.1.tgz#07e2a8006f23a3bc898b3f317e0a58cc8076b86e"
+  integrity sha512-so+DHzZKsoOcoXrILB4rqDkMDy7NLMErRdOxvzvOKb507YINKUP4Di+shbTZDhSE/pBZ+vr7XGIpcOO0VLSA+Q==
+
 "@rollup/pluginutils@^4.2.0":
   version "4.2.1"
   resolved "https://registry.yarnpkg.com/@rollup/pluginutils/-/pluginutils-4.2.1.tgz#e6c6c3aba0744edce3fb2074922d3776c0af2a6d"
@@ -3184,6 +3189,21 @@ react-remove-scroll@2.5.5:
     use-callback-ref "^1.3.0"
     use-sidecar "^1.1.2"
 
+react-router-dom@^6.20.1:
+  version "6.20.1"
+  resolved "https://registry.yarnpkg.com/react-router-dom/-/react-router-dom-6.20.1.tgz#e34f8075b9304221420de3609e072bb349824984"
+  integrity sha512-npzfPWcxfQN35psS7rJgi/EW0Gx6EsNjfdJSAk73U/HqMEJZ2k/8puxfwHFgDQhBGmS3+sjnGbMdMSV45axPQw==
+  dependencies:
+    "@remix-run/router" "1.13.1"
+    react-router "6.20.1"
+
+react-router@6.20.1:
+  version "6.20.1"
+  resolved "https://registry.yarnpkg.com/react-router/-/react-router-6.20.1.tgz#e8cc326031d235aaeec405bb234af77cf0fe75ef"
+  integrity sha512-ccvLrB4QeT5DlaxSFFYi/KR8UMQ4fcD8zBcR71Zp1kaYTC5oJKYAp1cbavzGrogwxca+ubjkd7XjFZKBW8CxPA==
+  dependencies:
+    "@remix-run/router" "1.13.1"
+
 react-style-singleton@^2.2.1:
   version "2.2.1"
   resolved "https://registry.yarnpkg.com/react-style-singleton/-/react-style-singleton-2.2.1.tgz#f99e420492b2d8f34d38308ff660b60d0b1205b4"