diff --git a/backend/.gitignore b/backend/.gitignore index d9005f2..128eab6 100644 --- a/backend/.gitignore +++ b/backend/.gitignore @@ -150,3 +150,7 @@ cython_debug/ # and can be added to the global gitignore or merged into this file. For a more nuclear # option (not recommended) you can uncomment the following to ignore the entire idea folder. #.idea/ + + +# Temporary eval output +evals diff --git a/backend/eval.py b/backend/eval.py new file mode 100644 index 0000000..ac286bf --- /dev/null +++ b/backend/eval.py @@ -0,0 +1,67 @@ +# Load environment variables first +from typing import Any, Coroutine +from dotenv import load_dotenv + +from eval_config import EVALS_DIR +from eval_utils import image_to_data_url + +load_dotenv() + +import os +from llm import stream_openai_response +from prompts import assemble_prompt +import asyncio + +from utils import pprint_prompt + + +async def generate_code_core(image_url: str, stack: str) -> str: + prompt_messages = assemble_prompt(image_url, stack) + openai_api_key = os.environ.get("OPENAI_API_KEY") + openai_base_url = None + + pprint_prompt(prompt_messages) + + async def process_chunk(content: str): + pass + + if not openai_api_key: + raise Exception("OpenAI API key not found") + + completion = await stream_openai_response( + prompt_messages, + api_key=openai_api_key, + base_url=openai_base_url, + callback=lambda x: process_chunk(x), + ) + + return completion + + +async def main(): + INPUT_DIR = EVALS_DIR + "/inputs" + OUTPUT_DIR = EVALS_DIR + "/outputs" + + # Get all the files in the directory (only grab pngs) + evals = [f for f in os.listdir(INPUT_DIR) if f.endswith(".png")] + + tasks: list[Coroutine[Any, Any, str]] = [] + for filename in evals: + filepath = os.path.join(INPUT_DIR, filename) + data_url = await image_to_data_url(filepath) + task = generate_code_core(data_url, "html_tailwind") + tasks.append(task) + + results = await asyncio.gather(*tasks) + + os.makedirs(OUTPUT_DIR, exist_ok=True) + + for filename, content in zip(evals, results): + # File name is derived from the original filename in evals + output_filename = f"{os.path.splitext(filename)[0]}.html" + output_filepath = os.path.join(OUTPUT_DIR, output_filename) + with open(output_filepath, "w") as file: + file.write(content) + + +asyncio.run(main()) diff --git a/backend/eval_config.py b/backend/eval_config.py new file mode 100644 index 0000000..62a3b8f --- /dev/null +++ b/backend/eval_config.py @@ -0,0 +1 @@ +EVALS_DIR = "./evals" diff --git a/backend/eval_utils.py b/backend/eval_utils.py new file mode 100644 index 0000000..6a52f88 --- /dev/null +++ b/backend/eval_utils.py @@ -0,0 +1,7 @@ +import base64 + + +async def image_to_data_url(filepath: str): + with open(filepath, "rb") as image_file: + encoded_string = base64.b64encode(image_file.read()).decode() + return f"data:image/png;base64,{encoded_string}" diff --git a/backend/main.py b/backend/main.py index 2781b57..a55167c 100644 --- a/backend/main.py +++ b/backend/main.py @@ -7,7 +7,7 @@ load_dotenv() import os from fastapi import FastAPI from fastapi.middleware.cors import CORSMiddleware -from routes import screenshot, generate_code, home +from routes import screenshot, generate_code, home, evals from config import IS_PROD # Setup Sentry (only relevant in prod) @@ -40,3 +40,4 @@ app.add_middleware( app.include_router(generate_code.router) app.include_router(screenshot.router) app.include_router(home.router) +app.include_router(evals.router) diff --git a/backend/routes/evals.py b/backend/routes/evals.py new file mode 100644 index 0000000..48a3a95 --- /dev/null +++ b/backend/routes/evals.py @@ -0,0 +1,46 @@ +import os +from fastapi import APIRouter +from pydantic import BaseModel +from eval_utils import image_to_data_url +from eval_config import EVALS_DIR + + +router = APIRouter() + + +class Eval(BaseModel): + input: str + output: str + + +@router.get("/evals") +async def get_evals(): + # Get all evals from EVALS_DIR + input_dir = EVALS_DIR + "/inputs" + output_dir = EVALS_DIR + "/outputs" + + evals: list[Eval] = [] + for file in os.listdir(input_dir): + if file.endswith(".png"): + input_file_path = os.path.join(input_dir, file) + input_file = await image_to_data_url(input_file_path) + + # Construct the corresponding output file name + output_file_name = file.replace(".png", ".html") + output_file_path = os.path.join(output_dir, output_file_name) + + # Check if the output file exists + if os.path.exists(output_file_path): + with open(output_file_path, "r") as f: + output_file_data = f.read() + else: + output_file_data = "Output file not found." + + evals.append( + Eval( + input=input_file, + output=output_file_data, + ) + ) + + return evals diff --git a/frontend/package.json b/frontend/package.json index f262933..af9d6e6 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -18,6 +18,7 @@ "@radix-ui/react-accordion": "^1.1.2", "@radix-ui/react-alert-dialog": "^1.0.5", "@radix-ui/react-checkbox": "^1.0.4", + "@radix-ui/react-collapsible": "^1.0.3", "@radix-ui/react-dialog": "^1.0.5", "@radix-ui/react-hover-card": "^1.0.7", "@radix-ui/react-icons": "^1.3.0", @@ -41,6 +42,7 @@ "react-dropzone": "^14.2.3", "react-hot-toast": "^2.4.1", "react-icons": "^4.12.0", + "react-router-dom": "^6.20.1", "tailwind-merge": "^2.0.0", "tailwindcss-animate": "^1.0.7", "thememirror": "^2.0.1", diff --git a/frontend/src/components/evals/EvalsPage.tsx b/frontend/src/components/evals/EvalsPage.tsx new file mode 100644 index 0000000..6e76a0d --- /dev/null +++ b/frontend/src/components/evals/EvalsPage.tsx @@ -0,0 +1,70 @@ +import React, { useEffect } from "react"; +import { HTTP_BACKEND_URL } from "../../config"; +import RatingPicker from "./RatingPicker"; + +interface Eval { + input: string; + output: string; +} + +function EvalsPage() { + const [evals, setEvals] = React.useState([]); + const [ratings, setRatings] = React.useState([]); + + const total = ratings.reduce((a, b) => a + b, 0); + const max = ratings.length * 4; + const score = ((total / max) * 100 || 0).toFixed(2); + + useEffect(() => { + if (evals.length > 0) return; + + fetch(`${HTTP_BACKEND_URL}/evals`) + .then((res) => res.json()) + .then((data) => { + setEvals(data); + setRatings(new Array(data.length).fill(0)); + }); + }, [evals]); + + return ( +
+ {/* Display total */} +
+ + Total: {total} out of {max} ({score}%) + +
+ +
+ {evals.map((e, index) => ( +
+
+
+ +
+
+ {/* Put output into an iframe */} + +
+
+
+ { + const newRatings = [...ratings]; + newRatings[index] = rating; + setRatings(newRatings); + }} + /> +
+
+ ))} +
+
+ ); +} + +export default EvalsPage; diff --git a/frontend/src/components/evals/RatingPicker.tsx b/frontend/src/components/evals/RatingPicker.tsx new file mode 100644 index 0000000..3b88d07 --- /dev/null +++ b/frontend/src/components/evals/RatingPicker.tsx @@ -0,0 +1,38 @@ +import React from "react"; + +interface Props { + onSelect: (rating: number) => void; +} + +function RatingPicker({ onSelect }: Props) { + const [selected, setSelected] = React.useState(null); + + const renderCircle = (number: number) => { + const isSelected = selected === number; + const bgColor = isSelected ? "bg-black" : "bg-gray-300"; + const textColor = isSelected ? "text-white" : "text-black"; + + return ( +
{ + setSelected(number); + onSelect(number); + }} + > + {number} +
+ ); + }; + + return ( +
+ {renderCircle(1)} + {renderCircle(2)} + {renderCircle(3)} + {renderCircle(4)} +
+ ); +} + +export default RatingPicker; diff --git a/frontend/src/components/history/HistoryDisplay.tsx b/frontend/src/components/history/HistoryDisplay.tsx index db0b74b..ce00db4 100644 --- a/frontend/src/components/history/HistoryDisplay.tsx +++ b/frontend/src/components/history/HistoryDisplay.tsx @@ -1,12 +1,16 @@ -import { History, HistoryItemType } from "./history_types"; +import { History } from "./history_types"; import toast from "react-hot-toast"; import classNames from "classnames"; -import { - HoverCard, - HoverCardTrigger, - HoverCardContent, -} from "../ui/hover-card"; + import { Badge } from "../ui/badge"; +import { renderHistory } from "./utils"; +import { + Collapsible, + CollapsibleContent, + CollapsibleTrigger, +} from "../ui/collapsible"; +import { Button } from "../ui/button"; +import { CaretSortIcon } from "@radix-ui/react-icons"; interface Props { history: History; @@ -15,82 +19,65 @@ interface Props { shouldDisableReverts: boolean; } -function displayHistoryItemType(itemType: HistoryItemType) { - switch (itemType) { - case "ai_create": - return "Create"; - case "ai_edit": - return "Edit"; - case "code_create": - return "Imported from code"; - default: { - const exhaustiveCheck: never = itemType; - throw new Error(`Unhandled case: ${exhaustiveCheck}`); - } - } -} - export default function HistoryDisplay({ history, currentVersion, revertToVersion, shouldDisableReverts, }: Props) { - return history.length === 0 ? null : ( + const renderedHistory = renderHistory(history, currentVersion); + + return renderedHistory.length === 0 ? null : (

Versions

    - {history.map((item, index) => ( + {renderedHistory.map((item, index) => (
  • - - +
    - shouldDisableReverts - ? toast.error( - "Please wait for code generation to complete before viewing an older version." - ) - : revertToVersion(index) - } > - {" "} -
    -

    - {item.type === "ai_edit" - ? item.inputs.prompt - : item.type === "ai_create" - ? "Create" - : "Imported from code"} -

    - {/*

    {displayHistoryItemType(item.type)}

    */} - {item.parentIndex !== null && - item.parentIndex !== index - 1 ? ( -

    - (parent: v{(item.parentIndex || 0) + 1}) -

    - ) : null} +
    + shouldDisableReverts + ? toast.error( + "Please wait for code generation to complete before viewing an older version." + ) + : revertToVersion(index) + } + > +
    +

    {item.summary}

    + {item.parentVersion !== null && ( +

    + (parent: {item.parentVersion}) +

    + )} +
    +

    v{index + 1}

    -

    v{index + 1}

    - - -
    - {item.type === "ai_edit" - ? item.inputs.prompt - : item.type === "ai_create" - ? "Create" - : "Imported from code"} + + + +
    + +
    Full prompt: {item.summary}
    +
    + {item.type}
    - {displayHistoryItemType(item.type)} -     - + +
    • ))}
diff --git a/frontend/src/components/history/history_types.ts b/frontend/src/components/history/history_types.ts index 832e379..8dcd219 100644 --- a/frontend/src/components/history/history_types.ts +++ b/frontend/src/components/history/history_types.ts @@ -32,3 +32,10 @@ export type CodeCreateInputs = { }; export type History = HistoryItem[]; + +export type RenderedHistoryItem = { + type: string; + summary: string; + parentVersion: string | null; + isActive: boolean; +}; diff --git a/frontend/src/components/history/utils.test.ts b/frontend/src/components/history/utils.test.ts index 2abaf90..330b8e5 100644 --- a/frontend/src/components/history/utils.test.ts +++ b/frontend/src/components/history/utils.test.ts @@ -1,5 +1,5 @@ import { expect, test } from "vitest"; -import { extractHistoryTree } from "./utils"; +import { extractHistoryTree, renderHistory } from "./utils"; import type { History } from "./history_types"; const basicLinearHistory: History = [ @@ -29,6 +29,18 @@ const basicLinearHistory: History = [ }, ]; +const basicLinearHistoryWithCode: History = [ + { + type: "code_create", + parentIndex: null, + code: "1. create", + inputs: { + code: "1. create", + }, + }, + ...basicLinearHistory.slice(1), +]; + const basicBranchingHistory: History = [ ...basicLinearHistory, { @@ -121,3 +133,98 @@ test("should correctly extract the history tree", () => { // Bad tree expect(() => extractHistoryTree(basicBadHistory, 1)).toThrow(); }); + +test("should correctly render the history tree", () => { + expect(renderHistory(basicLinearHistory, 2)).toEqual([ + { + isActive: false, + parentVersion: null, + summary: "Create", + type: "Create", + }, + { + isActive: false, + parentVersion: null, + summary: "use better icons", + type: "Edit", + }, + { + isActive: true, + parentVersion: null, + summary: "make text red", + type: "Edit", + }, + ]); + + // Current version is the first version + expect(renderHistory(basicLinearHistory, 0)).toEqual([ + { + isActive: true, + parentVersion: null, + summary: "Create", + type: "Create", + }, + { + isActive: false, + parentVersion: null, + summary: "use better icons", + type: "Edit", + }, + { + isActive: false, + parentVersion: null, + summary: "make text red", + type: "Edit", + }, + ]); + + // Render a history with code + expect(renderHistory(basicLinearHistoryWithCode, 0)).toEqual([ + { + isActive: true, + parentVersion: null, + summary: "Imported from code", + type: "Imported from code", + }, + { + isActive: false, + parentVersion: null, + summary: "use better icons", + type: "Edit", + }, + { + isActive: false, + parentVersion: null, + summary: "make text red", + type: "Edit", + }, + ]); + + // Render a non-linear history + expect(renderHistory(basicBranchingHistory, 3)).toEqual([ + { + isActive: false, + parentVersion: null, + summary: "Create", + type: "Create", + }, + { + isActive: false, + parentVersion: null, + summary: "use better icons", + type: "Edit", + }, + { + isActive: false, + parentVersion: null, + summary: "make text red", + type: "Edit", + }, + { + isActive: true, + parentVersion: "v2", + summary: "make text green", + type: "Edit", + }, + ]); +}); diff --git a/frontend/src/components/history/utils.ts b/frontend/src/components/history/utils.ts index 5e476af..785c20b 100644 --- a/frontend/src/components/history/utils.ts +++ b/frontend/src/components/history/utils.ts @@ -1,4 +1,9 @@ -import { History, HistoryItem } from "./history_types"; +import { + History, + HistoryItem, + HistoryItemType, + RenderedHistoryItem, +} from "./history_types"; export function extractHistoryTree( history: History, @@ -30,3 +35,62 @@ export function extractHistoryTree( return flatHistory; } + +function displayHistoryItemType(itemType: HistoryItemType) { + switch (itemType) { + case "ai_create": + return "Create"; + case "ai_edit": + return "Edit"; + case "code_create": + return "Imported from code"; + default: { + const exhaustiveCheck: never = itemType; + throw new Error(`Unhandled case: ${exhaustiveCheck}`); + } + } +} + +function summarizeHistoryItem(item: HistoryItem) { + const itemType = item.type; + switch (itemType) { + case "ai_create": + return "Create"; + case "ai_edit": + return item.inputs.prompt; + case "code_create": + return "Imported from code"; + default: { + const exhaustiveCheck: never = itemType; + throw new Error(`Unhandled case: ${exhaustiveCheck}`); + } + } +} + +export const renderHistory = ( + history: History, + currentVersion: number | null +) => { + const renderedHistory: RenderedHistoryItem[] = []; + + for (let i = 0; i < history.length; i++) { + const item = history[i]; + // Only show the parent version if it's not the previous version + // (i.e. it's the branching point) and if it's not the first version + const parentVersion = + item.parentIndex !== null && item.parentIndex !== i - 1 + ? `v${(item.parentIndex || 0) + 1}` + : null; + const type = displayHistoryItemType(item.type); + const isActive = i === currentVersion; + const summary = summarizeHistoryItem(item); + renderedHistory.push({ + isActive, + summary: summary, + parentVersion, + type, + }); + } + + return renderedHistory; +}; diff --git a/frontend/src/components/ui/collapsible.tsx b/frontend/src/components/ui/collapsible.tsx new file mode 100644 index 0000000..a23e7a2 --- /dev/null +++ b/frontend/src/components/ui/collapsible.tsx @@ -0,0 +1,9 @@ +import * as CollapsiblePrimitive from "@radix-ui/react-collapsible" + +const Collapsible = CollapsiblePrimitive.Root + +const CollapsibleTrigger = CollapsiblePrimitive.CollapsibleTrigger + +const CollapsibleContent = CollapsiblePrimitive.CollapsibleContent + +export { Collapsible, CollapsibleTrigger, CollapsibleContent } diff --git a/frontend/src/main.tsx b/frontend/src/main.tsx index 740829f..da58943 100644 --- a/frontend/src/main.tsx +++ b/frontend/src/main.tsx @@ -3,13 +3,20 @@ import ReactDOM from "react-dom/client"; import { Toaster } from "react-hot-toast"; import AppContainer from "./components/hosted/AppContainer.tsx"; import { ClerkProvider } from "@clerk/clerk-react"; +import EvalsPage from "./components/evals/EvalsPage.tsx"; +import { BrowserRouter as Router, Routes, Route } from "react-router-dom"; import { CLERK_PUBLISHABLE_KEY } from "./config.ts"; import "./index.css"; ReactDOM.createRoot(document.getElementById("root")!).render( - + + + } /> + } /> + + diff --git a/frontend/yarn.lock b/frontend/yarn.lock index f35b9bb..b229fc0 100644 --- a/frontend/yarn.lock +++ b/frontend/yarn.lock @@ -829,7 +829,7 @@ "@radix-ui/react-use-previous" "1.0.1" "@radix-ui/react-use-size" "1.0.1" -"@radix-ui/react-collapsible@1.0.3": +"@radix-ui/react-collapsible@1.0.3", "@radix-ui/react-collapsible@^1.0.3": version "1.0.3" resolved "https://registry.yarnpkg.com/@radix-ui/react-collapsible/-/react-collapsible-1.0.3.tgz#df0e22e7a025439f13f62d4e4a9e92c4a0df5b81" integrity sha512-UBmVDkmR6IvDsloHVN+3rtx4Mi5TFvylYXpluuv0f37dtaz3H99bp8No0LGXRigVpl3UAT4l9j6bIchh42S/Gg== @@ -1209,6 +1209,11 @@ dependencies: "@babel/runtime" "^7.13.10" +"@remix-run/router@1.13.1": + version "1.13.1" + resolved "https://registry.yarnpkg.com/@remix-run/router/-/router-1.13.1.tgz#07e2a8006f23a3bc898b3f317e0a58cc8076b86e" + integrity sha512-so+DHzZKsoOcoXrILB4rqDkMDy7NLMErRdOxvzvOKb507YINKUP4Di+shbTZDhSE/pBZ+vr7XGIpcOO0VLSA+Q== + "@rollup/pluginutils@^4.2.0": version "4.2.1" resolved "https://registry.yarnpkg.com/@rollup/pluginutils/-/pluginutils-4.2.1.tgz#e6c6c3aba0744edce3fb2074922d3776c0af2a6d" @@ -3184,6 +3189,21 @@ react-remove-scroll@2.5.5: use-callback-ref "^1.3.0" use-sidecar "^1.1.2" +react-router-dom@^6.20.1: + version "6.20.1" + resolved "https://registry.yarnpkg.com/react-router-dom/-/react-router-dom-6.20.1.tgz#e34f8075b9304221420de3609e072bb349824984" + integrity sha512-npzfPWcxfQN35psS7rJgi/EW0Gx6EsNjfdJSAk73U/HqMEJZ2k/8puxfwHFgDQhBGmS3+sjnGbMdMSV45axPQw== + dependencies: + "@remix-run/router" "1.13.1" + react-router "6.20.1" + +react-router@6.20.1: + version "6.20.1" + resolved "https://registry.yarnpkg.com/react-router/-/react-router-6.20.1.tgz#e8cc326031d235aaeec405bb234af77cf0fe75ef" + integrity sha512-ccvLrB4QeT5DlaxSFFYi/KR8UMQ4fcD8zBcR71Zp1kaYTC5oJKYAp1cbavzGrogwxca+ubjkd7XjFZKBW8CxPA== + dependencies: + "@remix-run/router" "1.13.1" + react-style-singleton@^2.2.1: version "2.2.1" resolved "https://registry.yarnpkg.com/react-style-singleton/-/react-style-singleton-2.2.1.tgz#f99e420492b2d8f34d38308ff660b60d0b1205b4"