From 9494f4c1cc9109d7ba3fc1f8642eaed14ce6ae73 Mon Sep 17 00:00:00 2001 From: Abi Raja Date: Thu, 14 Dec 2023 08:27:16 -0500 Subject: [PATCH 1/4] fix reset function to reset all state correctly --- frontend/src/App.tsx | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 27937ca..658b6ef 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -125,9 +125,11 @@ function App() { setGeneratedCode(""); setReferenceImages([]); setExecutionConsole([]); + setUpdateInstruction(""); + setIsImportedFromCode(false); setAppHistory([]); setCurrentVersion(null); - setIsImportedFromCode(false); + setShouldIncludeResultImage(false); }; const cancelCodeGeneration = () => { @@ -199,6 +201,7 @@ function App() { }); } }, + // On status update (line) => setExecutionConsole((prev) => [...prev, line]), // On cancel () => { From f1a9859384a1b7d7b777b75d41ec03dbdb3f2020 Mon Sep 17 00:00:00 2001 From: Abi Raja Date: Thu, 14 Dec 2023 08:29:04 -0500 Subject: [PATCH 2/4] fix bug with history not being updated if image generation fails --- backend/routes/generate_code.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/backend/routes/generate_code.py b/backend/routes/generate_code.py index d34f861..b260cc0 100644 --- a/backend/routes/generate_code.py +++ b/backend/routes/generate_code.py @@ -254,6 +254,9 @@ async def stream_code(websocket: WebSocket): except Exception as e: traceback.print_exc() print("Image generation failed", e) + # Send set code even if image generation fails since that triggers + # the frontend to update history + await websocket.send_json({"type": "setCode", "value": completion}) await websocket.send_json( {"type": "status", "value": "Image generation failed but code is complete."} ) From f676151edf74bbca6712d9246df3cbe14e77738d Mon Sep 17 00:00:00 2001 From: Abi Raja Date: Thu, 14 Dec 2023 09:20:06 -0500 Subject: [PATCH 3/4] fix bug with preview not updating immediately --- frontend/src/components/Preview.tsx | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/frontend/src/components/Preview.tsx b/frontend/src/components/Preview.tsx index 24f33c0..42cc7b0 100644 --- a/frontend/src/components/Preview.tsx +++ b/frontend/src/components/Preview.tsx @@ -1,4 +1,4 @@ -import { useEffect, useRef } from 'react'; +import { useEffect, useRef } from "react"; import classNames from "classnames"; import useThrottle from "../hooks/useThrottle"; @@ -8,7 +8,9 @@ interface Props { } function Preview({ code, device }: Props) { - const throttledCode = useThrottle(code, 200); + const throttledCode = code; + // Temporary disable throttling for the preview not updating when the code changes + // useThrottle(code, 200); const iframeRef = useRef(null); useEffect(() => { @@ -39,4 +41,4 @@ function Preview({ code, device }: Props) { ); } -export default Preview; \ No newline at end of file +export default Preview; From 9b728d034bf21cb40c90af692c8cbceb7658f931 Mon Sep 17 00:00:00 2001 From: Abi Raja Date: Thu, 14 Dec 2023 10:18:41 -0500 Subject: [PATCH 4/4] add experimental support for SVG --- backend/eval.py | 2 +- backend/imported_code_prompts.py | 12 +++++ backend/prompts.py | 21 +++++++- backend/screenshot_system_prompts.py | 18 +++++++ backend/test_prompts.py | 52 +++++++++++++++++++ .../src/components/OutputSettingsSection.tsx | 29 +++++++++-- frontend/src/components/Preview.tsx | 2 +- frontend/src/types.ts | 1 + 8 files changed, 129 insertions(+), 8 deletions(-) diff --git a/backend/eval.py b/backend/eval.py index ac286bf..000c6fe 100644 --- a/backend/eval.py +++ b/backend/eval.py @@ -49,7 +49,7 @@ async def main(): for filename in evals: filepath = os.path.join(INPUT_DIR, filename) data_url = await image_to_data_url(filepath) - task = generate_code_core(data_url, "html_tailwind") + task = generate_code_core(data_url, "svg") tasks.append(task) results = await asyncio.gather(*tasks) diff --git a/backend/imported_code_prompts.py b/backend/imported_code_prompts.py index 748c770..a8bfa6a 100644 --- a/backend/imported_code_prompts.py +++ b/backend/imported_code_prompts.py @@ -78,3 +78,15 @@ In terms of libraries, Return only the full code in tags. Do not include markdown "```" or "```html" at the start or end. """ + +IMPORTED_CODE_SVG_SYSTEM_PROMPT = """ +You are an expert at building SVGs. + +- Do not add comments in the code such as "" and "" in place of writing the full code. WRITE THE FULL CODE. +- Repeat elements as needed to match the screenshot. For example, if there are 15 items, the code should have 15 items. DO NOT LEAVE comments like "" or bad things will happen. +- For images, use placeholder images from https://placehold.co and include a detailed description of the image in the alt text so that an image generation AI can generate the image later. +- You can use Google Fonts + +Return only the full code in tags. +Do not include markdown "```" or "```svg" at the start or end. +""" diff --git a/backend/prompts.py b/backend/prompts.py index f053964..1fe4f9b 100644 --- a/backend/prompts.py +++ b/backend/prompts.py @@ -7,12 +7,14 @@ from imported_code_prompts import ( IMPORTED_CODE_IONIC_TAILWIND_SYSTEM_PROMPT, IMPORTED_CODE_REACT_TAILWIND_SYSTEM_PROMPT, IMPORTED_CODE_TAILWIND_SYSTEM_PROMPT, + IMPORTED_CODE_SVG_SYSTEM_PROMPT, ) from screenshot_system_prompts import ( BOOTSTRAP_SYSTEM_PROMPT, IONIC_TAILWIND_SYSTEM_PROMPT, REACT_TAILWIND_SYSTEM_PROMPT, TAILWIND_SYSTEM_PROMPT, + SVG_SYSTEM_PROMPT, ) @@ -20,6 +22,10 @@ USER_PROMPT = """ Generate code for a web page that looks exactly like this. """ +SVG_USER_PROMPT = """ +Generate code for a SVG that looks exactly like this. +""" + def assemble_imported_code_prompt( code: str, stack: str, result_image_data_url: Union[str, None] = None @@ -33,9 +39,16 @@ def assemble_imported_code_prompt( system_content = IMPORTED_CODE_BOOTSTRAP_SYSTEM_PROMPT elif stack == "ionic_tailwind": system_content = IMPORTED_CODE_IONIC_TAILWIND_SYSTEM_PROMPT + elif stack == "svg": + system_content = IMPORTED_CODE_SVG_SYSTEM_PROMPT else: raise Exception("Code config is not one of available options") + user_content = ( + "Here is the code of the app: " + code + if stack != "svg" + else "Here is the code of the SVG: " + code + ) return [ { "role": "system", @@ -43,7 +56,7 @@ def assemble_imported_code_prompt( }, { "role": "user", - "content": "Here is the code of the app: " + code, + "content": user_content, }, ] # TODO: Use result_image_data_url @@ -64,9 +77,13 @@ def assemble_prompt( system_content = BOOTSTRAP_SYSTEM_PROMPT elif generated_code_config == "ionic_tailwind": system_content = IONIC_TAILWIND_SYSTEM_PROMPT + elif generated_code_config == "svg": + system_content = SVG_SYSTEM_PROMPT else: raise Exception("Code config is not one of available options") + user_prompt = USER_PROMPT if generated_code_config != "svg" else SVG_USER_PROMPT + user_content: List[ChatCompletionContentPartParam] = [ { "type": "image_url", @@ -74,7 +91,7 @@ def assemble_prompt( }, { "type": "text", - "text": USER_PROMPT, + "text": user_prompt, }, ] diff --git a/backend/screenshot_system_prompts.py b/backend/screenshot_system_prompts.py index a48adaa..3308d03 100644 --- a/backend/screenshot_system_prompts.py +++ b/backend/screenshot_system_prompts.py @@ -110,3 +110,21 @@ In terms of libraries, Return only the full code in tags. Do not include markdown "```" or "```html" at the start or end. """ + + +SVG_SYSTEM_PROMPT = """ +You are an expert at building SVGs. +You take screenshots of a reference web page from the user, and then build a SVG that looks exactly like the screenshot. + +- Make sure the SVG looks exactly like the screenshot. +- Pay close attention to background color, text color, font size, font family, +padding, margin, border, etc. Match the colors and sizes exactly. +- Use the exact text from the screenshot. +- Do not add comments in the code such as "" and "" in place of writing the full code. WRITE THE FULL CODE. +- Repeat elements as needed to match the screenshot. For example, if there are 15 items, the code should have 15 items. DO NOT LEAVE comments like "" or bad things will happen. +- For images, use placeholder images from https://placehold.co and include a detailed description of the image in the alt text so that an image generation AI can generate the image later. +- You can use Google Fonts + +Return only the full code in tags. +Do not include markdown "```" or "```svg" at the start or end. +""" diff --git a/backend/test_prompts.py b/backend/test_prompts.py index 87f3281..303e3a9 100644 --- a/backend/test_prompts.py +++ b/backend/test_prompts.py @@ -113,6 +113,23 @@ Return only the full code in tags. Do not include markdown "```" or "```html" at the start or end. """ +SVG_SYSTEM_PROMPT = """ +You are an expert at building SVGs. +You take screenshots of a reference web page from the user, and then build a SVG that looks exactly like the screenshot. + +- Make sure the SVG looks exactly like the screenshot. +- Pay close attention to background color, text color, font size, font family, +padding, margin, border, etc. Match the colors and sizes exactly. +- Use the exact text from the screenshot. +- Do not add comments in the code such as "" and "" in place of writing the full code. WRITE THE FULL CODE. +- Repeat elements as needed to match the screenshot. For example, if there are 15 items, the code should have 15 items. DO NOT LEAVE comments like "" or bad things will happen. +- For images, use placeholder images from https://placehold.co and include a detailed description of the image in the alt text so that an image generation AI can generate the image later. +- You can use Google Fonts + +Return only the full code in tags. +Do not include markdown "```" or "```svg" at the start or end. +""" + IMPORTED_CODE_TAILWIND_SYSTEM_PROMPT = """ You are an expert Tailwind developer. @@ -194,27 +211,55 @@ Return only the full code in tags. Do not include markdown "```" or "```html" at the start or end. """ +IMPORTED_CODE_SVG_SYSTEM_PROMPT = """ +You are an expert at building SVGs. + +- Do not add comments in the code such as "" and "" in place of writing the full code. WRITE THE FULL CODE. +- Repeat elements as needed to match the screenshot. For example, if there are 15 items, the code should have 15 items. DO NOT LEAVE comments like "" or bad things will happen. +- For images, use placeholder images from https://placehold.co and include a detailed description of the image in the alt text so that an image generation AI can generate the image later. +- You can use Google Fonts + +Return only the full code in tags. +Do not include markdown "```" or "```svg" at the start or end. +""" + +USER_PROMPT = """ +Generate code for a web page that looks exactly like this. +""" + +SVG_USER_PROMPT = """ +Generate code for a SVG that looks exactly like this. +""" + def test_prompts(): tailwind_prompt = assemble_prompt( "image_data_url", "html_tailwind", "result_image_data_url" ) assert tailwind_prompt[0]["content"] == TAILWIND_SYSTEM_PROMPT + assert tailwind_prompt[1]["content"][2]["text"] == USER_PROMPT # type: ignore react_tailwind_prompt = assemble_prompt( "image_data_url", "react_tailwind", "result_image_data_url" ) assert react_tailwind_prompt[0]["content"] == REACT_TAILWIND_SYSTEM_PROMPT + assert react_tailwind_prompt[1]["content"][2]["text"] == USER_PROMPT # type: ignore bootstrap_prompt = assemble_prompt( "image_data_url", "bootstrap", "result_image_data_url" ) assert bootstrap_prompt[0]["content"] == BOOTSTRAP_SYSTEM_PROMPT + assert bootstrap_prompt[1]["content"][2]["text"] == USER_PROMPT # type: ignore ionic_tailwind = assemble_prompt( "image_data_url", "ionic_tailwind", "result_image_data_url" ) assert ionic_tailwind[0]["content"] == IONIC_TAILWIND_SYSTEM_PROMPT + assert ionic_tailwind[1]["content"][2]["text"] == USER_PROMPT # type: ignore + + svg_prompt = assemble_prompt("image_data_url", "svg", "result_image_data_url") + assert svg_prompt[0]["content"] == SVG_SYSTEM_PROMPT + assert svg_prompt[1]["content"][2]["text"] == SVG_USER_PROMPT # type: ignore def test_imported_code_prompts(): @@ -253,3 +298,10 @@ def test_imported_code_prompts(): {"role": "user", "content": "Here is the code of the app: code"}, ] assert ionic_tailwind == expected_ionic_tailwind + + svg = assemble_imported_code_prompt("code", "svg", "result_image_data_url") + expected_svg = [ + {"role": "system", "content": IMPORTED_CODE_SVG_SYSTEM_PROMPT}, + {"role": "user", "content": "Here is the code of the SVG: code"}, + ] + assert svg == expected_svg diff --git a/frontend/src/components/OutputSettingsSection.tsx b/frontend/src/components/OutputSettingsSection.tsx index fc116ea..f671b58 100644 --- a/frontend/src/components/OutputSettingsSection.tsx +++ b/frontend/src/components/OutputSettingsSection.tsx @@ -6,6 +6,7 @@ import { SelectTrigger, } from "./ui/select"; import { GeneratedCodeConfig } from "../types"; +import { Badge } from "./ui/badge"; function generateDisplayComponent(config: GeneratedCodeConfig) { switch (config) { @@ -36,9 +37,16 @@ function generateDisplayComponent(config: GeneratedCodeConfig) { Tailwind ); - default: - // TODO: Should never reach this out. Error out - return config; + case GeneratedCodeConfig.SVG: + return ( +
+ SVG +
+ ); + default: { + const exhaustiveCheck: never = config; + throw new Error(`Unhandled case: ${exhaustiveCheck}`); + } } } @@ -83,7 +91,20 @@ function OutputSettingsSection({ {generateDisplayComponent(GeneratedCodeConfig.BOOTSTRAP)} - {generateDisplayComponent(GeneratedCodeConfig.IONIC_TAILWIND)} +
+ {generateDisplayComponent(GeneratedCodeConfig.IONIC_TAILWIND)} + + Beta + +
+ + +
+ {generateDisplayComponent(GeneratedCodeConfig.SVG)} + + Beta + +
diff --git a/frontend/src/components/Preview.tsx b/frontend/src/components/Preview.tsx index 42cc7b0..d2eff16 100644 --- a/frontend/src/components/Preview.tsx +++ b/frontend/src/components/Preview.tsx @@ -1,6 +1,6 @@ import { useEffect, useRef } from "react"; import classNames from "classnames"; -import useThrottle from "../hooks/useThrottle"; +// import useThrottle from "../hooks/useThrottle"; interface Props { code: string; diff --git a/frontend/src/types.ts b/frontend/src/types.ts index 236116a..2820427 100644 --- a/frontend/src/types.ts +++ b/frontend/src/types.ts @@ -9,6 +9,7 @@ export enum GeneratedCodeConfig { REACT_TAILWIND = "react_tailwind", BOOTSTRAP = "bootstrap", IONIC_TAILWIND = "ionic_tailwind", + SVG = "svg", } export interface Settings {