From 7c1fb7a54d4cf135b447adca6559bdd4dc3fee2d Mon Sep 17 00:00:00 2001 From: Julian Toro Date: Fri, 2 Feb 2024 10:19:52 -0500 Subject: [PATCH 1/7] Add HTML/CSS system prompt and update stack descriptions --- backend/prompts/screenshot_system_prompts.py | 24 ++++++++++++++++++++ backend/prompts/types.py | 2 ++ frontend/src/lib/stacks.ts | 2 ++ 3 files changed, 28 insertions(+) diff --git a/backend/prompts/screenshot_system_prompts.py b/backend/prompts/screenshot_system_prompts.py index fca91ba..7de6dca 100644 --- a/backend/prompts/screenshot_system_prompts.py +++ b/backend/prompts/screenshot_system_prompts.py @@ -1,5 +1,28 @@ from prompts.types import SystemPrompts +HTML_CSS_SYSTEM_PROMPT = """ +You are an expert CSS developer +You take screenshots of a reference web page from the user, and then build single page apps +using CSS, HTML and JS. +You might also be given a screenshot(The second image) of a web page that you have already built, and asked to +update it to look more like the reference image(The first image). + +- Make sure the app looks exactly like the screenshot. +- Pay close attention to background color, text color, font size, font family, +padding, margin, border, etc. Match the colors and sizes exactly. +- Use the exact text from the screenshot. +- Do not add comments in the code such as "" and "" in place of writing the full code. WRITE THE FULL CODE. +- Repeat elements as needed to match the screenshot. For example, if there are 15 items, the code should have 15 items. DO NOT LEAVE comments like "" or bad things will happen. +- For images, use placeholder images from https://placehold.co and include a detailed description of the image in the alt text so that an image generation AI can generate the image later. + +In terms of libraries, + +- You can use Google Fonts +- Font Awesome for icons: + +Return only the full code in tags. +Do not include markdown "```" or "```html" at the start or end. +""" HTML_TAILWIND_SYSTEM_PROMPT = """ You are an expert Tailwind developer @@ -176,6 +199,7 @@ Do not include markdown "```" or "```svg" at the start or end. SYSTEM_PROMPTS = SystemPrompts( + html_css=HTML_CSS_SYSTEM_PROMPT, html_tailwind=HTML_TAILWIND_SYSTEM_PROMPT, react_tailwind=REACT_TAILWIND_SYSTEM_PROMPT, bootstrap=BOOTSTRAP_SYSTEM_PROMPT, diff --git a/backend/prompts/types.py b/backend/prompts/types.py index 9068443..7d4ca2d 100644 --- a/backend/prompts/types.py +++ b/backend/prompts/types.py @@ -2,6 +2,7 @@ from typing import Literal, TypedDict class SystemPrompts(TypedDict): + html_css: str html_tailwind: str react_tailwind: str bootstrap: str @@ -11,6 +12,7 @@ class SystemPrompts(TypedDict): Stack = Literal[ + "html_css", "html_tailwind", "react_tailwind", "bootstrap", diff --git a/frontend/src/lib/stacks.ts b/frontend/src/lib/stacks.ts index b8b372e..8e3a26a 100644 --- a/frontend/src/lib/stacks.ts +++ b/frontend/src/lib/stacks.ts @@ -1,5 +1,6 @@ // Keep in sync with backend (prompts/types.py) export enum Stack { + HTML_CSS = "html_css", HTML_TAILWIND = "html_tailwind", REACT_TAILWIND = "react_tailwind", BOOTSTRAP = "bootstrap", @@ -11,6 +12,7 @@ export enum Stack { export const STACK_DESCRIPTIONS: { [key in Stack]: { components: string[]; inBeta: boolean }; } = { + html_css: { components: ["HTML", "Css"], inBeta: false }, html_tailwind: { components: ["HTML", "Tailwind"], inBeta: false }, react_tailwind: { components: ["React", "Tailwind"], inBeta: false }, bootstrap: { components: ["Bootstrap"], inBeta: false }, From d4e182cfe01ebbb2394fefa4bc73b875f493af21 Mon Sep 17 00:00:00 2001 From: Abi Raja Date: Thu, 27 Jun 2024 14:58:14 +0800 Subject: [PATCH 2/7] update name of stack --- frontend/src/lib/stacks.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/lib/stacks.ts b/frontend/src/lib/stacks.ts index 8e3a26a..6fe22e0 100644 --- a/frontend/src/lib/stacks.ts +++ b/frontend/src/lib/stacks.ts @@ -12,7 +12,7 @@ export enum Stack { export const STACK_DESCRIPTIONS: { [key in Stack]: { components: string[]; inBeta: boolean }; } = { - html_css: { components: ["HTML", "Css"], inBeta: false }, + html_css: { components: ["HTML", "CSS"], inBeta: false }, html_tailwind: { components: ["HTML", "Tailwind"], inBeta: false }, react_tailwind: { components: ["React", "Tailwind"], inBeta: false }, bootstrap: { components: ["Bootstrap"], inBeta: false }, From 51e06a4c4ba9a9979b397235bf38a455ed717caf Mon Sep 17 00:00:00 2001 From: Abi Raja Date: Thu, 27 Jun 2024 15:03:42 +0800 Subject: [PATCH 3/7] fix order and update README --- README.md | 13 +++++++------ frontend/src/lib/stacks.ts | 3 ++- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index d744478..77c93ed 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,7 @@ https://github.com/abi/screenshot-to-code/assets/23818/6cebadae-2fe3-4986-ac6a-8 Supported stacks: - HTML + Tailwind +- HTML + CSS - React + Tailwind - Vue + Tailwind - Bootstrap @@ -24,7 +25,7 @@ Supported AI models: See the [Examples](#-examples) section below for more demos. -We also just added experimental support for taking a video/screen recording of a website in action and turning that into a functional prototype. +We also just added experimental support for taking a video/screen recording of a website in action and turning that into a functional prototype. ![google in app quick 3](https://github.com/abi/screenshot-to-code/assets/23818/8758ffa4-9483-4b9b-bb66-abd6d1594c33) @@ -42,12 +43,12 @@ We also just added experimental support for taking a video/screen recording of a ## 🛠 Getting Started -The app has a React/Vite frontend and a FastAPI backend. +The app has a React/Vite frontend and a FastAPI backend. Keys needed: -* [OpenAI API key with access to GPT-4](https://github.com/abi/screenshot-to-code/blob/main/Troubleshooting.md) -* Anthropic key (optional) - only if you want to use Claude Sonnet, or for experimental video support. +- [OpenAI API key with access to GPT-4](https://github.com/abi/screenshot-to-code/blob/main/Troubleshooting.md) +- Anthropic key (optional) - only if you want to use Claude Sonnet, or for experimental video support. Run the backend (I use Poetry for package management - `pip install poetry` if you don't have it): @@ -94,9 +95,9 @@ The app will be up and running at http://localhost:5173. Note that you can't dev - **I'm running into an error when setting up the backend. How can I fix it?** [Try this](https://github.com/abi/screenshot-to-code/issues/3#issuecomment-1814777959). If that still doesn't work, open an issue. - **How do I get an OpenAI API key?** See https://github.com/abi/screenshot-to-code/blob/main/Troubleshooting.md -- **How can I configure an OpenAI proxy?** - If you're not able to access the OpenAI API directly (due to e.g. country restrictions), you can try a VPN or you can configure the OpenAI base URL to use a proxy: Set OPENAI_BASE_URL in the `backend/.env` or directly in the UI in the settings dialog. Make sure the URL has "v1" in the path so it should look like this: `https://xxx.xxxxx.xxx/v1` +- **How can I configure an OpenAI proxy?** - If you're not able to access the OpenAI API directly (due to e.g. country restrictions), you can try a VPN or you can configure the OpenAI base URL to use a proxy: Set OPENAI_BASE_URL in the `backend/.env` or directly in the UI in the settings dialog. Make sure the URL has "v1" in the path so it should look like this: `https://xxx.xxxxx.xxx/v1` - **How can I update the backend host that my front-end connects to?** - Configure VITE_HTTP_BACKEND_URL and VITE_WS_BACKEND_URL in front/.env.local For example, set VITE_HTTP_BACKEND_URL=http://124.10.20.1:7001 -- **Seeing UTF-8 errors when running the backend?** - On windows, open the .env file with notepad++, then go to Encoding and select UTF-8. +- **Seeing UTF-8 errors when running the backend?** - On windows, open the .env file with notepad++, then go to Encoding and select UTF-8. - **How can I provide feedback?** For feedback, feature requests and bug reports, open an issue or ping me on [Twitter](https://twitter.com/_abi_). ## 📚 Examples diff --git a/frontend/src/lib/stacks.ts b/frontend/src/lib/stacks.ts index 6fe22e0..89515db 100644 --- a/frontend/src/lib/stacks.ts +++ b/frontend/src/lib/stacks.ts @@ -1,7 +1,8 @@ // Keep in sync with backend (prompts/types.py) +// Order here determines order in dropdown export enum Stack { - HTML_CSS = "html_css", HTML_TAILWIND = "html_tailwind", + HTML_CSS = "html_css", REACT_TAILWIND = "react_tailwind", BOOTSTRAP = "bootstrap", VUE_TAILWIND = "vue_tailwind", From ec1bada3e238d11b42f3a7223422145fc57c493b Mon Sep 17 00:00:00 2001 From: Abi Raja Date: Thu, 27 Jun 2024 16:01:28 +0800 Subject: [PATCH 4/7] fix type errors introduced with previous changes --- backend/prompts/test_prompts.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/backend/prompts/test_prompts.py b/backend/prompts/test_prompts.py index 42e89c3..4e3d7a7 100644 --- a/backend/prompts/test_prompts.py +++ b/backend/prompts/test_prompts.py @@ -1,3 +1,4 @@ +from llm import Llm from prompts import assemble_imported_code_prompt, assemble_prompt TAILWIND_SYSTEM_PROMPT = """ @@ -345,7 +346,7 @@ def test_prompts(): def test_imported_code_prompts(): tailwind_prompt = assemble_imported_code_prompt( - "code", "html_tailwind", "result_image_data_url" + "code", "html_tailwind", Llm.GPT_4O_2024_05_13 ) expected_tailwind_prompt = [ {"role": "system", "content": IMPORTED_CODE_TAILWIND_SYSTEM_PROMPT}, @@ -354,7 +355,7 @@ def test_imported_code_prompts(): assert tailwind_prompt == expected_tailwind_prompt react_tailwind_prompt = assemble_imported_code_prompt( - "code", "react_tailwind", "result_image_data_url" + "code", "react_tailwind", Llm.GPT_4O_2024_05_13 ) expected_react_tailwind_prompt = [ {"role": "system", "content": IMPORTED_CODE_REACT_TAILWIND_SYSTEM_PROMPT}, @@ -363,7 +364,7 @@ def test_imported_code_prompts(): assert react_tailwind_prompt == expected_react_tailwind_prompt bootstrap_prompt = assemble_imported_code_prompt( - "code", "bootstrap", "result_image_data_url" + "code", "bootstrap", Llm.GPT_4O_2024_05_13 ) expected_bootstrap_prompt = [ {"role": "system", "content": IMPORTED_CODE_BOOTSTRAP_SYSTEM_PROMPT}, @@ -372,7 +373,7 @@ def test_imported_code_prompts(): assert bootstrap_prompt == expected_bootstrap_prompt ionic_tailwind = assemble_imported_code_prompt( - "code", "ionic_tailwind", "result_image_data_url" + "code", "ionic_tailwind", Llm.GPT_4O_2024_05_13 ) expected_ionic_tailwind = [ {"role": "system", "content": IMPORTED_CODE_IONIC_TAILWIND_SYSTEM_PROMPT}, @@ -381,7 +382,7 @@ def test_imported_code_prompts(): assert ionic_tailwind == expected_ionic_tailwind vue_tailwind = assemble_imported_code_prompt( - "code", "vue_tailwind", "result_image_data_url" + "code", "vue_tailwind", Llm.GPT_4O_2024_05_13 ) expected_vue_tailwind = [ {"role": "system", "content": IMPORTED_CODE_VUE_TAILWIND_PROMPT}, @@ -389,7 +390,7 @@ def test_imported_code_prompts(): ] assert vue_tailwind == expected_vue_tailwind - svg = assemble_imported_code_prompt("code", "svg", "result_image_data_url") + svg = assemble_imported_code_prompt("code", "svg", Llm.GPT_4O_2024_05_13) expected_svg = [ {"role": "system", "content": IMPORTED_CODE_SVG_SYSTEM_PROMPT}, {"role": "user", "content": "Here is the code of the SVG: code"}, From 638053a54c6dbe9fca879ed8248e8d5816b4b1ee Mon Sep 17 00:00:00 2001 From: Abi Raja Date: Thu, 27 Jun 2024 16:02:06 +0800 Subject: [PATCH 5/7] add HTML + CSS prompt for imported code --- backend/prompts/imported_code_prompts.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/backend/prompts/imported_code_prompts.py b/backend/prompts/imported_code_prompts.py index 8babf78..8683110 100644 --- a/backend/prompts/imported_code_prompts.py +++ b/backend/prompts/imported_code_prompts.py @@ -18,6 +18,22 @@ Return only the full code in tags. Do not include markdown "```" or "```html" at the start or end. """ +IMPORTED_CODE_HTML_CSS_SYSTEM_PROMPT = """ +You are an expert CSS developer. + +- Do not add comments in the code such as "" and "" in place of writing the full code. WRITE THE FULL CODE. +- Repeat elements as needed. For example, if there are 15 items, the code should have 15 items. DO NOT LEAVE comments like "" or bad things will happen. +- For images, use placeholder images from https://placehold.co and include a detailed description of the image in the alt text so that an image generation AI can generate the image later. + +In terms of libraries, + +- You can use Google Fonts +- Font Awesome for icons: + +Return only the full code in tags. +Do not include markdown "```" or "```html" at the start or end. +""" + IMPORTED_CODE_REACT_TAILWIND_SYSTEM_PROMPT = """ You are an expert React/Tailwind developer @@ -128,6 +144,7 @@ Do not include markdown "```" or "```svg" at the start or end. IMPORTED_CODE_SYSTEM_PROMPTS = SystemPrompts( html_tailwind=IMPORTED_CODE_TAILWIND_SYSTEM_PROMPT, + html_css=IMPORTED_CODE_HTML_CSS_SYSTEM_PROMPT, react_tailwind=IMPORTED_CODE_REACT_TAILWIND_SYSTEM_PROMPT, bootstrap=IMPORTED_CODE_BOOTSTRAP_SYSTEM_PROMPT, ionic_tailwind=IMPORTED_CODE_IONIC_TAILWIND_SYSTEM_PROMPT, From 38a06517fb8bd085f0bd444a92a8e2c8bb14333c Mon Sep 17 00:00:00 2001 From: Abi Raja Date: Thu, 27 Jun 2024 16:08:41 +0800 Subject: [PATCH 6/7] add unit tests for the prompt assembly --- backend/prompts/screenshot_system_prompts.py | 47 +++++++++-------- backend/prompts/test_prompts.py | 55 ++++++++++++++++++++ 2 files changed, 79 insertions(+), 23 deletions(-) diff --git a/backend/prompts/screenshot_system_prompts.py b/backend/prompts/screenshot_system_prompts.py index 7de6dca..199fcde 100644 --- a/backend/prompts/screenshot_system_prompts.py +++ b/backend/prompts/screenshot_system_prompts.py @@ -1,28 +1,5 @@ from prompts.types import SystemPrompts -HTML_CSS_SYSTEM_PROMPT = """ -You are an expert CSS developer -You take screenshots of a reference web page from the user, and then build single page apps -using CSS, HTML and JS. -You might also be given a screenshot(The second image) of a web page that you have already built, and asked to -update it to look more like the reference image(The first image). - -- Make sure the app looks exactly like the screenshot. -- Pay close attention to background color, text color, font size, font family, -padding, margin, border, etc. Match the colors and sizes exactly. -- Use the exact text from the screenshot. -- Do not add comments in the code such as "" and "" in place of writing the full code. WRITE THE FULL CODE. -- Repeat elements as needed to match the screenshot. For example, if there are 15 items, the code should have 15 items. DO NOT LEAVE comments like "" or bad things will happen. -- For images, use placeholder images from https://placehold.co and include a detailed description of the image in the alt text so that an image generation AI can generate the image later. - -In terms of libraries, - -- You can use Google Fonts -- Font Awesome for icons: - -Return only the full code in tags. -Do not include markdown "```" or "```html" at the start or end. -""" HTML_TAILWIND_SYSTEM_PROMPT = """ You are an expert Tailwind developer @@ -49,6 +26,30 @@ Return only the full code in tags. Do not include markdown "```" or "```html" at the start or end. """ +HTML_CSS_SYSTEM_PROMPT = """ +You are an expert CSS developer +You take screenshots of a reference web page from the user, and then build single page apps +using CSS, HTML and JS. +You might also be given a screenshot(The second image) of a web page that you have already built, and asked to +update it to look more like the reference image(The first image). + +- Make sure the app looks exactly like the screenshot. +- Pay close attention to background color, text color, font size, font family, +padding, margin, border, etc. Match the colors and sizes exactly. +- Use the exact text from the screenshot. +- Do not add comments in the code such as "" and "" in place of writing the full code. WRITE THE FULL CODE. +- Repeat elements as needed to match the screenshot. For example, if there are 15 items, the code should have 15 items. DO NOT LEAVE comments like "" or bad things will happen. +- For images, use placeholder images from https://placehold.co and include a detailed description of the image in the alt text so that an image generation AI can generate the image later. + +In terms of libraries, + +- You can use Google Fonts +- Font Awesome for icons: + +Return only the full code in tags. +Do not include markdown "```" or "```html" at the start or end. +""" + BOOTSTRAP_SYSTEM_PROMPT = """ You are an expert Bootstrap developer You take screenshots of a reference web page from the user, and then build single page apps diff --git a/backend/prompts/test_prompts.py b/backend/prompts/test_prompts.py index 4e3d7a7..9175fd8 100644 --- a/backend/prompts/test_prompts.py +++ b/backend/prompts/test_prompts.py @@ -26,6 +26,30 @@ Return only the full code in tags. Do not include markdown "```" or "```html" at the start or end. """ +HTML_CSS_SYSTEM_PROMPT = """ +You are an expert CSS developer +You take screenshots of a reference web page from the user, and then build single page apps +using CSS, HTML and JS. +You might also be given a screenshot(The second image) of a web page that you have already built, and asked to +update it to look more like the reference image(The first image). + +- Make sure the app looks exactly like the screenshot. +- Pay close attention to background color, text color, font size, font family, +padding, margin, border, etc. Match the colors and sizes exactly. +- Use the exact text from the screenshot. +- Do not add comments in the code such as "" and "" in place of writing the full code. WRITE THE FULL CODE. +- Repeat elements as needed to match the screenshot. For example, if there are 15 items, the code should have 15 items. DO NOT LEAVE comments like "" or bad things will happen. +- For images, use placeholder images from https://placehold.co and include a detailed description of the image in the alt text so that an image generation AI can generate the image later. + +In terms of libraries, + +- You can use Google Fonts +- Font Awesome for icons: + +Return only the full code in tags. +Do not include markdown "```" or "```html" at the start or end. +""" + BOOTSTRAP_SYSTEM_PROMPT = """ You are an expert Bootstrap developer You take screenshots of a reference web page from the user, and then build single page apps @@ -190,6 +214,22 @@ Return only the full code in tags. Do not include markdown "```" or "```html" at the start or end. """ +IMPORTED_CODE_HTML_CSS_SYSTEM_PROMPT = """ +You are an expert CSS developer. + +- Do not add comments in the code such as "" and "" in place of writing the full code. WRITE THE FULL CODE. +- Repeat elements as needed. For example, if there are 15 items, the code should have 15 items. DO NOT LEAVE comments like "" or bad things will happen. +- For images, use placeholder images from https://placehold.co and include a detailed description of the image in the alt text so that an image generation AI can generate the image later. + +In terms of libraries, + +- You can use Google Fonts +- Font Awesome for icons: + +Return only the full code in tags. +Do not include markdown "```" or "```html" at the start or end. +""" + IMPORTED_CODE_REACT_TAILWIND_SYSTEM_PROMPT = """ You are an expert React/Tailwind developer @@ -315,6 +355,12 @@ def test_prompts(): assert tailwind_prompt[0].get("content") == TAILWIND_SYSTEM_PROMPT assert tailwind_prompt[1]["content"][2]["text"] == USER_PROMPT # type: ignore + html_css_prompt = assemble_prompt( + "image_data_url", "html_css", "result_image_data_url" + ) + assert html_css_prompt[0].get("content") == HTML_CSS_SYSTEM_PROMPT + assert html_css_prompt[1]["content"][2]["text"] == USER_PROMPT # type: ignore + react_tailwind_prompt = assemble_prompt( "image_data_url", "react_tailwind", "result_image_data_url" ) @@ -354,6 +400,15 @@ def test_imported_code_prompts(): ] assert tailwind_prompt == expected_tailwind_prompt + html_css_prompt = assemble_imported_code_prompt( + "code", "html_css", Llm.GPT_4O_2024_05_13 + ) + expected_html_css_prompt = [ + {"role": "system", "content": IMPORTED_CODE_HTML_CSS_SYSTEM_PROMPT}, + {"role": "user", "content": "Here is the code of the app: code"}, + ] + assert html_css_prompt == expected_html_css_prompt + react_tailwind_prompt = assemble_imported_code_prompt( "code", "react_tailwind", Llm.GPT_4O_2024_05_13 ) From 606f535bc287ec88cbb268fbac6a12a9068e3738 Mon Sep 17 00:00:00 2001 From: Abi Raja Date: Thu, 27 Jun 2024 16:24:06 +0800 Subject: [PATCH 7/7] enable Select and Edit for HTML + CSS stack --- frontend/src/App.tsx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index bef14d0..952d9d3 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -104,7 +104,8 @@ function App() { (selectedCodeGenerationModel === CodeGenerationModel.GPT_4O_2024_05_13 || selectedCodeGenerationModel === CodeGenerationModel.CLAUDE_3_5_SONNET_2024_06_20) && - settings.generatedCodeConfig === Stack.HTML_TAILWIND; + (settings.generatedCodeConfig === Stack.HTML_TAILWIND || + settings.generatedCodeConfig === Stack.HTML_CSS); // Indicate coding state using the browser tab's favicon and title useBrowserTabIndicator(appState === AppState.CODING);