Merge f6f8b2f707 into b904710cfc

.github/FUNDING.yml

@@ -0,0 +1 @@
+github: [abi]

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,21 @@
+---
+name: Bug report
+about: Create a report to help us improve
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**To Reproduce**
+Steps to reproduce the behavior:
+1. Go to '...'
+2. Click on '....'
+3. Scroll down to '....'
+4. See error
+
+**Screenshots of backend AND frontend terminal logs**
+If applicable, add screenshots to help explain your problem.

.github/ISSUE_TEMPLATE/custom.md

@@ -0,0 +1,10 @@
+---
+name: Custom issue template
+about: Describe this issue template's purpose here.
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,20 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+**Is your feature request related to a problem? Please describe.**
+A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+
+**Describe the solution you'd like**
+A clear and concise description of what you want to happen.
+
+**Describe alternatives you've considered**
+A clear and concise description of any alternative solutions or features you've considered.
+
+**Additional context**
+Add any other context or screenshots about the feature request here.

.gitignore

@@ -11,3 +11,6 @@ backend/backend/*
 # Env vars
 frontend/.env.local
 .env
+
+# Mac files
+.DS_Store

.vscode/settings.json

@@ -0,0 +1,5 @@
+{
+  "python.analysis.typeCheckingMode": "strict",
+  "python.analysis.extraPaths": ["./backend"],
+  "python.autoComplete.extraPaths": ["./backend"]
+}

Evaluation.md

@@ -0,0 +1,19 @@
+## Evaluating models and prompts
+
+Evaluation dataset consists of 16 screenshots. A Python script for running screenshot-to-code on the dataset and a UI for rating outputs is included. With this set up, we can compare and evaluate various models and prompts.
+
+### Running evals
+
+- Input screenshots should be located at `backend/evals_data/inputs` and the outputs will be `backend/evals_data/outputs`. If you want to modify this, modify `EVALS_DIR` in `backend/evals/config.py`. You can download the input screenshot dataset here: TODO.
+- Set a stack and model (`STACK` var, `MODEL` var) in `backend/run_evals.py`
+- Run `OPENAI_API_KEY=sk-... python run_evals.py` - this runs the screenshot-to-code on the input dataset in parallel but it will still take a few minutes to complete.
+- Once the script is done, you can find the outputs in `backend/evals_data/outputs`.
+
+### Rating evals
+
+In order to view and rate the outputs, visit your front-end at `/evals`.
+
+- Rate each output on a scale of 1-4
+- You can also print the page as PDF to share your results with others.
+
+Generally, I run three tests for each model/prompt + stack combo and take the average score out of those tests to evaluate.

README.md

@@ -1,29 +1,48 @@
 # screenshot-to-code
 
-This simple app converts a screenshot to code (HTML/Tailwind CSS, or React or Vue or Bootstrap). It uses GPT-4 Vision to generate the code and DALL-E 3 to generate similar-looking images. You can now also enter a URL to clone a live website!
+A simple tool to convert screenshots, mockups and Figma designs into clean, functional code using AI. **Now supporting GPT-4O!**
 
 https://github.com/abi/screenshot-to-code/assets/23818/6cebadae-2fe3-4986-ac6a-8fb9db030045
 
-See the [Examples](#examples) section below for more demos.
+Supported stacks:
 
-## 🚀 Try It Out!
+- HTML + Tailwind
+- React + Tailwind
+- Vue + Tailwind
+- Bootstrap
+- Ionic + Tailwind
+- SVG
 
-🆕 [Try it here](https://screenshottocode.com) (bring your own OpenAI key - **your key must have access to GPT-4 Vision. See [FAQ](#%EF%B8%8F-faqs) section below for details**). Or see [Getting Started](#-getting-started) below for local install instructions.
+Supported AI models:
 
-## 🌟 Recent Updates
+- GPT-4O - Best model!
+- GPT-4 Turbo (Apr 2024)
+- GPT-4 Vision (Nov 2023)
+- Claude 3 Sonnet
+- DALL-E 3 for image generation
 
-- Nov 28 - 🔥 🔥 🔥 Get output code in React or Bootstrap or TailwindCSS
+See the [Examples](#-examples) section below for more demos.
-- Nov 23 - Send in a screenshot of the current replicated version (sometimes improves quality of subsequent generations)
+
-- Nov 21 - Edit code in the code editor and preview changes live thanks to [@clean99](https://github.com/clean99)
+We also just added experimental support for taking a video/screen recording of a website in action and turning that into a functional prototype. 
-- Nov 20 - Paste in a URL to screenshot and clone (requires [ScreenshotOne free API key](https://screenshotone.com?via=screenshot-to-code))
+
-- Nov 19 - Support for dark/light code editor theme - thanks [@kachbit](https://github.com/kachbit)
+![google in app quick 3](https://github.com/abi/screenshot-to-code/assets/23818/8758ffa4-9483-4b9b-bb66-abd6d1594c33)
-- Nov 16 - Added a setting to disable DALL-E image generation if you don't need that
+
-- Nov 16 - View code directly within the app
+[Learn more about video here](https://github.com/abi/screenshot-to-code/wiki/Screen-Recording-to-Code).
-- Nov 15 - You can now instruct the AI to update the code as you wish. It is helpful if the AI messed up some styles or missed a section.
+
+[Follow me on Twitter for updates](https://twitter.com/_abi_).
+
+## Sponsors
+
+<a href="https://konghq.com/products/kong-konnect/register?utm_medium=referral&utm_source=github&utm_campaign=platform&utm_content=screenshot-to-code" target="_blank" title="Kong - powering the API world"><img src="https://picoapps.xyz/s2c-sponsors/Kong-GitHub-240x100.png"></a>
+
+
+## 🚀 Try It Out without no install
+
+[Try it live on the hosted version (paid)](https://screenshottocode.com).
 
 ## 🛠 Getting Started
 
-The app has a React/Vite frontend and a FastAPI backend. You will need an OpenAI API key with access to the GPT-4 Vision API.
+The app has a React/Vite frontend and a FastAPI backend. You will need an OpenAI API key with access to the GPT-4 Vision API or an Anthropic key if you want to use Claude Sonnet, or for experimental video support.
 
 Run the backend (I use Poetry for package management - `pip install poetry` if you don't have it):
 
@@ -35,6 +54,8 @@ poetry shell
 poetry run uvicorn main:app --reload --port 7001
 ```
 
+If you want to use Anthropic, add the `ANTHROPIC_API_KEY` to `backend/.env` with your API key from Anthropic.
+
 Run the frontend:
 
 ```bash
@@ -68,6 +89,9 @@ The app will be up and running at http://localhost:5173. Note that you can't dev
 
 - **I'm running into an error when setting up the backend. How can I fix it?** [Try this](https://github.com/abi/screenshot-to-code/issues/3#issuecomment-1814777959). If that still doesn't work, open an issue.
 - **How do I get an OpenAI API key?** See https://github.com/abi/screenshot-to-code/blob/main/Troubleshooting.md
+- **How can I configure an OpenAI proxy?** - If you're not able to access the OpenAI API directly (due to e.g. country restrictions), you can try a VPN or you can configure the OpenAI base URL to use a proxy: Set OPENAI_BASE_URL in the `backend/.env` or directly in the UI in the settings dialog. Make sure the URL has "v1" in the path so it should look like this:  `https://xxx.xxxxx.xxx/v1`
+- **How can I update the backend host that my front-end connects to?** - Configure VITE_HTTP_BACKEND_URL and VITE_WS_BACKEND_URL in front/.env.local For example, set VITE_HTTP_BACKEND_URL=http://124.10.20.1:7001
+- **Seeing UTF-8 errors when running the backend?** - On windows, open the .env file with notepad++, then go to Encoding and select UTF-8. 
 - **How can I provide feedback?** For feedback, feature requests and bug reports, open an issue or ping me on [Twitter](https://twitter.com/_abi_).
 
 ## 📚 Examples
@@ -88,6 +112,6 @@ https://github.com/abi/screenshot-to-code/assets/23818/3fec0f77-44e8-4fb3-a769-a
 
 ## 🌍 Hosted Version
 
-🆕 [Try it here](https://screenshottocode.com) (bring your own OpenAI key - **your key must have access to GPT-4 Vision. See [FAQ](#%EF%B8%8F-faqs) section for details**). Or see [Getting Started](#-getting-started) for local install instructions.
+🆕 [Try it here (paid)](https://screenshottocode.com). Or see [Getting Started](#-getting-started) for local install instructions to use with your own API keys.
 
 [!["Buy Me A Coffee"](https://www.buymeacoffee.com/assets/img/custom_images/orange_img.png)](https://www.buymeacoffee.com/abiraja)

Troubleshooting.md

@@ -1,16 +1,22 @@
-### Getting an OpenAI API key
+### Getting an OpenAI API key with GPT4-Vision model access
 
 You don't need a ChatGPT Pro account. Screenshot to code uses API keys from your OpenAI developer account. In order to get access to the GPT4 Vision model, log into your OpenAI account and then, follow these instructions:
 
 1. Open [OpenAI Dashboard](https://platform.openai.com/)
 1. Go to Settings > Billing
 1. Click at the Add payment details
-<img width="1030" alt="285636868-c80deb92-ab47-45cd-988f-deee67fbd44d" src="https://github.com/abi/screenshot-to-code/assets/23818/4e0f4b77-9578-4f9a-803c-c12b1502f3d7">
+<img width="900" alt="285636868-c80deb92-ab47-45cd-988f-deee67fbd44d" src="https://github.com/abi/screenshot-to-code/assets/23818/4e0f4b77-9578-4f9a-803c-c12b1502f3d7">
+
 4. You have to buy some credits. The minimum is $5.
-
 5. Go to Settings > Limits and check at the bottom of the page, your current tier has to be "Tier 1" to have GPT4 access
-<img width="785" alt="285636973-da38bd4d-8a78-4904-8027-ca67d729b933" src="https://github.com/abi/screenshot-to-code/assets/23818/8d07cd84-0cf9-4f88-bc00-80eba492eadf">
+<img width="900" alt="285636973-da38bd4d-8a78-4904-8027-ca67d729b933" src="https://github.com/abi/screenshot-to-code/assets/23818/8d07cd84-0cf9-4f88-bc00-80eba492eadf">
 
-Some users have also reported that it can take upto 30 minutes after your credit purchase for the GPT4 vision model to be activated.
+6. Navigate to OpenAI [api keys](https://platform.openai.com/api-keys) page and create and copy a new secret key.
+7. Go to Screenshot to code and paste it in the Settings dialog under OpenAI key (gear icon). Your key is only stored in your browser. Never stored on our servers.
 
-If you've followed these steps, and it still doesn't work, feel free to open a Github issue.
+## Still not working?
+
+- Some users have also reported that it can take upto 30 minutes after your credit purchase for the GPT4 vision model to be activated.
+- You need to add credits to your account AND set it to renew when credits run out in order to be upgraded to Tier 1. Make sure your "Settings > Limits" page shows that you are at Tier 1.
+
+If you've followed these steps, and it still doesn't work, feel free to open a Github issue. We only provide support for the open source version since we don't have debugging logs on the hosted version. If you're looking to use the hosted version, we recommend getting a paid subscription on screenshottocode.com

backend/.gitignore

@@ -150,3 +150,11 @@ cython_debug/
 #  and can be added to the global gitignore or merged into this file.  For a more nuclear
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
 #.idea/
+
+
+# Temporary eval output
+evals_data
+
+
+# Temporary video evals (Remove before merge)
+video_evals

backend/.pre-commit-config.yaml

@@ -0,0 +1,25 @@
+# See https://pre-commit.com for more information
+# See https://pre-commit.com/hooks.html for more hooks
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v3.2.0
+    hooks:
+      - id: end-of-file-fixer
+      - id: check-yaml
+      - id: check-added-large-files
+  - repo: local
+    hooks:
+      - id: poetry-pytest
+        name: Run pytest with Poetry
+        entry: poetry run --directory backend pytest
+        language: system
+        pass_filenames: false
+        always_run: true
+        files: ^backend/
+      # - id: poetry-pyright
+      #   name: Run pyright with Poetry
+      #   entry: poetry run --directory backend pyright
+      #   language: system
+      #   pass_filenames: false
+      #   always_run: true
+      #   files: ^backend/

backend/README.md

@@ -1,3 +1,7 @@
-Run tests
+# Run the type checker
 
-pytest test_prompts.py
+poetry run pyright
+
+# Run tests
+
+poetry run pytest

backend/access_token.py

@@ -1,25 +0,0 @@
-import json
-import os
-import httpx
-
-
-async def validate_access_token(access_code: str):
-    async with httpx.AsyncClient() as client:
-        url = "http://localhost:8000/screenshot_to_code/validate_access_token"
-        data = json.dumps(
-            {
-                "access_code": access_code,
-                "secret": os.environ.get("PICO_BACKEND_SECRET"),
-            }
-        )
-        headers = {"Content-Type": "application/json"}
-
-        response = await client.post(url, content=data, headers=headers)
-        response_data = response.json()
-
-        if response_data["success"]:
-            print("Access token is valid.")
-            return True
-        else:
-            print(f"Access token validation failed: {response_data['failure_reason']}")
-            return False

backend/config.py

@@ -0,0 +1,16 @@
+# Useful for debugging purposes when you don't want to waste GPT4-Vision credits
+# Setting to True will stream a mock response instead of calling the OpenAI API
+# TODO: Should only be set to true when value is 'True', not any abitrary truthy value
+import os
+
+ANTHROPIC_API_KEY = os.environ.get("ANTHROPIC_API_KEY", None)
+
+# Debugging-related
+
+SHOULD_MOCK_AI_RESPONSE = bool(os.environ.get("MOCK", False))
+IS_DEBUG_ENABLED = bool(os.environ.get("IS_DEBUG_ENABLED", False))
+DEBUG_DIR = os.environ.get("DEBUG_DIR", "")
+
+# Set to True when running in production (on the hosted version)
+# Used as a feature flag to enable or disable certain features
+IS_PROD = os.environ.get("IS_PROD", False)

backend/custom_types.py

@@ -0,0 +1,7 @@
+from typing import Literal
+
+
+InputMode = Literal[
+    "image",
+    "video",
+]

backend/debug/DebugFileWriter.py

@@ -0,0 +1,30 @@
+import os
+import logging
+import uuid
+
+from config import DEBUG_DIR, IS_DEBUG_ENABLED
+
+
+class DebugFileWriter:
+    def __init__(self):
+        if not IS_DEBUG_ENABLED:
+            return
+
+        try:
+            self.debug_artifacts_path = os.path.expanduser(
+                f"{DEBUG_DIR}/{str(uuid.uuid4())}"
+            )
+            os.makedirs(self.debug_artifacts_path, exist_ok=True)
+            print(f"Debugging artifacts will be stored in: {self.debug_artifacts_path}")
+        except:
+            logging.error("Failed to create debug directory")
+
+    def write_to_file(self, filename: str, content: str) -> None:
+        try:
+            with open(os.path.join(self.debug_artifacts_path, filename), "w") as file:
+                file.write(content)
+        except Exception as e:
+            logging.error(f"Failed to write to file: {e}")
+
+    def extract_html_content(self, text: str) -> str:
+        return str(text.split("<html>")[-1].rsplit("</html>", 1)[0] + "</html>")

backend/evals/config.py

@@ -0,0 +1 @@
+EVALS_DIR = "./evals_data"

backend/evals/core.py

@@ -0,0 +1,39 @@
+import os
+from config import ANTHROPIC_API_KEY
+
+from llm import Llm, stream_claude_response, stream_openai_response
+from prompts import assemble_prompt
+from prompts.types import Stack
+
+
+async def generate_code_core(image_url: str, stack: Stack, model: Llm) -> str:
+    prompt_messages = assemble_prompt(image_url, stack)
+    openai_api_key = os.environ.get("OPENAI_API_KEY")
+    anthropic_api_key = ANTHROPIC_API_KEY
+    openai_base_url = None
+
+    async def process_chunk(content: str):
+        pass
+
+    if model == Llm.CLAUDE_3_SONNET:
+        if not anthropic_api_key:
+            raise Exception("Anthropic API key not found")
+
+        completion = await stream_claude_response(
+            prompt_messages,
+            api_key=anthropic_api_key,
+            callback=lambda x: process_chunk(x),
+        )
+    else:
+        if not openai_api_key:
+            raise Exception("OpenAI API key not found")
+
+        completion = await stream_openai_response(
+            prompt_messages,
+            api_key=openai_api_key,
+            base_url=openai_base_url,
+            callback=lambda x: process_chunk(x),
+            model=model,
+        )
+
+    return completion

backend/evals/utils.py

@@ -0,0 +1,7 @@
+import base64
+
+
+async def image_to_data_url(filepath: str):
+    with open(filepath, "rb") as image_file:
+        encoded_string = base64.b64encode(image_file.read()).decode()
+    return f"data:image/png;base64,{encoded_string}"

backend/image_generation.py

@@ -1,28 +1,28 @@
 import asyncio
-import os
 import re
+from typing import Dict, List, Union
 from openai import AsyncOpenAI
 from bs4 import BeautifulSoup
 
 
-async def process_tasks(prompts, api_key):
+async def process_tasks(prompts: List[str], api_key: str, base_url: str):
-    tasks = [generate_image(prompt, api_key) for prompt in prompts]
+    tasks = [generate_image(prompt, api_key, base_url) for prompt in prompts]
     results = await asyncio.gather(*tasks, return_exceptions=True)
 
-    processed_results = []
+    processed_results: List[Union[str, None]] = []
     for result in results:
         if isinstance(result, Exception):
             print(f"An exception occurred: {result}")
             processed_results.append(None)
         else:
-            processed_results.append(result)
+            processed_results.append(result)  # type: ignore
 
     return processed_results
 
 
-async def generate_image(prompt, api_key):
+async def generate_image(prompt: str, api_key: str, base_url: str):
-    client = AsyncOpenAI(api_key=api_key)
+    client = AsyncOpenAI(api_key=api_key, base_url=base_url)
-    image_params = {
+    image_params: Dict[str, Union[str, int]] = {
         "model": "dall-e-3",
         "quality": "standard",
         "style": "natural",
@@ -30,11 +30,12 @@ async def generate_image(prompt, api_key):
         "size": "1024x1024",
         "prompt": prompt,
     }
-    res = await client.images.generate(**image_params)
+    res = await client.images.generate(**image_params)  # type: ignore
+    await client.close()
     return res.data[0].url
 
 
-def extract_dimensions(url):
+def extract_dimensions(url: str):
     # Regular expression to match numbers in the format '300x200'
     matches = re.findall(r"(\d+)x(\d+)", url)
 
@@ -47,11 +48,11 @@ def extract_dimensions(url):
         return (100, 100)
 
 
-def create_alt_url_mapping(code):
+def create_alt_url_mapping(code: str) -> Dict[str, str]:
     soup = BeautifulSoup(code, "html.parser")
     images = soup.find_all("img")
 
-    mapping = {}
+    mapping: Dict[str, str] = {}
 
     for image in images:
         if not image["src"].startswith("https://placehold.co"):
@@ -60,7 +61,9 @@ def create_alt_url_mapping(code):
     return mapping
 
 
-async def generate_images(code, api_key, image_cache):
+async def generate_images(
+    code: str, api_key: str, base_url: Union[str, None], image_cache: Dict[str, str]
+):
     # Find all images
     soup = BeautifulSoup(code, "html.parser")
     images = soup.find_all("img")
@@ -74,26 +77,26 @@ async def generate_images(code, api_key, image_cache):
             img["src"].startswith("https://placehold.co")
             and image_cache.get(img.get("alt")) is None
         ):
-            alts.append(img.get("alt", None))
+            alts.append(img.get("alt", None))  # type: ignore
 
     # Exclude images with no alt text
-    alts = [alt for alt in alts if alt is not None]
+    alts = [alt for alt in alts if alt is not None]  # type: ignore
 
     # Remove duplicates
-    prompts = list(set(alts))
+    prompts = list(set(alts))  # type: ignore
 
     # Return early if there are no images to replace
-    if len(prompts) == 0:
+    if len(prompts) == 0:  # type: ignore
         return code
 
     # Generate images
-    results = await process_tasks(prompts, api_key)
+    results = await process_tasks(prompts, api_key, base_url)  # type: ignore
 
     # Create a dict mapping alt text to image URL
-    mapped_image_urls = dict(zip(prompts, results))
+    mapped_image_urls = dict(zip(prompts, results))  # type: ignore
 
     # Merge with image_cache
-    mapped_image_urls = {**mapped_image_urls, **image_cache}
+    mapped_image_urls = {**mapped_image_urls, **image_cache}  # type: ignore
 
     # Replace old image URLs with the generated URLs
     for img in images:

backend/llm.py

@@ -1,30 +1,217 @@
-import os
+from enum import Enum
-from typing import Awaitable, Callable
+from typing import Any, Awaitable, Callable, List, cast
+from anthropic import AsyncAnthropic
 from openai import AsyncOpenAI
+from openai.types.chat import ChatCompletionMessageParam, ChatCompletionChunk
+from config import IS_DEBUG_ENABLED
+from debug.DebugFileWriter import DebugFileWriter
 
-MODEL_GPT_4_VISION = "gpt-4-vision-preview"
+from utils import pprint_prompt
+
+
+# Actual model versions that are passed to the LLMs and stored in our logs
+class Llm(Enum):
+    GPT_4_VISION = "gpt-4-vision-preview"
+    GPT_4_TURBO_2024_04_09 = "gpt-4-turbo-2024-04-09"
+    GPT_4O_2024_05_13 = "gpt-4o-2024-05-13"
+    CLAUDE_3_SONNET = "claude-3-sonnet-20240229"
+    CLAUDE_3_OPUS = "claude-3-opus-20240229"
+    CLAUDE_3_HAIKU = "claude-3-haiku-20240307"
+
+
+# Will throw errors if you send a garbage string
+def convert_frontend_str_to_llm(frontend_str: str) -> Llm:
+    if frontend_str == "gpt_4_vision":
+        return Llm.GPT_4_VISION
+    elif frontend_str == "claude_3_sonnet":
+        return Llm.CLAUDE_3_SONNET
+    else:
+        return Llm(frontend_str)
 
 
 async def stream_openai_response(
-    messages, api_key: str, callback: Callable[[str], Awaitable[None]]
+    messages: List[ChatCompletionMessageParam],
-):
+    api_key: str,
-    client = AsyncOpenAI(api_key=api_key)
+    base_url: str | None,
-
+    callback: Callable[[str], Awaitable[None]],
-    model = MODEL_GPT_4_VISION
+    model: Llm,
+) -> str:
+    client = AsyncOpenAI(api_key=api_key, base_url=base_url)
 
     # Base parameters
-    params = {"model": model, "messages": messages, "stream": True, "timeout": 600}
+    params = {
+        "model": model.value,
+        "messages": messages,
+        "stream": True,
+        "timeout": 600,
+        "temperature": 0.0,
+    }
 
-    # Add 'max_tokens' only if the model is a GPT4 vision model
+    # Add 'max_tokens' only if the model is a GPT4 vision or Turbo model
-    if model == MODEL_GPT_4_VISION:
+    if (
+        model == Llm.GPT_4_VISION
+        or model == Llm.GPT_4_TURBO_2024_04_09
+        or model == Llm.GPT_4O_2024_05_13
+    ):
         params["max_tokens"] = 4096
-        params["temperature"] = 0
 
-    completion = await client.chat.completions.create(**params)
+    stream = await client.chat.completions.create(**params)  # type: ignore
     full_response = ""
-    async for chunk in completion:
+    async for chunk in stream:  # type: ignore
+        assert isinstance(chunk, ChatCompletionChunk)
         content = chunk.choices[0].delta.content or ""
         full_response += content
         await callback(content)
 
+    await client.close()
+
     return full_response
+
+
+# TODO: Have a seperate function that translates OpenAI messages to Claude messages
+async def stream_claude_response(
+    messages: List[ChatCompletionMessageParam],
+    api_key: str,
+    callback: Callable[[str], Awaitable[None]],
+) -> str:
+
+    client = AsyncAnthropic(api_key=api_key)
+
+    # Base parameters
+    model = Llm.CLAUDE_3_SONNET
+    max_tokens = 4096
+    temperature = 0.0
+
+    # Translate OpenAI messages to Claude messages
+    system_prompt = cast(str, messages[0].get("content"))
+    claude_messages = [dict(message) for message in messages[1:]]
+    for message in claude_messages:
+        if not isinstance(message["content"], list):
+            continue
+
+        for content in message["content"]:  # type: ignore
+            if content["type"] == "image_url":
+                content["type"] = "image"
+
+                # Extract base64 data and media type from data URL
+                # Example base64 data URL: data:image/png;base64,iVBOR...
+                image_data_url = cast(str, content["image_url"]["url"])
+                media_type = image_data_url.split(";")[0].split(":")[1]
+                base64_data = image_data_url.split(",")[1]
+
+                # Remove OpenAI parameter
+                del content["image_url"]
+
+                content["source"] = {
+                    "type": "base64",
+                    "media_type": media_type,
+                    "data": base64_data,
+                }
+
+    # Stream Claude response
+    async with client.messages.stream(
+        model=model.value,
+        max_tokens=max_tokens,
+        temperature=temperature,
+        system=system_prompt,
+        messages=claude_messages,  # type: ignore
+    ) as stream:
+        async for text in stream.text_stream:
+            await callback(text)
+
+    # Return final message
+    response = await stream.get_final_message()
+
+    # Close the Anthropic client
+    await client.close()
+
+    return response.content[0].text
+
+
+async def stream_claude_response_native(
+    system_prompt: str,
+    messages: list[Any],
+    api_key: str,
+    callback: Callable[[str], Awaitable[None]],
+    include_thinking: bool = False,
+    model: Llm = Llm.CLAUDE_3_OPUS,
+) -> str:
+
+    client = AsyncAnthropic(api_key=api_key)
+
+    # Base model parameters
+    max_tokens = 4096
+    temperature = 0.0
+
+    # Multi-pass flow
+    current_pass_num = 1
+    max_passes = 2
+
+    prefix = "<thinking>"
+    response = None
+
+    # For debugging
+    full_stream = ""
+    debug_file_writer = DebugFileWriter()
+
+    while current_pass_num <= max_passes:
+        current_pass_num += 1
+
+        # Set up message depending on whether we have a <thinking> prefix
+        messages_to_send = (
+            messages + [{"role": "assistant", "content": prefix}]
+            if include_thinking
+            else messages
+        )
+
+        pprint_prompt(messages_to_send)
+
+        async with client.messages.stream(
+            model=model.value,
+            max_tokens=max_tokens,
+            temperature=temperature,
+            system=system_prompt,
+            messages=messages_to_send,  # type: ignore
+        ) as stream:
+            async for text in stream.text_stream:
+                print(text, end="", flush=True)
+                full_stream += text
+                await callback(text)
+
+        response = await stream.get_final_message()
+        response_text = response.content[0].text
+
+        # Write each pass's code to .html file and thinking to .txt file
+        if IS_DEBUG_ENABLED:
+            debug_file_writer.write_to_file(
+                f"pass_{current_pass_num - 1}.html",
+                debug_file_writer.extract_html_content(response_text),
+            )
+            debug_file_writer.write_to_file(
+                f"thinking_pass_{current_pass_num - 1}.txt",
+                response_text.split("</thinking>")[0],
+            )
+
+        # Set up messages array for next pass
+        messages += [
+            {"role": "assistant", "content": str(prefix) + response.content[0].text},
+            {
+                "role": "user",
+                "content": "You've done a good job with a first draft. Improve this further based on the original instructions so that the app is fully functional and looks like the original video of the app we're trying to replicate.",
+            },
+        ]
+
+        print(
+            f"Token usage: Input Tokens: {response.usage.input_tokens}, Output Tokens: {response.usage.output_tokens}"
+        )
+
+    # Close the Anthropic client
+    await client.close()
+
+    if IS_DEBUG_ENABLED:
+        debug_file_writer.write_to_file("full_stream.txt", full_stream)
+
+    if not response:
+        raise Exception("No HTML response found in AI response")
+    else:
+        return response.content[0].text

backend/main.py

@@ -4,23 +4,12 @@ from dotenv import load_dotenv
 load_dotenv()
 
 
-import json
+from fastapi import FastAPI
-import os
-import traceback
-from datetime import datetime
-from fastapi import FastAPI, WebSocket
 from fastapi.middleware.cors import CORSMiddleware
-from llm import stream_openai_response
+from routes import screenshot, generate_code, home, evals
-from mock import mock_completion
-from image_generation import create_alt_url_mapping, generate_images
-from prompts import assemble_prompt
-from routes import screenshot
-from access_token import validate_access_token
 
 app = FastAPI(openapi_url=None, docs_url=None, redoc_url=None)
 
-# Configure CORS
-
 # Configure CORS settings
 app.add_middleware(
     CORSMiddleware,
@@ -30,150 +19,8 @@ app.add_middleware(
     allow_headers=["*"],
 )
 
-
+# Add routes
-# Useful for debugging purposes when you don't want to waste GPT4-Vision credits
+app.include_router(generate_code.router)
-# Setting to True will stream a mock response instead of calling the OpenAI API
-# TODO: Should only be set to true when value is 'True', not any abitrary truthy value
-SHOULD_MOCK_AI_RESPONSE = bool(os.environ.get("MOCK", False))
-
-
 app.include_router(screenshot.router)
-
+app.include_router(home.router)
-
+app.include_router(evals.router)
-def write_logs(prompt_messages, completion):
-    # Get the logs path from environment, default to the current working directory
-    logs_path = os.environ.get("LOGS_PATH", os.getcwd())
-
-    # Create run_logs directory if it doesn't exist within the specified logs path
-    logs_directory = os.path.join(logs_path, "run_logs")
-    if not os.path.exists(logs_directory):
-        os.makedirs(logs_directory)
-
-    print("Writing to logs directory:", logs_directory)
-
-    # Generate a unique filename using the current timestamp within the logs directory
-    filename = datetime.now().strftime(f"{logs_directory}/messages_%Y%m%d_%H%M%S.json")
-
-    # Write the messages dict into a new file for each run
-    with open(filename, "w") as f:
-        f.write(json.dumps({"prompt": prompt_messages, "completion": completion}))
-
-
-@app.websocket("/generate-code")
-async def stream_code(websocket: WebSocket):
-    await websocket.accept()
-
-    print("Incoming websocket connection...")
-
-    params = await websocket.receive_json()
-
-    print("Received params")
-
-    # Read the output settings from the request. Fall back to default if not provided.
-    output_settings = {"css": "tailwind", "js": "vanilla"}
-    if params["outputSettings"] and params["outputSettings"]["css"]:
-        output_settings["css"] = params["outputSettings"]["css"]
-    if params["outputSettings"] and params["outputSettings"]["js"]:
-        output_settings["js"] = params["outputSettings"]["js"]
-    print("Using output settings:", output_settings)
-
-    # Get the OpenAI API key from the request. Fall back to environment variable if not provided.
-    # If neither is provided, we throw an error.
-    openai_api_key = None
-    if "accessCode" in params and params["accessCode"]:
-        print("Access code - using platform API key")
-        if await validate_access_token(params["accessCode"]):
-            openai_api_key = os.environ.get("PLATFORM_OPENAI_API_KEY")
-        else:
-            await websocket.send_json(
-                {
-                    "type": "error",
-                    "value": "Invalid access code or you're out of credits. Please try again.",
-                }
-            )
-            return
-    else:
-        if params["openAiApiKey"]:
-            openai_api_key = params["openAiApiKey"]
-            print("Using OpenAI API key from client-side settings dialog")
-        else:
-            openai_api_key = os.environ.get("OPENAI_API_KEY")
-            if openai_api_key:
-                print("Using OpenAI API key from environment variable")
-
-    if not openai_api_key:
-        print("OpenAI API key not found")
-        await websocket.send_json(
-            {
-                "type": "error",
-                "value": "No OpenAI API key found. Please add your API key in the settings dialog or add it to backend/.env file.",
-            }
-        )
-        return
-
-    should_generate_images = (
-        params["isImageGenerationEnabled"]
-        if "isImageGenerationEnabled" in params
-        else True
-    )
-
-    print("generating code...")
-    await websocket.send_json({"type": "status", "value": "Generating code..."})
-
-    async def process_chunk(content):
-        await websocket.send_json({"type": "chunk", "value": content})
-
-    if params.get("resultImage") and params["resultImage"]:
-        prompt_messages = assemble_prompt(
-            params["image"], output_settings, params["resultImage"]
-        )
-    else:
-        prompt_messages = assemble_prompt(params["image"], output_settings)
-
-    # Image cache for updates so that we don't have to regenerate images
-    image_cache = {}
-
-    if params["generationType"] == "update":
-        # Transform into message format
-        # TODO: Move this to frontend
-        for index, text in enumerate(params["history"]):
-            prompt_messages += [
-                {"role": "assistant" if index % 2 == 0 else "user", "content": text}
-            ]
-
-        image_cache = create_alt_url_mapping(params["history"][-2])
-
-    if SHOULD_MOCK_AI_RESPONSE:
-        completion = await mock_completion(process_chunk)
-    else:
-        completion = await stream_openai_response(
-            prompt_messages,
-            api_key=openai_api_key,
-            callback=lambda x: process_chunk(x),
-        )
-
-    # Write the messages dict into a log so that we can debug later
-    write_logs(prompt_messages, completion)
-
-    try:
-        if should_generate_images:
-            await websocket.send_json(
-                {"type": "status", "value": "Generating images..."}
-            )
-            updated_html = await generate_images(
-                completion, api_key=openai_api_key, image_cache=image_cache
-            )
-        else:
-            updated_html = completion
-        await websocket.send_json({"type": "setCode", "value": updated_html})
-        await websocket.send_json(
-            {"type": "status", "value": "Code generation complete."}
-        )
-    except Exception as e:
-        traceback.print_exc()
-        print("Image generation failed", e)
-        await websocket.send_json(
-            {"type": "status", "value": "Image generation failed but code is complete."}
-        )
-    finally:
-        await websocket.close()

backend/mock.py

@@ -1,207 +0,0 @@
-import asyncio
-
-
-async def mock_completion(process_chunk):
-    code_to_return = NO_IMAGES_NYTIMES_MOCK_CODE
-
-    for i in range(0, len(code_to_return), 10):
-        await process_chunk(code_to_return[i : i + 10])
-        await asyncio.sleep(0.01)
-
-    return code_to_return
-
-
-APPLE_MOCK_CODE = """<html lang="en">
-<head>
-    <meta charset="UTF-8">
-    <meta name="viewport" content="width=device-width, initial-scale=1.0">
-    <title>Product Showcase</title>
-    <script src="https://cdn.tailwindcss.com"></script>
-    <link href="https://fonts.googleapis.com/css2?family=Roboto:wght@400;500;700&display=swap" rel="stylesheet">
-    <style>
-        body {
-            font-family: 'Roboto', sans-serif;
-        }
-    </style>
-</head>
-<body class="bg-black text-white">
-    <nav class="py-6">
-        <div class="max-w-7xl mx-auto px-4 sm:px-6 lg:px-8 flex justify-between items-center">
-            <div class="flex items-center">
-                <img src="https://placehold.co/24x24" alt="Company Logo" class="mr-8">
-                <a href="#" class="text-white text-sm font-medium mr-4">Store</a>
-                <a href="#" class="text-white text-sm font-medium mr-4">Mac</a>
-                <a href="#" class="text-white text-sm font-medium mr-4">iPad</a>
-                <a href="#" class="text-white text-sm font-medium mr-4">iPhone</a>
-                <a href="#" class="text-white text-sm font-medium mr-4">Watch</a>
-                <a href="#" class="text-white text-sm font-medium mr-4">Vision</a>
-                <a href="#" class="text-white text-sm font-medium mr-4">AirPods</a>
-                <a href="#" class="text-white text-sm font-medium mr-4">TV & Home</a>
-                <a href="#" class="text-white text-sm font-medium mr-4">Entertainment</a>
-                <a href="#" class="text-white text-sm font-medium mr-4">Accessories</a>
-                <a href="#" class="text-white text-sm font-medium">Support</a>
-            </div>
-            <div class="flex items-center">
-                <a href="#" class="text-white text-sm font-medium mr-4"><i class="fas fa-search"></i></a>
-                <a href="#" class="text-white text-sm font-medium"><i class="fas fa-shopping-bag"></i></a>
-            </div>
-        </div>
-    </nav>
-
-    <main class="mt-8">
-        <div class="max-w-7xl mx-auto px-4 sm:px-6 lg:px-8">
-            <div class="text-center">
-                <img src="https://placehold.co/100x100" alt="Brand Logo" class="mx-auto mb-4">
-                <h1 class="text-5xl font-bold mb-4">WATCH SERIES 9</h1>
-                <p class="text-2xl font-medium mb-8">Smarter. Brighter. Mightier.</p>
-                <div class="flex justify-center space-x-4">
-                    <a href="#" class="text-blue-600 text-sm font-medium">Learn more ></a>
-                    <a href="#" class="text-blue-600 text-sm font-medium">Buy ></a>
-                </div>
-            </div>
-            <div class="flex justify-center mt-12">
-                <img src="https://placehold.co/500x300" alt="Product image of a smartwatch with a pink band and a circular interface displaying various health metrics." class="mr-8">
-                <img src="https://placehold.co/500x300" alt="Product image of a smartwatch with a blue band and a square interface showing a classic analog clock face." class="ml-8">
-            </div>
-        </div>
-    </main>
-</body>
-</html>"""
-
-NYTIMES_MOCK_CODE = """
-<html lang="en">
-<head>
-    <meta charset="UTF-8">
-    <meta name="viewport" content="width=device-width, initial-scale=1.0">
-    <title>The New York Times - News</title>
-    <script src="https://cdn.tailwindcss.com"></script>
-    <link href="https://fonts.googleapis.com/css2?family=Libre+Franklin:wght@300;400;700&display=swap" rel="stylesheet">
-    <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.15.3/css/all.min.css">
-    <style>
-        body {
-            font-family: 'Libre Franklin', sans-serif;
-        }
-    </style>
-</head>
-<body class="bg-gray-100">
-    <div class="container mx-auto px-4">
-        <header class="border-b border-gray-300 py-4">
-            <div class="flex justify-between items-center">
-                <div class="flex items-center space-x-4">
-                    <button class="text-gray-700"><i class="fas fa-bars"></i></button>
-                    <button class="text-gray-700"><i class="fas fa-search"></i></button>
-                    <div class="text-xs uppercase tracking-widest">Tuesday, November 14, 2023<br>Today's Paper</div>
-                </div>
-                <div>
-                    <img src="https://placehold.co/200x50?text=The+New+York+Times+Logo" alt="The New York Times Logo" class="h-8">
-                </div>
-                <div class="flex items-center space-x-4">
-                    <button class="bg-black text-white px-4 py-1 text-xs uppercase tracking-widest">Give the times</button>
-                    <div class="text-xs">Account</div>
-                </div>
-            </div>
-            <nav class="flex justify-between items-center py-4">
-                <div class="flex space-x-4">
-                    <a href="#" class="text-xs uppercase tracking-widest text-gray-700">U.S.</a>
-                    <!-- Add other navigation links as needed -->
-                </div>
-                <div class="flex space-x-4">
-                    <a href="#" class="text-xs uppercase tracking-widest text-gray-700">Cooking</a>
-                    <!-- Add other navigation links as needed -->
-                </div>
-            </nav>
-        </header>
-        <main>
-            <section class="py-6">
-                <div class="grid grid-cols-3 gap-4">
-                    <div class="col-span-2">
-                        <article class="mb-4">
-                            <h2 class="text-xl font-bold mb-2">Israeli Military Raids Gaza’s Largest Hospital</h2>
-                            <p class="text-gray-700 mb-2">Israeli troops have entered the Al-Shifa Hospital complex, where conditions have grown dire and Israel says Hamas fighters are embedded.</p>
-                            <a href="#" class="text-blue-600 text-sm">See more updates <i class="fas fa-external-link-alt"></i></a>
-                        </article>
-                        <!-- Repeat for each news item -->
-                    </div>
-                    <div class="col-span-1">
-                        <article class="mb-4">
-                            <img src="https://placehold.co/300x200?text=News+Image" alt="Flares and plumes of smoke over the northern Gaza skyline on Tuesday." class="mb-2">
-                            <h2 class="text-xl font-bold mb-2">From Elvis to Elopements, the Evolution of the Las Vegas Wedding</h2>
-                            <p class="text-gray-700 mb-2">The glittering city that attracts thousands of couples seeking unconventional nuptials has grown beyond the drive-through wedding.</p>
-                            <a href="#" class="text-blue-600 text-sm">8 MIN READ</a>
-                        </article>
-                        <!-- Repeat for each news item -->
-                    </div>
-                </div>
-            </section>
-        </main>
-    </div>
-</body>
-</html>
-"""
-
-NO_IMAGES_NYTIMES_MOCK_CODE = """
-<html lang="en">
-<head>
-    <meta charset="UTF-8">
-    <meta name="viewport" content="width=device-width, initial-scale=1.0">
-    <title>The New York Times - News</title>
-    <script src="https://cdn.tailwindcss.com"></script>
-    <link href="https://fonts.googleapis.com/css2?family=Libre+Franklin:wght@300;400;700&display=swap" rel="stylesheet">
-    <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.15.3/css/all.min.css">
-    <style>
-        body {
-            font-family: 'Libre Franklin', sans-serif;
-        }
-    </style>
-</head>
-<body class="bg-gray-100">
-    <div class="container mx-auto px-4">
-        <header class="border-b border-gray-300 py-4">
-            <div class="flex justify-between items-center">
-                <div class="flex items-center space-x-4">
-                    <button class="text-gray-700"><i class="fas fa-bars"></i></button>
-                    <button class="text-gray-700"><i class="fas fa-search"></i></button>
-                    <div class="text-xs uppercase tracking-widest">Tuesday, November 14, 2023<br>Today's Paper</div>
-                </div>
-                <div class="flex items-center space-x-4">
-                    <button class="bg-black text-white px-4 py-1 text-xs uppercase tracking-widest">Give the times</button>
-                    <div class="text-xs">Account</div>
-                </div>
-            </div>
-            <nav class="flex justify-between items-center py-4">
-                <div class="flex space-x-4">
-                    <a href="#" class="text-xs uppercase tracking-widest text-gray-700">U.S.</a>
-                    <!-- Add other navigation links as needed -->
-                </div>
-                <div class="flex space-x-4">
-                    <a href="#" class="text-xs uppercase tracking-widest text-gray-700">Cooking</a>
-                    <!-- Add other navigation links as needed -->
-                </div>
-            </nav>
-        </header>
-        <main>
-            <section class="py-6">
-                <div class="grid grid-cols-3 gap-4">
-                    <div class="col-span-2">
-                        <article class="mb-4">
-                            <h2 class="text-xl font-bold mb-2">Israeli Military Raids Gaza’s Largest Hospital</h2>
-                            <p class="text-gray-700 mb-2">Israeli troops have entered the Al-Shifa Hospital complex, where conditions have grown dire and Israel says Hamas fighters are embedded.</p>
-                            <a href="#" class="text-blue-600 text-sm">See more updates <i class="fas fa-external-link-alt"></i></a>
-                        </article>
-                        <!-- Repeat for each news item -->
-                    </div>
-                    <div class="col-span-1">
-                        <article class="mb-4">
-                            <h2 class="text-xl font-bold mb-2">From Elvis to Elopements, the Evolution of the Las Vegas Wedding</h2>
-                            <p class="text-gray-700 mb-2">The glittering city that attracts thousands of couples seeking unconventional nuptials has grown beyond the drive-through wedding.</p>
-                            <a href="#" class="text-blue-600 text-sm">8 MIN READ</a>
-                        </article>
-                        <!-- Repeat for each news item -->
-                    </div>
-                </div>
-            </section>
-        </main>
-    </div>
-</body>
-</html>
-"""

backend/prompts.py

@@ -1,127 +0,0 @@
-TAILWIND_SYSTEM_PROMPT = """
-You are an expert Tailwind developer
-You take screenshots of a reference web page from the user, and then build single page apps 
-using Tailwind, HTML and JS.
-You might also be given a screenshot(The second image) of a web page that you have already built, and asked to
-update it to look more like the reference image(The first image).
-
-- Make sure the app looks exactly like the screenshot.
-- Pay close attention to background color, text color, font size, font family, 
-padding, margin, border, etc. Match the colors and sizes exactly.
-- Use the exact text from the screenshot.
-- Do not add comments in the code such as "<!-- Add other navigation links as needed -->" and "<!-- ... other news items ... -->" in place of writing the full code. WRITE THE FULL CODE.
-- Repeat elements as needed to match the screenshot. For example, if there are 15 items, the code should have 15 items. DO NOT LEAVE comments like "<!-- Repeat for each news item -->" or bad things will happen.
-- For images, use placeholder images from https://placehold.co and include a detailed description of the image in the alt text so that an image generation AI can generate the image later.
-
-In terms of libraries,
-
-- Use this script to include Tailwind: <script src="https://cdn.tailwindcss.com"></script>
-- You can use Google Fonts
-- Font Awesome for icons: <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.15.3/css/all.min.css"></link>
-
-Return only the full code in <html></html> tags.
-Do not include markdown "```" or "```html" at the start or end.
-"""
-
-BOOTSTRAP_SYSTEM_PROMPT = """
-You are an expert Bootstrap developer
-You take screenshots of a reference web page from the user, and then build single page apps 
-using Bootstrap, HTML and JS.
-You might also be given a screenshot(The second image) of a web page that you have already built, and asked to
-update it to look more like the reference image(The first image).
-
-- Make sure the app looks exactly like the screenshot.
-- Pay close attention to background color, text color, font size, font family, 
-padding, margin, border, etc. Match the colors and sizes exactly.
-- Use the exact text from the screenshot.
-- Do not add comments in the code such as "<!-- Add other navigation links as needed -->" and "<!-- ... other news items ... -->" in place of writing the full code. WRITE THE FULL CODE.
-- Repeat elements as needed to match the screenshot. For example, if there are 15 items, the code should have 15 items. DO NOT LEAVE comments like "<!-- Repeat for each news item -->" or bad things will happen.
-- For images, use placeholder images from https://placehold.co and include a detailed description of the image in the alt text so that an image generation AI can generate the image later.
-
-In terms of libraries,
-
-- Use this script to include Bootstrap: <link href="https://cdn.jsdelivr.net/npm/bootstrap@5.3.2/dist/css/bootstrap.min.css" rel="stylesheet" integrity="sha384-T3c6CoIi6uLrA9TneNEoa7RxnatzjcDSCmG1MXxSR1GAsXEV/Dwwykc2MPK8M2HN" crossorigin="anonymous">
-- You can use Google Fonts
-- Font Awesome for icons: <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.15.3/css/all.min.css"></link>
-
-Return only the full code in <html></html> tags.
-Do not include markdown "```" or "```html" at the start or end.
-"""
-
-REACT_TAILWIND_SYSTEM_PROMPT = """
-You are an expert React/Tailwind developer
-You take screenshots of a reference web page from the user, and then build single page apps 
-using React and Tailwind CSS.
-You might also be given a screenshot(The second image) of a web page that you have already built, and asked to
-update it to look more like the reference image(The first image).
-
-- Make sure the app looks exactly like the screenshot.
-- Pay close attention to background color, text color, font size, font family, 
-padding, margin, border, etc. Match the colors and sizes exactly.
-- Use the exact text from the screenshot.
-- Do not add comments in the code such as "<!-- Add other navigation links as needed -->" and "<!-- ... other news items ... -->" in place of writing the full code. WRITE THE FULL CODE.
-- Repeat elements as needed to match the screenshot. For example, if there are 15 items, the code should have 15 items. DO NOT LEAVE comments like "<!-- Repeat for each news item -->" or bad things will happen.
-- For images, use placeholder images from https://placehold.co and include a detailed description of the image in the alt text so that an image generation AI can generate the image later.
-
-In terms of libraries,
-
-- Use these script to include React so that it can run on a standalone page:
-    <script src="https://unpkg.com/react/umd/react.development.js"></script>
-    <script src="https://unpkg.com/react-dom/umd/react-dom.development.js"></script>
-    <script src="https://unpkg.com/@babel/standalone/babel.js"></script>
-- Use this script to include Tailwind: <script src="https://cdn.tailwindcss.com"></script>
-- You can use Google Fonts
-- Font Awesome for icons: <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.15.3/css/all.min.css"></link>
-
-Return only the full code in <html></html> tags.
-Do not include markdown "```" or "```html" at the start or end.
-"""
-
-USER_PROMPT = """
-Generate code for a web page that looks exactly like this.
-"""
-
-
-def assemble_prompt(image_data_url, output_settings: dict, result_image_data_url=None):
-    # Set the system prompt based on the output settings
-    chosen_prompt_name = "tailwind"
-    system_content = TAILWIND_SYSTEM_PROMPT
-    if output_settings["css"] == "bootstrap":
-        chosen_prompt_name = "bootstrap"
-        system_content = BOOTSTRAP_SYSTEM_PROMPT
-    if output_settings["js"] == "react":
-        chosen_prompt_name = "react-tailwind"
-        system_content = REACT_TAILWIND_SYSTEM_PROMPT
-
-    print("Using system prompt:", chosen_prompt_name)
-
-    user_content = [
-        {
-            "type": "image_url",
-            "image_url": {"url": image_data_url, "detail": "high"},
-        },
-        {
-            "type": "text",
-            "text": USER_PROMPT,
-        },
-    ]
-
-    # Include the result image if it exists
-    if result_image_data_url:
-        user_content.insert(
-            1,
-            {
-                "type": "image_url",
-                "image_url": {"url": result_image_data_url, "detail": "high"},
-            },
-        )
-    return [
-        {
-            "role": "system",
-            "content": system_content,
-        },
-        {
-            "role": "user",
-            "content": user_content,
-        },
-    ]

backend/prompts/__init__.py

@@ -0,0 +1,79 @@
+from typing import List, NoReturn, Union
+
+from openai.types.chat import ChatCompletionMessageParam, ChatCompletionContentPartParam
+
+from prompts.imported_code_prompts import IMPORTED_CODE_SYSTEM_PROMPTS
+from prompts.screenshot_system_prompts import SYSTEM_PROMPTS
+from prompts.types import Stack
+
+
+USER_PROMPT = """
+Generate code for a web page that looks exactly like this.
+"""
+
+SVG_USER_PROMPT = """
+Generate code for a SVG that looks exactly like this.
+"""
+
+
+def assemble_imported_code_prompt(
+    code: str, stack: Stack, result_image_data_url: Union[str, None] = None
+) -> List[ChatCompletionMessageParam]:
+    system_content = IMPORTED_CODE_SYSTEM_PROMPTS[stack]
+
+    user_content = (
+        "Here is the code of the app: " + code
+        if stack != "svg"
+        else "Here is the code of the SVG: " + code
+    )
+    return [
+        {
+            "role": "system",
+            "content": system_content,
+        },
+        {
+            "role": "user",
+            "content": user_content,
+        },
+    ]
+    # TODO: Use result_image_data_url
+
+
+def assemble_prompt(
+    image_data_url: str,
+    stack: Stack,
+    result_image_data_url: Union[str, None] = None,
+) -> List[ChatCompletionMessageParam]:
+    system_content = SYSTEM_PROMPTS[stack]
+    user_prompt = USER_PROMPT if stack != "svg" else SVG_USER_PROMPT
+
+    user_content: List[ChatCompletionContentPartParam] = [
+        {
+            "type": "image_url",
+            "image_url": {"url": image_data_url, "detail": "high"},
+        },
+        {
+            "type": "text",
+            "text": user_prompt,
+        },
+    ]
+
+    # Include the result image if it exists
+    if result_image_data_url:
+        user_content.insert(
+            1,
+            {
+                "type": "image_url",
+                "image_url": {"url": result_image_data_url, "detail": "high"},
+            },
+        )
+    return [
+        {
+            "role": "system",
+            "content": system_content,
+        },
+        {
+            "role": "user",
+            "content": user_content,
+        },
+    ]

backend/prompts/claude_prompts.py

@@ -0,0 +1,114 @@
+# Not used yet
+# References:
+# https://github.com/hundredblocks/transcription_demo
+# https://docs.anthropic.com/claude/docs/prompt-engineering
+# https://github.com/anthropics/anthropic-cookbook/blob/main/multimodal/best_practices_for_vision.ipynb
+
+VIDEO_PROMPT = """
+You are an expert at building single page, funtional apps using HTML, Jquery and Tailwind CSS.
+You also have perfect vision and pay great attention to detail.
+
+You will be given screenshots in order at consistent intervals from a video of a user interacting with a web app. You need to re-create the same app exactly such that the same user interactions will produce the same results in the app you build.
+
+- Make sure the app looks exactly like the screenshot.
+- Pay close attention to background color, text color, font size, font family, 
+padding, margin, border, etc. Match the colors and sizes exactly.
+- For images, use placeholder images from https://placehold.co and include a detailed description of the image in the alt text so that an image generation AI can generate the image later.
+- If some fuctionality requires a backend call, just mock the data instead.
+- MAKE THE APP FUNCTIONAL using Javascript. Allow the user to interact with the app and get the same behavior as the video.
+
+In terms of libraries,
+
+- Use this script to include Tailwind: <script src="https://cdn.tailwindcss.com"></script>
+- You can use Google Fonts
+- Font Awesome for icons: <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.15.3/css/all.min.css"></link>
+- Use jQuery: <script src="https://code.jquery.com/jquery-3.7.1.min.js"></script>
+
+Before generating the code for the app, think step-by-step: first, about the user flow depicated in the video and then about you how would you build it and how you would structure the code. Do the thinking within <thinking></thinking> tags. Then, provide your code within <html></html> tags.
+"""
+
+VIDEO_PROMPT_ALPINE_JS = """
+You are an expert at building single page, funtional apps using HTML, Alpine.js and Tailwind CSS.
+You also have perfect vision and pay great attention to detail.
+
+You will be given screenshots in order at consistent intervals from a video of a user interacting with a web app. You need to re-create the same app exactly such that the same user interactions will produce the same results in the app you build.
+
+- Make sure the app looks exactly like the screenshot.
+- Pay close attention to background color, text color, font size, font family, 
+padding, margin, border, etc. Match the colors and sizes exactly.
+- For images, use placeholder images from https://placehold.co and include a detailed description of the image in the alt text so that an image generation AI can generate the image later.
+- If some fuctionality requires a backend call, just mock the data instead.
+- MAKE THE APP FUNCTIONAL using Javascript. Allow the user to interact with the app and get the same behavior as the video.
+
+In terms of libraries,
+
+- Use this script to include Tailwind: <script src="https://cdn.tailwindcss.com"></script>
+- You can use Google Fonts
+- Font Awesome for icons: <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.15.3/css/all.min.css"></link>
+- Use Alpine.js: <script defer src="https://cdn.jsdelivr.net/npm/alpinejs@3.x.x/dist/cdn.min.js"></script>
+
+Before generating the code for the app, think step-by-step: first, about the user flow depicated in the video and then about you how would you build it and how you would structure the code. Do the thinking within <thinking></thinking> tags. Then, provide your code within <html></html> tags.
+"""
+
+
+HTML_TAILWIND_CLAUDE_SYSTEM_PROMPT = """
+You have perfect vision and pay great attention to detail which makes you an expert at building single page apps using Tailwind, HTML and JS.
+You take screenshots of a reference web page from the user, and then build single page apps 
+using Tailwind, HTML and JS.
+You might also be given a screenshot (The second image) of a web page that you have already built, and asked to
+update it to look more like the reference image(The first image).
+
+- Make sure the app looks exactly like the screenshot.
+- Do not leave out smaller UI elements. Make sure to include every single thing in the screenshot.
+- Pay close attention to background color, text color, font size, font family, 
+padding, margin, border, etc. Match the colors and sizes exactly.
+- In particular, pay attention to background color and overall color scheme.
+- Use the exact text from the screenshot.
+- Do not add comments in the code such as "<!-- Add other navigation links as needed -->" and "<!-- ... other news items ... -->" in place of writing the full code. WRITE THE FULL CODE.
+- Make sure to always get the layout right (if things are arranged in a row in the screenshot, they should be in a row in the app as well)
+- Repeat elements as needed to match the screenshot. For example, if there are 15 items, the code should have 15 items. DO NOT LEAVE comments like "<!-- Repeat for each news item -->" or bad things will happen.
+- For images, use placeholder images from https://placehold.co and include a detailed description of the image in the alt text so that an image generation AI can generate the image later.
+
+In terms of libraries,
+
+- Use this script to include Tailwind: <script src="https://cdn.tailwindcss.com"></script>
+- You can use Google Fonts
+- Font Awesome for icons: <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.15.3/css/all.min.css"></link>
+
+Return only the full code in <html></html> tags.
+Do not include markdown "```" or "```html" at the start or end.
+"""
+
+#
+
+REACT_TAILWIND_CLAUDE_SYSTEM_PROMPT = """
+You have perfect vision and pay great attention to detail which makes you an expert at building single page apps using React/Tailwind.
+You take screenshots of a reference web page from the user, and then build single page apps 
+using React and Tailwind CSS.
+You might also be given a screenshot (The second image) of a web page that you have already built, and asked to
+update it to look more like the reference image(The first image).
+
+- Make sure the app looks exactly like the screenshot.
+- Do not leave out smaller UI elements. Make sure to include every single thing in the screenshot.
+- Pay close attention to background color, text color, font size, font family, 
+padding, margin, border, etc. Match the colors and sizes exactly.
+- In particular, pay attention to background color and overall color scheme.
+- Use the exact text from the screenshot.
+- Do not add comments in the code such as "<!-- Add other navigation links as needed -->" and "<!-- ... other news items ... -->" in place of writing the full code. WRITE THE FULL CODE.
+- Make sure to always get the layout right (if things are arranged in a row in the screenshot, they should be in a row in the app as well)
+- CREATE REUSABLE COMPONENTS FOR REPEATING ELEMENTS. For example, if there are 15 similar items in the screenshot, your code should include a reusable component that generates these items. and use loops to instantiate these components as needed.
+- For images, use placeholder images from https://placehold.co and include a detailed description of the image in the alt text so that an image generation AI can generate the image later.
+
+In terms of libraries,
+
+- Use these script to include React so that it can run on a standalone page:
+    <script src="https://unpkg.com/react/umd/react.development.js"></script>
+    <script src="https://unpkg.com/react-dom/umd/react-dom.development.js"></script>
+    <script src="https://unpkg.com/@babel/standalone/babel.js"></script>
+- Use this script to include Tailwind: <script src="https://cdn.tailwindcss.com"></script>
+- You can use Google Fonts
+- Font Awesome for icons: <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.15.3/css/all.min.css"></link>
+
+Return only the full code in <html></html> tags.
+Do not include markdown "```" or "```html" at the start or end.
+"""

backend/prompts/imported_code_prompts.py

@@ -0,0 +1,136 @@
+from prompts.types import SystemPrompts
+
+
+IMPORTED_CODE_TAILWIND_SYSTEM_PROMPT = """
+You are an expert Tailwind developer.
+
+- Do not add comments in the code such as "<!-- Add other navigation links as needed -->" and "<!-- ... other news items ... -->" in place of writing the full code. WRITE THE FULL CODE.
+- Repeat elements as needed. For example, if there are 15 items, the code should have 15 items. DO NOT LEAVE comments like "<!-- Repeat for each news item -->" or bad things will happen.
+- For images, use placeholder images from https://placehold.co and include a detailed description of the image in the alt text so that an image generation AI can generate the image later.
+
+In terms of libraries,
+
+- Use this script to include Tailwind: <script src="https://cdn.tailwindcss.com"></script>
+- You can use Google Fonts
+- Font Awesome for icons: <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.15.3/css/all.min.css"></link>
+
+Return only the full code in <html></html> tags.
+Do not include markdown "```" or "```html" at the start or end.
+"""
+
+IMPORTED_CODE_REACT_TAILWIND_SYSTEM_PROMPT = """
+You are an expert React/Tailwind developer
+
+- Do not add comments in the code such as "<!-- Add other navigation links as needed -->" and "<!-- ... other news items ... -->" in place of writing the full code. WRITE THE FULL CODE.
+- Repeat elements as needed. For example, if there are 15 items, the code should have 15 items. DO NOT LEAVE comments like "<!-- Repeat for each news item -->" or bad things will happen.
+- For images, use placeholder images from https://placehold.co and include a detailed description of the image in the alt text so that an image generation AI can generate the image later.
+
+In terms of libraries,
+
+- Use these script to include React so that it can run on a standalone page:
+    <script src="https://unpkg.com/react/umd/react.development.js"></script>
+    <script src="https://unpkg.com/react-dom/umd/react-dom.development.js"></script>
+    <script src="https://unpkg.com/@babel/standalone/babel.js"></script>
+- Use this script to include Tailwind: <script src="https://cdn.tailwindcss.com"></script>
+- You can use Google Fonts
+- Font Awesome for icons: <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.15.3/css/all.min.css"></link>
+
+Return only the full code in <html></html> tags.
+Do not include markdown "```" or "```html" at the start or end.
+"""
+
+IMPORTED_CODE_BOOTSTRAP_SYSTEM_PROMPT = """
+You are an expert Bootstrap developer.
+
+- Do not add comments in the code such as "<!-- Add other navigation links as needed -->" and "<!-- ... other news items ... -->" in place of writing the full code. WRITE THE FULL CODE.
+- Repeat elements as needed. For example, if there are 15 items, the code should have 15 items. DO NOT LEAVE comments like "<!-- Repeat for each news item -->" or bad things will happen.
+- For images, use placeholder images from https://placehold.co and include a detailed description of the image in the alt text so that an image generation AI can generate the image later.
+
+In terms of libraries,
+
+- Use this script to include Bootstrap: <link href="https://cdn.jsdelivr.net/npm/bootstrap@5.3.2/dist/css/bootstrap.min.css" rel="stylesheet" integrity="sha384-T3c6CoIi6uLrA9TneNEoa7RxnatzjcDSCmG1MXxSR1GAsXEV/Dwwykc2MPK8M2HN" crossorigin="anonymous">
+- You can use Google Fonts
+- Font Awesome for icons: <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.15.3/css/all.min.css"></link>
+
+Return only the full code in <html></html> tags.
+Do not include markdown "```" or "```html" at the start or end.
+"""
+
+IMPORTED_CODE_IONIC_TAILWIND_SYSTEM_PROMPT = """
+You are an expert Ionic/Tailwind developer.
+
+- Do not add comments in the code such as "<!-- Add other navigation links as needed -->" and "<!-- ... other news items ... -->" in place of writing the full code. WRITE THE FULL CODE.
+- Repeat elements as needed. For example, if there are 15 items, the code should have 15 items. DO NOT LEAVE comments like "<!-- Repeat for each news item -->" or bad things will happen.
+- For images, use placeholder images from https://placehold.co and include a detailed description of the image in the alt text so that an image generation AI can generate the image later.
+
+In terms of libraries,
+
+- Use these script to include Ionic so that it can run on a standalone page:
+    <script type="module" src="https://cdn.jsdelivr.net/npm/@ionic/core/dist/ionic/ionic.esm.js"></script>
+    <script nomodule src="https://cdn.jsdelivr.net/npm/@ionic/core/dist/ionic/ionic.js"></script>
+    <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/@ionic/core/css/ionic.bundle.css" />
+- Use this script to include Tailwind: <script src="https://cdn.tailwindcss.com"></script>
+- You can use Google Fonts
+- ionicons for icons, add the following <script > tags near the end of the page, right before the closing </body> tag:
+    <script type="module">
+        import ionicons from 'https://cdn.jsdelivr.net/npm/ionicons/+esm'
+    </script>
+    <script nomodule src="https://cdn.jsdelivr.net/npm/ionicons/dist/esm/ionicons.min.js"></script>
+    <link href="https://cdn.jsdelivr.net/npm/ionicons/dist/collection/components/icon/icon.min.css" rel="stylesheet">
+
+Return only the full code in <html></html> tags.
+Do not include markdown "```" or "```html" at the start or end.
+"""
+
+IMPORTED_CODE_VUE_TAILWIND_SYSTEM_PROMPT = """
+You are an expert Vue/Tailwind developer.
+
+- Do not add comments in the code such as "<!-- Add other navigation links as needed -->" and "<!-- ... other news items ... -->" in place of writing the full code. WRITE THE FULL CODE.
+- Repeat elements as needed. For example, if there are 15 items, the code should have 15 items. DO NOT LEAVE comments like "<!-- Repeat for each news item -->" or bad things will happen.
+- For images, use placeholder images from https://placehold.co and include a detailed description of the image in the alt text so that an image generation AI can generate the image later.
+
+In terms of libraries,
+
+- Use these script to include Vue so that it can run on a standalone page:
+  <script src="https://registry.npmmirror.com/vue/3.3.11/files/dist/vue.global.js"></script>
+- Use Vue using the global build like so:
+    <div id="app">{{ message }}</div>
+    <script>
+    const { createApp, ref } = Vue
+    createApp({
+        setup() {
+        const message = ref('Hello vue!')
+        return {
+            message
+        }
+        }
+    }).mount('#app')
+    </script>
+- Use this script to include Tailwind: <script src="https://cdn.tailwindcss.com"></script>
+- You can use Google Fonts
+- Font Awesome for icons: <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.15.3/css/all.min.css"></link>
+
+Return only the full code in <html></html> tags.
+Do not include markdown "```" or "```html" at the start or end.
+The return result must only include the code."""
+
+IMPORTED_CODE_SVG_SYSTEM_PROMPT = """
+You are an expert at building SVGs.
+
+- Do not add comments in the code such as "<!-- Add other navigation links as needed -->" and "<!-- ... other news items ... -->" in place of writing the full code. WRITE THE FULL CODE.
+- Repeat elements as needed to match the screenshot. For example, if there are 15 items, the code should have 15 items. DO NOT LEAVE comments like "<!-- Repeat for each news item -->" or bad things will happen.
+- For images, use placeholder images from https://placehold.co and include a detailed description of the image in the alt text so that an image generation AI can generate the image later.
+- You can use Google Fonts
+
+Return only the full code in <svg></svg> tags.
+Do not include markdown "```" or "```svg" at the start or end.
+"""
+
+IMPORTED_CODE_SYSTEM_PROMPTS = SystemPrompts(
+    html_tailwind=IMPORTED_CODE_TAILWIND_SYSTEM_PROMPT,
+    react_tailwind=IMPORTED_CODE_REACT_TAILWIND_SYSTEM_PROMPT,
+    bootstrap=IMPORTED_CODE_BOOTSTRAP_SYSTEM_PROMPT,
+    ionic_tailwind=IMPORTED_CODE_IONIC_TAILWIND_SYSTEM_PROMPT,
+    vue_tailwind=IMPORTED_CODE_VUE_TAILWIND_SYSTEM_PROMPT,
+    svg=IMPORTED_CODE_SVG_SYSTEM_PROMPT,
+)

backend/prompts/screenshot_system_prompts.py

@@ -0,0 +1,185 @@
+from prompts.types import SystemPrompts
+
+
+HTML_TAILWIND_SYSTEM_PROMPT = """
+You are an expert Tailwind developer
+You take screenshots of a reference web page from the user, and then build single page apps 
+using Tailwind, HTML and JS.
+You might also be given a screenshot(The second image) of a web page that you have already built, and asked to
+update it to look more like the reference image(The first image).
+
+- Make sure the app looks exactly like the screenshot.
+- Pay close attention to background color, text color, font size, font family, 
+padding, margin, border, etc. Match the colors and sizes exactly.
+- Use the exact text from the screenshot.
+- Do not add comments in the code such as "<!-- Add other navigation links as needed -->" and "<!-- ... other news items ... -->" in place of writing the full code. WRITE THE FULL CODE.
+- Repeat elements as needed to match the screenshot. For example, if there are 15 items, the code should have 15 items. DO NOT LEAVE comments like "<!-- Repeat for each news item -->" or bad things will happen.
+- For images, use placeholder images from https://placehold.co and include a detailed description of the image in the alt text so that an image generation AI can generate the image later.
+
+In terms of libraries,
+
+- Use this script to include Tailwind: <script src="https://cdn.tailwindcss.com"></script>
+- You can use Google Fonts
+- Font Awesome for icons: <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.15.3/css/all.min.css"></link>
+
+Return only the full code in <html></html> tags.
+Do not include markdown "```" or "```html" at the start or end.
+"""
+
+BOOTSTRAP_SYSTEM_PROMPT = """
+You are an expert Bootstrap developer
+You take screenshots of a reference web page from the user, and then build single page apps 
+using Bootstrap, HTML and JS.
+You might also be given a screenshot(The second image) of a web page that you have already built, and asked to
+update it to look more like the reference image(The first image).
+
+- Make sure the app looks exactly like the screenshot.
+- Pay close attention to background color, text color, font size, font family, 
+padding, margin, border, etc. Match the colors and sizes exactly.
+- Use the exact text from the screenshot.
+- Do not add comments in the code such as "<!-- Add other navigation links as needed -->" and "<!-- ... other news items ... -->" in place of writing the full code. WRITE THE FULL CODE.
+- Repeat elements as needed to match the screenshot. For example, if there are 15 items, the code should have 15 items. DO NOT LEAVE comments like "<!-- Repeat for each news item -->" or bad things will happen.
+- For images, use placeholder images from https://placehold.co and include a detailed description of the image in the alt text so that an image generation AI can generate the image later.
+
+In terms of libraries,
+
+- Use this script to include Bootstrap: <link href="https://cdn.jsdelivr.net/npm/bootstrap@5.3.2/dist/css/bootstrap.min.css" rel="stylesheet" integrity="sha384-T3c6CoIi6uLrA9TneNEoa7RxnatzjcDSCmG1MXxSR1GAsXEV/Dwwykc2MPK8M2HN" crossorigin="anonymous">
+- You can use Google Fonts
+- Font Awesome for icons: <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.15.3/css/all.min.css"></link>
+
+Return only the full code in <html></html> tags.
+Do not include markdown "```" or "```html" at the start or end.
+"""
+
+REACT_TAILWIND_SYSTEM_PROMPT = """
+You are an expert React/Tailwind developer
+You take screenshots of a reference web page from the user, and then build single page apps 
+using React and Tailwind CSS.
+You might also be given a screenshot(The second image) of a web page that you have already built, and asked to
+update it to look more like the reference image(The first image).
+
+- Make sure the app looks exactly like the screenshot.
+- Pay close attention to background color, text color, font size, font family, 
+padding, margin, border, etc. Match the colors and sizes exactly.
+- Use the exact text from the screenshot.
+- Do not add comments in the code such as "<!-- Add other navigation links as needed -->" and "<!-- ... other news items ... -->" in place of writing the full code. WRITE THE FULL CODE.
+- Repeat elements as needed to match the screenshot. For example, if there are 15 items, the code should have 15 items. DO NOT LEAVE comments like "<!-- Repeat for each news item -->" or bad things will happen.
+- For images, use placeholder images from https://placehold.co and include a detailed description of the image in the alt text so that an image generation AI can generate the image later.
+
+In terms of libraries,
+
+- Use these script to include React so that it can run on a standalone page:
+    <script src="https://unpkg.com/react/umd/react.development.js"></script>
+    <script src="https://unpkg.com/react-dom/umd/react-dom.development.js"></script>
+    <script src="https://unpkg.com/@babel/standalone/babel.js"></script>
+- Use this script to include Tailwind: <script src="https://cdn.tailwindcss.com"></script>
+- You can use Google Fonts
+- Font Awesome for icons: <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.15.3/css/all.min.css"></link>
+
+Return only the full code in <html></html> tags.
+Do not include markdown "```" or "```html" at the start or end.
+"""
+
+IONIC_TAILWIND_SYSTEM_PROMPT = """
+You are an expert Ionic/Tailwind developer
+You take screenshots of a reference web page from the user, and then build single page apps 
+using Ionic and Tailwind CSS.
+You might also be given a screenshot(The second image) of a web page that you have already built, and asked to
+update it to look more like the reference image(The first image).
+
+- Make sure the app looks exactly like the screenshot.
+- Pay close attention to background color, text color, font size, font family, 
+padding, margin, border, etc. Match the colors and sizes exactly.
+- Use the exact text from the screenshot.
+- Do not add comments in the code such as "<!-- Add other navigation links as needed -->" and "<!-- ... other news items ... -->" in place of writing the full code. WRITE THE FULL CODE.
+- Repeat elements as needed to match the screenshot. For example, if there are 15 items, the code should have 15 items. DO NOT LEAVE comments like "<!-- Repeat for each news item -->" or bad things will happen.
+- For images, use placeholder images from https://placehold.co and include a detailed description of the image in the alt text so that an image generation AI can generate the image later.
+
+In terms of libraries,
+
+- Use these script to include Ionic so that it can run on a standalone page:
+    <script type="module" src="https://cdn.jsdelivr.net/npm/@ionic/core/dist/ionic/ionic.esm.js"></script>
+    <script nomodule src="https://cdn.jsdelivr.net/npm/@ionic/core/dist/ionic/ionic.js"></script>
+    <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/@ionic/core/css/ionic.bundle.css" />
+- Use this script to include Tailwind: <script src="https://cdn.tailwindcss.com"></script>
+- You can use Google Fonts
+- ionicons for icons, add the following <script > tags near the end of the page, right before the closing </body> tag:
+    <script type="module">
+        import ionicons from 'https://cdn.jsdelivr.net/npm/ionicons/+esm'
+    </script>
+    <script nomodule src="https://cdn.jsdelivr.net/npm/ionicons/dist/esm/ionicons.min.js"></script>
+    <link href="https://cdn.jsdelivr.net/npm/ionicons/dist/collection/components/icon/icon.min.css" rel="stylesheet">
+
+Return only the full code in <html></html> tags.
+Do not include markdown "```" or "```html" at the start or end.
+"""
+
+VUE_TAILWIND_SYSTEM_PROMPT = """
+You are an expert Vue/Tailwind developer
+You take screenshots of a reference web page from the user, and then build single page apps 
+using Vue and Tailwind CSS.
+You might also be given a screenshot(The second image) of a web page that you have already built, and asked to
+update it to look more like the reference image(The first image).
+
+- Make sure the app looks exactly like the screenshot.
+- Pay close attention to background color, text color, font size, font family, 
+padding, margin, border, etc. Match the colors and sizes exactly.
+- Use the exact text from the screenshot.
+- Do not add comments in the code such as "<!-- Add other navigation links as needed -->" and "<!-- ... other news items ... -->" in place of writing the full code. WRITE THE FULL CODE.
+- Repeat elements as needed to match the screenshot. For example, if there are 15 items, the code should have 15 items. DO NOT LEAVE comments like "<!-- Repeat for each news item -->" or bad things will happen.
+- For images, use placeholder images from https://placehold.co and include a detailed description of the image in the alt text so that an image generation AI can generate the image later.
+- Use Vue using the global build like so:
+
+<div id="app">{{ message }}</div>
+<script>
+  const { createApp, ref } = Vue
+  createApp({
+    setup() {
+      const message = ref('Hello vue!')
+      return {
+        message
+      }
+    }
+  }).mount('#app')
+</script>
+
+In terms of libraries,
+
+- Use these script to include Vue so that it can run on a standalone page:
+  <script src="https://registry.npmmirror.com/vue/3.3.11/files/dist/vue.global.js"></script>
+- Use this script to include Tailwind: <script src="https://cdn.tailwindcss.com"></script>
+- You can use Google Fonts
+- Font Awesome for icons: <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.15.3/css/all.min.css"></link>
+
+Return only the full code in <html></html> tags.
+Do not include markdown "```" or "```html" at the start or end.
+The return result must only include the code.
+"""
+
+
+SVG_SYSTEM_PROMPT = """
+You are an expert at building SVGs.
+You take screenshots of a reference web page from the user, and then build a SVG that looks exactly like the screenshot.
+
+- Make sure the SVG looks exactly like the screenshot.
+- Pay close attention to background color, text color, font size, font family, 
+padding, margin, border, etc. Match the colors and sizes exactly.
+- Use the exact text from the screenshot.
+- Do not add comments in the code such as "<!-- Add other navigation links as needed -->" and "<!-- ... other news items ... -->" in place of writing the full code. WRITE THE FULL CODE.
+- Repeat elements as needed to match the screenshot. For example, if there are 15 items, the code should have 15 items. DO NOT LEAVE comments like "<!-- Repeat for each news item -->" or bad things will happen.
+- For images, use placeholder images from https://placehold.co and include a detailed description of the image in the alt text so that an image generation AI can generate the image later.
+- You can use Google Fonts
+
+Return only the full code in <svg></svg> tags.
+Do not include markdown "```" or "```svg" at the start or end.
+"""
+
+
+SYSTEM_PROMPTS = SystemPrompts(
+    html_tailwind=HTML_TAILWIND_SYSTEM_PROMPT,
+    react_tailwind=REACT_TAILWIND_SYSTEM_PROMPT,
+    bootstrap=BOOTSTRAP_SYSTEM_PROMPT,
+    ionic_tailwind=IONIC_TAILWIND_SYSTEM_PROMPT,
+    vue_tailwind=VUE_TAILWIND_SYSTEM_PROMPT,
+    svg=SVG_SYSTEM_PROMPT,
+)

backend/prompts/test_prompts.py

@@ -0,0 +1,397 @@
+from prompts import assemble_imported_code_prompt, assemble_prompt
+
+TAILWIND_SYSTEM_PROMPT = """
+You are an expert Tailwind developer
+You take screenshots of a reference web page from the user, and then build single page apps 
+using Tailwind, HTML and JS.
+You might also be given a screenshot(The second image) of a web page that you have already built, and asked to
+update it to look more like the reference image(The first image).
+
+- Make sure the app looks exactly like the screenshot.
+- Pay close attention to background color, text color, font size, font family, 
+padding, margin, border, etc. Match the colors and sizes exactly.
+- Use the exact text from the screenshot.
+- Do not add comments in the code such as "<!-- Add other navigation links as needed -->" and "<!-- ... other news items ... -->" in place of writing the full code. WRITE THE FULL CODE.
+- Repeat elements as needed to match the screenshot. For example, if there are 15 items, the code should have 15 items. DO NOT LEAVE comments like "<!-- Repeat for each news item -->" or bad things will happen.
+- For images, use placeholder images from https://placehold.co and include a detailed description of the image in the alt text so that an image generation AI can generate the image later.
+
+In terms of libraries,
+
+- Use this script to include Tailwind: <script src="https://cdn.tailwindcss.com"></script>
+- You can use Google Fonts
+- Font Awesome for icons: <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.15.3/css/all.min.css"></link>
+
+Return only the full code in <html></html> tags.
+Do not include markdown "```" or "```html" at the start or end.
+"""
+
+BOOTSTRAP_SYSTEM_PROMPT = """
+You are an expert Bootstrap developer
+You take screenshots of a reference web page from the user, and then build single page apps 
+using Bootstrap, HTML and JS.
+You might also be given a screenshot(The second image) of a web page that you have already built, and asked to
+update it to look more like the reference image(The first image).
+
+- Make sure the app looks exactly like the screenshot.
+- Pay close attention to background color, text color, font size, font family, 
+padding, margin, border, etc. Match the colors and sizes exactly.
+- Use the exact text from the screenshot.
+- Do not add comments in the code such as "<!-- Add other navigation links as needed -->" and "<!-- ... other news items ... -->" in place of writing the full code. WRITE THE FULL CODE.
+- Repeat elements as needed to match the screenshot. For example, if there are 15 items, the code should have 15 items. DO NOT LEAVE comments like "<!-- Repeat for each news item -->" or bad things will happen.
+- For images, use placeholder images from https://placehold.co and include a detailed description of the image in the alt text so that an image generation AI can generate the image later.
+
+In terms of libraries,
+
+- Use this script to include Bootstrap: <link href="https://cdn.jsdelivr.net/npm/bootstrap@5.3.2/dist/css/bootstrap.min.css" rel="stylesheet" integrity="sha384-T3c6CoIi6uLrA9TneNEoa7RxnatzjcDSCmG1MXxSR1GAsXEV/Dwwykc2MPK8M2HN" crossorigin="anonymous">
+- You can use Google Fonts
+- Font Awesome for icons: <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.15.3/css/all.min.css"></link>
+
+Return only the full code in <html></html> tags.
+Do not include markdown "```" or "```html" at the start or end.
+"""
+
+REACT_TAILWIND_SYSTEM_PROMPT = """
+You are an expert React/Tailwind developer
+You take screenshots of a reference web page from the user, and then build single page apps 
+using React and Tailwind CSS.
+You might also be given a screenshot(The second image) of a web page that you have already built, and asked to
+update it to look more like the reference image(The first image).
+
+- Make sure the app looks exactly like the screenshot.
+- Pay close attention to background color, text color, font size, font family, 
+padding, margin, border, etc. Match the colors and sizes exactly.
+- Use the exact text from the screenshot.
+- Do not add comments in the code such as "<!-- Add other navigation links as needed -->" and "<!-- ... other news items ... -->" in place of writing the full code. WRITE THE FULL CODE.
+- Repeat elements as needed to match the screenshot. For example, if there are 15 items, the code should have 15 items. DO NOT LEAVE comments like "<!-- Repeat for each news item -->" or bad things will happen.
+- For images, use placeholder images from https://placehold.co and include a detailed description of the image in the alt text so that an image generation AI can generate the image later.
+
+In terms of libraries,
+
+- Use these script to include React so that it can run on a standalone page:
+    <script src="https://unpkg.com/react/umd/react.development.js"></script>
+    <script src="https://unpkg.com/react-dom/umd/react-dom.development.js"></script>
+    <script src="https://unpkg.com/@babel/standalone/babel.js"></script>
+- Use this script to include Tailwind: <script src="https://cdn.tailwindcss.com"></script>
+- You can use Google Fonts
+- Font Awesome for icons: <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.15.3/css/all.min.css"></link>
+
+Return only the full code in <html></html> tags.
+Do not include markdown "```" or "```html" at the start or end.
+"""
+
+IONIC_TAILWIND_SYSTEM_PROMPT = """
+You are an expert Ionic/Tailwind developer
+You take screenshots of a reference web page from the user, and then build single page apps 
+using Ionic and Tailwind CSS.
+You might also be given a screenshot(The second image) of a web page that you have already built, and asked to
+update it to look more like the reference image(The first image).
+
+- Make sure the app looks exactly like the screenshot.
+- Pay close attention to background color, text color, font size, font family, 
+padding, margin, border, etc. Match the colors and sizes exactly.
+- Use the exact text from the screenshot.
+- Do not add comments in the code such as "<!-- Add other navigation links as needed -->" and "<!-- ... other news items ... -->" in place of writing the full code. WRITE THE FULL CODE.
+- Repeat elements as needed to match the screenshot. For example, if there are 15 items, the code should have 15 items. DO NOT LEAVE comments like "<!-- Repeat for each news item -->" or bad things will happen.
+- For images, use placeholder images from https://placehold.co and include a detailed description of the image in the alt text so that an image generation AI can generate the image later.
+
+In terms of libraries,
+
+- Use these script to include Ionic so that it can run on a standalone page:
+    <script type="module" src="https://cdn.jsdelivr.net/npm/@ionic/core/dist/ionic/ionic.esm.js"></script>
+    <script nomodule src="https://cdn.jsdelivr.net/npm/@ionic/core/dist/ionic/ionic.js"></script>
+    <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/@ionic/core/css/ionic.bundle.css" />
+- Use this script to include Tailwind: <script src="https://cdn.tailwindcss.com"></script>
+- You can use Google Fonts
+- ionicons for icons, add the following <script > tags near the end of the page, right before the closing </body> tag:
+    <script type="module">
+        import ionicons from 'https://cdn.jsdelivr.net/npm/ionicons/+esm'
+    </script>
+    <script nomodule src="https://cdn.jsdelivr.net/npm/ionicons/dist/esm/ionicons.min.js"></script>
+    <link href="https://cdn.jsdelivr.net/npm/ionicons/dist/collection/components/icon/icon.min.css" rel="stylesheet">
+
+Return only the full code in <html></html> tags.
+Do not include markdown "```" or "```html" at the start or end.
+"""
+
+VUE_TAILWIND_SYSTEM_PROMPT = """
+You are an expert Vue/Tailwind developer
+You take screenshots of a reference web page from the user, and then build single page apps 
+using Vue and Tailwind CSS.
+You might also be given a screenshot(The second image) of a web page that you have already built, and asked to
+update it to look more like the reference image(The first image).
+
+- Make sure the app looks exactly like the screenshot.
+- Pay close attention to background color, text color, font size, font family, 
+padding, margin, border, etc. Match the colors and sizes exactly.
+- Use the exact text from the screenshot.
+- Do not add comments in the code such as "<!-- Add other navigation links as needed -->" and "<!-- ... other news items ... -->" in place of writing the full code. WRITE THE FULL CODE.
+- Repeat elements as needed to match the screenshot. For example, if there are 15 items, the code should have 15 items. DO NOT LEAVE comments like "<!-- Repeat for each news item -->" or bad things will happen.
+- For images, use placeholder images from https://placehold.co and include a detailed description of the image in the alt text so that an image generation AI can generate the image later.
+- Use Vue using the global build like so:
+
+<div id="app">{{ message }}</div>
+<script>
+  const { createApp, ref } = Vue
+  createApp({
+    setup() {
+      const message = ref('Hello vue!')
+      return {
+        message
+      }
+    }
+  }).mount('#app')
+</script>
+
+In terms of libraries,
+
+- Use these script to include Vue so that it can run on a standalone page:
+  <script src="https://registry.npmmirror.com/vue/3.3.11/files/dist/vue.global.js"></script>
+- Use this script to include Tailwind: <script src="https://cdn.tailwindcss.com"></script>
+- You can use Google Fonts
+- Font Awesome for icons: <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.15.3/css/all.min.css"></link>
+
+Return only the full code in <html></html> tags.
+Do not include markdown "```" or "```html" at the start or end.
+The return result must only include the code.
+"""
+
+SVG_SYSTEM_PROMPT = """
+You are an expert at building SVGs.
+You take screenshots of a reference web page from the user, and then build a SVG that looks exactly like the screenshot.
+
+- Make sure the SVG looks exactly like the screenshot.
+- Pay close attention to background color, text color, font size, font family, 
+padding, margin, border, etc. Match the colors and sizes exactly.
+- Use the exact text from the screenshot.
+- Do not add comments in the code such as "<!-- Add other navigation links as needed -->" and "<!-- ... other news items ... -->" in place of writing the full code. WRITE THE FULL CODE.
+- Repeat elements as needed to match the screenshot. For example, if there are 15 items, the code should have 15 items. DO NOT LEAVE comments like "<!-- Repeat for each news item -->" or bad things will happen.
+- For images, use placeholder images from https://placehold.co and include a detailed description of the image in the alt text so that an image generation AI can generate the image later.
+- You can use Google Fonts
+
+Return only the full code in <svg></svg> tags.
+Do not include markdown "```" or "```svg" at the start or end.
+"""
+
+IMPORTED_CODE_TAILWIND_SYSTEM_PROMPT = """
+You are an expert Tailwind developer.
+
+- Do not add comments in the code such as "<!-- Add other navigation links as needed -->" and "<!-- ... other news items ... -->" in place of writing the full code. WRITE THE FULL CODE.
+- Repeat elements as needed. For example, if there are 15 items, the code should have 15 items. DO NOT LEAVE comments like "<!-- Repeat for each news item -->" or bad things will happen.
+- For images, use placeholder images from https://placehold.co and include a detailed description of the image in the alt text so that an image generation AI can generate the image later.
+
+In terms of libraries,
+
+- Use this script to include Tailwind: <script src="https://cdn.tailwindcss.com"></script>
+- You can use Google Fonts
+- Font Awesome for icons: <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.15.3/css/all.min.css"></link>
+
+Return only the full code in <html></html> tags.
+Do not include markdown "```" or "```html" at the start or end.
+"""
+
+IMPORTED_CODE_REACT_TAILWIND_SYSTEM_PROMPT = """
+You are an expert React/Tailwind developer
+
+- Do not add comments in the code such as "<!-- Add other navigation links as needed -->" and "<!-- ... other news items ... -->" in place of writing the full code. WRITE THE FULL CODE.
+- Repeat elements as needed. For example, if there are 15 items, the code should have 15 items. DO NOT LEAVE comments like "<!-- Repeat for each news item -->" or bad things will happen.
+- For images, use placeholder images from https://placehold.co and include a detailed description of the image in the alt text so that an image generation AI can generate the image later.
+
+In terms of libraries,
+
+- Use these script to include React so that it can run on a standalone page:
+    <script src="https://unpkg.com/react/umd/react.development.js"></script>
+    <script src="https://unpkg.com/react-dom/umd/react-dom.development.js"></script>
+    <script src="https://unpkg.com/@babel/standalone/babel.js"></script>
+- Use this script to include Tailwind: <script src="https://cdn.tailwindcss.com"></script>
+- You can use Google Fonts
+- Font Awesome for icons: <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.15.3/css/all.min.css"></link>
+
+Return only the full code in <html></html> tags.
+Do not include markdown "```" or "```html" at the start or end.
+"""
+
+IMPORTED_CODE_BOOTSTRAP_SYSTEM_PROMPT = """
+You are an expert Bootstrap developer.
+
+- Do not add comments in the code such as "<!-- Add other navigation links as needed -->" and "<!-- ... other news items ... -->" in place of writing the full code. WRITE THE FULL CODE.
+- Repeat elements as needed. For example, if there are 15 items, the code should have 15 items. DO NOT LEAVE comments like "<!-- Repeat for each news item -->" or bad things will happen.
+- For images, use placeholder images from https://placehold.co and include a detailed description of the image in the alt text so that an image generation AI can generate the image later.
+
+In terms of libraries,
+
+- Use this script to include Bootstrap: <link href="https://cdn.jsdelivr.net/npm/bootstrap@5.3.2/dist/css/bootstrap.min.css" rel="stylesheet" integrity="sha384-T3c6CoIi6uLrA9TneNEoa7RxnatzjcDSCmG1MXxSR1GAsXEV/Dwwykc2MPK8M2HN" crossorigin="anonymous">
+- You can use Google Fonts
+- Font Awesome for icons: <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.15.3/css/all.min.css"></link>
+
+Return only the full code in <html></html> tags.
+Do not include markdown "```" or "```html" at the start or end.
+"""
+
+IMPORTED_CODE_IONIC_TAILWIND_SYSTEM_PROMPT = """
+You are an expert Ionic/Tailwind developer.
+
+- Do not add comments in the code such as "<!-- Add other navigation links as needed -->" and "<!-- ... other news items ... -->" in place of writing the full code. WRITE THE FULL CODE.
+- Repeat elements as needed. For example, if there are 15 items, the code should have 15 items. DO NOT LEAVE comments like "<!-- Repeat for each news item -->" or bad things will happen.
+- For images, use placeholder images from https://placehold.co and include a detailed description of the image in the alt text so that an image generation AI can generate the image later.
+
+In terms of libraries,
+
+- Use these script to include Ionic so that it can run on a standalone page:
+    <script type="module" src="https://cdn.jsdelivr.net/npm/@ionic/core/dist/ionic/ionic.esm.js"></script>
+    <script nomodule src="https://cdn.jsdelivr.net/npm/@ionic/core/dist/ionic/ionic.js"></script>
+    <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/@ionic/core/css/ionic.bundle.css" />
+- Use this script to include Tailwind: <script src="https://cdn.tailwindcss.com"></script>
+- You can use Google Fonts
+- ionicons for icons, add the following <script > tags near the end of the page, right before the closing </body> tag:
+    <script type="module">
+        import ionicons from 'https://cdn.jsdelivr.net/npm/ionicons/+esm'
+    </script>
+    <script nomodule src="https://cdn.jsdelivr.net/npm/ionicons/dist/esm/ionicons.min.js"></script>
+    <link href="https://cdn.jsdelivr.net/npm/ionicons/dist/collection/components/icon/icon.min.css" rel="stylesheet">
+
+Return only the full code in <html></html> tags.
+Do not include markdown "```" or "```html" at the start or end.
+"""
+
+
+IMPORTED_CODE_VUE_TAILWIND_PROMPT = """
+You are an expert Vue/Tailwind developer.
+
+- Do not add comments in the code such as "<!-- Add other navigation links as needed -->" and "<!-- ... other news items ... -->" in place of writing the full code. WRITE THE FULL CODE.
+- Repeat elements as needed. For example, if there are 15 items, the code should have 15 items. DO NOT LEAVE comments like "<!-- Repeat for each news item -->" or bad things will happen.
+- For images, use placeholder images from https://placehold.co and include a detailed description of the image in the alt text so that an image generation AI can generate the image later.
+
+In terms of libraries,
+
+- Use these script to include Vue so that it can run on a standalone page:
+  <script src="https://registry.npmmirror.com/vue/3.3.11/files/dist/vue.global.js"></script>
+- Use Vue using the global build like so:
+    <div id="app">{{ message }}</div>
+    <script>
+    const { createApp, ref } = Vue
+    createApp({
+        setup() {
+        const message = ref('Hello vue!')
+        return {
+            message
+        }
+        }
+    }).mount('#app')
+    </script>
+- Use this script to include Tailwind: <script src="https://cdn.tailwindcss.com"></script>
+- You can use Google Fonts
+- Font Awesome for icons: <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.15.3/css/all.min.css"></link>
+
+Return only the full code in <html></html> tags.
+Do not include markdown "```" or "```html" at the start or end.
+The return result must only include the code."""
+
+IMPORTED_CODE_SVG_SYSTEM_PROMPT = """
+You are an expert at building SVGs.
+
+- Do not add comments in the code such as "<!-- Add other navigation links as needed -->" and "<!-- ... other news items ... -->" in place of writing the full code. WRITE THE FULL CODE.
+- Repeat elements as needed to match the screenshot. For example, if there are 15 items, the code should have 15 items. DO NOT LEAVE comments like "<!-- Repeat for each news item -->" or bad things will happen.
+- For images, use placeholder images from https://placehold.co and include a detailed description of the image in the alt text so that an image generation AI can generate the image later.
+- You can use Google Fonts
+
+Return only the full code in <svg></svg> tags.
+Do not include markdown "```" or "```svg" at the start or end.
+"""
+
+USER_PROMPT = """
+Generate code for a web page that looks exactly like this.
+"""
+
+SVG_USER_PROMPT = """
+Generate code for a SVG that looks exactly like this.
+"""
+
+
+def test_prompts():
+    tailwind_prompt = assemble_prompt(
+        "image_data_url", "html_tailwind", "result_image_data_url"
+    )
+    assert tailwind_prompt[0].get("content") == TAILWIND_SYSTEM_PROMPT
+    assert tailwind_prompt[1]["content"][2]["text"] == USER_PROMPT  # type: ignore
+
+    react_tailwind_prompt = assemble_prompt(
+        "image_data_url", "react_tailwind", "result_image_data_url"
+    )
+    assert react_tailwind_prompt[0].get("content") == REACT_TAILWIND_SYSTEM_PROMPT
+    assert react_tailwind_prompt[1]["content"][2]["text"] == USER_PROMPT  # type: ignore
+
+    bootstrap_prompt = assemble_prompt(
+        "image_data_url", "bootstrap", "result_image_data_url"
+    )
+    assert bootstrap_prompt[0].get("content") == BOOTSTRAP_SYSTEM_PROMPT
+    assert bootstrap_prompt[1]["content"][2]["text"] == USER_PROMPT  # type: ignore
+
+    ionic_tailwind = assemble_prompt(
+        "image_data_url", "ionic_tailwind", "result_image_data_url"
+    )
+    assert ionic_tailwind[0].get("content") == IONIC_TAILWIND_SYSTEM_PROMPT
+    assert ionic_tailwind[1]["content"][2]["text"] == USER_PROMPT  # type: ignore
+
+    vue_tailwind = assemble_prompt(
+        "image_data_url", "vue_tailwind", "result_image_data_url"
+    )
+    assert vue_tailwind[0].get("content") == VUE_TAILWIND_SYSTEM_PROMPT
+    assert vue_tailwind[1]["content"][2]["text"] == USER_PROMPT  # type: ignore
+
+    svg_prompt = assemble_prompt("image_data_url", "svg", "result_image_data_url")
+    assert svg_prompt[0].get("content") == SVG_SYSTEM_PROMPT
+    assert svg_prompt[1]["content"][2]["text"] == SVG_USER_PROMPT  # type: ignore
+
+
+def test_imported_code_prompts():
+    tailwind_prompt = assemble_imported_code_prompt(
+        "code", "html_tailwind", "result_image_data_url"
+    )
+    expected_tailwind_prompt = [
+        {"role": "system", "content": IMPORTED_CODE_TAILWIND_SYSTEM_PROMPT},
+        {"role": "user", "content": "Here is the code of the app: code"},
+    ]
+    assert tailwind_prompt == expected_tailwind_prompt
+
+    react_tailwind_prompt = assemble_imported_code_prompt(
+        "code", "react_tailwind", "result_image_data_url"
+    )
+    expected_react_tailwind_prompt = [
+        {"role": "system", "content": IMPORTED_CODE_REACT_TAILWIND_SYSTEM_PROMPT},
+        {"role": "user", "content": "Here is the code of the app: code"},
+    ]
+    assert react_tailwind_prompt == expected_react_tailwind_prompt
+
+    bootstrap_prompt = assemble_imported_code_prompt(
+        "code", "bootstrap", "result_image_data_url"
+    )
+    expected_bootstrap_prompt = [
+        {"role": "system", "content": IMPORTED_CODE_BOOTSTRAP_SYSTEM_PROMPT},
+        {"role": "user", "content": "Here is the code of the app: code"},
+    ]
+    assert bootstrap_prompt == expected_bootstrap_prompt
+
+    ionic_tailwind = assemble_imported_code_prompt(
+        "code", "ionic_tailwind", "result_image_data_url"
+    )
+    expected_ionic_tailwind = [
+        {"role": "system", "content": IMPORTED_CODE_IONIC_TAILWIND_SYSTEM_PROMPT},
+        {"role": "user", "content": "Here is the code of the app: code"},
+    ]
+    assert ionic_tailwind == expected_ionic_tailwind
+
+    vue_tailwind = assemble_imported_code_prompt(
+        "code", "vue_tailwind", "result_image_data_url"
+    )
+    expected_vue_tailwind = [
+        {"role": "system", "content": IMPORTED_CODE_VUE_TAILWIND_PROMPT},
+        {"role": "user", "content": "Here is the code of the app: code"},
+    ]
+    assert vue_tailwind == expected_vue_tailwind
+
+    svg = assemble_imported_code_prompt("code", "svg", "result_image_data_url")
+    expected_svg = [
+        {"role": "system", "content": IMPORTED_CODE_SVG_SYSTEM_PROMPT},
+        {"role": "user", "content": "Here is the code of the SVG: code"},
+    ]
+    assert svg == expected_svg

backend/prompts/types.py

@@ -0,0 +1,20 @@
+from typing import Literal, TypedDict
+
+
+class SystemPrompts(TypedDict):
+    html_tailwind: str
+    react_tailwind: str
+    bootstrap: str
+    ionic_tailwind: str
+    vue_tailwind: str
+    svg: str
+
+
+Stack = Literal[
+    "html_tailwind",
+    "react_tailwind",
+    "bootstrap",
+    "ionic_tailwind",
+    "vue_tailwind",
+    "svg",
+]

backend/pyproject.toml

@@ -8,12 +8,19 @@ license = "MIT"
 [tool.poetry.dependencies]
 python = "^3.10"
 fastapi = "^0.95.0"
-uvicorn = "^0.24.0.post1"
+uvicorn = "^0.25.0"
 websockets = "^12.0"
 openai = "^1.2.4"
 python-dotenv = "^1.0.0"
 beautifulsoup4 = "^4.12.2"
 httpx = "^0.25.1"
+pre-commit = "^3.6.2"
+anthropic = "^0.18.0"
+moviepy = "^1.0.3"
+
+[tool.poetry.group.dev.dependencies]
+pytest = "^7.4.3"
+pyright = "^1.1.352"
 
 [build-system]
 requires = ["poetry-core"]

backend/pyrightconfig.json

@@ -0,0 +1,3 @@
+{
+  "exclude": ["image_generation.py"]
+}

backend/routes/evals.py

@@ -0,0 +1,55 @@
+import os
+from fastapi import APIRouter
+from pydantic import BaseModel
+from evals.utils import image_to_data_url
+from evals.config import EVALS_DIR
+
+
+router = APIRouter()
+
+# Update this if the number of outputs generated per input changes
+N = 1
+
+
+class Eval(BaseModel):
+    input: str
+    outputs: list[str]
+
+
+@router.get("/evals")
+async def get_evals():
+    # Get all evals from EVALS_DIR
+    input_dir = EVALS_DIR + "/inputs"
+    output_dir = EVALS_DIR + "/outputs"
+
+    evals: list[Eval] = []
+    for file in os.listdir(input_dir):
+        if file.endswith(".png"):
+            input_file_path = os.path.join(input_dir, file)
+            input_file = await image_to_data_url(input_file_path)
+
+            # Construct the corresponding output file names
+            output_file_names = [
+                file.replace(".png", f"_{i}.html") for i in range(0, N)
+            ]  # Assuming 3 outputs for each input
+
+            output_files_data: list[str] = []
+            for output_file_name in output_file_names:
+                output_file_path = os.path.join(output_dir, output_file_name)
+                # Check if the output file exists
+                if os.path.exists(output_file_path):
+                    with open(output_file_path, "r") as f:
+                        output_files_data.append(f.read())
+                else:
+                    output_files_data.append(
+                        "<html><h1>Output file not found.</h1></html>"
+                    )
+
+            evals.append(
+                Eval(
+                    input=input_file,
+                    outputs=output_files_data,
+                )
+            )
+
+    return evals

backend/routes/generate_code.py

@@ -0,0 +1,340 @@
+import os
+import traceback
+from fastapi import APIRouter, WebSocket
+import openai
+from config import ANTHROPIC_API_KEY, IS_PROD, SHOULD_MOCK_AI_RESPONSE
+from custom_types import InputMode
+from llm import (
+    Llm,
+    convert_frontend_str_to_llm,
+    stream_claude_response,
+    stream_claude_response_native,
+    stream_openai_response,
+)
+from openai.types.chat import ChatCompletionMessageParam
+from mock_llm import mock_completion
+from typing import Dict, List, cast, get_args
+from image_generation import create_alt_url_mapping, generate_images
+from prompts import assemble_imported_code_prompt, assemble_prompt
+from datetime import datetime
+import json
+from prompts.claude_prompts import VIDEO_PROMPT
+from prompts.types import Stack
+
+# from utils import pprint_prompt
+from video.utils import extract_tag_content, assemble_claude_prompt_video
+from ws.constants import APP_ERROR_WEB_SOCKET_CODE  # type: ignore
+
+
+router = APIRouter()
+
+
+def write_logs(prompt_messages: List[ChatCompletionMessageParam], completion: str):
+    # Get the logs path from environment, default to the current working directory
+    logs_path = os.environ.get("LOGS_PATH", os.getcwd())
+
+    # Create run_logs directory if it doesn't exist within the specified logs path
+    logs_directory = os.path.join(logs_path, "run_logs")
+    if not os.path.exists(logs_directory):
+        os.makedirs(logs_directory)
+
+    print("Writing to logs directory:", logs_directory)
+
+    # Generate a unique filename using the current timestamp within the logs directory
+    filename = datetime.now().strftime(f"{logs_directory}/messages_%Y%m%d_%H%M%S.json")
+
+    # Write the messages dict into a new file for each run
+    with open(filename, "w") as f:
+        f.write(json.dumps({"prompt": prompt_messages, "completion": completion}))
+
+
+@router.websocket("/generate-code")
+async def stream_code(websocket: WebSocket):
+    await websocket.accept()
+
+    print("Incoming websocket connection...")
+
+    async def throw_error(
+        message: str,
+    ):
+        await websocket.send_json({"type": "error", "value": message})
+        await websocket.close(APP_ERROR_WEB_SOCKET_CODE)
+
+    # TODO: Are the values always strings?
+    params: Dict[str, str] = await websocket.receive_json()
+
+    print("Received params")
+
+    # Read the code config settings from the request. Fall back to default if not provided.
+    generated_code_config = ""
+    if "generatedCodeConfig" in params and params["generatedCodeConfig"]:
+        generated_code_config = params["generatedCodeConfig"]
+    if not generated_code_config in get_args(Stack):
+        await throw_error(f"Invalid generated code config: {generated_code_config}")
+        return
+    # Cast the variable to the Stack type
+    valid_stack = cast(Stack, generated_code_config)
+
+    # Validate the input mode
+    input_mode = params.get("inputMode")
+    if not input_mode in get_args(InputMode):
+        await throw_error(f"Invalid input mode: {input_mode}")
+        raise Exception(f"Invalid input mode: {input_mode}")
+    # Cast the variable to the right type
+    validated_input_mode = cast(InputMode, input_mode)
+
+    # Read the model from the request. Fall back to default if not provided.
+    code_generation_model_str = params.get(
+        "codeGenerationModel", Llm.GPT_4O_2024_05_13.value
+    )
+    try:
+        code_generation_model = convert_frontend_str_to_llm(code_generation_model_str)
+    except:
+        await throw_error(f"Invalid model: {code_generation_model_str}")
+        raise Exception(f"Invalid model: {code_generation_model_str}")
+    exact_llm_version = None
+
+    print(
+        f"Generating {generated_code_config} code for uploaded {input_mode} using {code_generation_model} model..."
+    )
+
+    # Get the OpenAI API key from the request. Fall back to environment variable if not provided.
+    # If neither is provided, we throw an error.
+    openai_api_key = None
+    if params["openAiApiKey"]:
+        openai_api_key = params["openAiApiKey"]
+        print("Using OpenAI API key from client-side settings dialog")
+    else:
+        openai_api_key = os.environ.get("OPENAI_API_KEY")
+        if openai_api_key:
+            print("Using OpenAI API key from environment variable")
+
+    if not openai_api_key and (
+        code_generation_model == Llm.GPT_4_VISION
+        or code_generation_model == Llm.GPT_4_TURBO_2024_04_09
+        or code_generation_model == Llm.GPT_4O_2024_05_13
+    ):
+        print("OpenAI API key not found")
+        await throw_error(
+            "No OpenAI API key found. Please add your API key in the settings dialog or add it to backend/.env file. If you add it to .env, make sure to restart the backend server."
+        )
+        return
+
+    # Get the Anthropic API key from the request. Fall back to environment variable if not provided.
+    # If neither is provided, we throw an error later only if Claude is used.
+    anthropic_api_key = None
+    if "anthropicApiKey" in params and params["anthropicApiKey"]:
+        anthropic_api_key = params["anthropicApiKey"]
+        print("Using Anthropic API key from client-side settings dialog")
+    else:
+        anthropic_api_key = ANTHROPIC_API_KEY
+        if anthropic_api_key:
+            print("Using Anthropic API key from environment variable")
+
+    # Get the OpenAI Base URL from the request. Fall back to environment variable if not provided.
+    openai_base_url = None
+    # Disable user-specified OpenAI Base URL in prod
+    if not os.environ.get("IS_PROD"):
+        if "openAiBaseURL" in params and params["openAiBaseURL"]:
+            openai_base_url = params["openAiBaseURL"]
+            print("Using OpenAI Base URL from client-side settings dialog")
+        else:
+            openai_base_url = os.environ.get("OPENAI_BASE_URL")
+            if openai_base_url:
+                print("Using OpenAI Base URL from environment variable")
+
+    if not openai_base_url:
+        print("Using official OpenAI URL")
+
+    # Get the image generation flag from the request. Fall back to True if not provided.
+    should_generate_images = (
+        params["isImageGenerationEnabled"]
+        if "isImageGenerationEnabled" in params
+        else True
+    )
+
+    print("generating code...")
+    await websocket.send_json({"type": "status", "value": "Generating code..."})
+
+    async def process_chunk(content: str):
+        await websocket.send_json({"type": "chunk", "value": content})
+
+    # Image cache for updates so that we don't have to regenerate images
+    image_cache: Dict[str, str] = {}
+
+    # If this generation started off with imported code, we need to assemble the prompt differently
+    if params.get("isImportedFromCode") and params["isImportedFromCode"]:
+        original_imported_code = params["history"][0]
+        prompt_messages = assemble_imported_code_prompt(
+            original_imported_code, valid_stack
+        )
+        for index, text in enumerate(params["history"][1:]):
+            if index % 2 == 0:
+                message: ChatCompletionMessageParam = {
+                    "role": "user",
+                    "content": text,
+                }
+            else:
+                message: ChatCompletionMessageParam = {
+                    "role": "assistant",
+                    "content": text,
+                }
+            prompt_messages.append(message)
+    else:
+        # Assemble the prompt
+        try:
+            if params.get("resultImage") and params["resultImage"]:
+                prompt_messages = assemble_prompt(
+                    params["image"], valid_stack, params["resultImage"]
+                )
+            else:
+                prompt_messages = assemble_prompt(params["image"], valid_stack)
+        except:
+            await websocket.send_json(
+                {
+                    "type": "error",
+                    "value": "Error assembling prompt. Contact support at support@picoapps.xyz",
+                }
+            )
+            await websocket.close()
+            return
+
+        if params["generationType"] == "update":
+            # Transform the history tree into message format
+            # TODO: Move this to frontend
+            for index, text in enumerate(params["history"]):
+                if index % 2 == 0:
+                    message: ChatCompletionMessageParam = {
+                        "role": "assistant",
+                        "content": text,
+                    }
+                else:
+                    message: ChatCompletionMessageParam = {
+                        "role": "user",
+                        "content": text,
+                    }
+                prompt_messages.append(message)
+
+            image_cache = create_alt_url_mapping(params["history"][-2])
+
+    if validated_input_mode == "video":
+        video_data_url = params["image"]
+        prompt_messages = await assemble_claude_prompt_video(video_data_url)
+
+    # pprint_prompt(prompt_messages)  # type: ignore
+
+    if SHOULD_MOCK_AI_RESPONSE:
+        completion = await mock_completion(
+            process_chunk, input_mode=validated_input_mode
+        )
+    else:
+        try:
+            if validated_input_mode == "video":
+                if not anthropic_api_key:
+                    await throw_error(
+                        "Video only works with Anthropic models. No Anthropic API key found. Please add the environment variable ANTHROPIC_API_KEY to backend/.env or in the settings dialog"
+                    )
+                    raise Exception("No Anthropic key")
+
+                completion = await stream_claude_response_native(
+                    system_prompt=VIDEO_PROMPT,
+                    messages=prompt_messages,  # type: ignore
+                    api_key=anthropic_api_key,
+                    callback=lambda x: process_chunk(x),
+                    model=Llm.CLAUDE_3_OPUS,
+                    include_thinking=True,
+                )
+                exact_llm_version = Llm.CLAUDE_3_OPUS
+            elif code_generation_model == Llm.CLAUDE_3_SONNET:
+                if not anthropic_api_key:
+                    await throw_error(
+                        "No Anthropic API key found. Please add the environment variable ANTHROPIC_API_KEY to backend/.env or in the settings dialog"
+                    )
+                    raise Exception("No Anthropic key")
+
+                completion = await stream_claude_response(
+                    prompt_messages,  # type: ignore
+                    api_key=anthropic_api_key,
+                    callback=lambda x: process_chunk(x),
+                )
+                exact_llm_version = code_generation_model
+            else:
+                completion = await stream_openai_response(
+                    prompt_messages,  # type: ignore
+                    api_key=openai_api_key,
+                    base_url=openai_base_url,
+                    callback=lambda x: process_chunk(x),
+                    model=code_generation_model,
+                )
+                exact_llm_version = code_generation_model
+        except openai.AuthenticationError as e:
+            print("[GENERATE_CODE] Authentication failed", e)
+            error_message = (
+                "Incorrect OpenAI key. Please make sure your OpenAI API key is correct, or create a new OpenAI API key on your OpenAI dashboard."
+                + (
+                    " Alternatively, you can purchase code generation credits directly on this website."
+                    if IS_PROD
+                    else ""
+                )
+            )
+            return await throw_error(error_message)
+        except openai.NotFoundError as e:
+            print("[GENERATE_CODE] Model not found", e)
+            error_message = (
+                e.message
+                + ". Please make sure you have followed the instructions correctly to obtain an OpenAI key with GPT vision access: https://github.com/abi/screenshot-to-code/blob/main/Troubleshooting.md"
+                + (
+                    " Alternatively, you can purchase code generation credits directly on this website."
+                    if IS_PROD
+                    else ""
+                )
+            )
+            return await throw_error(error_message)
+        except openai.RateLimitError as e:
+            print("[GENERATE_CODE] Rate limit exceeded", e)
+            error_message = (
+                "OpenAI error - 'You exceeded your current quota, please check your plan and billing details.'"
+                + (
+                    " Alternatively, you can purchase code generation credits directly on this website."
+                    if IS_PROD
+                    else ""
+                )
+            )
+            return await throw_error(error_message)
+
+    if validated_input_mode == "video":
+        completion = extract_tag_content("html", completion)
+
+    print("Exact used model for generation: ", exact_llm_version)
+
+    # Write the messages dict into a log so that we can debug later
+    write_logs(prompt_messages, completion)  # type: ignore
+
+    try:
+        if should_generate_images:
+            await websocket.send_json(
+                {"type": "status", "value": "Generating images..."}
+            )
+            updated_html = await generate_images(
+                completion,
+                api_key=openai_api_key,
+                base_url=openai_base_url,
+                image_cache=image_cache,
+            )
+        else:
+            updated_html = completion
+        await websocket.send_json({"type": "setCode", "value": updated_html})
+        await websocket.send_json(
+            {"type": "status", "value": "Code generation complete."}
+        )
+    except Exception as e:
+        traceback.print_exc()
+        print("Image generation failed", e)
+        # Send set code even if image generation fails since that triggers
+        # the frontend to update history
+        await websocket.send_json({"type": "setCode", "value": completion})
+        await websocket.send_json(
+            {"type": "status", "value": "Image generation failed but code is complete."}
+        )
+
+    await websocket.close()

backend/routes/home.py

@@ -0,0 +1,12 @@
+from fastapi import APIRouter
+from fastapi.responses import HTMLResponse
+
+
+router = APIRouter()
+
+
+@router.get("/")
+async def get_status():
+    return HTMLResponse(
+        content="<h3>Your backend is running correctly. Please open the front-end URL (default is http://localhost:5173) to use screenshot-to-code.</h3>"
+    )

backend/routes/screenshot.py

@@ -11,7 +11,9 @@ def bytes_to_data_url(image_bytes: bytes, mime_type: str) -> str:
     return f"data:{mime_type};base64,{base64_image}"
 
 
-async def capture_screenshot(target_url, api_key, device="desktop") -> bytes:
+async def capture_screenshot(
+    target_url: str, api_key: str, device: str = "desktop"
+) -> bytes:
     api_base_url = "https://api.screenshotone.com/take"
 
     params = {

backend/run_evals.py

@@ -0,0 +1,52 @@
+# Load environment variables first
+from dotenv import load_dotenv
+
+from llm import Llm
+
+load_dotenv()
+
+import os
+from typing import Any, Coroutine
+import asyncio
+
+from evals.config import EVALS_DIR
+from evals.core import generate_code_core
+from evals.utils import image_to_data_url
+
+STACK = "ionic_tailwind"
+MODEL = Llm.GPT_4O_2024_05_13
+N = 1  # Number of outputs to generate
+
+
+async def main():
+    INPUT_DIR = EVALS_DIR + "/inputs"
+    OUTPUT_DIR = EVALS_DIR + "/outputs"
+
+    # Get all the files in the directory (only grab pngs)
+    evals = [f for f in os.listdir(INPUT_DIR) if f.endswith(".png")]
+
+    tasks: list[Coroutine[Any, Any, str]] = []
+    for filename in evals:
+        filepath = os.path.join(INPUT_DIR, filename)
+        data_url = await image_to_data_url(filepath)
+        for _ in range(N):  # Generate N tasks for each input
+            task = generate_code_core(image_url=data_url, stack=STACK, model=MODEL)
+            tasks.append(task)
+
+    results = await asyncio.gather(*tasks)
+
+    os.makedirs(OUTPUT_DIR, exist_ok=True)
+
+    for i, content in enumerate(results):
+        # Calculate index for filename and output number
+        eval_index = i // N
+        output_number = i % N
+        filename = evals[eval_index]
+        # File name is derived from the original filename in evals with an added output number
+        output_filename = f"{os.path.splitext(filename)[0]}_{output_number}.html"
+        output_filepath = os.path.join(OUTPUT_DIR, output_filename)
+        with open(output_filepath, "w") as file:
+            file.write(content)
+
+
+asyncio.run(main())

backend/start.py

@@ -0,0 +1,4 @@
+import uvicorn
+
+if __name__ == "__main__":
+    uvicorn.run("main:app", port=7001, reload=True)

backend/test_llm.py

@@ -0,0 +1,41 @@
+import unittest
+from llm import convert_frontend_str_to_llm, Llm
+
+
+class TestConvertFrontendStrToLlm(unittest.TestCase):
+    def test_convert_valid_strings(self):
+        self.assertEqual(
+            convert_frontend_str_to_llm("gpt_4_vision"),
+            Llm.GPT_4_VISION,
+            "Should convert 'gpt_4_vision' to Llm.GPT_4_VISION",
+        )
+        self.assertEqual(
+            convert_frontend_str_to_llm("claude_3_sonnet"),
+            Llm.CLAUDE_3_SONNET,
+            "Should convert 'claude_3_sonnet' to Llm.CLAUDE_3_SONNET",
+        )
+        self.assertEqual(
+            convert_frontend_str_to_llm("claude-3-opus-20240229"),
+            Llm.CLAUDE_3_OPUS,
+            "Should convert 'claude-3-opus-20240229' to Llm.CLAUDE_3_OPUS",
+        )
+        self.assertEqual(
+            convert_frontend_str_to_llm("gpt-4-turbo-2024-04-09"),
+            Llm.GPT_4_TURBO_2024_04_09,
+            "Should convert 'gpt-4-turbo-2024-04-09' to Llm.GPT_4_TURBO_2024_04_09",
+        )
+        self.assertEqual(
+            convert_frontend_str_to_llm("gpt-4o-2024-05-13"),
+            Llm.GPT_4O_2024_05_13,
+            "Should convert 'gpt-4o-2024-05-13' to Llm.GPT_4O_2024_05_13",
+        )
+
+    def test_convert_invalid_string_raises_exception(self):
+        with self.assertRaises(ValueError):
+            convert_frontend_str_to_llm("invalid_string")
+        with self.assertRaises(ValueError):
+            convert_frontend_str_to_llm("another_invalid_string")
+
+
+if __name__ == "__main__":
+    unittest.main()

backend/test_prompts.py

@@ -1,97 +0,0 @@
-from prompts import assemble_prompt
-
-TAILWIND_SYSTEM_PROMPT = """
-You are an expert Tailwind developer
-You take screenshots of a reference web page from the user, and then build single page apps 
-using Tailwind, HTML and JS.
-You might also be given a screenshot(The second image) of a web page that you have already built, and asked to
-update it to look more like the reference image(The first image).
-
-- Make sure the app looks exactly like the screenshot.
-- Pay close attention to background color, text color, font size, font family, 
-padding, margin, border, etc. Match the colors and sizes exactly.
-- Use the exact text from the screenshot.
-- Do not add comments in the code such as "<!-- Add other navigation links as needed -->" and "<!-- ... other news items ... -->" in place of writing the full code. WRITE THE FULL CODE.
-- Repeat elements as needed to match the screenshot. For example, if there are 15 items, the code should have 15 items. DO NOT LEAVE comments like "<!-- Repeat for each news item -->" or bad things will happen.
-- For images, use placeholder images from https://placehold.co and include a detailed description of the image in the alt text so that an image generation AI can generate the image later.
-
-In terms of libraries,
-
-- Use this script to include Tailwind: <script src="https://cdn.tailwindcss.com"></script>
-- You can use Google Fonts
-- Font Awesome for icons: <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.15.3/css/all.min.css"></link>
-
-Return only the full code in <html></html> tags.
-Do not include markdown "```" or "```html" at the start or end.
-"""
-
-BOOTSTRAP_SYSTEM_PROMPT = """
-You are an expert Bootstrap developer
-You take screenshots of a reference web page from the user, and then build single page apps 
-using Bootstrap, HTML and JS.
-You might also be given a screenshot(The second image) of a web page that you have already built, and asked to
-update it to look more like the reference image(The first image).
-
-- Make sure the app looks exactly like the screenshot.
-- Pay close attention to background color, text color, font size, font family, 
-padding, margin, border, etc. Match the colors and sizes exactly.
-- Use the exact text from the screenshot.
-- Do not add comments in the code such as "<!-- Add other navigation links as needed -->" and "<!-- ... other news items ... -->" in place of writing the full code. WRITE THE FULL CODE.
-- Repeat elements as needed to match the screenshot. For example, if there are 15 items, the code should have 15 items. DO NOT LEAVE comments like "<!-- Repeat for each news item -->" or bad things will happen.
-- For images, use placeholder images from https://placehold.co and include a detailed description of the image in the alt text so that an image generation AI can generate the image later.
-
-In terms of libraries,
-
-- Use this script to include Bootstrap: <link href="https://cdn.jsdelivr.net/npm/bootstrap@5.3.2/dist/css/bootstrap.min.css" rel="stylesheet" integrity="sha384-T3c6CoIi6uLrA9TneNEoa7RxnatzjcDSCmG1MXxSR1GAsXEV/Dwwykc2MPK8M2HN" crossorigin="anonymous">
-- You can use Google Fonts
-- Font Awesome for icons: <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.15.3/css/all.min.css"></link>
-
-Return only the full code in <html></html> tags.
-Do not include markdown "```" or "```html" at the start or end.
-"""
-
-REACT_TAILWIND_SYSTEM_PROMPT = """
-You are an expert React/Tailwind developer
-You take screenshots of a reference web page from the user, and then build single page apps 
-using React and Tailwind CSS.
-You might also be given a screenshot(The second image) of a web page that you have already built, and asked to
-update it to look more like the reference image(The first image).
-
-- Make sure the app looks exactly like the screenshot.
-- Pay close attention to background color, text color, font size, font family, 
-padding, margin, border, etc. Match the colors and sizes exactly.
-- Use the exact text from the screenshot.
-- Do not add comments in the code such as "<!-- Add other navigation links as needed -->" and "<!-- ... other news items ... -->" in place of writing the full code. WRITE THE FULL CODE.
-- Repeat elements as needed to match the screenshot. For example, if there are 15 items, the code should have 15 items. DO NOT LEAVE comments like "<!-- Repeat for each news item -->" or bad things will happen.
-- For images, use placeholder images from https://placehold.co and include a detailed description of the image in the alt text so that an image generation AI can generate the image later.
-
-In terms of libraries,
-
-- Use these script to include React so that it can run on a standalone page:
-    <script src="https://unpkg.com/react/umd/react.development.js"></script>
-    <script src="https://unpkg.com/react-dom/umd/react-dom.development.js"></script>
-    <script src="https://unpkg.com/@babel/standalone/babel.js"></script>
-- Use this script to include Tailwind: <script src="https://cdn.tailwindcss.com"></script>
-- You can use Google Fonts
-- Font Awesome for icons: <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.15.3/css/all.min.css"></link>
-
-Return only the full code in <html></html> tags.
-Do not include markdown "```" or "```html" at the start or end.
-"""
-
-
-def test_prompts():
-    tailwind_prompt = assemble_prompt(
-        "image_data_url", {"css": "tailwind", "js": "vanilla"}, "result_image_data_url"
-    )
-    assert tailwind_prompt[0]["content"] == TAILWIND_SYSTEM_PROMPT
-
-    bootstrap_prompt = assemble_prompt(
-        "image_data_url", {"css": "bootstrap", "js": "vanilla"}, "result_image_data_url"
-    )
-    assert bootstrap_prompt[0]["content"] == BOOTSTRAP_SYSTEM_PROMPT
-
-    react_tailwind_prompt = assemble_prompt(
-        "image_data_url", {"css": "tailwind", "js": "react"}, "result_image_data_url"
-    )
-    assert react_tailwind_prompt[0]["content"] == REACT_TAILWIND_SYSTEM_PROMPT

backend/utils.py

@@ -1,20 +1,30 @@
 import copy
+import json
+from typing import List
+from openai.types.chat import ChatCompletionMessageParam
 
 
-def truncate_data_strings(data):
+def pprint_prompt(prompt_messages: List[ChatCompletionMessageParam]):
+    print(json.dumps(truncate_data_strings(prompt_messages), indent=4))
+
+
+def truncate_data_strings(data: List[ChatCompletionMessageParam]):  # type: ignore
     # Deep clone the data to avoid modifying the original object
     cloned_data = copy.deepcopy(data)
 
     if isinstance(cloned_data, dict):
-        for key, value in cloned_data.items():
+        for key, value in cloned_data.items():  # type: ignore
             # Recursively call the function if the value is a dictionary or a list
             if isinstance(value, (dict, list)):
-                cloned_data[key] = truncate_data_strings(value)
+                cloned_data[key] = truncate_data_strings(value)  # type: ignore
-            # Truncate the string if it starts with 'data:'
+            # Truncate the string if it it's long and add ellipsis and length
-            elif isinstance(value, str) and value.startswith("data:"):
+            elif isinstance(value, str):
-                cloned_data[key] = value[:20]
+                cloned_data[key] = value[:40]  # type: ignore
-    elif isinstance(cloned_data, list):
+                if len(value) > 40:
-        # Process each item in the list
+                    cloned_data[key] += "..." + f" ({len(value)} chars)"  # type: ignore
-        cloned_data = [truncate_data_strings(item) for item in cloned_data]
 
-    return cloned_data
+    elif isinstance(cloned_data, list):  # type: ignore
+        # Process each item in the list
+        cloned_data = [truncate_data_strings(item) for item in cloned_data]  # type: ignore
+
+    return cloned_data  # type: ignore

backend/video/utils.py

@@ -0,0 +1,134 @@
+# Extract HTML content from the completion string
+import base64
+import io
+import mimetypes
+import os
+import tempfile
+import uuid
+from typing import Any, Union, cast
+from moviepy.editor import VideoFileClip  # type: ignore
+from PIL import Image
+import math
+
+
+DEBUG = True
+TARGET_NUM_SCREENSHOTS = (
+    20  # Should be max that Claude supports (20) - reduce to save tokens on testing
+)
+
+
+async def assemble_claude_prompt_video(video_data_url: str) -> list[Any]:
+    images = split_video_into_screenshots(video_data_url)
+
+    # Save images to tmp if we're debugging
+    if DEBUG:
+        save_images_to_tmp(images)
+
+    # Validate number of images
+    print(f"Number of frames extracted from video: {len(images)}")
+    if len(images) > 20:
+        print(f"Too many screenshots: {len(images)}")
+        raise ValueError("Too many screenshots extracted from video")
+
+    # Convert images to the message format for Claude
+    content_messages: list[dict[str, Union[dict[str, str], str]]] = []
+    for image in images:
+
+        # Convert Image to buffer
+        buffered = io.BytesIO()
+        image.save(buffered, format="JPEG")
+
+        # Encode bytes as base64
+        base64_data = base64.b64encode(buffered.getvalue()).decode("utf-8")
+        media_type = "image/jpeg"
+
+        content_messages.append(
+            {
+                "type": "image",
+                "source": {
+                    "type": "base64",
+                    "media_type": media_type,
+                    "data": base64_data,
+                },
+            }
+        )
+
+    return [
+        {
+            "role": "user",
+            "content": content_messages,
+        },
+    ]
+
+
+# Returns a list of images/frame (RGB format)
+def split_video_into_screenshots(video_data_url: str) -> list[Image.Image]:
+    target_num_screenshots = TARGET_NUM_SCREENSHOTS
+
+    # Decode the base64 URL to get the video bytes
+    video_encoded_data = video_data_url.split(",")[1]
+    video_bytes = base64.b64decode(video_encoded_data)
+
+    mime_type = video_data_url.split(";")[0].split(":")[1]
+    suffix = mimetypes.guess_extension(mime_type)
+
+    with tempfile.NamedTemporaryFile(suffix=suffix, delete=True) as temp_video_file:
+        print(temp_video_file.name)
+        temp_video_file.write(video_bytes)
+        temp_video_file.flush()
+        clip = VideoFileClip(temp_video_file.name)
+        images: list[Image.Image] = []
+        total_frames = cast(int, clip.reader.nframes)  # type: ignore
+
+        # Calculate frame skip interval by dividing total frames by the target number of screenshots
+        # Ensuring a minimum skip of 1 frame
+        frame_skip = max(1, math.ceil(total_frames / target_num_screenshots))
+
+        # Iterate over each frame in the clip
+        for i, frame in enumerate(clip.iter_frames()):
+            # Save every nth frame
+            if i % frame_skip == 0:
+                frame_image = Image.fromarray(frame)  # type: ignore
+                images.append(frame_image)
+                # Ensure that we don't capture more than the desired number of frames
+                if len(images) >= target_num_screenshots:
+                    break
+
+        # Close the video file to release resources
+        clip.close()
+
+        return images
+
+
+# Save a list of PIL images to a random temporary directory
+def save_images_to_tmp(images: list[Image.Image]):
+
+    # Create a unique temporary directory
+    unique_dir_name = f"screenshots_{uuid.uuid4()}"
+    tmp_screenshots_dir = os.path.join(tempfile.gettempdir(), unique_dir_name)
+    os.makedirs(tmp_screenshots_dir, exist_ok=True)
+
+    for idx, image in enumerate(images):
+        # Generate a unique image filename using index
+        image_filename = f"screenshot_{idx}.jpg"
+        tmp_filepath = os.path.join(tmp_screenshots_dir, image_filename)
+        image.save(tmp_filepath, format="JPEG")
+
+    print("Saved to " + tmp_screenshots_dir)
+
+
+def extract_tag_content(tag: str, text: str) -> str:
+    """
+    Extracts content for a given tag from the provided text.
+
+    :param tag: The tag to search for.
+    :param text: The text to search within.
+    :return: The content found within the tag, if any.
+    """
+    tag_start = f"<{tag}>"
+    tag_end = f"</{tag}>"
+    start_idx = text.find(tag_start)
+    end_idx = text.find(tag_end, start_idx)
+    if start_idx != -1 and end_idx != -1:
+        return text[start_idx : end_idx + len(tag_end)]
+    return ""

backend/video_to_app.py

@@ -0,0 +1,122 @@
+# Load environment variables first
+
+from dotenv import load_dotenv
+
+load_dotenv()
+
+
+import base64
+import mimetypes
+import time
+import subprocess
+import os
+import asyncio
+from datetime import datetime
+from prompts.claude_prompts import VIDEO_PROMPT
+from utils import pprint_prompt
+from config import ANTHROPIC_API_KEY
+from video.utils import extract_tag_content, assemble_claude_prompt_video
+from llm import (
+    Llm,
+    stream_claude_response_native,
+)
+
+STACK = "html_tailwind"
+
+VIDEO_DIR = "./video_evals/videos"
+SCREENSHOTS_DIR = "./video_evals/screenshots"
+OUTPUTS_DIR = "./video_evals/outputs"
+
+
+async def main():
+    video_filename = "shortest.mov"
+    is_followup = False
+
+    if not ANTHROPIC_API_KEY:
+        raise ValueError("ANTHROPIC_API_KEY is not set")
+
+    # Get previous HTML
+    previous_html = ""
+    if is_followup:
+        previous_html_file = max(
+            [
+                os.path.join(OUTPUTS_DIR, f)
+                for f in os.listdir(OUTPUTS_DIR)
+                if f.endswith(".html")
+            ],
+            key=os.path.getctime,
+        )
+        with open(previous_html_file, "r") as file:
+            previous_html = file.read()
+
+    video_file = os.path.join(VIDEO_DIR, video_filename)
+    mime_type = mimetypes.guess_type(video_file)[0]
+    with open(video_file, "rb") as file:
+        video_content = file.read()
+    video_data_url = (
+        f"data:{mime_type};base64,{base64.b64encode(video_content).decode('utf-8')}"
+    )
+
+    prompt_messages = await assemble_claude_prompt_video(video_data_url)
+
+    # Tell the model to continue
+    # {"role": "assistant", "content": SECOND_MESSAGE},
+    # {"role": "user", "content": "continue"},
+
+    if is_followup:
+        prompt_messages += [
+            {"role": "assistant", "content": previous_html},
+            {
+                "role": "user",
+                "content": "You've done a good job with a first draft. Improve this further based on the original instructions so that the app is fully functional like in the original video.",
+            },
+        ]  # type: ignore
+
+    async def process_chunk(content: str):
+        print(content, end="", flush=True)
+
+    response_prefix = "<thinking>"
+
+    pprint_prompt(prompt_messages)  # type: ignore
+
+    start_time = time.time()
+
+    completion = await stream_claude_response_native(
+        system_prompt=VIDEO_PROMPT,
+        messages=prompt_messages,
+        api_key=ANTHROPIC_API_KEY,
+        callback=lambda x: process_chunk(x),
+        model=Llm.CLAUDE_3_OPUS,
+        include_thinking=True,
+    )
+
+    end_time = time.time()
+
+    # Prepend the response prefix to the completion
+    completion = response_prefix + completion
+
+    # Extract the outputs
+    html_content = extract_tag_content("html", completion)
+    thinking = extract_tag_content("thinking", completion)
+
+    print(thinking)
+    print(f"Operation took {end_time - start_time} seconds")
+
+    os.makedirs(OUTPUTS_DIR, exist_ok=True)
+
+    # Generate a unique filename based on the current time
+    timestamp = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
+    filename = f"video_test_output_{timestamp}.html"
+    output_path = os.path.join(OUTPUTS_DIR, filename)
+
+    # Write the HTML content to the file
+    with open(output_path, "w") as file:
+        file.write(html_content)
+
+    print(f"Output file path: {output_path}")
+
+    # Show a notification
+    subprocess.run(["osascript", "-e", 'display notification "Coding Complete"'])
+
+
+asyncio.run(main())

backend/ws/constants.py

@@ -0,0 +1,2 @@
+# WebSocket protocol (RFC 6455) allows for the use of custom close codes in the range 4000-4999
+APP_ERROR_WEB_SOCKET_CODE = 4332

blog/evaluating-claude.md

@@ -0,0 +1,59 @@
+# Claude 3 for converting screenshots to code
+
+Claude 3 dropped yesterday, claiming to rival GPT-4 on a wide variety of tasks. I maintain a very popular open source project called “screenshot-to-code” (this one!) that uses GPT-4 vision to convert screenshots/designs into clean code. Naturally, I was excited to see how good Claude 3 was at this task.
+
+**TLDR:** Claude 3 is on par with GPT-4 vision for screenshot to code, better in some ways but worse in others.
+
+## Evaluation Setup
+
+I don’t know of a public benchmark for “screenshot to code” so I created simple evaluation setup for the purposes of testing:
+
+- **Evaluation Dataset**: 16 screenshots with a mix of UI elements, landing pages, dashboards and popular websites.
+<img width="784" alt="Screenshot 2024-03-05 at 3 05 52 PM" src="https://github.com/abi/screenshot-to-code/assets/23818/c32af2db-eb5a-44c1-9a19-2f0c3dd11ab4">
+
+- **Evaluation Metric**: Replication accuracy, as in “How close does the generated code look to the screenshot?” While there are other metrics that are important like code quality, speed and so on, this is by far the #1 thing most users of the repo care about.
+- **Evaluation Mechanism**: Each output is subjectively rated by a human on a rating scale from 0 to 4. 4 = very close to an exact replica while 0 = nothing like the screenshot. With 16 screenshots, the maximum any model can score is 64.
+
+
+To make the evaluation process easy, I created [a Python script](https://github.com/abi/screenshot-to-code/blob/main/backend/run_evals.py) that runs code for all the inputs in parallel. I also made a simple UI to do a side-by-side comparison of the input and output.
+
+![Google Chrome](https://github.com/abi/screenshot-to-code/assets/23818/38126f8f-205d-4ed1-b8cf-039e81dcc3d0)
+
+
+## Results
+
+Quick note about what kind of code we’ll be generating: currently, screenshot-to-code supports generating code in HTML + Tailwind, React, Vue, and several other frameworks. Stacks can impact the replication accuracy quite a bit. For example, because Bootstrap uses a relatively restrictive set of user elements, generations using Bootstrap tend to have a distinct "Bootstrap" style.
+
+I only ran the evals on HTML/Tailwind here which is the stack where GPT-4 vision tends to perform the best.
+
+Here are the results (average of 3 runs for each model):
+
+- GPT-4 Vision obtains a score of **65.10%** - this is what we’re trying to beat
+- Claude 3 Sonnet receives a score of **70.31%**, which is a bit better.
+- Surprisingly, Claude 3 Opus which is supposed to be the smarter and slower model scores worse than both GPT-4 vision and Claude 3 Sonnet, comes in at **61.46%**.
+
+Overall, a very strong showing for Claude 3. Obviously, there's a lot of subjectivity involved in this evaluation but Claude 3 is definitely on par with GPT-4 Vision, if not better.
+
+You can see the [side-by-side comparison for a run of Claude 3 Sonnet here](https://github.com/abi/screenshot-to-code-files/blob/main/sonnet%20results.png). And for [a run of GPT-4 Vision here](https://github.com/abi/screenshot-to-code-files/blob/main/gpt%204%20vision%20results.png).
+
+Other notes:
+
+- The prompts used are optimized for GPT-4 vision. Adjusting the prompts a bit for Claude did yield a small improvement. But nothing game-changing and potentially not worth the trade-off of maintaining two sets of prompts.
+- All the models excel at code quality - the quality is usually comparable to a human or better.
+- Claude 3 is much less lazy than GPT-4 Vision. When asked to recreate Hacker News, GPT-4 Vision will only create two items in the list and leave comments in this code like `<!-- Repeat for each news item -->` and `<!-- ... other news items ... -->`.
+<img width="699" alt="Screenshot 2024-03-05 at 9 25 04 PM" src="https://github.com/abi/screenshot-to-code/assets/23818/04b03155-45e0-40b0-8de0-b1f0b4382bee">
+
+While Claude 3 Sonnet can sometimes be lazy too, most of the time, it does what you ask it to do.
+
+<img width="904" alt="Screenshot 2024-03-05 at 9 30 23 PM" src="https://github.com/abi/screenshot-to-code/assets/23818/b7c7d1ba-47c1-414d-928f-6989e81cf41d">
+
+- For some reason, all the models struggle with side-by-side "flex" layouts
+<img width="1090" alt="Screenshot 2024-03-05 at 9 20 58 PM" src="https://github.com/abi/screenshot-to-code/assets/23818/8957bb3a-da66-467d-997d-1c7cc24e6d9a">
+
+- Claude 3 Sonnet is a lot faster
+- Claude 3 gets background and text colors wrong quite often! (like in the Hacker News image above)
+- My suspicion is that Claude 3 Opus results can be improved to be on par with the other models through better prompting
+  
+Overall, I'm very impressed with Claude 3 Sonnet for this use case. I've added it as an alternative to GPT-4 Vision in the open source repo (hosted version update coming soon).
+
+If you’d like to contribute to this effort, I have some documentation on [running these evals yourself here](https://github.com/abi/screenshot-to-code/blob/main/Evaluation.md). I'm also working on a better evaluation mechanism with Elo ratings and would love some help on that.

design-docs.md

@@ -0,0 +1,5 @@
+## Version History
+
+Version history is stored as a tree on the client-side.
+
+![Screenshot to Code](https://github.com/abi/screenshot-to-code/assets/23818/e35644aa-b90a-4aa7-8027-b8732796fd7c)

frontend/.gitignore

@@ -25,3 +25,6 @@ dist-ssr
 
 # Env files
 .env*
+
+# Test files
+src/tests/results/

frontend/Dockerfile

@@ -16,4 +16,4 @@ COPY ./ /app/
 EXPOSE 5173
 
 # Command to run the application
-CMD ["yarn", "dev", "--host", "0.0.0.0"]
+CMD ["yarn", "dev", "--host", "0.0.0.0"]

frontend/components.json

@@ -13,4 +13,4 @@
     "components": "@/components",
     "utils": "@/lib/utils"
   }
-}
+}

frontend/index.html

@@ -2,11 +2,7 @@
 <html lang="en">
   <head>
     <meta charset="UTF-8" />
-    <link
+    <link rel="icon" type="image/png" href="/favicon/main.png" />
-      rel="icon"
-      type="image/svg+xml"
-      href="https://picoapps.xyz/favicon.png"
-    />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
 
     <!-- Google Fonts -->
@@ -21,6 +17,34 @@
     <%- injectHead %>
 
     <title>Screenshot to Code</title>
+
+    <!-- Open Graph Meta Tags -->
+    <meta property="og:title" content="Screenshot to Code" />
+    <meta
+      property="og:description"
+      content="Convert any screenshot or design to clean code"
+    />
+    <meta
+      property="og:image"
+      content="https://screenshottocode.com/brand/twitter-summary-card.png"
+    />
+    <meta property="og:image:width" content="1200" />
+    <meta property="og:image:height" content="628" />
+    <meta property="og:url" content="https://screenshottocode.com" />
+    <meta property="og:type" content="website" />
+    <!-- Twitter Card tags -->
+    <meta name="twitter:card" content="summary_large_image" />
+    <meta name="twitter:site" content="@picoapps" />
+    <!-- Keep in sync with og:title, og:description and og:image -->
+    <meta name="twitter:title" content="Screenshot to Code" />
+    <meta
+      name="twitter:description"
+      content="Convert any screenshot or design to clean code"
+    />
+    <meta
+      name="twitter:image"
+      content="https://screenshottocode.com/brand/twitter-summary-card.png"
+    />
   </head>
   <body>
     <div id="root"></div>

frontend/jest.config.js

@@ -0,0 +1,9 @@
+export default {
+  preset: "ts-jest",
+  testEnvironment: "node",
+  setupFiles: ["<rootDir>/src/setupTests.ts"],
+  transform: {
+    "^.+\\.tsx?$": "ts-jest",
+  },
+  testTimeout: 30000,
+};

frontend/package.json

@@ -9,17 +9,22 @@
     "build": "tsc && vite build",
     "build-hosted": "tsc && vite build --mode prod",
     "lint": "eslint . --ext ts,tsx --report-unused-disable-directives --max-warnings 0",
-    "preview": "vite preview"
+    "preview": "vite preview",
+    "test": "jest"
   },
   "dependencies": {
     "@codemirror/lang-html": "^6.4.6",
     "@radix-ui/react-accordion": "^1.1.2",
     "@radix-ui/react-alert-dialog": "^1.0.5",
     "@radix-ui/react-checkbox": "^1.0.4",
+    "@radix-ui/react-collapsible": "^1.0.3",
     "@radix-ui/react-dialog": "^1.0.5",
+    "@radix-ui/react-hover-card": "^1.0.7",
     "@radix-ui/react-icons": "^1.3.0",
     "@radix-ui/react-label": "^2.0.2",
     "@radix-ui/react-popover": "^1.0.7",
+    "@radix-ui/react-progress": "^1.0.3",
+    "@radix-ui/react-scroll-area": "^1.0.5",
     "@radix-ui/react-select": "^2.0.0",
     "@radix-ui/react-separator": "^1.0.3",
     "@radix-ui/react-slot": "^1.0.2",
@@ -36,27 +41,37 @@
     "react-dropzone": "^14.2.3",
     "react-hot-toast": "^2.4.1",
     "react-icons": "^4.12.0",
+    "react-router-dom": "^6.20.1",
     "tailwind-merge": "^2.0.0",
     "tailwindcss-animate": "^1.0.7",
     "thememirror": "^2.0.1",
-    "vite-plugin-checker": "^0.6.2"
+    "vite-plugin-checker": "^0.6.2",
+    "webm-duration-fix": "^1.0.4",
+    "zustand": "^4.5.2"
   },
   "devDependencies": {
+    "@types/jest": "^29.5.12",
     "@types/node": "^20.9.0",
+    "@types/puppeteer": "^7.0.4",
     "@types/react": "^18.2.15",
     "@types/react-dom": "^18.2.7",
     "@typescript-eslint/eslint-plugin": "^6.0.0",
     "@typescript-eslint/parser": "^6.0.0",
     "@vitejs/plugin-react": "^4.0.3",
     "autoprefixer": "^10.4.16",
+    "dotenv": "^16.4.5",
     "eslint": "^8.45.0",
     "eslint-plugin-react-hooks": "^4.6.0",
     "eslint-plugin-react-refresh": "^0.4.3",
+    "jest": "^29.7.0",
     "postcss": "^8.4.31",
+    "puppeteer": "^22.6.4",
     "tailwindcss": "^3.3.5",
+    "ts-jest": "^29.1.2",
     "typescript": "^5.0.2",
     "vite": "^4.4.5",
-    "vite-plugin-html": "^3.2.0"
+    "vite-plugin-html": "^3.2.0",
+    "vitest": "^1.0.1"
   },
   "engines": {
     "node": ">=14.18.0"

frontend/src/.env.jest.example

@@ -0,0 +1,2 @@
+TEST_SCREENSHOTONE_API_KEY=
+TEST_ROOT_PATH=

frontend/src/App.tsx

@@ -1,8 +1,8 @@
-import { useRef, useState } from "react";
+import { useEffect, useRef, useState } from "react";
 import ImageUpload from "./components/ImageUpload";
 import CodePreview from "./components/CodePreview";
 import Preview from "./components/Preview";
-import { CodeGenerationParams, generateCode } from "./generateCode";
+import { generateCode } from "./generateCode";
 import Spinner from "./components/Spinner";
 import classNames from "classnames";
 import {
@@ -18,14 +18,7 @@ import { Button } from "@/components/ui/button";
 import { Textarea } from "@/components/ui/textarea";
 import { Tabs, TabsContent, TabsList, TabsTrigger } from "./components/ui/tabs";
 import SettingsDialog from "./components/SettingsDialog";
-import {
+import { AppState, CodeGenerationParams, EditorTheme, Settings } from "./types";
-  Settings,
-  EditorTheme,
-  AppState,
-  CSSOption,
-  OutputSettings,
-  JSFrameworkOption,
-} from "./types";
 import { IS_RUNNING_ON_CLOUD } from "./config";
 import { PicoBadge } from "./components/PicoBadge";
 import { OnboardingNote } from "./components/OnboardingNote";
@@ -36,34 +29,86 @@ import html2canvas from "html2canvas";
 import { USER_CLOSE_WEB_SOCKET_CODE } from "./constants";
 import CodeTab from "./components/CodeTab";
 import OutputSettingsSection from "./components/OutputSettingsSection";
+import { History } from "./components/history/history_types";
+import HistoryDisplay from "./components/history/HistoryDisplay";
+import { extractHistoryTree } from "./components/history/utils";
+import toast from "react-hot-toast";
+import ImportCodeSection from "./components/ImportCodeSection";
+import { Stack } from "./lib/stacks";
+import { CodeGenerationModel } from "./lib/models";
+import ModelSettingsSection from "./components/ModelSettingsSection";
+import { extractHtml } from "./components/preview/extractHtml";
+import useBrowserTabIndicator from "./hooks/useBrowserTabIndicator";
+import TipLink from "./components/core/TipLink";
+
+const IS_OPENAI_DOWN = false;
 
 function App() {
   const [appState, setAppState] = useState<AppState>(AppState.INITIAL);
   const [generatedCode, setGeneratedCode] = useState<string>("");
+
+  const [inputMode, setInputMode] = useState<"image" | "video">("image");
+
   const [referenceImages, setReferenceImages] = useState<string[]>([]);
   const [executionConsole, setExecutionConsole] = useState<string[]>([]);
   const [updateInstruction, setUpdateInstruction] = useState("");
-  const [history, setHistory] = useState<string[]>([]);
+  const [isImportedFromCode, setIsImportedFromCode] = useState<boolean>(false);
+
+  // Settings
   const [settings, setSettings] = usePersistedState<Settings>(
     {
       openAiApiKey: null,
+      openAiBaseURL: null,
+      anthropicApiKey: null,
       screenshotOneApiKey: null,
       isImageGenerationEnabled: true,
       editorTheme: EditorTheme.COBALT,
+      generatedCodeConfig: Stack.HTML_TAILWIND,
+      codeGenerationModel: CodeGenerationModel.GPT_4O_2024_05_13,
+      // Only relevant for hosted version
       isTermOfServiceAccepted: false,
-      accessCode: null,
     },
     "setting"
   );
-  const [outputSettings, setOutputSettings] = useState<OutputSettings>({
+
-    css: CSSOption.TAILWIND,
+  // Code generation model from local storage or the default value
-    js: JSFrameworkOption.NO_FRAMEWORK,
+  const selectedCodeGenerationModel =
-  });
+    settings.codeGenerationModel || CodeGenerationModel.GPT_4_VISION;
+
+  // App history
+  const [appHistory, setAppHistory] = useState<History>([]);
+  // Tracks the currently shown version from app history
+  const [currentVersion, setCurrentVersion] = useState<number | null>(null);
+
   const [shouldIncludeResultImage, setShouldIncludeResultImage] =
     useState<boolean>(false);
 
   const wsRef = useRef<WebSocket>(null);
 
+  const showReactWarning =
+    selectedCodeGenerationModel ===
+      CodeGenerationModel.GPT_4_TURBO_2024_04_09 &&
+    settings.generatedCodeConfig === Stack.REACT_TAILWIND;
+
+  const showGpt4OMessage =
+    selectedCodeGenerationModel !== CodeGenerationModel.GPT_4O_2024_05_13 &&
+    appState === AppState.INITIAL;
+
+  // Indicate coding state using the browser tab's favicon and title
+  useBrowserTabIndicator(appState === AppState.CODING);
+
+  // When the user already has the settings in local storage, newly added keys
+  // do not get added to the settings so if it's falsy, we populate it with the default
+  // value
+  useEffect(() => {
+    if (!settings.generatedCodeConfig) {
+      setSettings((prev) => ({
+        ...prev,
+        generatedCodeConfig: Stack.HTML_TAILWIND,
+      }));
+    }
+  }, [settings.generatedCodeConfig, setSettings]);
+
   const takeScreenshot = async (): Promise<string> => {
     const iframeElement = document.querySelector(
       "#preview-desktop"
@@ -99,66 +144,187 @@ function App() {
     setGeneratedCode("");
     setReferenceImages([]);
     setExecutionConsole([]);
-    setHistory([]);
+    setUpdateInstruction("");
+    setIsImportedFromCode(false);
+    setAppHistory([]);
+    setCurrentVersion(null);
+    setShouldIncludeResultImage(false);
   };
 
-  const stop = () => {
+  const regenerate = () => {
+    if (currentVersion === null) {
+      toast.error(
+        "No current version set. Please open a Github issue as this shouldn't happen."
+      );
+      return;
+    }
+
+    // Retrieve the previous command
+    const previousCommand = appHistory[currentVersion];
+    if (previousCommand.type !== "ai_create") {
+      toast.error("Only the first version can be regenerated.");
+      return;
+    }
+
+    // Re-run the create
+    doCreate(referenceImages, inputMode);
+  };
+
+  const cancelCodeGeneration = () => {
     wsRef.current?.close?.(USER_CLOSE_WEB_SOCKET_CODE);
     // make sure stop can correct the state even if the websocket is already closed
-    setAppState(AppState.CODE_READY);
+    cancelCodeGenerationAndReset();
   };
 
-  function doGenerateCode(params: CodeGenerationParams) {
+  const previewCode =
+    inputMode === "video" && appState === AppState.CODING
+      ? extractHtml(generatedCode)
+      : generatedCode;
+
+  const cancelCodeGenerationAndReset = () => {
+    // When this is the first version, reset the entire app state
+    if (currentVersion === null) {
+      reset();
+    } else {
+      // Otherwise, revert to the last version
+      setGeneratedCode(appHistory[currentVersion].code);
+      setAppState(AppState.CODE_READY);
+    }
+  };
+
+  function doGenerateCode(
+    params: CodeGenerationParams,
+    parentVersion: number | null
+  ) {
     setExecutionConsole([]);
     setAppState(AppState.CODING);
 
     // Merge settings with params
-    const updatedParams = { ...params, ...settings, outputSettings };
+    const updatedParams = { ...params, ...settings };
 
     generateCode(
       wsRef,
       updatedParams,
+      // On change
       (token) => setGeneratedCode((prev) => prev + token),
-      (code) => setGeneratedCode(code),
+      // On set code
+      (code) => {
+        setGeneratedCode(code);
+        if (params.generationType === "create") {
+          setAppHistory([
+            {
+              type: "ai_create",
+              parentIndex: null,
+              code,
+              inputs: { image_url: referenceImages[0] },
+            },
+          ]);
+          setCurrentVersion(0);
+        } else {
+          setAppHistory((prev) => {
+            // Validate parent version
+            if (parentVersion === null) {
+              toast.error(
+                "No parent version set. Contact support or open a Github issue."
+              );
+              return prev;
+            }
+
+            const newHistory: History = [
+              ...prev,
+              {
+                type: "ai_edit",
+                parentIndex: parentVersion,
+                code,
+                inputs: {
+                  prompt: updateInstruction,
+                },
+              },
+            ];
+            setCurrentVersion(newHistory.length - 1);
+            return newHistory;
+          });
+        }
+      },
+      // On status update
       (line) => setExecutionConsole((prev) => [...prev, line]),
-      () => setAppState(AppState.CODE_READY)
+      // On cancel
+      () => {
+        cancelCodeGenerationAndReset();
+      },
+      // On complete
+      () => {
+        setAppState(AppState.CODE_READY);
+      }
     );
   }
 
   // Initial version creation
-  function doCreate(referenceImages: string[]) {
+  function doCreate(referenceImages: string[], inputMode: "image" | "video") {
     // Reset any existing state
     reset();
 
     setReferenceImages(referenceImages);
+    setInputMode(inputMode);
     if (referenceImages.length > 0) {
-      doGenerateCode({
+      doGenerateCode(
-        generationType: "create",
+        {
-        image: referenceImages[0],
+          generationType: "create",
-      });
+          image: referenceImages[0],
+          inputMode,
+        },
+        currentVersion
+      );
     }
   }
 
   // Subsequent updates
   async function doUpdate() {
-    const updatedHistory = [...history, generatedCode, updateInstruction];
+    if (currentVersion === null) {
-    if (shouldIncludeResultImage) {
+      toast.error(
-      const resultImage = await takeScreenshot();
+        "No current version set. Contact support or open a Github issue."
-      doGenerateCode({
+      );
-        generationType: "update",
+      return;
-        image: referenceImages[0],
+    }
-        resultImage: resultImage,
+
-        history: updatedHistory,
+    let historyTree;
-      });
+    try {
-    } else {
+      historyTree = extractHistoryTree(appHistory, currentVersion);
-      doGenerateCode({
+    } catch {
-        generationType: "update",
+      toast.error(
-        image: referenceImages[0],
+        "Version history is invalid. This shouldn't happen. Please contact support or open a Github issue."
-        history: updatedHistory,
+      );
-      });
+      return;
+    }
+
+    const updatedHistory = [...historyTree, updateInstruction];
+
+    if (shouldIncludeResultImage) {
+      const resultImage = await takeScreenshot();
+      doGenerateCode(
+        {
+          generationType: "update",
+          inputMode,
+          image: referenceImages[0],
+          resultImage: resultImage,
+          history: updatedHistory,
+          isImportedFromCode,
+        },
+        currentVersion
+      );
+    } else {
+      doGenerateCode(
+        {
+          generationType: "update",
+          inputMode,
+          image: referenceImages[0],
+          history: updatedHistory,
+          isImportedFromCode,
+        },
+        currentVersion
+      );
     }
 
-    setHistory(updatedHistory);
     setGeneratedCode("");
     setUpdateInstruction("");
   }
@@ -170,8 +336,41 @@ function App() {
     }));
   };
 
+  function setStack(stack: Stack) {
+    setSettings((prev) => ({
+      ...prev,
+      generatedCodeConfig: stack,
+    }));
+  }
+
+  function setCodeGenerationModel(codeGenerationModel: CodeGenerationModel) {
+    setSettings((prev) => ({
+      ...prev,
+      codeGenerationModel,
+    }));
+  }
+
+  function importFromCode(code: string, stack: Stack) {
+    setIsImportedFromCode(true);
+
+    // Set up this project
+    setGeneratedCode(code);
+    setStack(stack);
+    setAppHistory([
+      {
+        type: "code_create",
+        parentIndex: null,
+        code,
+        inputs: { code },
+      },
+    ]);
+    setCurrentVersion(0);
+
+    setAppState(AppState.CODE_READY);
+  }
+
   return (
-    <div className="mt-2">
+    <div className="mt-2 dark:bg-black dark:text-white">
       {IS_RUNNING_ON_CLOUD && <PicoBadge />}
       {IS_RUNNING_ON_CLOUD && (
         <TermsOfServiceDialog
@@ -179,40 +378,91 @@ function App() {
           onOpenChange={handleTermDialogOpenChange}
         />
       )}
-
       <div className="lg:fixed lg:inset-y-0 lg:z-40 lg:flex lg:w-96 lg:flex-col">
-        <div className="flex grow flex-col gap-y-2 overflow-y-auto border-r border-gray-200 bg-white px-6">
+        <div
+          className="flex grow flex-col gap-y-2 overflow-y-auto border-r 
+          border-gray-200 bg-white px-6 dark:bg-zinc-950 dark:text-white"
+        >
           <div className="flex items-center justify-between mt-10 mb-2">
             <h1 className="text-2xl ">Screenshot to Code</h1>
             <SettingsDialog settings={settings} setSettings={setSettings} />
           </div>
 
           <OutputSettingsSection
-            outputSettings={outputSettings}
+            stack={settings.generatedCodeConfig}
-            setOutputSettings={setOutputSettings}
+            setStack={(config) => setStack(config)}
             shouldDisableUpdates={
               appState === AppState.CODING || appState === AppState.CODE_READY
             }
           />
 
+          <ModelSettingsSection
+            codeGenerationModel={selectedCodeGenerationModel}
+            setCodeGenerationModel={setCodeGenerationModel}
+            shouldDisableUpdates={
+              appState === AppState.CODING || appState === AppState.CODE_READY
+            }
+          />
+
+          {showReactWarning && (
+            <div className="text-sm bg-yellow-200 rounded p-2">
+              Sorry - React is not currently working with GPT-4 Turbo. Please
+              use GPT-4 Vision or Claude Sonnet. We are working on a fix.
+            </div>
+          )}
+
+          {showGpt4OMessage && (
+            <div className="rounded-lg p-2 bg-fuchsia-200">
+              <p className="text-gray-800 text-sm">
+                Now supporting GPT-4o. Higher quality and 2x faster. Give it a
+                try!
+              </p>
+            </div>
+          )}
+
+          {appState !== AppState.CODE_READY && <TipLink />}
+
           {IS_RUNNING_ON_CLOUD && !settings.openAiApiKey && <OnboardingNote />}
 
+          {IS_OPENAI_DOWN && (
+            <div className="bg-black text-white dark:bg-white dark:text-black p-3 rounded">
+              OpenAI API is currently down. Try back in 30 minutes or later. We
+              apologize for the inconvenience.
+            </div>
+          )}
+
           {(appState === AppState.CODING ||
             appState === AppState.CODE_READY) && (
             <>
               {/* Show code preview only when coding */}
               {appState === AppState.CODING && (
                 <div className="flex flex-col">
+                  {/* Speed disclaimer for video mode */}
+                  {inputMode === "video" && (
+                    <div
+                      className="bg-yellow-100 border-l-4 border-yellow-500 text-yellow-700
+                    p-2 text-xs mb-4 mt-1"
+                    >
+                      Code generation from videos can take 3-4 minutes. We do
+                      multiple passes to get the best result. Please be patient.
+                    </div>
+                  )}
+
                   <div className="flex items-center gap-x-1">
                     <Spinner />
                     {executionConsole.slice(-1)[0]}
                   </div>
-                  <div className="flex mt-4 w-full">
+
-                    <Button onClick={stop} className="w-full">
+                  <CodePreview code={generatedCode} />
-                      Stop
+
+                  <div className="flex w-full">
+                    <Button
+                      onClick={cancelCodeGeneration}
+                      className="w-full dark:text-white dark:bg-gray-700"
+                    >
+                      Cancel
                     </Button>
                   </div>
-                  <CodePreview code={generatedCode} />
                 </div>
               )}
 
@@ -225,52 +475,69 @@ function App() {
                       value={updateInstruction}
                     />
                     <div className="flex justify-between items-center gap-x-2">
-                      <div className="font-500 text-xs text-slate-700">
+                      <div className="font-500 text-xs text-slate-700 dark:text-white">
                         Include screenshot of current version?
                       </div>
                       <Switch
                         checked={shouldIncludeResultImage}
                         onCheckedChange={setShouldIncludeResultImage}
+                        className="dark:bg-gray-700"
                       />
                     </div>
-                    <Button onClick={doUpdate}>Update</Button>
+                    <Button
+                      onClick={doUpdate}
+                      className="dark:text-white dark:bg-gray-700 update-btn"
+                    >
+                      Update
+                    </Button>
                   </div>
-                  <div className="flex items-center gap-x-2 mt-2">
+                  <div className="flex items-center justify-end gap-x-2 mt-2">
                     <Button
-                      onClick={downloadCode}
+                      onClick={regenerate}
-                      className="flex items-center gap-x-2"
+                      className="flex items-center gap-x-2 dark:text-white dark:bg-gray-700 regenerate-btn"
                     >
-                      <FaDownload /> Download
+                      🔄 Regenerate
-                    </Button>
-                    <Button
-                      onClick={reset}
-                      className="flex items-center gap-x-2"
-                    >
-                      <FaUndo />
-                      Reset
                     </Button>
                   </div>
+                  <div className="flex justify-end items-center mt-2">
+                    <TipLink />
+                  </div>
                 </div>
               )}
 
               {/* Reference image display */}
               <div className="flex gap-x-2 mt-2">
-                <div className="flex flex-col">
+                {referenceImages.length > 0 && (
-                  <div
+                  <div className="flex flex-col">
-                    className={classNames({
+                    <div
-                      "scanning relative": appState === AppState.CODING,
+                      className={classNames({
-                    })}
+                        "scanning relative": appState === AppState.CODING,
-                  >
+                      })}
-                    <img
+                    >
-                      className="w-[340px] border border-gray-200 rounded-md"
+                      {inputMode === "image" && (
-                      src={referenceImages[0]}
+                        <img
-                      alt="Reference"
+                          className="w-[340px] border border-gray-200 rounded-md"
-                    />
+                          src={referenceImages[0]}
+                          alt="Reference"
+                        />
+                      )}
+                      {inputMode === "video" && (
+                        <video
+                          muted
+                          autoPlay
+                          loop
+                          className="w-[340px] border border-gray-200 rounded-md"
+                          src={referenceImages[0]}
+                        />
+                      )}
+                    </div>
+                    <div className="text-gray-400 uppercase text-sm text-center mt-1">
+                      {inputMode === "video"
+                        ? "Original Video"
+                        : "Original Screenshot"}
+                    </div>
                   </div>
-                  <div className="text-gray-400 uppercase text-sm text-center mt-1">
+                )}
-                    Original Screenshot
-                  </div>
-                </div>
                 <div className="bg-gray-400 px-4 py-2 rounded text-sm hidden">
                   <h2 className="text-lg mb-4 border-b border-gray-800">
                     Console
@@ -287,6 +554,23 @@ function App() {
               </div>
             </>
           )}
+          {
+            <HistoryDisplay
+              history={appHistory}
+              currentVersion={currentVersion}
+              revertToVersion={(index) => {
+                if (
+                  index < 0 ||
+                  index >= appHistory.length ||
+                  !appHistory[index]
+                )
+                  return;
+                setCurrentVersion(index);
+                setGeneratedCode(appHistory[index].code);
+              }}
+              shouldDisableReverts={appState === AppState.CODING}
+            />
+          }
         </div>
       </div>
 
@@ -298,35 +582,58 @@ function App() {
               doCreate={doCreate}
               screenshotOneApiKey={settings.screenshotOneApiKey}
             />
+            <ImportCodeSection importFromCode={importFromCode} />
           </div>
         )}
 
         {(appState === AppState.CODING || appState === AppState.CODE_READY) && (
           <div className="ml-4">
             <Tabs defaultValue="desktop">
-              <div className="flex justify-end mr-8 mb-4">
+              <div className="flex justify-between mr-8 mb-4">
-                <TabsList>
+                <div className="flex items-center gap-x-2">
-                  <TabsTrigger value="desktop" className="flex gap-x-2">
+                  {appState === AppState.CODE_READY && (
-                    <FaDesktop /> Desktop
+                    <>
-                  </TabsTrigger>
+                      <Button
-                  <TabsTrigger value="mobile" className="flex gap-x-2">
+                        onClick={reset}
-                    <FaMobile /> Mobile
+                        className="flex items-center ml-4 gap-x-2 dark:text-white dark:bg-gray-700"
-                  </TabsTrigger>
+                      >
-                  <TabsTrigger value="code" className="flex gap-x-2">
+                        <FaUndo />
-                    <FaCode />
+                        Reset
-                    Code
+                      </Button>
-                  </TabsTrigger>
+                      <Button
-                </TabsList>
+                        onClick={downloadCode}
+                        variant="secondary"
+                        className="flex items-center gap-x-2 mr-4 dark:text-white dark:bg-gray-700 download-btn"
+                      >
+                        <FaDownload /> Download
+                      </Button>
+                    </>
+                  )}
+                </div>
+                <div className="flex items-center">
+                  <TabsList>
+                    <TabsTrigger value="desktop" className="flex gap-x-2">
+                      <FaDesktop /> Desktop
+                    </TabsTrigger>
+                    <TabsTrigger value="mobile" className="flex gap-x-2">
+                      <FaMobile /> Mobile
+                    </TabsTrigger>
+                    <TabsTrigger value="code" className="flex gap-x-2">
+                      <FaCode />
+                      Code
+                    </TabsTrigger>
+                  </TabsList>
+                </div>
               </div>
               <TabsContent value="desktop">
-                <Preview code={generatedCode} device="desktop" />
+                <Preview code={previewCode} device="desktop" />
               </TabsContent>
               <TabsContent value="mobile">
-                <Preview code={generatedCode} device="mobile" />
+                <Preview code={previewCode} device="mobile" />
               </TabsContent>
               <TabsContent value="code">
                 <CodeTab
-                  code={generatedCode}
+                  code={previewCode}
                   setCode={setGeneratedCode}
                   settings={settings}
                 />

frontend/src/components/ImageUpload.tsx

@@ -1,7 +1,12 @@
-import { useState, useEffect, useMemo, useCallback } from "react";
+import { useState, useEffect, useMemo } from "react";
+// useCallback
 import { useDropzone } from "react-dropzone";
 // import { PromptImage } from "../../../types";
 import { toast } from "react-hot-toast";
+import { URLS } from "../urls";
+import { Badge } from "./ui/badge";
+import ScreenRecorder from "./recording/ScreenRecorder";
+import { ScreenRecorderState } from "../types";
 
 const baseStyle = {
   flex: 1,
@@ -50,19 +55,31 @@ type FileWithPreview = {
 } & File;
 
 interface Props {
-  setReferenceImages: (referenceImages: string[]) => void;
+  setReferenceImages: (
+    referenceImages: string[],
+    inputMode: "image" | "video"
+  ) => void;
 }
 
 function ImageUpload({ setReferenceImages }: Props) {
   const [files, setFiles] = useState<FileWithPreview[]>([]);
+  // TODO: Switch to Zustand
+  const [screenRecorderState, setScreenRecorderState] =
+    useState<ScreenRecorderState>(ScreenRecorderState.INITIAL);
+
   const { getRootProps, getInputProps, isFocused, isDragAccept, isDragReject } =
     useDropzone({
       maxFiles: 1,
-      maxSize: 1024 * 1024 * 5, // 5 MB
+      maxSize: 1024 * 1024 * 20, // 20 MB
       accept: {
+        // Image formats
         "image/png": [".png"],
         "image/jpeg": [".jpeg"],
         "image/jpg": [".jpg"],
+        // Video formats
+        "video/quicktime": [".mov"],
+        "video/mp4": [".mp4"],
+        "video/webm": [".webm"],
       },
       onDrop: (acceptedFiles) => {
         // Set up the preview thumbnail images
@@ -77,7 +94,14 @@ function ImageUpload({ setReferenceImages }: Props) {
         // Convert images to data URLs and set the prompt images state
         Promise.all(acceptedFiles.map((file) => fileToDataURL(file)))
           .then((dataUrls) => {
-            setReferenceImages(dataUrls.map((dataUrl) => dataUrl as string));
+            if (dataUrls.length > 0) {
+              setReferenceImages(
+                dataUrls.map((dataUrl) => dataUrl as string),
+                (dataUrls[0] as string).startsWith("data:video")
+                  ? "video"
+                  : "image"
+              );
+            }
           })
           .catch((error) => {
             toast.error("Error reading files" + error);
@@ -89,39 +113,39 @@ function ImageUpload({ setReferenceImages }: Props) {
       },
     });
 
-  const pasteEvent = useCallback(
+  // const pasteEvent = useCallback(
-    (event: ClipboardEvent) => {
+  //   (event: ClipboardEvent) => {
-      const clipboardData = event.clipboardData;
+  //     const clipboardData = event.clipboardData;
-      if (!clipboardData) return;
+  //     if (!clipboardData) return;
 
-      const items = clipboardData.items;
+  //     const items = clipboardData.items;
-      const files = [];
+  //     const files = [];
-      for (let i = 0; i < items.length; i++) {
+  //     for (let i = 0; i < items.length; i++) {
-        const file = items[i].getAsFile();
+  //       const file = items[i].getAsFile();
-        if (file && file.type.startsWith("image/")) {
+  //       if (file && file.type.startsWith("image/")) {
-          files.push(file);
+  //         files.push(file);
-        }
+  //       }
-      }
+  //     }
 
-      // Convert images to data URLs and set the prompt images state
+  //     // Convert images to data URLs and set the prompt images state
-      Promise.all(files.map((file) => fileToDataURL(file)))
+  //     Promise.all(files.map((file) => fileToDataURL(file)))
-        .then((dataUrls) => {
+  //       .then((dataUrls) => {
-          if (dataUrls.length > 0) {
+  //         if (dataUrls.length > 0) {
-            setReferenceImages(dataUrls.map((dataUrl) => dataUrl as string));
+  //           setReferenceImages(dataUrls.map((dataUrl) => dataUrl as string));
-          }
+  //         }
-        })
+  //       })
-        .catch((error) => {
+  //       .catch((error) => {
-          // TODO: Display error to user
+  //         // TODO: Display error to user
-          console.error("Error reading files:", error);
+  //         console.error("Error reading files:", error);
-        });
+  //       });
-    },
+  //   },
-    [setReferenceImages]
+  //   [setReferenceImages]
-  );
+  // );
 
   // TODO: Make sure we don't listen to paste events in text input components
-  useEffect(() => {
+  // useEffect(() => {
-    window.addEventListener("paste", pasteEvent);
+  //   window.addEventListener("paste", pasteEvent);
-  }, [pasteEvent]);
+  // }, [pasteEvent]);
 
   useEffect(() => {
     return () => files.forEach((file) => URL.revokeObjectURL(file.preview));
@@ -139,15 +163,34 @@ function ImageUpload({ setReferenceImages }: Props) {
 
   return (
     <section className="container">
-      {/* eslint-disable-next-line @typescript-eslint/no-explicit-any */}
+      {screenRecorderState === ScreenRecorderState.INITIAL && (
-      <div {...getRootProps({ style: style as any })}>
+        /* eslint-disable-next-line @typescript-eslint/no-explicit-any */
-        <input {...getInputProps()} />
+        <div {...getRootProps({ style: style as any })}>
-        <p className="text-slate-700 text-lg">
+          <input {...getInputProps()} className="file-input" />
-          Drag & drop a screenshot here, <br />
+          <p className="text-slate-700 text-lg">
-          or paste from clipboard, <br />
+            Drag & drop a screenshot here, <br />
-          or click to upload
+            or click to upload
-        </p>
+          </p>
-      </div>
+        </div>
+      )}
+      {screenRecorderState === ScreenRecorderState.INITIAL && (
+        <div className="text-center text-sm text-slate-800 mt-4">
+          <Badge>New!</Badge> Upload a screen recording (.mp4, .mov) or record
+          your screen to clone a whole app (experimental).{" "}
+          <a
+            className="underline"
+            href={URLS["intro-to-video"]}
+            target="_blank"
+          >
+            Learn more.
+          </a>
+        </div>
+      )}
+      <ScreenRecorder
+        screenRecorderState={screenRecorderState}
+        setScreenRecorderState={setScreenRecorderState}
+        generateCode={setReferenceImages}
+      />
     </section>
   );
 }

frontend/src/components/ImportCodeSection.tsx

@@ -0,0 +1,76 @@
+import { useState } from "react";
+import { Button } from "./ui/button";
+import {
+  Dialog,
+  DialogContent,
+  DialogDescription,
+  DialogFooter,
+  DialogHeader,
+  DialogTitle,
+  DialogTrigger,
+} from "./ui/dialog";
+import { Textarea } from "./ui/textarea";
+import OutputSettingsSection from "./OutputSettingsSection";
+import toast from "react-hot-toast";
+import { Stack } from "../lib/stacks";
+
+interface Props {
+  importFromCode: (code: string, stack: Stack) => void;
+}
+
+function ImportCodeSection({ importFromCode }: Props) {
+  const [code, setCode] = useState("");
+  const [stack, setStack] = useState<Stack | undefined>(undefined);
+
+  const doImport = () => {
+    if (code === "") {
+      toast.error("Please paste in some code");
+      return;
+    }
+
+    if (stack === undefined) {
+      toast.error("Please select your stack");
+      return;
+    }
+
+    importFromCode(code, stack);
+  };
+  return (
+    <Dialog>
+      <DialogTrigger asChild>
+        <Button className="import-from-code-btn" variant="secondary">
+          Import from Code
+        </Button>
+      </DialogTrigger>
+      <DialogContent className="sm:max-w-[425px]">
+        <DialogHeader>
+          <DialogTitle>Paste in your HTML code</DialogTitle>
+          <DialogDescription>
+            Make sure that the code you're importing is valid HTML.
+          </DialogDescription>
+        </DialogHeader>
+
+        <Textarea
+          value={code}
+          onChange={(e) => setCode(e.target.value)}
+          className="w-full h-64"
+        />
+
+        <OutputSettingsSection
+          stack={stack}
+          setStack={(config: Stack) => setStack(config)}
+          label="Stack:"
+          shouldDisableUpdates={false}
+        />
+
+        <DialogFooter>
+          <Button className="import-btn" type="submit" onClick={doImport}>
+            Import
+          </Button>
+        </DialogFooter>
+      </DialogContent>
+    </Dialog>
+  );
+}
+
+export default ImportCodeSection;

frontend/src/components/ModelSettingsSection.tsx

@@ -0,0 +1,65 @@
+import {
+  Select,
+  SelectContent,
+  SelectGroup,
+  SelectItem,
+  SelectTrigger,
+} from "./ui/select";
+import {
+  CODE_GENERATION_MODEL_DESCRIPTIONS,
+  CodeGenerationModel,
+} from "../lib/models";
+import { Badge } from "./ui/badge";
+
+interface Props {
+  codeGenerationModel: CodeGenerationModel;
+  setCodeGenerationModel: (codeGenerationModel: CodeGenerationModel) => void;
+  shouldDisableUpdates?: boolean;
+}
+
+function ModelSettingsSection({
+  codeGenerationModel,
+  setCodeGenerationModel,
+  shouldDisableUpdates = false,
+}: Props) {
+  return (
+    <div className="flex flex-col gap-y-2 justify-between text-sm">
+      <div className="grid grid-cols-3 items-center gap-4">
+        <span>AI Model:</span>
+        <Select
+          value={codeGenerationModel}
+          onValueChange={(value: string) =>
+            setCodeGenerationModel(value as CodeGenerationModel)
+          }
+          disabled={shouldDisableUpdates}
+        >
+          <SelectTrigger className="col-span-2" id="output-settings-js">
+            <span className="font-semibold">
+              {CODE_GENERATION_MODEL_DESCRIPTIONS[codeGenerationModel].name}
+            </span>
+          </SelectTrigger>
+          <SelectContent>
+            <SelectGroup>
+              {Object.values(CodeGenerationModel).map((model) => (
+                <SelectItem key={model} value={model}>
+                  <div className="flex items-center">
+                    <span className="font-semibold">
+                      {CODE_GENERATION_MODEL_DESCRIPTIONS[model].name}
+                    </span>
+                    {CODE_GENERATION_MODEL_DESCRIPTIONS[model].inBeta && (
+                      <Badge className="ml-2" variant="secondary">
+                        Beta
+                      </Badge>
+                    )}
+                  </div>
+                </SelectItem>
+              ))}
+            </SelectGroup>
+          </SelectContent>
+        </Select>
+      </div>
+    </div>
+  );
+}
+
+export default ModelSettingsSection;

frontend/src/components/OnboardingNote.tsx

@@ -2,8 +2,15 @@ export function OnboardingNote() {
   return (
     <div className="flex flex-col space-y-4 bg-green-700 p-2 rounded text-stone-200 text-sm">
       <span>
-        To use Screenshot to Code, you need an OpenAI API key with GPT4 vision
+        To use Screenshot to Code,{" "}
-        access.{" "}
+        <a
+          className="inline underline hover:opacity-70"
+          href="https://buy.stripe.com/8wM6sre70gBW1nqaEE"
+          target="_blank"
+        >
+          buy some credits (100 generations for $36)
+        </a>{" "}
+        or use your own OpenAI API key with GPT4 vision access.{" "}
         <a
           href="https://github.com/abi/screenshot-to-code/blob/main/Troubleshooting.md"
           className="inline underline hover:opacity-70"
@@ -11,18 +18,8 @@ export function OnboardingNote() {
         >
           Follow these instructions to get yourself a key.
         </a>{" "}
-        Then, paste it in the Settings dialog (gear icon above).
+        and paste it in the Settings dialog (gear icon above). Your key is only
-      </span>
+        stored in your browser. Never stored on our servers.
-      <span>
-        Your key is only stored in your browser. Never stored on our servers. If
-        you prefer, you can also run this app completely locally.{" "}
-        <a
-          href="https://github.com/abi/screenshot-to-code"
-          className="inline underline hover:opacity-70"
-          target="_blank"
-        >
-          See the Github project for instructions.
-        </a>
       </span>
     </div>
   );

frontend/src/components/OutputSettingsSection.tsx

@@ -1,3 +1,4 @@
+import React from "react";
 import {
   Select,
   SelectContent,
@@ -5,195 +6,67 @@ import {
   SelectItem,
   SelectTrigger,
 } from "./ui/select";
-import { CSSOption, JSFrameworkOption, OutputSettings } from "../types";
+import { Badge } from "./ui/badge";
-import toast from "react-hot-toast";
+import { Stack, STACK_DESCRIPTIONS } from "../lib/stacks";
-import { Label } from "@radix-ui/react-label";
-import { Button } from "./ui/button";
-import { Popover, PopoverTrigger, PopoverContent } from "./ui/popover";
 
-function displayCSSOption(option: CSSOption) {
+function generateDisplayComponent(stack: Stack) {
-  switch (option) {
+  const stackComponents = STACK_DESCRIPTIONS[stack].components;
-    case CSSOption.TAILWIND:
-      return "Tailwind";
-    case CSSOption.BOOTSTRAP:
-      return "Bootstrap";
-    default:
-      return option;
-  }
-}
 
-function displayJSOption(option: JSFrameworkOption) {
+  return (
-  switch (option) {
+    <div>
-    case JSFrameworkOption.REACT:
+      {stackComponents.map((component, index) => (
-      return "React";
+        <React.Fragment key={index}>
-    case JSFrameworkOption.NO_FRAMEWORK:
+          <span className="font-semibold">{component}</span>
-      return "No Framework";
+          {index < stackComponents.length - 1 && " + "}
-    default:
+        </React.Fragment>
-      return option;
+      ))}
-  }
+    </div>
-}
+  );
-
-function convertStringToCSSOption(option: string) {
-  switch (option) {
-    case "tailwind":
-      return CSSOption.TAILWIND;
-    case "bootstrap":
-      return CSSOption.BOOTSTRAP;
-    default:
-      throw new Error(`Unknown CSS option: ${option}`);
-  }
-}
-
-function generateDisplayString(settings: OutputSettings) {
-  if (
-    settings.js === JSFrameworkOption.REACT &&
-    settings.css === CSSOption.TAILWIND
-  ) {
-    return (
-      <div>
-        Generating <span className="font-bold">React</span> +{" "}
-        <span className="font-bold">Tailwind</span> code
-      </div>
-    );
-  } else if (
-    settings.js === JSFrameworkOption.NO_FRAMEWORK &&
-    settings.css === CSSOption.TAILWIND
-  ) {
-    return (
-      <div className="text-gray-800">
-        Generating <span className="font-bold">HTML</span> +{" "}
-        <span className="font-bold">Tailwind</span> code
-      </div>
-    );
-  } else if (
-    settings.js === JSFrameworkOption.NO_FRAMEWORK &&
-    settings.css === CSSOption.BOOTSTRAP
-  ) {
-    return (
-      <div>
-        Generating <span className="font-bold">HTML</span> +{" "}
-        <span className="font-bold">Bootstrap</span> code
-      </div>
-    );
-  }
 }
 
 interface Props {
-  outputSettings: OutputSettings;
+  stack: Stack | undefined;
-  setOutputSettings: React.Dispatch<React.SetStateAction<OutputSettings>>;
+  setStack: (config: Stack) => void;
+  label?: string;
   shouldDisableUpdates?: boolean;
 }
 
 function OutputSettingsSection({
-  outputSettings,
+  stack,
-  setOutputSettings,
+  setStack,
+  label = "Generating:",
   shouldDisableUpdates = false,
 }: Props) {
-  const onCSSValueChange = (value: string) => {
-    setOutputSettings((prev) => {
-      if (prev.js === JSFrameworkOption.REACT) {
-        if (value !== CSSOption.TAILWIND) {
-          toast.error(
-            'React only supports Tailwind CSS. Change JS framework to "No Framework" to use Bootstrap.'
-          );
-        }
-        return {
-          css: CSSOption.TAILWIND,
-          js: JSFrameworkOption.REACT,
-        };
-      } else {
-        return {
-          ...prev,
-          css: convertStringToCSSOption(value),
-        };
-      }
-    });
-  };
-
-  const onJsFrameworkChange = (value: string) => {
-    if (value === JSFrameworkOption.REACT) {
-      setOutputSettings(() => ({
-        css: CSSOption.TAILWIND,
-        js: value as JSFrameworkOption,
-      }));
-    } else {
-      setOutputSettings((prev) => ({
-        ...prev,
-        js: value as JSFrameworkOption,
-      }));
-    }
-  };
-
   return (
     <div className="flex flex-col gap-y-2 justify-between text-sm">
-      {generateDisplayString(outputSettings)}{" "}
+      <div className="grid grid-cols-3 items-center gap-4">
-      {!shouldDisableUpdates && (
+        <span>{label}</span>
-        <Popover>
+        <Select
-          <PopoverTrigger asChild>
+          value={stack}
-            <Button variant="outline">Customize</Button>
+          onValueChange={(value: string) => setStack(value as Stack)}
-          </PopoverTrigger>
+          disabled={shouldDisableUpdates}
-          <PopoverContent className="w-80 text-sm">
+        >
-            <div className="grid gap-4">
+          <SelectTrigger className="col-span-2" id="output-settings-js">
-              <div className="space-y-2">
+            {stack ? generateDisplayComponent(stack) : "Select a stack"}
-                <h4 className="font-medium leading-none">Code Settings</h4>
+          </SelectTrigger>
-                <p className="text-muted-foreground">
+          <SelectContent>
-                  Customize your code output
+            <SelectGroup>
-                </p>
+              {Object.values(Stack).map((stack) => (
-              </div>
+                <SelectItem key={stack} value={stack}>
-              <div className="grid gap-2">
+                  <div className="flex items-center">
-                <div className="grid grid-cols-3 items-center gap-4">
+                    {generateDisplayComponent(stack)}
-                  <Label htmlFor="output-settings-js">JS</Label>
+                    {STACK_DESCRIPTIONS[stack].inBeta && (
-                  <Select
+                      <Badge className="ml-2" variant="secondary">
-                    value={outputSettings.js}
+                        Beta
-                    onValueChange={onJsFrameworkChange}
+                      </Badge>
-                  >
+                    )}
-                    <SelectTrigger
+                  </div>
-                      className="col-span-2 h-8"
+                </SelectItem>
-                      id="output-settings-js"
+              ))}
-                    >
+            </SelectGroup>
-                      {displayJSOption(outputSettings.js)}
+          </SelectContent>
-                    </SelectTrigger>
+        </Select>
-                    <SelectContent>
+      </div>
-                      <SelectGroup>
-                        <SelectItem value={JSFrameworkOption.NO_FRAMEWORK}>
-                          No Framework
-                        </SelectItem>
-                        <SelectItem value={JSFrameworkOption.REACT}>
-                          React
-                        </SelectItem>
-                      </SelectGroup>
-                    </SelectContent>
-                  </Select>
-                </div>
-                <div className="grid grid-cols-3 items-center gap-4">
-                  <Label htmlFor="output-settings-css">CSS</Label>
-                  <Select
-                    value={outputSettings.css}
-                    onValueChange={onCSSValueChange}
-                  >
-                    <SelectTrigger
-                      className="col-span-2 h-8"
-                      id="output-settings-css"
-                    >
-                      {displayCSSOption(outputSettings.css)}
-                    </SelectTrigger>
-                    <SelectContent>
-                      <SelectGroup>
-                        <SelectItem value={CSSOption.TAILWIND}>
-                          Tailwind
-                        </SelectItem>
-                        <SelectItem value={CSSOption.BOOTSTRAP}>
-                          Bootstrap
-                        </SelectItem>
-                      </SelectGroup>
-                    </SelectContent>
-                  </Select>
-                </div>
-              </div>
-            </div>
-          </PopoverContent>
-        </Popover>
-      )}
     </div>
   );
 }

frontend/src/components/Preview.tsx

@@ -1,4 +1,4 @@
-import { useEffect, useRef } from 'react';
+import { useEffect, useRef } from "react";
 import classNames from "classnames";
 import useThrottle from "../hooks/useThrottle";
 
@@ -8,15 +8,14 @@ interface Props {
 }
 
 function Preview({ code, device }: Props) {
-  const throttledCode = useThrottle(code, 200);
   const iframeRef = useRef<HTMLIFrameElement | null>(null);
 
+  // Don't update code more often than every 200ms.
+  const throttledCode = useThrottle(code, 200);
+
   useEffect(() => {
-    const iframe = iframeRef.current;
+    if (iframeRef.current) {
-    if (iframe && iframe.contentDocument) {
+      iframeRef.current.srcdoc = throttledCode;
-      iframe.contentDocument.open();
-      iframe.contentDocument.write(throttledCode);
-      iframe.contentDocument.close();
     }
   }, [throttledCode]);
 
@@ -39,4 +38,4 @@ function Preview({ code, device }: Props) {
   );
 }
 
-export default Preview;
+export default Preview;

frontend/src/components/SettingsDialog.tsx

@@ -16,6 +16,12 @@ import { Input } from "./ui/input";
 import { Select, SelectContent, SelectItem, SelectTrigger } from "./ui/select";
 import { capitalize } from "../lib/utils";
 import { IS_RUNNING_ON_CLOUD } from "../config";
+import {
+  Accordion,
+  AccordionContent,
+  AccordionItem,
+  AccordionTrigger,
+} from "./ui/accordion";
 
 interface Props {
   settings: Settings;
@@ -40,34 +46,10 @@ function SettingsDialog({ settings, setSettings }: Props) {
           <DialogTitle className="mb-4">Settings</DialogTitle>
         </DialogHeader>
 
-        {/* Access code */}
-        {IS_RUNNING_ON_CLOUD && (
-          <div className="flex flex-col space-y-4 bg-slate-300 p-4 rounded">
-            <Label htmlFor="access-code">
-              <div>Access Code</div>
-              <div className="font-light mt-1 leading-relaxed">
-                Buy an access code.
-              </div>
-            </Label>
-
-            <Input
-              id="access-code"
-              placeholder="Enter your Screenshot to Code access code"
-              value={settings.accessCode || ""}
-              onChange={(e) =>
-                setSettings((s) => ({
-                  ...s,
-                  accessCode: e.target.value,
-                }))
-              }
-            />
-          </div>
-        )}
-
         <div className="flex items-center space-x-2">
           <Label htmlFor="image-generation">
             <div>DALL-E Placeholder Image Generation</div>
-            <div className="font-light mt-2">
+            <div className="font-light mt-2 text-xs">
               More fun with it but if you want to save money, turn it off.
             </div>
           </Label>
@@ -82,74 +64,160 @@ function SettingsDialog({ settings, setSettings }: Props) {
             }
           />
         </div>
-        <div className="flex flex-col space-y-4">
+        <div className="flex flex-col space-y-6">
-          <Label htmlFor="openai-api-key">
-            <div>OpenAI API key</div>
-            <div className="font-light mt-2 leading-relaxed">
-              Only stored in your browser. Never stored on servers. Overrides
-              your .env config.
-            </div>
-          </Label>
-
-          <Input
-            id="openai-api-key"
-            placeholder="OpenAI API key"
-            value={settings.openAiApiKey || ""}
-            onChange={(e) =>
-              setSettings((s) => ({
-                ...s,
-                openAiApiKey: e.target.value,
-              }))
-            }
-          />
-
-          <Label htmlFor="screenshot-one-api-key">
-            <div>
-              ScreenshotOne API key (optional - only needed if you want to use
-              URLs directly instead of taking the screenshot yourself)
-            </div>
-            <div className="font-light mt-2 leading-relaxed">
-              Only stored in your browser. Never stored on servers.{" "}
-              <a
-                href="https://screenshotone.com?via=screenshot-to-code"
-                className="underline"
-                target="_blank"
-              >
-                Get 100 screenshots/mo for free.
-              </a>
-            </div>
-          </Label>
-
-          <Input
-            id="screenshot-one-api-key"
-            placeholder="ScreenshotOne API key"
-            value={settings.screenshotOneApiKey || ""}
-            onChange={(e) =>
-              setSettings((s) => ({
-                ...s,
-                screenshotOneApiKey: e.target.value,
-              }))
-            }
-          />
-
-          <Label htmlFor="editor-theme">
-            <div>Editor Theme</div>
-          </Label>
           <div>
-            <Select // Use the custom Select component here
+            <Label htmlFor="openai-api-key">
-              name="editor-theme"
+              <div>OpenAI API key</div>
-              value={settings.editorTheme}
+              <div className="font-light mt-1 mb-2 text-xs leading-relaxed">
-              onValueChange={(value) => handleThemeChange(value as EditorTheme)}
+                Only stored in your browser. Never stored on servers. Overrides
-            >
+                your .env config.
-              <SelectTrigger className="w-[180px]">
+              </div>
-                {capitalize(settings.editorTheme)}
+            </Label>
-              </SelectTrigger>
+
-              <SelectContent>
+            <Input
-                <SelectItem value="cobalt">Cobalt</SelectItem>
+              id="openai-api-key"
-                <SelectItem value="espresso">Espresso</SelectItem>
+              placeholder="OpenAI API key"
-              </SelectContent>
+              value={settings.openAiApiKey || ""}
-            </Select>
+              onChange={(e) =>
+                setSettings((s) => ({
+                  ...s,
+                  openAiApiKey: e.target.value,
+                }))
+              }
+            />
           </div>
+
+          {!IS_RUNNING_ON_CLOUD && (
+            <div>
+              <Label htmlFor="openai-api-key">
+                <div>OpenAI Base URL (optional)</div>
+                <div className="font-light mt-2 leading-relaxed">
+                  Replace with a proxy URL if you don't want to use the default.
+                </div>
+              </Label>
+
+              <Input
+                id="openai-base-url"
+                placeholder="OpenAI Base URL"
+                value={settings.openAiBaseURL || ""}
+                onChange={(e) =>
+                  setSettings((s) => ({
+                    ...s,
+                    openAiBaseURL: e.target.value,
+                  }))
+                }
+              />
+            </div>
+          )}
+
+          <div>
+            <Label htmlFor="anthropic-api-key">
+              <div>Anthropic API key</div>
+              <div className="font-light mt-1 text-xs leading-relaxed">
+                Only stored in your browser. Never stored on servers. Overrides
+                your .env config.
+              </div>
+            </Label>
+
+            <Input
+              id="anthropic-api-key"
+              placeholder="Anthropic API key"
+              value={settings.anthropicApiKey || ""}
+              onChange={(e) =>
+                setSettings((s) => ({
+                  ...s,
+                  anthropicApiKey: e.target.value,
+                }))
+              }
+            />
+          </div>
+
+          <Accordion type="single" collapsible className="w-full">
+            <AccordionItem value="item-1">
+              <AccordionTrigger>Screenshot by URL Config</AccordionTrigger>
+              <AccordionContent>
+                <Label htmlFor="screenshot-one-api-key">
+                  <div className="leading-normal font-normal text-xs">
+                    If you want to use URLs directly instead of taking the
+                    screenshot yourself, add a ScreenshotOne API key.{" "}
+                    <a
+                      href="https://screenshotone.com?via=screenshot-to-code"
+                      className="underline"
+                      target="_blank"
+                    >
+                      Get 100 screenshots/mo for free.
+                    </a>
+                  </div>
+                </Label>
+
+                <Input
+                  id="screenshot-one-api-key"
+                  className="mt-2"
+                  placeholder="ScreenshotOne API key"
+                  value={settings.screenshotOneApiKey || ""}
+                  onChange={(e) =>
+                    setSettings((s) => ({
+                      ...s,
+                      screenshotOneApiKey: e.target.value,
+                    }))
+                  }
+                />
+              </AccordionContent>
+            </AccordionItem>
+          </Accordion>
+
+          <Accordion type="single" collapsible className="w-full">
+            <AccordionItem value="item-1">
+              <AccordionTrigger>Theme Settings</AccordionTrigger>
+              <AccordionContent className="space-y-4 flex flex-col">
+                <div className="flex items-center justify-between">
+                  <Label htmlFor="app-theme">
+                    <div>App Theme</div>
+                  </Label>
+                  <div>
+                    <button
+                      className="flex rounded-md border border-input bg-transparent px-3 py-1 text-sm shadow-sm transition-colors file:border-0 file:bg-transparent file:text-sm file:font-medium placeholder:text-muted-foreground focus-visible:outline-none focus-visible:ring-1 focus-visible:ring-ring disabled:cursor-not-allowed disabled:opacity-50t"
+                      onClick={() => {
+                        document
+                          .querySelector("div.mt-2")
+                          ?.classList.toggle("dark"); // enable dark mode for sidebar
+                        document.body.classList.toggle("dark");
+                        document
+                          .querySelector('div[role="presentation"]')
+                          ?.classList.toggle("dark"); // enable dark mode for upload container
+                      }}
+                    >
+                      Toggle dark mode
+                    </button>
+                  </div>
+                </div>
+                <div className="flex items-center justify-between">
+                  <Label htmlFor="editor-theme">
+                    <div>
+                      Code Editor Theme - requires page refresh to update
+                    </div>
+                  </Label>
+                  <div>
+                    <Select // Use the custom Select component here
+                      name="editor-theme"
+                      value={settings.editorTheme}
+                      onValueChange={(value) =>
+                        handleThemeChange(value as EditorTheme)
+                      }
+                    >
+                      <SelectTrigger className="w-[180px]">
+                        {capitalize(settings.editorTheme)}
+                      </SelectTrigger>
+                      <SelectContent>
+                        <SelectItem value="cobalt">Cobalt</SelectItem>
+                        <SelectItem value="espresso">Espresso</SelectItem>
+                      </SelectContent>
+                    </Select>
+                  </div>
+                </div>
+              </AccordionContent>
+            </AccordionItem>
+          </Accordion>
         </div>
 
         <DialogFooter>

frontend/src/components/TermsOfServiceDialog.tsx

@@ -11,6 +11,8 @@ import { Input } from "./ui/input";
 import toast from "react-hot-toast";
 import { PICO_BACKEND_FORM_SECRET } from "../config";
 
+const LOGOS = ["microsoft", "amazon", "mit", "stanford", "bytedance", "baidu"];
+
 const TermsOfServiceDialog: React.FC<{
   open: boolean;
   onOpenChange: (open: boolean) => void;
@@ -31,7 +33,7 @@ const TermsOfServiceDialog: React.FC<{
     <AlertDialog open={open} onOpenChange={onOpenChange}>
       <AlertDialogContent>
         <AlertDialogHeader>
-          <AlertDialogTitle className="mb-2">
+          <AlertDialogTitle className="mb-2 text-xl">
             Enter your email to get started
           </AlertDialogTitle>
         </AlertDialogHeader>
@@ -45,8 +47,8 @@ const TermsOfServiceDialog: React.FC<{
             }}
           />
         </div>
-        <div className="flex items-center space-x-2">
+        <div className="flex flex-col space-y-3 text-sm">
-          <span>
+          <p>
             By providing your email, you consent to receiving occasional product
             updates, and you accept the{" "}
             <a
@@ -56,8 +58,11 @@ const TermsOfServiceDialog: React.FC<{
             >
               terms of service
             </a>
-            . <br />
+            .{" "}
-            <br />
+          </p>
+
+          <p>
+            {" "}
             Prefer to run it yourself locally? This project is open source.{" "}
             <a
               href="https://github.com/abi/screenshot-to-code"
@@ -66,7 +71,7 @@ const TermsOfServiceDialog: React.FC<{
             >
               Download the code and get started on Github.
             </a>
-          </span>
+          </p>
         </div>
 
         <AlertDialogFooter>
@@ -80,9 +85,32 @@ const TermsOfServiceDialog: React.FC<{
               }
             }}
           >
-            Agree
+            Agree & Continue
           </AlertDialogAction>
         </AlertDialogFooter>
+
+        {/* Logos */}
+        <div>
+          <div
+            className="mx-auto grid max-w-lg items-center gap-x-2 
+          gap-y-10 sm:max-w-xl grid-cols-6 lg:mx-0 lg:max-w-none mt-10"
+          >
+            {LOGOS.map((companyName) => (
+              <img
+                key={companyName}
+                className="col-span-1 max-h-12 w-full object-contain grayscale opacity-50 hover:opacity-100"
+                src={`https://picoapps.xyz/logos/${companyName}.png`}
+                alt={companyName}
+                width={120}
+                height={48}
+              />
+            ))}
+          </div>
+          <div className="text-gray-500 text-xs mt-4 text-center">
+            Designers and engineers from these organizations use Screenshot to
+            Code to build interfaces faster.
+          </div>
+        </div>
       </AlertDialogContent>
     </AlertDialog>
   );

frontend/src/components/UrlInputSection.tsx

@@ -6,7 +6,7 @@ import { toast } from "react-hot-toast";
 
 interface Props {
   screenshotOneApiKey: string | null;
-  doCreate: (urls: string[]) => void;
+  doCreate: (urls: string[], inputMode: "image" | "video") => void;
 }
 
 export function UrlInputSection({ doCreate, screenshotOneApiKey }: Props) {
@@ -46,7 +46,7 @@ export function UrlInputSection({ doCreate, screenshotOneApiKey }: Props) {
         }
 
         const res = await response.json();
-        doCreate([res.url]);
+        doCreate([res.url], "image");
       } catch (error) {
         console.error(error);
         toast.error(
@@ -59,7 +59,7 @@ export function UrlInputSection({ doCreate, screenshotOneApiKey }: Props) {
   }
 
   return (
-    <div className="w-[400px] gap-y-2 flex flex-col">
+    <div className="max-w-[90%] min-w-[40%] gap-y-2 flex flex-col">
       <div className="text-gray-500 text-sm">Or screenshot a URL...</div>
       <Input
         placeholder="Enter URL"
@@ -69,7 +69,7 @@ export function UrlInputSection({ doCreate, screenshotOneApiKey }: Props) {
       <Button
         onClick={takeScreenshot}
         disabled={isLoading}
-        className="bg-slate-400"
+        className="bg-slate-400 capture-btn"
       >
         {isLoading ? "Capturing..." : "Capture"}
       </Button>

frontend/src/components/core/TipLink.tsx

@@ -0,0 +1,16 @@
+import { URLS } from "../../urls";
+
+function TipLink() {
+  return (
+    <a
+      className="text-xs underline text-gray-500 text-right"
+      href={URLS.tips}
+      target="_blank"
+      rel="noopener"
+    >
+      Tips for better results
+    </a>
+  );
+}
+
+export default TipLink;

frontend/src/components/evals/EvalsPage.tsx

@@ -0,0 +1,74 @@
+import React, { useEffect } from "react";
+import { HTTP_BACKEND_URL } from "../../config";
+import RatingPicker from "./RatingPicker";
+
+interface Eval {
+  input: string;
+  outputs: string[];
+}
+
+function EvalsPage() {
+  const [evals, setEvals] = React.useState<Eval[]>([]);
+  const [ratings, setRatings] = React.useState<number[]>([]);
+
+  const total = ratings.reduce((a, b) => a + b, 0);
+  const max = ratings.length * 4;
+  const score = ((total / max) * 100 || 0).toFixed(2);
+
+  useEffect(() => {
+    if (evals.length > 0) return;
+
+    fetch(`${HTTP_BACKEND_URL}/evals`)
+      .then((res) => res.json())
+      .then((data) => {
+        setEvals(data);
+        setRatings(new Array(data.length).fill(0));
+      });
+  }, [evals]);
+
+  return (
+    <div className="mx-auto">
+      {/* Display total */}
+      <div className="flex items-center justify-center w-full h-12 bg-zinc-950">
+        <span className="text-2xl font-semibold text-white">
+          Total: {total} out of {max} ({score}%)
+        </span>
+      </div>
+
+      <div className="flex flex-col gap-y-4 mt-4 mx-auto justify-center">
+        {evals.map((e, index) => (
+          <div className="flex flex-col justify-center" key={index}>
+            <h2 className="font-bold text-lg ml-4">{index}</h2>
+            <div className="flex gap-x-2 justify-center ml-4">
+              {/* Update w if N changes to a fixed number like w-[600px] */}
+              <div className="w-1/2 p-1 border">
+                <img src={e.input} alt={`Input for eval ${index}`} />
+              </div>
+              {e.outputs.map((output, outputIndex) => (
+                <div className="w-1/2 p-1 border" key={outputIndex}>
+                  {/* Put output into an iframe */}
+                  <iframe
+                    srcDoc={output}
+                    className="w-[1200px] h-[800px] transform scale-[0.60]"
+                    style={{ transformOrigin: "top left" }}
+                  ></iframe>
+                </div>
+              ))}
+            </div>
+            <div className="ml-8 mt-4 flex justify-center">
+              <RatingPicker
+                onSelect={(rating) => {
+                  const newRatings = [...ratings];
+                  newRatings[index] = rating;
+                  setRatings(newRatings);
+                }}
+              />
+            </div>
+          </div>
+        ))}
+      </div>
+    </div>
+  );
+}
+
+export default EvalsPage;

frontend/src/components/evals/RatingPicker.tsx

@@ -0,0 +1,38 @@
+import React from "react";
+
+interface Props {
+  onSelect: (rating: number) => void;
+}
+
+function RatingPicker({ onSelect }: Props) {
+  const [selected, setSelected] = React.useState<number | null>(null);
+
+  const renderCircle = (number: number) => {
+    const isSelected = selected === number;
+    const bgColor = isSelected ? "bg-black" : "bg-gray-300";
+    const textColor = isSelected ? "text-white" : "text-black";
+
+    return (
+      <div
+        className={`flex items-center justify-center w-8 h-8 ${bgColor} rounded-full cursor-pointer`}
+        onClick={() => {
+          setSelected(number);
+          onSelect(number);
+        }}
+      >
+        <span className={`text-lg font-semibold ${textColor}`}>{number}</span>
+      </div>
+    );
+  };
+
+  return (
+    <div className="flex space-x-4">
+      {renderCircle(1)}
+      {renderCircle(2)}
+      {renderCircle(3)}
+      {renderCircle(4)}
+    </div>
+  );
+}
+
+export default RatingPicker;

frontend/src/components/history/HistoryDisplay.tsx

@@ -0,0 +1,86 @@
+import { History } from "./history_types";
+import toast from "react-hot-toast";
+import classNames from "classnames";
+
+import { Badge } from "../ui/badge";
+import { renderHistory } from "./utils";
+import {
+  Collapsible,
+  CollapsibleContent,
+  CollapsibleTrigger,
+} from "../ui/collapsible";
+import { Button } from "../ui/button";
+import { CaretSortIcon } from "@radix-ui/react-icons";
+
+interface Props {
+  history: History;
+  currentVersion: number | null;
+  revertToVersion: (version: number) => void;
+  shouldDisableReverts: boolean;
+}
+
+export default function HistoryDisplay({
+  history,
+  currentVersion,
+  revertToVersion,
+  shouldDisableReverts,
+}: Props) {
+  const renderedHistory = renderHistory(history, currentVersion);
+
+  return renderedHistory.length === 0 ? null : (
+    <div className="flex flex-col h-screen">
+      <h1 className="font-bold mb-2">Versions</h1>
+      <ul className="space-y-0 flex flex-col-reverse">
+        {renderedHistory.map((item, index) => (
+          <li key={index}>
+            <Collapsible>
+              <div
+                className={classNames(
+                  "flex items-center justify-between space-x-2 w-full pr-2",
+                  "border-b cursor-pointer",
+                  {
+                    " hover:bg-black hover:text-white": !item.isActive,
+                    "bg-slate-500 text-white": item.isActive,
+                  }
+                )}
+              >
+                <div
+                  className="flex justify-between truncate flex-1 p-2"
+                  onClick={() =>
+                    shouldDisableReverts
+                      ? toast.error(
+                          "Please wait for code generation to complete before viewing an older version."
+                        )
+                      : revertToVersion(index)
+                  }
+                >
+                  <div className="flex gap-x-1 truncate">
+                    <h2 className="text-sm truncate">{item.summary}</h2>
+                    {item.parentVersion !== null && (
+                      <h2 className="text-sm">
+                        (parent: {item.parentVersion})
+                      </h2>
+                    )}
+                  </div>
+                  <h2 className="text-sm">v{index + 1}</h2>
+                </div>
+                <CollapsibleTrigger asChild>
+                  <Button variant="ghost" size="sm" className="h-6">
+                    <CaretSortIcon className="h-4 w-4" />
+                    <span className="sr-only">Toggle</span>
+                  </Button>
+                </CollapsibleTrigger>
+              </div>
+              <CollapsibleContent className="w-full bg-slate-300 p-2">
+                <div>Full prompt: {item.summary}</div>
+                <div className="flex justify-end">
+                  <Badge>{item.type}</Badge>
+                </div>
+              </CollapsibleContent>
+            </Collapsible>
+          </li>
+        ))}
+      </ul>
+    </div>
+  );
+}

frontend/src/components/history/history_types.ts

@@ -0,0 +1,41 @@
+export type HistoryItemType = "ai_create" | "ai_edit" | "code_create";
+
+type CommonHistoryItem = {
+  parentIndex: null | number;
+  code: string;
+};
+
+export type HistoryItem =
+  | ({
+      type: "ai_create";
+      inputs: AiCreateInputs;
+    } & CommonHistoryItem)
+  | ({
+      type: "ai_edit";
+      inputs: AiEditInputs;
+    } & CommonHistoryItem)
+  | ({
+      type: "code_create";
+      inputs: CodeCreateInputs;
+    } & CommonHistoryItem);
+
+export type AiCreateInputs = {
+  image_url: string;
+};
+
+export type AiEditInputs = {
+  prompt: string;
+};
+
+export type CodeCreateInputs = {
+  code: string;
+};
+
+export type History = HistoryItem[];
+
+export type RenderedHistoryItem = {
+  type: string;
+  summary: string;
+  parentVersion: string | null;
+  isActive: boolean;
+};

frontend/src/components/history/utils.test.ts

@@ -0,0 +1,231 @@
+import { extractHistoryTree, renderHistory } from "./utils";
+import type { History } from "./history_types";
+
+const basicLinearHistory: History = [
+  {
+    type: "ai_create",
+    parentIndex: null,
+    code: "<html>1. create</html>",
+    inputs: {
+      image_url: "",
+    },
+  },
+  {
+    type: "ai_edit",
+    parentIndex: 0,
+    code: "<html>2. edit with better icons</html>",
+    inputs: {
+      prompt: "use better icons",
+    },
+  },
+  {
+    type: "ai_edit",
+    parentIndex: 1,
+    code: "<html>3. edit with better icons and red text</html>",
+    inputs: {
+      prompt: "make text red",
+    },
+  },
+];
+
+const basicLinearHistoryWithCode: History = [
+  {
+    type: "code_create",
+    parentIndex: null,
+    code: "<html>1. create</html>",
+    inputs: {
+      code: "<html>1. create</html>",
+    },
+  },
+  ...basicLinearHistory.slice(1),
+];
+
+const basicBranchingHistory: History = [
+  ...basicLinearHistory,
+  {
+    type: "ai_edit",
+    parentIndex: 1,
+    code: "<html>4. edit with better icons and green text</html>",
+    inputs: {
+      prompt: "make text green",
+    },
+  },
+];
+
+const longerBranchingHistory: History = [
+  ...basicBranchingHistory,
+  {
+    type: "ai_edit",
+    parentIndex: 3,
+    code: "<html>5. edit with better icons and green, bold text</html>",
+    inputs: {
+      prompt: "make text bold",
+    },
+  },
+];
+
+const basicBadHistory: History = [
+  {
+    type: "ai_create",
+    parentIndex: null,
+    code: "<html>1. create</html>",
+    inputs: {
+      image_url: "",
+    },
+  },
+  {
+    type: "ai_edit",
+    parentIndex: 2, // <- Bad parent index
+    code: "<html>2. edit with better icons</html>",
+    inputs: {
+      prompt: "use better icons",
+    },
+  },
+];
+
+describe("History Utils", () => {
+  test("should correctly extract the history tree", () => {
+    expect(extractHistoryTree(basicLinearHistory, 2)).toEqual([
+      "<html>1. create</html>",
+      "use better icons",
+      "<html>2. edit with better icons</html>",
+      "make text red",
+      "<html>3. edit with better icons and red text</html>",
+    ]);
+
+    expect(extractHistoryTree(basicLinearHistory, 0)).toEqual([
+      "<html>1. create</html>",
+    ]);
+
+    // Test branching
+    expect(extractHistoryTree(basicBranchingHistory, 3)).toEqual([
+      "<html>1. create</html>",
+      "use better icons",
+      "<html>2. edit with better icons</html>",
+      "make text green",
+      "<html>4. edit with better icons and green text</html>",
+    ]);
+
+    expect(extractHistoryTree(longerBranchingHistory, 4)).toEqual([
+      "<html>1. create</html>",
+      "use better icons",
+      "<html>2. edit with better icons</html>",
+      "make text green",
+      "<html>4. edit with better icons and green text</html>",
+      "make text bold",
+      "<html>5. edit with better icons and green, bold text</html>",
+    ]);
+
+    expect(extractHistoryTree(longerBranchingHistory, 2)).toEqual([
+      "<html>1. create</html>",
+      "use better icons",
+      "<html>2. edit with better icons</html>",
+      "make text red",
+      "<html>3. edit with better icons and red text</html>",
+    ]);
+
+    // Errors
+
+    // Bad index
+    expect(() => extractHistoryTree(basicLinearHistory, 100)).toThrow();
+    expect(() => extractHistoryTree(basicLinearHistory, -2)).toThrow();
+
+    // Bad tree
+    expect(() => extractHistoryTree(basicBadHistory, 1)).toThrow();
+  });
+
+  test("should correctly render the history tree", () => {
+    expect(renderHistory(basicLinearHistory, 2)).toEqual([
+      {
+        isActive: false,
+        parentVersion: null,
+        summary: "Create",
+        type: "Create",
+      },
+      {
+        isActive: false,
+        parentVersion: null,
+        summary: "use better icons",
+        type: "Edit",
+      },
+      {
+        isActive: true,
+        parentVersion: null,
+        summary: "make text red",
+        type: "Edit",
+      },
+    ]);
+
+    // Current version is the first version
+    expect(renderHistory(basicLinearHistory, 0)).toEqual([
+      {
+        isActive: true,
+        parentVersion: null,
+        summary: "Create",
+        type: "Create",
+      },
+      {
+        isActive: false,
+        parentVersion: null,
+        summary: "use better icons",
+        type: "Edit",
+      },
+      {
+        isActive: false,
+        parentVersion: null,
+        summary: "make text red",
+        type: "Edit",
+      },
+    ]);
+
+    // Render a history with code
+    expect(renderHistory(basicLinearHistoryWithCode, 0)).toEqual([
+      {
+        isActive: true,
+        parentVersion: null,
+        summary: "Imported from code",
+        type: "Imported from code",
+      },
+      {
+        isActive: false,
+        parentVersion: null,
+        summary: "use better icons",
+        type: "Edit",
+      },
+      {
+        isActive: false,
+        parentVersion: null,
+        summary: "make text red",
+        type: "Edit",
+      },
+    ]);
+
+    // Render a non-linear history
+    expect(renderHistory(basicBranchingHistory, 3)).toEqual([
+      {
+        isActive: false,
+        parentVersion: null,
+        summary: "Create",
+        type: "Create",
+      },
+      {
+        isActive: false,
+        parentVersion: null,
+        summary: "use better icons",
+        type: "Edit",
+      },
+      {
+        isActive: false,
+        parentVersion: null,
+        summary: "make text red",
+        type: "Edit",
+      },
+      {
+        isActive: true,
+        parentVersion: "v2",
+        summary: "make text green",
+        type: "Edit",
+      },
+    ]);
+  });
+});

frontend/src/components/history/utils.ts

@@ -0,0 +1,96 @@
+import {
+  History,
+  HistoryItem,
+  HistoryItemType,
+  RenderedHistoryItem,
+} from "./history_types";
+
+export function extractHistoryTree(
+  history: History,
+  version: number
+): string[] {
+  const flatHistory: string[] = [];
+
+  let currentIndex: number | null = version;
+  while (currentIndex !== null) {
+    const item: HistoryItem = history[currentIndex];
+
+    if (item) {
+      if (item.type === "ai_create") {
+        // Don't include the image for ai_create
+        flatHistory.unshift(item.code);
+      } else if (item.type === "ai_edit") {
+        flatHistory.unshift(item.code);
+        flatHistory.unshift(item.inputs.prompt);
+      } else if (item.type === "code_create") {
+        flatHistory.unshift(item.code);
+      }
+
+      // Move to the parent of the current item
+      currentIndex = item.parentIndex;
+    } else {
+      throw new Error("Malformed history: missing parent index");
+    }
+  }
+
+  return flatHistory;
+}
+
+function displayHistoryItemType(itemType: HistoryItemType) {
+  switch (itemType) {
+    case "ai_create":
+      return "Create";
+    case "ai_edit":
+      return "Edit";
+    case "code_create":
+      return "Imported from code";
+    default: {
+      const exhaustiveCheck: never = itemType;
+      throw new Error(`Unhandled case: ${exhaustiveCheck}`);
+    }
+  }
+}
+
+function summarizeHistoryItem(item: HistoryItem) {
+  const itemType = item.type;
+  switch (itemType) {
+    case "ai_create":
+      return "Create";
+    case "ai_edit":
+      return item.inputs.prompt;
+    case "code_create":
+      return "Imported from code";
+    default: {
+      const exhaustiveCheck: never = itemType;
+      throw new Error(`Unhandled case: ${exhaustiveCheck}`);
+    }
+  }
+}
+
+export const renderHistory = (
+  history: History,
+  currentVersion: number | null
+) => {
+  const renderedHistory: RenderedHistoryItem[] = [];
+
+  for (let i = 0; i < history.length; i++) {
+    const item = history[i];
+    // Only show the parent version if it's not the previous version
+    // (i.e. it's the branching point) and if it's not the first version
+    const parentVersion =
+      item.parentIndex !== null && item.parentIndex !== i - 1
+        ? `v${(item.parentIndex || 0) + 1}`
+        : null;
+    const type = displayHistoryItemType(item.type);
+    const isActive = i === currentVersion;
+    const summary = summarizeHistoryItem(item);
+    renderedHistory.push({
+      isActive,
+      summary: summary,
+      parentVersion,
+      type,
+    });
+  }
+
+  return renderedHistory;
+};

frontend/src/components/preview/extractHtml.ts

@@ -0,0 +1,16 @@
+// Not robust enough to support <html lang='en'> for instance
+export function extractHtml(code: string): string {
+  const lastHtmlStartIndex = code.lastIndexOf("<html>");
+  let htmlEndIndex = code.indexOf("</html>", lastHtmlStartIndex);
+
+  if (lastHtmlStartIndex !== -1) {
+    // If "</html>" is found, adjust htmlEndIndex to include the "</html>" tag
+    if (htmlEndIndex !== -1) {
+      htmlEndIndex += "</html>".length;
+      return code.slice(lastHtmlStartIndex, htmlEndIndex);
+    }
+    // If "</html>" is not found, return the rest of the string starting from the last "<html>"
+    return code.slice(lastHtmlStartIndex);
+  }
+  return "";
+}

frontend/src/components/preview/simpleHash.ts

@@ -0,0 +1,10 @@
+
+export function simpleHash(str: string, seed = 0) {
+  let hash = seed;
+  for (let i = 0; i < str.length; i++) {
+    const char = str.charCodeAt(i);
+    hash = (hash << 5) - hash + char;
+    hash |= 0; // Convert to 32bit integer
+  }
+  return hash;
+}

frontend/src/components/recording/ScreenRecorder.tsx

@@ -0,0 +1,145 @@
+import { useState } from "react";
+import { Button } from "../ui/button";
+import { ScreenRecorderState } from "../../types";
+import { blobToBase64DataUrl } from "./utils";
+import fixWebmDuration from "webm-duration-fix";
+import toast from "react-hot-toast";
+
+interface Props {
+  screenRecorderState: ScreenRecorderState;
+  setScreenRecorderState: (state: ScreenRecorderState) => void;
+  generateCode: (
+    referenceImages: string[],
+    inputMode: "image" | "video"
+  ) => void;
+}
+
+function ScreenRecorder({
+  screenRecorderState,
+  setScreenRecorderState,
+  generateCode,
+}: Props) {
+  const [mediaStream, setMediaStream] = useState<MediaStream | null>(null);
+  const [mediaRecorder, setMediaRecorder] = useState<MediaRecorder | null>(
+    null
+  );
+  const [screenRecordingDataUrl, setScreenRecordingDataUrl] = useState<
+    string | null
+  >(null);
+
+  const startScreenRecording = async () => {
+    try {
+      // Get the screen recording stream
+      const stream = await navigator.mediaDevices.getDisplayMedia({
+        video: true,
+        audio: { echoCancellation: true },
+      });
+      setMediaStream(stream);
+
+      // TODO: Test across different browsers
+      // Create the media recorder
+      const options = { mimeType: "video/webm" };
+      const mediaRecorder = new MediaRecorder(stream, options);
+      setMediaRecorder(mediaRecorder);
+
+      const chunks: BlobPart[] = [];
+
+      // Accumalate chunks as data is available
+      mediaRecorder.ondataavailable = (e: BlobEvent) => chunks.push(e.data);
+
+      // When media recorder is stopped, create a data URL
+      mediaRecorder.onstop = async () => {
+        // TODO: Do I need to fix duration if it's not a webm?
+        const completeBlob = await fixWebmDuration(
+          new Blob(chunks, {
+            type: options.mimeType,
+          })
+        );
+
+        const dataUrl = await blobToBase64DataUrl(completeBlob);
+
+        setScreenRecordingDataUrl(dataUrl);
+        setScreenRecorderState(ScreenRecorderState.FINISHED);
+      };
+
+      // Start recording
+      mediaRecorder.start();
+      setScreenRecorderState(ScreenRecorderState.RECORDING);
+    } catch (error) {
+      toast.error("Could not start screen recording");
+      throw error;
+    }
+  };
+
+  const stopScreenRecording = () => {
+    // Stop the recorder
+    if (mediaRecorder) {
+      mediaRecorder.stop();
+      setMediaRecorder(null);
+    }
+
+    // Stop the screen sharing stream
+    if (mediaStream) {
+      mediaStream.getTracks().forEach((track) => {
+        track.stop();
+      });
+    }
+  };
+
+  const kickoffGeneration = () => {
+    if (screenRecordingDataUrl) {
+      generateCode([screenRecordingDataUrl], "video");
+    } else {
+      toast.error("Screen recording does not exist. Please try again.");
+      throw new Error("No screen recording data url");
+    }
+  };
+
+  return (
+    <div className="flex items-center justify-center my-3">
+      {screenRecorderState === ScreenRecorderState.INITIAL && (
+        <Button onClick={startScreenRecording}>Record Screen</Button>
+      )}
+
+      {screenRecorderState === ScreenRecorderState.RECORDING && (
+        <div className="flex items-center flex-col gap-y-4">
+          <div className="flex items-center mr-2 text-xl gap-x-1">
+            <span className="block h-10 w-10 bg-red-600 rounded-full mr-1 animate-pulse"></span>
+            <span>Recording...</span>
+          </div>
+          <Button onClick={stopScreenRecording}>Finish Recording</Button>
+        </div>
+      )}
+
+      {screenRecorderState === ScreenRecorderState.FINISHED && (
+        <div className="flex items-center flex-col gap-y-4">
+          <div className="flex items-center mr-2 text-xl gap-x-1">
+            <span>Screen Recording Captured.</span>
+          </div>
+          {screenRecordingDataUrl && (
+            <video
+              muted
+              autoPlay
+              loop
+              className="w-[340px] border border-gray-200 rounded-md"
+              src={screenRecordingDataUrl}
+            />
+          )}
+          <div className="flex gap-x-2">
+            <Button
+              variant="secondary"
+              onClick={() =>
+                setScreenRecorderState(ScreenRecorderState.INITIAL)
+              }
+            >
+              Re-record
+            </Button>
+            <Button onClick={kickoffGeneration}>Generate</Button>
+          </div>
+        </div>
+      )}
+    </div>
+  );
+}
+
+export default ScreenRecorder;

frontend/src/components/recording/utils.ts

@@ -0,0 +1,31 @@
+export function downloadBlob(blob: Blob) {
+  // Create a URL for the blob object
+  const videoURL = URL.createObjectURL(blob);
+
+  // Create a temporary anchor element and trigger the download
+  const a = document.createElement("a");
+  a.href = videoURL;
+  a.download = "recording.webm";
+  document.body.appendChild(a);
+  a.click();
+  document.body.removeChild(a);
+
+  // Clear object URL
+  URL.revokeObjectURL(videoURL);
+}
+
+export function blobToBase64DataUrl(blob: Blob): Promise<string> {
+  return new Promise((resolve, reject) => {
+    const reader = new FileReader();
+    reader.onloadend = () => {
+      if (reader.result) {
+        resolve(reader.result as string);
+      } else {
+        reject(new Error("FileReader did not return a result."));
+      }
+    };
+    reader.onerror = () =>
+      reject(new Error("FileReader encountered an error."));
+    reader.readAsDataURL(blob);
+  });
+}

frontend/src/components/ui/collapsible.tsx

@@ -0,0 +1,9 @@
+import * as CollapsiblePrimitive from "@radix-ui/react-collapsible"
+
+const Collapsible = CollapsiblePrimitive.Root
+
+const CollapsibleTrigger = CollapsiblePrimitive.CollapsibleTrigger
+
+const CollapsibleContent = CollapsiblePrimitive.CollapsibleContent
+
+export { Collapsible, CollapsibleTrigger, CollapsibleContent }

frontend/src/components/ui/hover-card.tsx

@@ -0,0 +1,27 @@
+import * as React from "react"
+import * as HoverCardPrimitive from "@radix-ui/react-hover-card"
+
+import { cn } from "@/lib/utils"
+
+const HoverCard = HoverCardPrimitive.Root
+
+const HoverCardTrigger = HoverCardPrimitive.Trigger
+
+const HoverCardContent = React.forwardRef<
+  React.ElementRef<typeof HoverCardPrimitive.Content>,
+  React.ComponentPropsWithoutRef<typeof HoverCardPrimitive.Content>
+>(({ className, align = "center", sideOffset = 4, ...props }, ref) => (
+  <HoverCardPrimitive.Content
+    ref={ref}
+    align={align}
+    sideOffset={sideOffset}
+    className={cn(
+      "z-50 w-64 rounded-md border bg-popover p-4 text-popover-foreground shadow-md outline-none data-[state=open]:animate-in data-[state=closed]:animate-out data-[state=closed]:fade-out-0 data-[state=open]:fade-in-0 data-[state=closed]:zoom-out-95 data-[state=open]:zoom-in-95 data-[side=bottom]:slide-in-from-top-2 data-[side=left]:slide-in-from-right-2 data-[side=right]:slide-in-from-left-2 data-[side=top]:slide-in-from-bottom-2",
+      className
+    )}
+    {...props}
+  />
+))
+HoverCardContent.displayName = HoverCardPrimitive.Content.displayName
+
+export { HoverCard, HoverCardTrigger, HoverCardContent }

frontend/src/components/ui/progress.tsx

@@ -0,0 +1,26 @@
+import * as React from "react"
+import * as ProgressPrimitive from "@radix-ui/react-progress"
+
+import { cn } from "@/lib/utils"
+
+const Progress = React.forwardRef<
+  React.ElementRef<typeof ProgressPrimitive.Root>,
+  React.ComponentPropsWithoutRef<typeof ProgressPrimitive.Root>
+>(({ className, value, ...props }, ref) => (
+  <ProgressPrimitive.Root
+    ref={ref}
+    className={cn(
+      "relative h-2 w-full overflow-hidden rounded-full bg-primary/20",
+      className
+    )}
+    {...props}
+  >
+    <ProgressPrimitive.Indicator
+      className="h-full w-full flex-1 bg-primary transition-all"
+      style={{ transform: `translateX(-${100 - (value || 0)}%)` }}
+    />
+  </ProgressPrimitive.Root>
+))
+Progress.displayName = ProgressPrimitive.Root.displayName
+
+export { Progress }

frontend/src/components/ui/scroll-area.tsx

@@ -0,0 +1,46 @@
+import * as React from "react"
+import * as ScrollAreaPrimitive from "@radix-ui/react-scroll-area"
+
+import { cn } from "@/lib/utils"
+
+const ScrollArea = React.forwardRef<
+  React.ElementRef<typeof ScrollAreaPrimitive.Root>,
+  React.ComponentPropsWithoutRef<typeof ScrollAreaPrimitive.Root>
+>(({ className, children, ...props }, ref) => (
+  <ScrollAreaPrimitive.Root
+    ref={ref}
+    className={cn("relative overflow-hidden", className)}
+    {...props}
+  >
+    <ScrollAreaPrimitive.Viewport className="h-full w-full rounded-[inherit]">
+      {children}
+    </ScrollAreaPrimitive.Viewport>
+    <ScrollBar />
+    <ScrollAreaPrimitive.Corner />
+  </ScrollAreaPrimitive.Root>
+))
+ScrollArea.displayName = ScrollAreaPrimitive.Root.displayName
+
+const ScrollBar = React.forwardRef<
+  React.ElementRef<typeof ScrollAreaPrimitive.ScrollAreaScrollbar>,
+  React.ComponentPropsWithoutRef<typeof ScrollAreaPrimitive.ScrollAreaScrollbar>
+>(({ className, orientation = "vertical", ...props }, ref) => (
+  <ScrollAreaPrimitive.ScrollAreaScrollbar
+    ref={ref}
+    orientation={orientation}
+    className={cn(
+      "flex touch-none select-none transition-colors",
+      orientation === "vertical" &&
+        "h-full w-2.5 border-l border-l-transparent p-[1px]",
+      orientation === "horizontal" &&
+        "h-2.5 flex-col border-t border-t-transparent p-[1px]",
+      className
+    )}
+    {...props}
+  >
+    <ScrollAreaPrimitive.ScrollAreaThumb className="relative flex-1 rounded-full bg-border" />
+  </ScrollAreaPrimitive.ScrollAreaScrollbar>
+))
+ScrollBar.displayName = ScrollAreaPrimitive.ScrollAreaScrollbar.displayName
+
+export { ScrollArea, ScrollBar }

frontend/src/constants.ts

@@ -1 +1,3 @@
+//  WebSocket protocol (RFC 6455) allows for the use of custom close codes in the range 4000-4999
+export const APP_ERROR_WEB_SOCKET_CODE = 4332;
 export const USER_CLOSE_WEB_SOCKET_CODE = 4333;

frontend/src/generateCode.ts

@@ -1,26 +1,23 @@
 import toast from "react-hot-toast";
 import { WS_BACKEND_URL } from "./config";
-import { USER_CLOSE_WEB_SOCKET_CODE } from "./constants";
+import {
+  APP_ERROR_WEB_SOCKET_CODE,
+  USER_CLOSE_WEB_SOCKET_CODE,
+} from "./constants";
+import { FullGenerationSettings } from "./types";
 
 const ERROR_MESSAGE =
   "Error generating code. Check the Developer Console AND the backend logs for details. Feel free to open a Github issue.";
 
-const STOP_MESSAGE = "Code generation stopped";
+const CANCEL_MESSAGE = "Code generation cancelled";
-
-export interface CodeGenerationParams {
-  generationType: "create" | "update";
-  image: string;
-  resultImage?: string;
-  history?: string[];
-  // isImageGenerationEnabled: boolean; // TODO: Merge with Settings type in types.ts
-}
 
 export function generateCode(
   wsRef: React.MutableRefObject<WebSocket | null>,
-  params: CodeGenerationParams,
+  params: FullGenerationSettings,
   onChange: (chunk: string) => void,
   onSetCode: (code: string) => void,
   onStatusUpdate: (status: string) => void,
+  onCancel: () => void,
   onComplete: () => void
 ) {
   const wsUrl = `${WS_BACKEND_URL}/generate-code`;
@@ -46,15 +43,22 @@ export function generateCode(
       toast.error(response.value);
     }
   });
+
   ws.addEventListener("close", (event) => {
     console.log("Connection closed", event.code, event.reason);
     if (event.code === USER_CLOSE_WEB_SOCKET_CODE) {
-      toast.success(STOP_MESSAGE);
+      toast.success(CANCEL_MESSAGE);
+      onCancel();
+    } else if (event.code === APP_ERROR_WEB_SOCKET_CODE) {
+      console.error("Known server error", event);
+      onCancel();
     } else if (event.code !== 1000) {
-      console.error("WebSocket error code", event);
+      console.error("Unknown server or connection error", event);
       toast.error(ERROR_MESSAGE);
+      onCancel();
+    } else {
+      onComplete();
     }
-    onComplete();
   });
 
   ws.addEventListener("error", (error) => {

frontend/src/hooks/useBrowserTabIndicator.ts

@@ -0,0 +1,29 @@
+import { useEffect } from "react";
+
+const CODING_SETTINGS = {
+  title: "Coding...",
+  favicon: "/favicon/coding.png",
+};
+const DEFAULT_SETTINGS = {
+  title: "Screenshot to Code",
+  favicon: "/favicon/main.png",
+};
+
+const useBrowserTabIndicator = (isCoding: boolean) => {
+  useEffect(() => {
+    const settings = isCoding ? CODING_SETTINGS : DEFAULT_SETTINGS;
+
+    // Set favicon
+    const faviconEl = document.querySelector(
+      "link[rel='icon']"
+    ) as HTMLLinkElement | null;
+    if (faviconEl) {
+      faviconEl.href = settings.favicon;
+    }
+
+    // Set title
+    document.title = settings.title;
+  }, [isCoding]);
+};
+
+export default useBrowserTabIndicator;

frontend/src/index.css

@@ -61,9 +61,21 @@
 
     --radius: 0.5rem;
   }
+  
+  body.dark {
+    background-color: black;
+  }
+
+  div[role="presentation"].dark {
+    background-color: #09090b !important;
+  }
+
+  iframe {
+    background-color: white !important; 
+  }
 
   .dark {
-    --background: 222.2 84% 4.9%;
+    --background: 222.2 0% 0%;
     --foreground: 210 40% 98%;
 
     --card: 222.2 84% 4.9%;

frontend/src/lib/models.ts

@@ -0,0 +1,18 @@
+// Keep in sync with backend (llm.py)
+// Order here matches dropdown order
+export enum CodeGenerationModel {
+  GPT_4O_2024_05_13 = "gpt-4o-2024-05-13",
+  GPT_4_TURBO_2024_04_09 = "gpt-4-turbo-2024-04-09",
+  GPT_4_VISION = "gpt_4_vision",
+  CLAUDE_3_SONNET = "claude_3_sonnet",
+}
+
+// Will generate a static error if a model in the enum above is not in the descriptions
+export const CODE_GENERATION_MODEL_DESCRIPTIONS: {
+  [key in CodeGenerationModel]: { name: string; inBeta: boolean };
+} = {
+  "gpt-4o-2024-05-13": { name: "GPT-4o 🌟", inBeta: false },
+  "gpt-4-turbo-2024-04-09": { name: "GPT-4 Turbo (Apr 2024)", inBeta: false },
+  gpt_4_vision: { name: "GPT-4 Vision (Nov 2023)", inBeta: false },
+  claude_3_sonnet: { name: "Claude 3 Sonnet", inBeta: false },
+};

frontend/src/lib/stacks.ts

@@ -0,0 +1,20 @@
+// Keep in sync with backend (prompts/types.py)
+export enum Stack {
+  HTML_TAILWIND = "html_tailwind",
+  REACT_TAILWIND = "react_tailwind",
+  BOOTSTRAP = "bootstrap",
+  VUE_TAILWIND = "vue_tailwind",
+  IONIC_TAILWIND = "ionic_tailwind",
+  SVG = "svg",
+}
+
+export const STACK_DESCRIPTIONS: {
+  [key in Stack]: { components: string[]; inBeta: boolean };
+} = {
+  html_tailwind: { components: ["HTML", "Tailwind"], inBeta: false },
+  react_tailwind: { components: ["React", "Tailwind"], inBeta: false },
+  bootstrap: { components: ["Bootstrap"], inBeta: false },
+  vue_tailwind: { components: ["Vue", "Tailwind"], inBeta: true },
+  ionic_tailwind: { components: ["Ionic", "Tailwind"], inBeta: true },
+  svg: { components: ["SVG"], inBeta: true },
+};

frontend/src/main.tsx

@@ -3,10 +3,17 @@ import ReactDOM from "react-dom/client";
 import App from "./App.tsx";
 import "./index.css";
 import { Toaster } from "react-hot-toast";
+import EvalsPage from "./components/evals/EvalsPage.tsx";
+import { BrowserRouter as Router, Routes, Route } from "react-router-dom";
 
 ReactDOM.createRoot(document.getElementById("root")!).render(
   <React.StrictMode>
-    <App />
+    <Router>
-    <Toaster />
+      <Routes>
+        <Route path="/" element={<App />} />
+        <Route path="/evals" element={<EvalsPage />} />
+      </Routes>
+    </Router>
+    <Toaster toastOptions={{ className: "dark:bg-zinc-950 dark:text-white" }} />
   </React.StrictMode>
 );

frontend/src/setupTests.ts

@@ -0,0 +1,3 @@
+// So jest test runner can read env vars from .env file
+import { config } from "dotenv";
+config({ path: ".env.jest" });

frontend/src/store/app-store.ts

@@ -0,0 +1,10 @@
+import { create } from "zustand";
+
+// Store for app-wide state
+interface AppStore {
+  inputMode: "image" | "video";
+}
+
+export const useStore = create<AppStore>(() => ({
+  inputMode: "image",
+}));

frontend/src/tests/qa.test.ts

@@ -0,0 +1,274 @@
+import puppeteer, { Browser, Page, ElementHandle } from "puppeteer";
+import { Stack } from "../lib/stacks";
+import { CodeGenerationModel } from "../lib/models";
+
+const TESTS_ROOT_PATH = process.env.TEST_ROOT_PATH;
+
+// Fixtures
+const FIXTURES_PATH = `${TESTS_ROOT_PATH}/fixtures`;
+const SIMPLE_SCREENSHOT = FIXTURES_PATH + "/simple_button.png";
+const SCREENSHOT_WITH_IMAGES = `${FIXTURES_PATH}/simple_ui_with_image.png`;
+
+// Results
+const RESULTS_DIR = `${TESTS_ROOT_PATH}/results`;
+
+describe("e2e tests", () => {
+  let browser: Browser;
+  let page: Page;
+
+  const DEBUG = false;
+  const IS_HEADLESS = true;
+
+  const stacks = Object.values(Stack).slice(0, DEBUG ? 1 : undefined);
+  const models = Object.values(CodeGenerationModel).slice(
+    0,
+    DEBUG ? 1 : undefined
+  );
+
+  beforeAll(async () => {
+    browser = await puppeteer.launch({ headless: IS_HEADLESS });
+    page = await browser.newPage();
+    await page.goto("http://localhost:5173/");
+
+    // Set screen size
+    await page.setViewport({ width: 1080, height: 1024 });
+
+    // TODO: Does this need to be moved?
+    // const client = await page.createCDPSession();
+    // Set download behavior path
+    // await client.send("Page.setDownloadBehavior", {
+    //   behavior: "allow",
+    //   downloadPath: DOWNLOAD_PATH,
+    // });
+  });
+
+  afterAll(async () => {
+    await browser.close();
+  });
+
+  // Create tests
+  models.forEach((model) => {
+    stacks.forEach((stack) => {
+      it(
+        `Create for : ${model} & ${stack}`,
+        async () => {
+          const app = new App(
+            page,
+            stack,
+            model,
+            `create_screenshot_${model}_${stack}`
+          );
+          await app.init();
+          // Generate from screenshot
+          await app.uploadImage(SCREENSHOT_WITH_IMAGES);
+        },
+        60 * 1000
+      );
+
+      it(
+        `Create from URL for : ${model} & ${stack}`,
+        async () => {
+          const app = new App(
+            page,
+            stack,
+            model,
+            `create_url_${model}_${stack}`
+          );
+          await app.init();
+          // Generate from screenshot
+          await app.generateFromUrl("https://a.picoapps.xyz/design-fear");
+        },
+        60 * 1000
+      );
+    });
+  });
+
+  // Update tests - for every model (doesn’t need to be repeated for each stack - fix to HTML Tailwind only)
+  models.forEach((model) => {
+    ["html_tailwind"].forEach((stack) => {
+      it(
+        `update: ${model}`,
+        async () => {
+          const app = new App(page, stack, model, `update_${model}_${stack}`);
+          await app.init();
+
+          // Generate from screenshot
+          await app.uploadImage(SIMPLE_SCREENSHOT);
+          // Regenerate works for v1
+          await app.regenerate();
+          // Make an update
+          await app.edit("make the button background blue", "v2");
+          // Make another update
+          await app.edit("make the text italic", "v3");
+          // Branch off v2 and make an update
+          await app.clickVersion("v2");
+          await app.edit("make the text yellow", "v4");
+        },
+        90 * 1000
+      );
+    });
+  });
+
+  // Start from code tests - for every model
+  models.forEach((model) => {
+    ["html_tailwind"].forEach((stack) => {
+      it.skip(
+        `Start from code: ${model}`,
+        async () => {
+          const app = new App(
+            page,
+            stack,
+            model,
+            `start_from_code_${model}_${stack}`
+          );
+          await app.init();
+
+          await app.importFromCode();
+
+          // Regenerate works for v1
+          // await app.regenerate();
+          // // Make an update
+          // await app.edit("make the header blue", "v2");
+          // // Make another update
+          // await app.edit("make all text italic", "v3");
+          // // Branch off v2 and make an update
+          // await app.clickVersion("v2");
+          // await app.edit("make all text red", "v4");
+        },
+        90 * 1000
+      );
+    });
+  });
+});
+
+class App {
+  private screenshotPathPrefix: string;
+  private page: Page;
+  private stack: string;
+  private model: string;
+
+  constructor(page: Page, stack: string, model: string, testId: string) {
+    this.page = page;
+    this.stack = stack;
+    this.model = model;
+    this.screenshotPathPrefix = `${RESULTS_DIR}/${testId}`;
+  }
+
+  async init() {
+    await this.setupLocalStorage();
+  }
+
+  async setupLocalStorage() {
+    const setting = {
+      openAiApiKey: null,
+      openAiBaseURL: null,
+      screenshotOneApiKey: process.env.TEST_SCREENSHOTONE_API_KEY,
+      isImageGenerationEnabled: true,
+      editorTheme: "cobalt",
+      generatedCodeConfig: this.stack,
+      codeGenerationModel: this.model,
+      isTermOfServiceAccepted: false,
+      accessCode: null,
+    };
+
+    await this.page.evaluate((setting) => {
+      localStorage.setItem("setting", JSON.stringify(setting));
+    }, setting);
+
+    // Reload the page to apply the local storage
+    await this.page.reload();
+  }
+
+  async _screenshot(step: string) {
+    await this.page.screenshot({
+      path: `${this.screenshotPathPrefix}_${step}.png`,
+    });
+  }
+
+  async _waitUntilVersionIsReady(version: string) {
+    await this.page.waitForNetworkIdle();
+    await this.page.waitForFunction(
+      (version) => document.body.innerText.includes(version),
+      {
+        timeout: 30000,
+      },
+      version
+    );
+    // Wait for 3s so that the HTML and JS has time to render before screenshotting
+    await new Promise((resolve) => setTimeout(resolve, 3000));
+  }
+
+  async generateFromUrl(url: string) {
+    // Type in the URL
+    await this.page.type('input[placeholder="Enter URL"]', url);
+    await this._screenshot("typed_url");
+
+    // Click the capture button and wait for the code to be generated
+    await this.page.click("button.capture-btn");
+    await this._waitUntilVersionIsReady("v1");
+    await this._screenshot("url_result");
+  }
+
+  // Uploads a screenshot and generates the image
+  async uploadImage(screenshotPath: string) {
+    // Upload file
+    const fileInput = (await this.page.$(
+      ".file-input"
+    )) as ElementHandle<HTMLInputElement>;
+    if (!fileInput) {
+      throw new Error("File input element not found");
+    }
+    await fileInput.uploadFile(screenshotPath);
+    await this._screenshot("image_uploaded");
+
+    // Click the generate button and wait for the code to be generated
+    await this._waitUntilVersionIsReady("v1");
+    await this._screenshot("image_results");
+  }
+
+  // Makes a text edit and waits for a new version
+  async edit(edit: string, version: string) {
+    // Type in the edit
+    await this.page.type(
+      'textarea[placeholder="Tell the AI what to change..."]',
+      edit
+    );
+    await this._screenshot(`typed_${version}`);
+
+    // Click the update button and wait for the code to be generated
+    await this.page.click(".update-btn");
+    await this._waitUntilVersionIsReady(version);
+    await this._screenshot(`done_${version}`);
+  }
+
+  async clickVersion(version: string) {
+    await this.page.evaluate((version) => {
+      document.querySelectorAll("div").forEach((div) => {
+        if (div.innerText.includes(version)) {
+          div.click();
+        }
+      });
+    }, version);
+  }
+
+  async regenerate() {
+    await this.page.click(".regenerate-btn");
+    await this._waitUntilVersionIsReady("v1");
+    await this._screenshot("regenerate_results");
+  }
+
+  // Work in progress
+  async importFromCode() {
+    await this.page.click(".import-from-code-btn");
+
+    await this.page.type("textarea", "<html>hello world</html>");
+
+    await this.page.select("#output-settings-js", "HTML + Tailwind");
+
+    await this._screenshot("typed_code");
+
+    await this.page.click(".import-btn");
+
+    await this._waitUntilVersionIsReady("v1");
+  }
+}

frontend/src/types.ts

@@ -1,31 +1,22 @@
+import { Stack } from "./lib/stacks";
+import { CodeGenerationModel } from "./lib/models";
+
 export enum EditorTheme {
   ESPRESSO = "espresso",
   COBALT = "cobalt",
 }
 
-export enum CSSOption {
-  TAILWIND = "tailwind",
-  BOOTSTRAP = "bootstrap",
-}
-
-export enum JSFrameworkOption {
-  NO_FRAMEWORK = "vanilla",
-  REACT = "react",
-  VUE = "vue",
-}
-
-export interface OutputSettings {
-  css: CSSOption;
-  js: JSFrameworkOption;
-}
-
 export interface Settings {
   openAiApiKey: string | null;
+  openAiBaseURL: string | null;
   screenshotOneApiKey: string | null;
   isImageGenerationEnabled: boolean;
   editorTheme: EditorTheme;
-  isTermOfServiceAccepted: boolean; // Only relevant for hosted version
+  generatedCodeConfig: Stack;
-  accessCode: string | null; // Only relevant for hosted version
+  codeGenerationModel: CodeGenerationModel;
+  // Only relevant for hosted version
+  isTermOfServiceAccepted: boolean;
+  anthropicApiKey: string | null; // Added property for anthropic API key
 }
 
 export enum AppState {
@@ -33,3 +24,20 @@ export enum AppState {
   CODING = "CODING",
   CODE_READY = "CODE_READY",
 }
+
+export enum ScreenRecorderState {
+  INITIAL = "initial",
+  RECORDING = "recording",
+  FINISHED = "finished",
+}
+
+export interface CodeGenerationParams {
+  generationType: "create" | "update";
+  inputMode: "image" | "video";
+  image: string;
+  resultImage?: string;
+  history?: string[];
+  isImportedFromCode?: boolean;
+}
+
+export type FullGenerationSettings = CodeGenerationParams & Settings;

frontend/src/urls.ts

@@ -0,0 +1,5 @@
+export const URLS = {
+  "intro-to-video":
+    "https://github.com/abi/screenshot-to-code/wiki/Screen-Recording-to-Code",
+  tips: "https://git.new/s5ywP0e",
+};