Github/screenshot-to-code
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #2 from wearzdk/up-stream

Browse Source 
Up stream
This commit is contained in:
wear工程师 2024-01-16 19:03:28 +08:00 committed by GitHub
commit c9e99f068c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
47 changed files with 1882 additions and 644 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.vscode/settings.json vendored Normal file
View File

@ -0,0 +1,3 @@
{
"python.analysis.typeCheckingMode": "strict"
}

11
README.md
View File

@ -1,6 +1,6 @@
# screenshot-to-code
This simple app converts a screenshot to code (HTML/Tailwind CSS, or React or Vue or Bootstrap). It uses GPT-4 Vision to generate the code and DALL-E 3 to generate similar-looking images. You can now also enter a URL to clone a live website!
This simple app converts a screenshot to code (HTML/Tailwind CSS, or React or Bootstrap or Vue). It uses GPT-4 Vision to generate the code and DALL-E 3 to generate similar-looking images. You can now also enter a URL to clone a live website!
https://github.com/abi/screenshot-to-code/assets/23818/6cebadae-2fe3-4986-ac6a-8fb9db030045
@ -12,6 +12,7 @@ See the [Examples](#-examples) section below for more demos.
## 🌟 Recent Updates
- Dec 11 - Start a new project from existing code (allows you to come back to an older project)
- Dec 7 - 🔥 🔥 🔥 View a history of your edits, and branch off them
- Nov 30 - Dark mode, output code in Ionic (thanks [@dialmedu](https://github.com/dialmedu)), set OpenAI base URL
- Nov 28 - 🔥 🔥 🔥 Customize your stack: React or Bootstrap or TailwindCSS
@ -37,6 +38,12 @@ poetry shell
poetry run uvicorn main:app --reload --port 7001
```
You can also run the backend (when you're in `backend`):
```bash
poetry run pyright
```
Run the frontend:
```bash
@ -57,7 +64,7 @@ MOCK=true poetry run uvicorn main:app --reload --port 7001
## Configuration
* You can configure the OpenAI base URL if you need to use a proxy: Set OPENAI_BASE_URL in the `backend/.env` or directly in the UI in the settings dialog
- You can configure the OpenAI base URL if you need to use a proxy: Set OPENAI_BASE_URL in the `backend/.env` or directly in the UI in the settings dialog
## Docker

4
backend/.gitignore vendored
View File

@ -150,3 +150,7 @@ cython_debug/
# and can be added to the global gitignore or merged into this file. For a more nuclear
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
#.idea/
# Temporary eval output
evals_data

8
backend/README.md
View File

@ -1,3 +1,7 @@
Run tests
# Run the type checker
pytest test_prompts.py
poetry run pyright
# Run tests
poetry run pytest

11
backend/config.py Normal file
View File

@ -0,0 +1,11 @@
# Useful for debugging purposes when you don't want to waste GPT4-Vision credits
# Setting to True will stream a mock response instead of calling the OpenAI API
# TODO: Should only be set to true when value is 'True', not any abitrary truthy value
import os
SHOULD_MOCK_AI_RESPONSE = bool(os.environ.get("MOCK", False))
# Set to True when running in production (on the hosted version)
# Used as a feature flag to enable or disable certain features
IS_PROD = os.environ.get("IS_PROD", False)

0
backend/evals/__init__.py Normal file
View File

1
backend/evals/config.py Normal file
View File

@ -0,0 +1 @@
EVALS_DIR = "./evals_data"

29
backend/evals/core.py Normal file
View File

@ -0,0 +1,29 @@
import os
from llm import stream_openai_response
from prompts import assemble_prompt
from prompts.types import Stack
from utils import pprint_prompt
async def generate_code_core(image_url: str, stack: Stack) -> str:
prompt_messages = assemble_prompt(image_url, stack)
openai_api_key = os.environ.get("OPENAI_API_KEY")
openai_base_url = None
pprint_prompt(prompt_messages)
async def process_chunk(content: str):
pass
if not openai_api_key:
raise Exception("OpenAI API key not found")
completion = await stream_openai_response(
prompt_messages,
api_key=openai_api_key,
base_url=openai_base_url,
callback=lambda x: process_chunk(x),
)
return completion

7
backend/evals/utils.py Normal file
View File

@ -0,0 +1,7 @@
import base64
async def image_to_data_url(filepath: str):
with open(filepath, "rb") as image_file:
encoded_string = base64.b64encode(image_file.read()).decode()
return f"data:image/png;base64,{encoded_string}"

20
backend/image_generation.py
View File

@ -1,15 +1,15 @@
import asyncio
import os
import re
from typing import Dict, List, Union
from openai import AsyncOpenAI
from bs4 import BeautifulSoup
async def process_tasks(prompts, api_key, base_url):
async def process_tasks(prompts: List[str], api_key: str, base_url: str):
tasks = [generate_image(prompt, api_key, base_url) for prompt in prompts]
results = await asyncio.gather(*tasks, return_exceptions=True)
processed_results = []
processed_results: List[Union[str, None]] = []
for result in results:
if isinstance(result, Exception):
print(f"An exception occurred: {result}")
@ -20,9 +20,9 @@ async def process_tasks(prompts, api_key, base_url):
return processed_results
async def generate_image(prompt, api_key, base_url):
async def generate_image(prompt: str, api_key: str, base_url: str):
client = AsyncOpenAI(api_key=api_key, base_url=base_url)
image_params = {
image_params: Dict[str, Union[str, int]] = {
"model": "dall-e-3",
"quality": "standard",
"style": "natural",
@ -35,7 +35,7 @@ async def generate_image(prompt, api_key, base_url):
return res.data[0].url
def extract_dimensions(url):
def extract_dimensions(url: str):
# Regular expression to match numbers in the format '300x200'
matches = re.findall(r"(\d+)x(\d+)", url)
@ -48,11 +48,11 @@ def extract_dimensions(url):
return (100, 100)
def create_alt_url_mapping(code):
def create_alt_url_mapping(code: str) -> Dict[str, str]:
soup = BeautifulSoup(code, "html.parser")
images = soup.find_all("img")
mapping = {}
mapping: Dict[str, str] = {}
for image in images:
if not image["src"].startswith("https://placehold.co"):
@ -61,7 +61,9 @@ def create_alt_url_mapping(code):
return mapping
async def generate_images(code, api_key, base_url, image_cache):
async def generate_images(
code: str, api_key: str, base_url: Union[str, None], image_cache: Dict[str, str]
):
# Find all images
soup = BeautifulSoup(code, "html.parser")
images = soup.find_all("img")

13
backend/llm.py
View File

@ -1,16 +1,16 @@
import os
from typing import Awaitable, Callable
from typing import Awaitable, Callable, List
from openai import AsyncOpenAI
from openai.types.chat import ChatCompletionMessageParam, ChatCompletionChunk
MODEL_GPT_4_VISION = "gpt-4-vision-preview"
async def stream_openai_response(
messages,
messages: List[ChatCompletionMessageParam],
api_key: str,
base_url: str | None,
callback: Callable[[str], Awaitable[None]],
):
) -> str:
client = AsyncOpenAI(api_key=api_key, base_url=base_url)
model = MODEL_GPT_4_VISION
@ -23,9 +23,10 @@ async def stream_openai_response(
params["max_tokens"] = 4096
params["temperature"] = 0
completion = await client.chat.completions.create(**params)
stream = await client.chat.completions.create(**params) # type: ignore
full_response = ""
async for chunk in completion:
async for chunk in stream: # type: ignore
assert isinstance(chunk, ChatCompletionChunk)
content = chunk.choices[0].delta.content or ""
full_response += content
await callback(content)

265
backend/main.py
View File

@ -1,26 +1,12 @@
# Load environment variables first
from dotenv import load_dotenv
load_dotenv()
import json
import os
import traceback
from datetime import datetime
from fastapi import FastAPI, WebSocket
from fastapi.staticfiles import StaticFiles
from fastapi import FastAPI
from fastapi.middleware.cors import CORSMiddleware
from fastapi.responses import FileResponse, HTMLResponse
import openai
from llm import stream_openai_response
from mock import mock_completion
from utils import pprint_prompt
from image_generation import create_alt_url_mapping, generate_images
from prompts import assemble_prompt
from routes import screenshot
from access_token import validate_access_token
from routes import screenshot, generate_code, home, evals
app = FastAPI(openapi_url=None, docs_url=None, redoc_url=None)
@ -33,247 +19,8 @@ app.add_middleware(
allow_headers=["*"],
)
# Useful for debugging purposes when you don't want to waste GPT4-Vision credits
# Setting to True will stream a mock response instead of calling the OpenAI API
# TODO: Should only be set to true when value is 'True', not any abitrary truthy value
SHOULD_MOCK_AI_RESPONSE = bool(os.environ.get("MOCK", False))
# Set to True when running in production (on the hosted version)
# Used as a feature flag to enable or disable certain features
IS_PROD = os.environ.get("IS_PROD", False)
# if webui folder exists, we are in build mode
IS_BUILD = os.path.exists("webui")
# Add routes
app.include_router(generate_code.router)
app.include_router(screenshot.router)
@app.get("/")
async def get_status():
if IS_BUILD:
# if in build mode, return webui/index.html
return FileResponse("webui/index.html")
return HTMLResponse(
content="<h3>Your backend is running correctly. Please open the front-end URL (default is http://localhost:5173) to use screenshot-to-code.</h3>"
)
def write_logs(prompt_messages, completion):
# Get the logs path from environment, default to the current working directory
logs_path = os.environ.get("LOGS_PATH", os.getcwd())
# Create run_logs directory if it doesn't exist within the specified logs path
logs_directory = os.path.join(logs_path, "run_logs")
if not os.path.exists(logs_directory):
os.makedirs(logs_directory)
print("Writing to logs directory:", logs_directory)
# Generate a unique filename using the current timestamp within the logs directory
filename = datetime.now().strftime(f"{logs_directory}/messages_%Y%m%d_%H%M%S.json")
# Write the messages dict into a new file for each run
with open(filename, "w") as f:
f.write(json.dumps({"prompt": prompt_messages, "completion": completion}))
@app.websocket("/generate-code")
async def stream_code(websocket: WebSocket):
await websocket.accept()
print("Incoming websocket connection...")
async def throw_error(
message: str,
):
await websocket.send_json({"type": "error", "value": message})
await websocket.close()
params = await websocket.receive_json()
print("Received params")
# Read the code config settings from the request. Fall back to default if not provided.
generated_code_config = ""
if "generatedCodeConfig" in params and params["generatedCodeConfig"]:
generated_code_config = params["generatedCodeConfig"]
print(f"Generating {generated_code_config} code")
# Get the OpenAI API key from the request. Fall back to environment variable if not provided.
# If neither is provided, we throw an error.
openai_api_key = None
if "accessCode" in params and params["accessCode"]:
print("Access code - using platform API key")
res = await validate_access_token(params["accessCode"])
if res["success"]:
openai_api_key = os.environ.get("PLATFORM_OPENAI_API_KEY")
else:
await websocket.send_json(
{
"type": "error",
"value": res["failure_reason"],
}
)
return
else:
if params["openAiApiKey"]:
openai_api_key = params["openAiApiKey"]
print("Using OpenAI API key from client-side settings dialog")
else:
openai_api_key = os.environ.get("OPENAI_API_KEY")
if openai_api_key:
print("Using OpenAI API key from environment variable")
if not openai_api_key:
print("OpenAI API key not found")
await websocket.send_json(
{
"type": "error",
"value": "No OpenAI API key found. Please add your API key in the settings dialog or add it to backend/.env file.",
}
)
return
# Get the OpenAI Base URL from the request. Fall back to environment variable if not provided.
openai_base_url = None
# Disable user-specified OpenAI Base URL in prod
if not os.environ.get("IS_PROD"):
if "openAiBaseURL" in params and params["openAiBaseURL"]:
openai_base_url = params["openAiBaseURL"]
print("Using OpenAI Base URL from client-side settings dialog")
else:
openai_base_url = os.environ.get("OPENAI_BASE_URL")
if openai_base_url:
print("Using OpenAI Base URL from environment variable")
if not openai_base_url:
print("Using official OpenAI URL")
# Get the image generation flag from the request. Fall back to True if not provided.
should_generate_images = (
params["isImageGenerationEnabled"]
if "isImageGenerationEnabled" in params
else True
)
print("generating code...")
await websocket.send_json({"type": "status", "value": "Generating code..."})
async def process_chunk(content):
await websocket.send_json({"type": "chunk", "value": content})
# Assemble the prompt
try:
if params.get("resultImage") and params["resultImage"]:
prompt_messages = assemble_prompt(
params["image"], generated_code_config, params["resultImage"]
)
else:
prompt_messages = assemble_prompt(params["image"], generated_code_config)
except:
await websocket.send_json(
{
"type": "error",
"value": "Error assembling prompt. Contact support at support@picoapps.xyz",
}
)
await websocket.close()
return
# Image cache for updates so that we don't have to regenerate images
image_cache = {}
if params["generationType"] == "update":
# Transform into message format
# TODO: Move this to frontend
for index, text in enumerate(params["history"]):
prompt_messages += [
{"role": "assistant" if index % 2 == 0 else "user", "content": text}
]
image_cache = create_alt_url_mapping(params["history"][-2])
if SHOULD_MOCK_AI_RESPONSE:
completion = await mock_completion(process_chunk)
else:
try:
completion = await stream_openai_response(
prompt_messages,
api_key=openai_api_key,
base_url=openai_base_url,
callback=lambda x: process_chunk(x),
)
except openai.AuthenticationError as e:
print("[GENERATE_CODE] Authentication failed", e)
error_message = (
"Incorrect OpenAI key. Please make sure your OpenAI API key is correct, or create a new OpenAI API key on your OpenAI dashboard."
+ (
" Alternatively, you can purchase code generation credits directly on this website."
if IS_PROD
else ""
)
)
return await throw_error(error_message)
except openai.NotFoundError as e:
print("[GENERATE_CODE] Model not found", e)
error_message = (
e.message
+ ". Please make sure you have followed the instructions correctly to obtain an OpenAI key with GPT vision access: https://github.com/abi/screenshot-to-code/blob/main/Troubleshooting.md"
+ (
" Alternatively, you can purchase code generation credits directly on this website."
if IS_PROD
else ""
)
)
return await throw_error(error_message)
except openai.RateLimitError as e:
print("[GENERATE_CODE] Rate limit exceeded", e)
error_message = (
"OpenAI error - 'You exceeded your current quota, please check your plan and billing details.'"
+ (
" Alternatively, you can purchase code generation credits directly on this website."
if IS_PROD
else ""
)
)
return await throw_error(error_message)
# Write the messages dict into a log so that we can debug later
write_logs(prompt_messages, completion)
try:
if should_generate_images:
await websocket.send_json(
{"type": "status", "value": "Generating images..."}
)
updated_html = await generate_images(
completion,
api_key=openai_api_key,
base_url=openai_base_url,
image_cache=image_cache,
)
else:
updated_html = completion
await websocket.send_json({"type": "setCode", "value": updated_html})
await websocket.send_json(
{"type": "status", "value": "Code generation complete."}
)
except Exception as e:
traceback.print_exc()
print("Image generation failed", e)
await websocket.send_json(
{"type": "status", "value": "Image generation failed but code is complete."}
)
await websocket.close()
# Serve the webui folder as static files
if IS_BUILD:
app.mount("/", StaticFiles(directory="webui", html=True), name="webui")
if __name__ == "__main__":
import uvicorn
uvicorn.run(app, host="0.0.0.0", port=8000)
app.include_router(home.router)
app.include_router(evals.router)

3
backend/mock.py → backend/mock_llm.py
View File

@ -1,7 +1,8 @@
import asyncio
from typing import Awaitable, Callable
async def mock_completion(process_chunk):
async def mock_completion(process_chunk: Callable[[str], Awaitable[None]]) -> str:
code_to_return = NO_IMAGES_NYTIMES_MOCK_CODE
for i in range(0, len(code_to_return), 10):

65
backend/poetry.lock generated
View File

@ -200,19 +200,31 @@ files = [
]
[[package]]
name = "macholib"
version = "1.16.3"
description = "Mach-O header analysis and editing"
name = "iniconfig"
version = "2.0.0"
description = "brain-dead simple config-ini parsing"
category = "dev"
optional = false
python-versions = "*"
python-versions = ">=3.7"
files = [
{file = "macholib-1.16.3-py2.py3-none-any.whl", hash = "sha256:0e315d7583d38b8c77e815b1ecbdbf504a8258d8b3e17b61165c6feb60d18f2c"},
{file = "macholib-1.16.3.tar.gz", hash = "sha256:07ae9e15e8e4cd9a788013d81f5908b3609aa76f9b1421bae9c4d7606ec86a30"},
{file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"},
{file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"},
]
[[package]]
name = "nodeenv"
version = "1.8.0"
description = "Node.js virtual environment builder"
category = "dev"
optional = false
python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*"
files = [
{file = "nodeenv-1.8.0-py2.py3-none-any.whl", hash = "sha256:df865724bb3c3adc86b3876fa209771517b0cfe596beff01a92700e0e8be4cec"},
{file = "nodeenv-1.8.0.tar.gz", hash = "sha256:d51e0c37e64fbf47d017feac3145cdbb58836d7eee8c6f6d3b6880c5456227d2"},
]
[package.dependencies]
altgraph = ">=0.17"
setuptools = "*"
[[package]]
name = "openai"
version = "1.3.7"
@ -240,6 +252,7 @@ datalib = ["numpy (>=1)", "pandas (>=1.2.3)", "pandas-stubs (>=1.1.0.11)"]
name = "packaging"
version = "23.2"
description = "Core utilities for Python packages"
category = "dev"
optional = false
python-versions = ">=3.7"
files = [
@ -248,16 +261,21 @@ files = [
]
[[package]]
name = "pefile"
version = "2023.2.7"
description = "Python PE parsing module"
name = "pluggy"
version = "1.3.0"
description = "plugin and hook calling mechanisms for python"
category = "dev"
optional = false
python-versions = ">=3.6.0"
python-versions = ">=3.8"
files = [
{file = "pefile-2023.2.7-py3-none-any.whl", hash = "sha256:da185cd2af68c08a6cd4481f7325ed600a88f6a813bad9dea07ab3ef73d8d8d6"},
{file = "pefile-2023.2.7.tar.gz", hash = "sha256:82e6114004b3d6911c77c3953e3838654b04511b8b66e8583db70c65998017dc"},
{file = "pluggy-1.3.0-py3-none-any.whl", hash = "sha256:d89c696a773f8bd377d18e5ecda92b7a3793cbe66c87060a6fb58c7b6e1061f7"},
{file = "pluggy-1.3.0.tar.gz", hash = "sha256:cf61ae8f126ac6f7c451172cf30e3e43d3ca77615509771b3a984a0730651e12"},
]
[package.extras]
dev = ["pre-commit", "tox"]
testing = ["pytest", "pytest-benchmark"]
[[package]]
name = "pydantic"
version = "1.10.13"
@ -354,7 +372,6 @@ files = [
{file = "pyinstaller-hooks-contrib-2023.10.tar.gz", hash = "sha256:4b4a998036abb713774cb26534ca06b7e6e09e4c628196017a10deb11a48747f"},
{file = "pyinstaller_hooks_contrib-2023.10-py2.py3-none-any.whl", hash = "sha256:6dc1786a8f452941245d5bb85893e2a33632ebdcbc4c23eea41f2ee08281b0c0"},
]
[[package]]
name = "python-dotenv"
version = "1.0.0"
@ -435,6 +452,18 @@ anyio = ">=3.4.0,<5"
[package.extras]
full = ["httpx (>=0.22.0)", "itsdangerous", "jinja2", "python-multipart", "pyyaml"]
[[package]]
name = "tomli"
version = "2.0.1"
description = "A lil' TOML parser"
category = "dev"
optional = false
python-versions = ">=3.7"
files = [
{file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"},
{file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"},
]
[[package]]
name = "tqdm"
version = "4.66.1"
@ -468,13 +497,13 @@ files = [
[[package]]
name = "uvicorn"
version = "0.24.0.post1"
version = "0.25.0"
description = "The lightning-fast ASGI server."
optional = false
python-versions = ">=3.8"
files = [
{file = "uvicorn-0.24.0.post1-py3-none-any.whl", hash = "sha256:7c84fea70c619d4a710153482c0d230929af7bcf76c7bfa6de151f0a3a80121e"},
{file = "uvicorn-0.24.0.post1.tar.gz", hash = "sha256:09c8e5a79dc466bdf28dead50093957db184de356fcdc48697bad3bde4c2588e"},
{file = "uvicorn-0.25.0-py3-none-any.whl", hash = "sha256:ce107f5d9bd02b4636001a77a4e74aab5e1e2b146868ebbad565237145af444c"},
{file = "uvicorn-0.25.0.tar.gz", hash = "sha256:6dddbad1d7ee0f5140aba5ec138ddc9612c5109399903828b4874c9937f009c2"},
]
[package.dependencies]

79
backend/prompts/__init__.py Normal file
View File

@ -0,0 +1,79 @@
from typing import List, NoReturn, Union
from openai.types.chat import ChatCompletionMessageParam, ChatCompletionContentPartParam
from prompts.imported_code_prompts import IMPORTED_CODE_SYSTEM_PROMPTS
from prompts.screenshot_system_prompts import SYSTEM_PROMPTS
from prompts.types import Stack
USER_PROMPT = """
Generate code for a web page that looks exactly like this.
"""
SVG_USER_PROMPT = """
Generate code for a SVG that looks exactly like this.
"""
def assemble_imported_code_prompt(
code: str, stack: Stack, result_image_data_url: Union[str, None] = None
) -> List[ChatCompletionMessageParam]:
system_content = IMPORTED_CODE_SYSTEM_PROMPTS[stack]
user_content = (
"Here is the code of the app: " + code
if stack != "svg"
else "Here is the code of the SVG: " + code
)
return [
{
"role": "system",
"content": system_content,
},
{
"role": "user",
"content": user_content,
},
]
# TODO: Use result_image_data_url
def assemble_prompt(
image_data_url: str,
stack: Stack,
result_image_data_url: Union[str, None] = None,
) -> List[ChatCompletionMessageParam]:
system_content = SYSTEM_PROMPTS[stack]
user_prompt = USER_PROMPT if stack != "svg" else SVG_USER_PROMPT
user_content: List[ChatCompletionContentPartParam] = [
{
"type": "image_url",
"image_url": {"url": image_data_url, "detail": "high"},
},
{
"type": "text",
"text": user_prompt,
},
]
# Include the result image if it exists
if result_image_data_url:
user_content.insert(
1,
{
"type": "image_url",
"image_url": {"url": result_image_data_url, "detail": "high"},
},
)
return [
{
"role": "system",
"content": system_content,
},
{
"role": "user",
"content": user_content,
},
]

150
backend/test_prompts.py → backend/prompts/imported_code_prompts.py
View File

@ -1,18 +1,11 @@
from prompts import assemble_prompt
from prompts.types import SystemPrompts
TAILWIND_SYSTEM_PROMPT = """
You are an expert Tailwind developer
You take screenshots of a reference web page from the user, and then build single page apps
using Tailwind, HTML and JS.
You might also be given a screenshot(The second image) of a web page that you have already built, and asked to
update it to look more like the reference image(The first image).
- Make sure the app looks exactly like the screenshot.
- Pay close attention to background color, text color, font size, font family,
padding, margin, border, etc. Match the colors and sizes exactly.
- Use the exact text from the screenshot.
IMPORTED_CODE_TAILWIND_SYSTEM_PROMPT = """
You are an expert Tailwind developer.
- Do not add comments in the code such as "<!-- Add other navigation links as needed -->" and "<!-- ... other news items ... -->" in place of writing the full code. WRITE THE FULL CODE.
- Repeat elements as needed to match the screenshot. For example, if there are 15 items, the code should have 15 items. DO NOT LEAVE comments like "<!-- Repeat for each news item -->" or bad things will happen.
- Repeat elements as needed. For example, if there are 15 items, the code should have 15 items. DO NOT LEAVE comments like "<!-- Repeat for each news item -->" or bad things will happen.
- For images, use placeholder images from https://placehold.co and include a detailed description of the image in the alt text so that an image generation AI can generate the image later.
In terms of libraries,
@ -25,44 +18,11 @@ Return only the full code in <html></html> tags.
Do not include markdown "```" or "```html" at the start or end.
"""
BOOTSTRAP_SYSTEM_PROMPT = """
You are an expert Bootstrap developer
You take screenshots of a reference web page from the user, and then build single page apps
using Bootstrap, HTML and JS.
You might also be given a screenshot(The second image) of a web page that you have already built, and asked to
update it to look more like the reference image(The first image).
- Make sure the app looks exactly like the screenshot.
- Pay close attention to background color, text color, font size, font family,
padding, margin, border, etc. Match the colors and sizes exactly.
- Use the exact text from the screenshot.
- Do not add comments in the code such as "<!-- Add other navigation links as needed -->" and "<!-- ... other news items ... -->" in place of writing the full code. WRITE THE FULL CODE.
- Repeat elements as needed to match the screenshot. For example, if there are 15 items, the code should have 15 items. DO NOT LEAVE comments like "<!-- Repeat for each news item -->" or bad things will happen.
- For images, use placeholder images from https://placehold.co and include a detailed description of the image in the alt text so that an image generation AI can generate the image later.
In terms of libraries,
- Use this script to include Bootstrap: <link href="https://cdn.jsdelivr.net/npm/bootstrap@5.3.2/dist/css/bootstrap.min.css" rel="stylesheet" integrity="sha384-T3c6CoIi6uLrA9TneNEoa7RxnatzjcDSCmG1MXxSR1GAsXEV/Dwwykc2MPK8M2HN" crossorigin="anonymous">
- You can use Google Fonts
- Font Awesome for icons: <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.15.3/css/all.min.css"></link>
Return only the full code in <html></html> tags.
Do not include markdown "```" or "```html" at the start or end.
"""
REACT_TAILWIND_SYSTEM_PROMPT = """
IMPORTED_CODE_REACT_TAILWIND_SYSTEM_PROMPT = """
You are an expert React/Tailwind developer
You take screenshots of a reference web page from the user, and then build single page apps
using React and Tailwind CSS.
You might also be given a screenshot(The second image) of a web page that you have already built, and asked to
update it to look more like the reference image(The first image).
- Make sure the app looks exactly like the screenshot.
- Pay close attention to background color, text color, font size, font family,
padding, margin, border, etc. Match the colors and sizes exactly.
- Use the exact text from the screenshot.
- Do not add comments in the code such as "<!-- Add other navigation links as needed -->" and "<!-- ... other news items ... -->" in place of writing the full code. WRITE THE FULL CODE.
- Repeat elements as needed to match the screenshot. For example, if there are 15 items, the code should have 15 items. DO NOT LEAVE comments like "<!-- Repeat for each news item -->" or bad things will happen.
- Repeat elements as needed. For example, if there are 15 items, the code should have 15 items. DO NOT LEAVE comments like "<!-- Repeat for each news item -->" or bad things will happen.
- For images, use placeholder images from https://placehold.co and include a detailed description of the image in the alt text so that an image generation AI can generate the image later.
In terms of libraries,
@ -79,19 +39,28 @@ Return only the full code in <html></html> tags.
Do not include markdown "```" or "```html" at the start or end.
"""
IONIC_TAILWIND_SYSTEM_PROMPT = """
You are an expert Ionic/Tailwind developer
You take screenshots of a reference web page from the user, and then build single page apps
using Ionic and Tailwind CSS.
You might also be given a screenshot(The second image) of a web page that you have already built, and asked to
update it to look more like the reference image(The first image).
IMPORTED_CODE_BOOTSTRAP_SYSTEM_PROMPT = """
You are an expert Bootstrap developer.
- Make sure the app looks exactly like the screenshot.
- Pay close attention to background color, text color, font size, font family,
padding, margin, border, etc. Match the colors and sizes exactly.
- Use the exact text from the screenshot.
- Do not add comments in the code such as "<!-- Add other navigation links as needed -->" and "<!-- ... other news items ... -->" in place of writing the full code. WRITE THE FULL CODE.
- Repeat elements as needed to match the screenshot. For example, if there are 15 items, the code should have 15 items. DO NOT LEAVE comments like "<!-- Repeat for each news item -->" or bad things will happen.
- Repeat elements as needed. For example, if there are 15 items, the code should have 15 items. DO NOT LEAVE comments like "<!-- Repeat for each news item -->" or bad things will happen.
- For images, use placeholder images from https://placehold.co and include a detailed description of the image in the alt text so that an image generation AI can generate the image later.
In terms of libraries,
- Use this script to include Bootstrap: <link href="https://cdn.jsdelivr.net/npm/bootstrap@5.3.2/dist/css/bootstrap.min.css" rel="stylesheet" integrity="sha384-T3c6CoIi6uLrA9TneNEoa7RxnatzjcDSCmG1MXxSR1GAsXEV/Dwwykc2MPK8M2HN" crossorigin="anonymous">
- You can use Google Fonts
- Font Awesome for icons: <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.15.3/css/all.min.css"></link>
Return only the full code in <html></html> tags.
Do not include markdown "```" or "```html" at the start or end.
"""
IMPORTED_CODE_IONIC_TAILWIND_SYSTEM_PROMPT = """
You are an expert Ionic/Tailwind developer.
- Do not add comments in the code such as "<!-- Add other navigation links as needed -->" and "<!-- ... other news items ... -->" in place of writing the full code. WRITE THE FULL CODE.
- Repeat elements as needed. For example, if there are 15 items, the code should have 15 items. DO NOT LEAVE comments like "<!-- Repeat for each news item -->" or bad things will happen.
- For images, use placeholder images from https://placehold.co and include a detailed description of the image in the alt text so that an image generation AI can generate the image later.
In terms of libraries,
@ -113,24 +82,55 @@ Return only the full code in <html></html> tags.
Do not include markdown "```" or "```html" at the start or end.
"""
IMPORTED_CODE_VUE_TAILWIND_SYSTEM_PROMPT = """
You are an expert Vue/Tailwind developer.
def test_prompts():
tailwind_prompt = assemble_prompt(
"image_data_url", "html_tailwind", "result_image_data_url"
)
assert tailwind_prompt[0]["content"] == TAILWIND_SYSTEM_PROMPT
- Do not add comments in the code such as "<!-- Add other navigation links as needed -->" and "<!-- ... other news items ... -->" in place of writing the full code. WRITE THE FULL CODE.
- Repeat elements as needed. For example, if there are 15 items, the code should have 15 items. DO NOT LEAVE comments like "<!-- Repeat for each news item -->" or bad things will happen.
- For images, use placeholder images from https://placehold.co and include a detailed description of the image in the alt text so that an image generation AI can generate the image later.
react_tailwind_prompt = assemble_prompt(
"image_data_url", "react_tailwind", "result_image_data_url"
)
assert react_tailwind_prompt[0]["content"] == REACT_TAILWIND_SYSTEM_PROMPT
In terms of libraries,
bootstrap_prompt = assemble_prompt(
"image_data_url", "bootstrap", "result_image_data_url"
)
assert bootstrap_prompt[0]["content"] == BOOTSTRAP_SYSTEM_PROMPT
- Use these script to include Vue so that it can run on a standalone page:
<script src="https://registry.npmmirror.com/vue/3.3.11/files/dist/vue.global.js"></script>
- Use Vue using the global build like so:
<div id="app">{{ message }}</div>
<script>
const { createApp, ref } = Vue
createApp({
setup() {
const message = ref('Hello vue!')
return {
message
}
}
}).mount('#app')
</script>
- Use this script to include Tailwind: <script src="https://cdn.tailwindcss.com"></script>
- You can use Google Fonts
- Font Awesome for icons: <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.15.3/css/all.min.css"></link>
ionic_tailwind = assemble_prompt(
"image_data_url", "ionic_tailwind", "result_image_data_url"
)
assert ionic_tailwind[0]["content"] == IONIC_TAILWIND_SYSTEM_PROMPT
Return only the full code in <html></html> tags.
Do not include markdown "```" or "```html" at the start or end.
The return result must only include the code."""
IMPORTED_CODE_SVG_SYSTEM_PROMPT = """
You are an expert at building SVGs.
- Do not add comments in the code such as "<!-- Add other navigation links as needed -->" and "<!-- ... other news items ... -->" in place of writing the full code. WRITE THE FULL CODE.
- Repeat elements as needed to match the screenshot. For example, if there are 15 items, the code should have 15 items. DO NOT LEAVE comments like "<!-- Repeat for each news item -->" or bad things will happen.
- For images, use placeholder images from https://placehold.co and include a detailed description of the image in the alt text so that an image generation AI can generate the image later.
- You can use Google Fonts
Return only the full code in <svg></svg> tags.
Do not include markdown "```" or "```svg" at the start or end.
"""
IMPORTED_CODE_SYSTEM_PROMPTS = SystemPrompts(
html_tailwind=IMPORTED_CODE_TAILWIND_SYSTEM_PROMPT,
react_tailwind=IMPORTED_CODE_REACT_TAILWIND_SYSTEM_PROMPT,
bootstrap=IMPORTED_CODE_BOOTSTRAP_SYSTEM_PROMPT,
ionic_tailwind=IMPORTED_CODE_IONIC_TAILWIND_SYSTEM_PROMPT,
vue_tailwind=IMPORTED_CODE_VUE_TAILWIND_SYSTEM_PROMPT,
svg=IMPORTED_CODE_SVG_SYSTEM_PROMPT,
)

115
backend/prompts.py → backend/prompts/screenshot_system_prompts.py
View File

@ -1,4 +1,7 @@
TAILWIND_SYSTEM_PROMPT = """
from prompts.types import SystemPrompts
HTML_TAILWIND_SYSTEM_PROMPT = """
You are an expert Tailwind developer
You take screenshots of a reference web page from the user, and then build single page apps
using Tailwind, HTML and JS.
@ -111,54 +114,72 @@ Return only the full code in <html></html> tags.
Do not include markdown "```" or "```html" at the start or end.
"""
USER_PROMPT = """
Generate code for a web page that looks exactly like this.
VUE_TAILWIND_SYSTEM_PROMPT = """
You are an expert Vue/Tailwind developer
You take screenshots of a reference web page from the user, and then build single page apps
using Vue and Tailwind CSS.
You might also be given a screenshot(The second image) of a web page that you have already built, and asked to
update it to look more like the reference image(The first image).
- Make sure the app looks exactly like the screenshot.
- Pay close attention to background color, text color, font size, font family,
padding, margin, border, etc. Match the colors and sizes exactly.
- Use the exact text from the screenshot.
- Do not add comments in the code such as "<!-- Add other navigation links as needed -->" and "<!-- ... other news items ... -->" in place of writing the full code. WRITE THE FULL CODE.
- Repeat elements as needed to match the screenshot. For example, if there are 15 items, the code should have 15 items. DO NOT LEAVE comments like "<!-- Repeat for each news item -->" or bad things will happen.
- For images, use placeholder images from https://placehold.co and include a detailed description of the image in the alt text so that an image generation AI can generate the image later.
- Use Vue using the global build like so:
<div id="app">{{ message }}</div>
<script>
const { createApp, ref } = Vue
createApp({
setup() {
const message = ref('Hello vue!')
return {
message
}
}
}).mount('#app')
</script>
In terms of libraries,
- Use these script to include Vue so that it can run on a standalone page:
<script src="https://registry.npmmirror.com/vue/3.3.11/files/dist/vue.global.js"></script>
- Use this script to include Tailwind: <script src="https://cdn.tailwindcss.com"></script>
- You can use Google Fonts
- Font Awesome for icons: <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.15.3/css/all.min.css"></link>
Return only the full code in <html></html> tags.
Do not include markdown "```" or "```html" at the start or end.
The return result must only include the code.
"""
def assemble_prompt(
image_data_url, generated_code_config: str, result_image_data_url=None
):
# Set the system prompt based on the output settings
system_content = TAILWIND_SYSTEM_PROMPT
if generated_code_config == "html_tailwind":
system_content = TAILWIND_SYSTEM_PROMPT
elif generated_code_config == "react_tailwind":
system_content = REACT_TAILWIND_SYSTEM_PROMPT
elif generated_code_config == "bootstrap":
system_content = BOOTSTRAP_SYSTEM_PROMPT
elif generated_code_config == "ionic_tailwind":
system_content = IONIC_TAILWIND_SYSTEM_PROMPT
else:
raise Exception("Code config is not one of available options")
SVG_SYSTEM_PROMPT = """
You are an expert at building SVGs.
You take screenshots of a reference web page from the user, and then build a SVG that looks exactly like the screenshot.
user_content = [
{
"type": "image_url",
"image_url": {"url": image_data_url, "detail": "high"},
},
{
"type": "text",
"text": USER_PROMPT,
},
]
- Make sure the SVG looks exactly like the screenshot.
- Pay close attention to background color, text color, font size, font family,
padding, margin, border, etc. Match the colors and sizes exactly.
- Use the exact text from the screenshot.
- Do not add comments in the code such as "<!-- Add other navigation links as needed -->" and "<!-- ... other news items ... -->" in place of writing the full code. WRITE THE FULL CODE.
- Repeat elements as needed to match the screenshot. For example, if there are 15 items, the code should have 15 items. DO NOT LEAVE comments like "<!-- Repeat for each news item -->" or bad things will happen.
- For images, use placeholder images from https://placehold.co and include a detailed description of the image in the alt text so that an image generation AI can generate the image later.
- You can use Google Fonts
# Include the result image if it exists
if result_image_data_url:
user_content.insert(
1,
{
"type": "image_url",
"image_url": {"url": result_image_data_url, "detail": "high"},
},
)
return [
{
"role": "system",
"content": system_content,
},
{
"role": "user",
"content": user_content,
},
]
Return only the full code in <svg></svg> tags.
Do not include markdown "```" or "```svg" at the start or end.
"""
SYSTEM_PROMPTS = SystemPrompts(
html_tailwind=HTML_TAILWIND_SYSTEM_PROMPT,
react_tailwind=REACT_TAILWIND_SYSTEM_PROMPT,
bootstrap=BOOTSTRAP_SYSTEM_PROMPT,
ionic_tailwind=IONIC_TAILWIND_SYSTEM_PROMPT,
vue_tailwind=VUE_TAILWIND_SYSTEM_PROMPT,
svg=SVG_SYSTEM_PROMPT,
)

397
backend/prompts/test_prompts.py Normal file
View File

@ -0,0 +1,397 @@
from prompts import assemble_imported_code_prompt, assemble_prompt
TAILWIND_SYSTEM_PROMPT = """
You are an expert Tailwind developer
You take screenshots of a reference web page from the user, and then build single page apps
using Tailwind, HTML and JS.
You might also be given a screenshot(The second image) of a web page that you have already built, and asked to
update it to look more like the reference image(The first image).
- Make sure the app looks exactly like the screenshot.
- Pay close attention to background color, text color, font size, font family,
padding, margin, border, etc. Match the colors and sizes exactly.
- Use the exact text from the screenshot.
- Do not add comments in the code such as "<!-- Add other navigation links as needed -->" and "<!-- ... other news items ... -->" in place of writing the full code. WRITE THE FULL CODE.
- Repeat elements as needed to match the screenshot. For example, if there are 15 items, the code should have 15 items. DO NOT LEAVE comments like "<!-- Repeat for each news item -->" or bad things will happen.
- For images, use placeholder images from https://placehold.co and include a detailed description of the image in the alt text so that an image generation AI can generate the image later.
In terms of libraries,
- Use this script to include Tailwind: <script src="https://cdn.tailwindcss.com"></script>
- You can use Google Fonts
- Font Awesome for icons: <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.15.3/css/all.min.css"></link>
Return only the full code in <html></html> tags.
Do not include markdown "```" or "```html" at the start or end.
"""
BOOTSTRAP_SYSTEM_PROMPT = """
You are an expert Bootstrap developer
You take screenshots of a reference web page from the user, and then build single page apps
using Bootstrap, HTML and JS.
You might also be given a screenshot(The second image) of a web page that you have already built, and asked to
update it to look more like the reference image(The first image).
- Make sure the app looks exactly like the screenshot.
- Pay close attention to background color, text color, font size, font family,
padding, margin, border, etc. Match the colors and sizes exactly.
- Use the exact text from the screenshot.
- Do not add comments in the code such as "<!-- Add other navigation links as needed -->" and "<!-- ... other news items ... -->" in place of writing the full code. WRITE THE FULL CODE.
- Repeat elements as needed to match the screenshot. For example, if there are 15 items, the code should have 15 items. DO NOT LEAVE comments like "<!-- Repeat for each news item -->" or bad things will happen.
- For images, use placeholder images from https://placehold.co and include a detailed description of the image in the alt text so that an image generation AI can generate the image later.
In terms of libraries,
- Use this script to include Bootstrap: <link href="https://cdn.jsdelivr.net/npm/bootstrap@5.3.2/dist/css/bootstrap.min.css" rel="stylesheet" integrity="sha384-T3c6CoIi6uLrA9TneNEoa7RxnatzjcDSCmG1MXxSR1GAsXEV/Dwwykc2MPK8M2HN" crossorigin="anonymous">
- You can use Google Fonts
- Font Awesome for icons: <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.15.3/css/all.min.css"></link>
Return only the full code in <html></html> tags.
Do not include markdown "```" or "```html" at the start or end.
"""
REACT_TAILWIND_SYSTEM_PROMPT = """
You are an expert React/Tailwind developer
You take screenshots of a reference web page from the user, and then build single page apps
using React and Tailwind CSS.
You might also be given a screenshot(The second image) of a web page that you have already built, and asked to
update it to look more like the reference image(The first image).
- Make sure the app looks exactly like the screenshot.
- Pay close attention to background color, text color, font size, font family,
padding, margin, border, etc. Match the colors and sizes exactly.
- Use the exact text from the screenshot.
- Do not add comments in the code such as "<!-- Add other navigation links as needed -->" and "<!-- ... other news items ... -->" in place of writing the full code. WRITE THE FULL CODE.
- Repeat elements as needed to match the screenshot. For example, if there are 15 items, the code should have 15 items. DO NOT LEAVE comments like "<!-- Repeat for each news item -->" or bad things will happen.
- For images, use placeholder images from https://placehold.co and include a detailed description of the image in the alt text so that an image generation AI can generate the image later.
In terms of libraries,
- Use these script to include React so that it can run on a standalone page:
<script src="https://unpkg.com/react/umd/react.development.js"></script>
<script src="https://unpkg.com/react-dom/umd/react-dom.development.js"></script>
<script src="https://unpkg.com/@babel/standalone/babel.js"></script>
- Use this script to include Tailwind: <script src="https://cdn.tailwindcss.com"></script>
- You can use Google Fonts
- Font Awesome for icons: <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.15.3/css/all.min.css"></link>
Return only the full code in <html></html> tags.
Do not include markdown "```" or "```html" at the start or end.
"""
IONIC_TAILWIND_SYSTEM_PROMPT = """
You are an expert Ionic/Tailwind developer
You take screenshots of a reference web page from the user, and then build single page apps
using Ionic and Tailwind CSS.
You might also be given a screenshot(The second image) of a web page that you have already built, and asked to
update it to look more like the reference image(The first image).
- Make sure the app looks exactly like the screenshot.
- Pay close attention to background color, text color, font size, font family,
padding, margin, border, etc. Match the colors and sizes exactly.
- Use the exact text from the screenshot.
- Do not add comments in the code such as "<!-- Add other navigation links as needed -->" and "<!-- ... other news items ... -->" in place of writing the full code. WRITE THE FULL CODE.
- Repeat elements as needed to match the screenshot. For example, if there are 15 items, the code should have 15 items. DO NOT LEAVE comments like "<!-- Repeat for each news item -->" or bad things will happen.
- For images, use placeholder images from https://placehold.co and include a detailed description of the image in the alt text so that an image generation AI can generate the image later.
In terms of libraries,
- Use these script to include Ionic so that it can run on a standalone page:
<script type="module" src="https://cdn.jsdelivr.net/npm/@ionic/core/dist/ionic/ionic.esm.js"></script>
<script nomodule src="https://cdn.jsdelivr.net/npm/@ionic/core/dist/ionic/ionic.js"></script>
<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/@ionic/core/css/ionic.bundle.css" />
- Use this script to include Tailwind: <script src="https://cdn.tailwindcss.com"></script>
- You can use Google Fonts
- ionicons for icons, add the following <script > tags near the end of the page, right before the closing </body> tag:
<script type="module">
import ionicons from 'https://cdn.jsdelivr.net/npm/ionicons/+esm'
</script>
<script nomodule src="https://cdn.jsdelivr.net/npm/ionicons/dist/esm/ionicons.min.js"></script>
<link href="https://cdn.jsdelivr.net/npm/ionicons/dist/collection/components/icon/icon.min.css" rel="stylesheet">
Return only the full code in <html></html> tags.
Do not include markdown "```" or "```html" at the start or end.
"""
VUE_TAILWIND_SYSTEM_PROMPT = """
You are an expert Vue/Tailwind developer
You take screenshots of a reference web page from the user, and then build single page apps
using Vue and Tailwind CSS.
You might also be given a screenshot(The second image) of a web page that you have already built, and asked to
update it to look more like the reference image(The first image).
- Make sure the app looks exactly like the screenshot.
- Pay close attention to background color, text color, font size, font family,
padding, margin, border, etc. Match the colors and sizes exactly.
- Use the exact text from the screenshot.
- Do not add comments in the code such as "<!-- Add other navigation links as needed -->" and "<!-- ... other news items ... -->" in place of writing the full code. WRITE THE FULL CODE.
- Repeat elements as needed to match the screenshot. For example, if there are 15 items, the code should have 15 items. DO NOT LEAVE comments like "<!-- Repeat for each news item -->" or bad things will happen.
- For images, use placeholder images from https://placehold.co and include a detailed description of the image in the alt text so that an image generation AI can generate the image later.
- Use Vue using the global build like so:
<div id="app">{{ message }}</div>
<script>
const { createApp, ref } = Vue
createApp({
setup() {
const message = ref('Hello vue!')
return {
message
}
}
}).mount('#app')
</script>
In terms of libraries,
- Use these script to include Vue so that it can run on a standalone page:
<script src="https://registry.npmmirror.com/vue/3.3.11/files/dist/vue.global.js"></script>
- Use this script to include Tailwind: <script src="https://cdn.tailwindcss.com"></script>
- You can use Google Fonts
- Font Awesome for icons: <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.15.3/css/all.min.css"></link>
Return only the full code in <html></html> tags.
Do not include markdown "```" or "```html" at the start or end.
The return result must only include the code.
"""
SVG_SYSTEM_PROMPT = """
You are an expert at building SVGs.
You take screenshots of a reference web page from the user, and then build a SVG that looks exactly like the screenshot.
- Make sure the SVG looks exactly like the screenshot.
- Pay close attention to background color, text color, font size, font family,
padding, margin, border, etc. Match the colors and sizes exactly.
- Use the exact text from the screenshot.
- Do not add comments in the code such as "<!-- Add other navigation links as needed -->" and "<!-- ... other news items ... -->" in place of writing the full code. WRITE THE FULL CODE.
- Repeat elements as needed to match the screenshot. For example, if there are 15 items, the code should have 15 items. DO NOT LEAVE comments like "<!-- Repeat for each news item -->" or bad things will happen.
- For images, use placeholder images from https://placehold.co and include a detailed description of the image in the alt text so that an image generation AI can generate the image later.
- You can use Google Fonts
Return only the full code in <svg></svg> tags.
Do not include markdown "```" or "```svg" at the start or end.
"""
IMPORTED_CODE_TAILWIND_SYSTEM_PROMPT = """
You are an expert Tailwind developer.
- Do not add comments in the code such as "<!-- Add other navigation links as needed -->" and "<!-- ... other news items ... -->" in place of writing the full code. WRITE THE FULL CODE.
- Repeat elements as needed. For example, if there are 15 items, the code should have 15 items. DO NOT LEAVE comments like "<!-- Repeat for each news item -->" or bad things will happen.
- For images, use placeholder images from https://placehold.co and include a detailed description of the image in the alt text so that an image generation AI can generate the image later.
In terms of libraries,
- Use this script to include Tailwind: <script src="https://cdn.tailwindcss.com"></script>
- You can use Google Fonts
- Font Awesome for icons: <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.15.3/css/all.min.css"></link>
Return only the full code in <html></html> tags.
Do not include markdown "```" or "```html" at the start or end.
"""
IMPORTED_CODE_REACT_TAILWIND_SYSTEM_PROMPT = """
You are an expert React/Tailwind developer
- Do not add comments in the code such as "<!-- Add other navigation links as needed -->" and "<!-- ... other news items ... -->" in place of writing the full code. WRITE THE FULL CODE.
- Repeat elements as needed. For example, if there are 15 items, the code should have 15 items. DO NOT LEAVE comments like "<!-- Repeat for each news item -->" or bad things will happen.
- For images, use placeholder images from https://placehold.co and include a detailed description of the image in the alt text so that an image generation AI can generate the image later.
In terms of libraries,
- Use these script to include React so that it can run on a standalone page:
<script src="https://unpkg.com/react/umd/react.development.js"></script>
<script src="https://unpkg.com/react-dom/umd/react-dom.development.js"></script>
<script src="https://unpkg.com/@babel/standalone/babel.js"></script>
- Use this script to include Tailwind: <script src="https://cdn.tailwindcss.com"></script>
- You can use Google Fonts
- Font Awesome for icons: <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.15.3/css/all.min.css"></link>
Return only the full code in <html></html> tags.
Do not include markdown "```" or "```html" at the start or end.
"""
IMPORTED_CODE_BOOTSTRAP_SYSTEM_PROMPT = """
You are an expert Bootstrap developer.
- Do not add comments in the code such as "<!-- Add other navigation links as needed -->" and "<!-- ... other news items ... -->" in place of writing the full code. WRITE THE FULL CODE.
- Repeat elements as needed. For example, if there are 15 items, the code should have 15 items. DO NOT LEAVE comments like "<!-- Repeat for each news item -->" or bad things will happen.
- For images, use placeholder images from https://placehold.co and include a detailed description of the image in the alt text so that an image generation AI can generate the image later.
In terms of libraries,
- Use this script to include Bootstrap: <link href="https://cdn.jsdelivr.net/npm/bootstrap@5.3.2/dist/css/bootstrap.min.css" rel="stylesheet" integrity="sha384-T3c6CoIi6uLrA9TneNEoa7RxnatzjcDSCmG1MXxSR1GAsXEV/Dwwykc2MPK8M2HN" crossorigin="anonymous">
- You can use Google Fonts
- Font Awesome for icons: <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.15.3/css/all.min.css"></link>
Return only the full code in <html></html> tags.
Do not include markdown "```" or "```html" at the start or end.
"""
IMPORTED_CODE_IONIC_TAILWIND_SYSTEM_PROMPT = """
You are an expert Ionic/Tailwind developer.
- Do not add comments in the code such as "<!-- Add other navigation links as needed -->" and "<!-- ... other news items ... -->" in place of writing the full code. WRITE THE FULL CODE.
- Repeat elements as needed. For example, if there are 15 items, the code should have 15 items. DO NOT LEAVE comments like "<!-- Repeat for each news item -->" or bad things will happen.
- For images, use placeholder images from https://placehold.co and include a detailed description of the image in the alt text so that an image generation AI can generate the image later.
In terms of libraries,
- Use these script to include Ionic so that it can run on a standalone page:
<script type="module" src="https://cdn.jsdelivr.net/npm/@ionic/core/dist/ionic/ionic.esm.js"></script>
<script nomodule src="https://cdn.jsdelivr.net/npm/@ionic/core/dist/ionic/ionic.js"></script>
<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/@ionic/core/css/ionic.bundle.css" />
- Use this script to include Tailwind: <script src="https://cdn.tailwindcss.com"></script>
- You can use Google Fonts
- ionicons for icons, add the following <script > tags near the end of the page, right before the closing </body> tag:
<script type="module">
import ionicons from 'https://cdn.jsdelivr.net/npm/ionicons/+esm'
</script>
<script nomodule src="https://cdn.jsdelivr.net/npm/ionicons/dist/esm/ionicons.min.js"></script>
<link href="https://cdn.jsdelivr.net/npm/ionicons/dist/collection/components/icon/icon.min.css" rel="stylesheet">
Return only the full code in <html></html> tags.
Do not include markdown "```" or "```html" at the start or end.
"""
IMPORTED_CODE_VUE_TAILWIND_PROMPT = """
You are an expert Vue/Tailwind developer.
- Do not add comments in the code such as "<!-- Add other navigation links as needed -->" and "<!-- ... other news items ... -->" in place of writing the full code. WRITE THE FULL CODE.
- Repeat elements as needed. For example, if there are 15 items, the code should have 15 items. DO NOT LEAVE comments like "<!-- Repeat for each news item -->" or bad things will happen.
- For images, use placeholder images from https://placehold.co and include a detailed description of the image in the alt text so that an image generation AI can generate the image later.
In terms of libraries,
- Use these script to include Vue so that it can run on a standalone page:
<script src="https://registry.npmmirror.com/vue/3.3.11/files/dist/vue.global.js"></script>
- Use Vue using the global build like so:
<div id="app">{{ message }}</div>
<script>
const { createApp, ref } = Vue
createApp({
setup() {
const message = ref('Hello vue!')
return {
message
}
}
}).mount('#app')
</script>
- Use this script to include Tailwind: <script src="https://cdn.tailwindcss.com"></script>
- You can use Google Fonts
- Font Awesome for icons: <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.15.3/css/all.min.css"></link>
Return only the full code in <html></html> tags.
Do not include markdown "```" or "```html" at the start or end.
The return result must only include the code."""
IMPORTED_CODE_SVG_SYSTEM_PROMPT = """
You are an expert at building SVGs.
- Do not add comments in the code such as "<!-- Add other navigation links as needed -->" and "<!-- ... other news items ... -->" in place of writing the full code. WRITE THE FULL CODE.
- Repeat elements as needed to match the screenshot. For example, if there are 15 items, the code should have 15 items. DO NOT LEAVE comments like "<!-- Repeat for each news item -->" or bad things will happen.
- For images, use placeholder images from https://placehold.co and include a detailed description of the image in the alt text so that an image generation AI can generate the image later.
- You can use Google Fonts
Return only the full code in <svg></svg> tags.
Do not include markdown "```" or "```svg" at the start or end.
"""
USER_PROMPT = """
Generate code for a web page that looks exactly like this.
"""
SVG_USER_PROMPT = """
Generate code for a SVG that looks exactly like this.
"""
def test_prompts():
tailwind_prompt = assemble_prompt(
"image_data_url", "html_tailwind", "result_image_data_url"
)
assert tailwind_prompt[0]["content"] == TAILWIND_SYSTEM_PROMPT
assert tailwind_prompt[1]["content"][2]["text"] == USER_PROMPT # type: ignore
react_tailwind_prompt = assemble_prompt(
"image_data_url", "react_tailwind", "result_image_data_url"
)
assert react_tailwind_prompt[0]["content"] == REACT_TAILWIND_SYSTEM_PROMPT
assert react_tailwind_prompt[1]["content"][2]["text"] == USER_PROMPT # type: ignore
bootstrap_prompt = assemble_prompt(
"image_data_url", "bootstrap", "result_image_data_url"
)
assert bootstrap_prompt[0]["content"] == BOOTSTRAP_SYSTEM_PROMPT
assert bootstrap_prompt[1]["content"][2]["text"] == USER_PROMPT # type: ignore
ionic_tailwind = assemble_prompt(
"image_data_url", "ionic_tailwind", "result_image_data_url"
)
assert ionic_tailwind[0]["content"] == IONIC_TAILWIND_SYSTEM_PROMPT
assert ionic_tailwind[1]["content"][2]["text"] == USER_PROMPT # type: ignore
vue_tailwind = assemble_prompt(
"image_data_url", "vue_tailwind", "result_image_data_url"
)
assert vue_tailwind[0]["content"] == VUE_TAILWIND_SYSTEM_PROMPT
assert vue_tailwind[1]["content"][2]["text"] == USER_PROMPT # type: ignore
svg_prompt = assemble_prompt("image_data_url", "svg", "result_image_data_url")
assert svg_prompt[0]["content"] == SVG_SYSTEM_PROMPT
assert svg_prompt[1]["content"][2]["text"] == SVG_USER_PROMPT # type: ignore
def test_imported_code_prompts():
tailwind_prompt = assemble_imported_code_prompt(
"code", "html_tailwind", "result_image_data_url"
)
expected_tailwind_prompt = [
{"role": "system", "content": IMPORTED_CODE_TAILWIND_SYSTEM_PROMPT},
{"role": "user", "content": "Here is the code of the app: code"},
]
assert tailwind_prompt == expected_tailwind_prompt
react_tailwind_prompt = assemble_imported_code_prompt(
"code", "react_tailwind", "result_image_data_url"
)
expected_react_tailwind_prompt = [
{"role": "system", "content": IMPORTED_CODE_REACT_TAILWIND_SYSTEM_PROMPT},
{"role": "user", "content": "Here is the code of the app: code"},
]
assert react_tailwind_prompt == expected_react_tailwind_prompt
bootstrap_prompt = assemble_imported_code_prompt(
"code", "bootstrap", "result_image_data_url"
)
expected_bootstrap_prompt = [
{"role": "system", "content": IMPORTED_CODE_BOOTSTRAP_SYSTEM_PROMPT},
{"role": "user", "content": "Here is the code of the app: code"},
]
assert bootstrap_prompt == expected_bootstrap_prompt
ionic_tailwind = assemble_imported_code_prompt(
"code", "ionic_tailwind", "result_image_data_url"
)
expected_ionic_tailwind = [
{"role": "system", "content": IMPORTED_CODE_IONIC_TAILWIND_SYSTEM_PROMPT},
{"role": "user", "content": "Here is the code of the app: code"},
]
assert ionic_tailwind == expected_ionic_tailwind
vue_tailwind = assemble_imported_code_prompt(
"code", "vue_tailwind", "result_image_data_url"
)
expected_vue_tailwind = [
{"role": "system", "content": IMPORTED_CODE_VUE_TAILWIND_PROMPT},
{"role": "user", "content": "Here is the code of the app: code"},
]
assert vue_tailwind == expected_vue_tailwind
svg = assemble_imported_code_prompt("code", "svg", "result_image_data_url")
expected_svg = [
{"role": "system", "content": IMPORTED_CODE_SVG_SYSTEM_PROMPT},
{"role": "user", "content": "Here is the code of the SVG: code"},
]
assert svg == expected_svg

20
backend/prompts/types.py Normal file
View File

@ -0,0 +1,20 @@
from typing import Literal, TypedDict
class SystemPrompts(TypedDict):
html_tailwind: str
react_tailwind: str
bootstrap: str
ionic_tailwind: str
vue_tailwind: str
svg: str
Stack = Literal[
"html_tailwind",
"react_tailwind",
"bootstrap",
"ionic_tailwind",
"vue_tailwind",
"svg",
]

4
backend/pyproject.toml
View File

@ -8,7 +8,7 @@ license = "MIT"
[tool.poetry.dependencies]
python = "^3.10, <3.13"
fastapi = "^0.95.0"
uvicorn = "^0.24.0.post1"
uvicorn = "^0.25.0"
websockets = "^12.0"
openai = "^1.2.4"
python-dotenv = "^1.0.0"
@ -16,6 +16,8 @@ beautifulsoup4 = "^4.12.2"
httpx = "^0.25.1"
[tool.poetry.group.dev.dependencies]
pytest = "^7.4.3"
pyright = "^1.1.345"
pyinstaller = "^6.2.0"
[build-system]

3
backend/pyrightconfig.json Normal file
View File

@ -0,0 +1,3 @@
{
"exclude": ["image_generation.py"]
}

46
backend/routes/evals.py Normal file
View File

@ -0,0 +1,46 @@
import os
from fastapi import APIRouter
from pydantic import BaseModel
from evals.utils import image_to_data_url
from evals.config import EVALS_DIR
router = APIRouter()
class Eval(BaseModel):
input: str
output: str
@router.get("/evals")
async def get_evals():
# Get all evals from EVALS_DIR
input_dir = EVALS_DIR + "/inputs"
output_dir = EVALS_DIR + "/outputs"
evals: list[Eval] = []
for file in os.listdir(input_dir):
if file.endswith(".png"):
input_file_path = os.path.join(input_dir, file)
input_file = await image_to_data_url(input_file_path)
# Construct the corresponding output file name
output_file_name = file.replace(".png", ".html")
output_file_path = os.path.join(output_dir, output_file_name)
# Check if the output file exists
if os.path.exists(output_file_path):
with open(output_file_path, "r") as f:
output_file_data = f.read()
else:
output_file_data = "Output file not found."
evals.append(
Eval(
input=input_file,
output=output_file_data,
)
)
return evals

270
backend/routes/generate_code.py Normal file
View File

@ -0,0 +1,270 @@
import os
import traceback
from fastapi import APIRouter, WebSocket
import openai
from config import IS_PROD, SHOULD_MOCK_AI_RESPONSE
from llm import stream_openai_response
from openai.types.chat import ChatCompletionMessageParam
from mock_llm import mock_completion
from typing import Dict, List, cast, get_args
from image_generation import create_alt_url_mapping, generate_images
from prompts import assemble_imported_code_prompt, assemble_prompt
from access_token import validate_access_token
from datetime import datetime
import json
from prompts.types import Stack
from utils import pprint_prompt # type: ignore
router = APIRouter()
def write_logs(prompt_messages: List[ChatCompletionMessageParam], completion: str):
# Get the logs path from environment, default to the current working directory
logs_path = os.environ.get("LOGS_PATH", os.getcwd())
# Create run_logs directory if it doesn't exist within the specified logs path
logs_directory = os.path.join(logs_path, "run_logs")
if not os.path.exists(logs_directory):
os.makedirs(logs_directory)
print("Writing to logs directory:", logs_directory)
# Generate a unique filename using the current timestamp within the logs directory
filename = datetime.now().strftime(f"{logs_directory}/messages_%Y%m%d_%H%M%S.json")
# Write the messages dict into a new file for each run
with open(filename, "w") as f:
f.write(json.dumps({"prompt": prompt_messages, "completion": completion}))
@router.websocket("/generate-code")
async def stream_code(websocket: WebSocket):
await websocket.accept()
print("Incoming websocket connection...")
async def throw_error(
message: str,
):
await websocket.send_json({"type": "error", "value": message})
await websocket.close()
# TODO: Are the values always strings?
params: Dict[str, str] = await websocket.receive_json()
print("Received params")
# Read the code config settings from the request. Fall back to default if not provided.
generated_code_config = ""
if "generatedCodeConfig" in params and params["generatedCodeConfig"]:
generated_code_config = params["generatedCodeConfig"]
print(f"Generating {generated_code_config} code")
# Get the OpenAI API key from the request. Fall back to environment variable if not provided.
# If neither is provided, we throw an error.
openai_api_key = None
if "accessCode" in params and params["accessCode"]:
print("Access code - using platform API key")
res = await validate_access_token(params["accessCode"])
if res["success"]:
openai_api_key = os.environ.get("PLATFORM_OPENAI_API_KEY")
else:
await websocket.send_json(
{
"type": "error",
"value": res["failure_reason"],
}
)
return
else:
if params["openAiApiKey"]:
openai_api_key = params["openAiApiKey"]
print("Using OpenAI API key from client-side settings dialog")
else:
openai_api_key = os.environ.get("OPENAI_API_KEY")
if openai_api_key:
print("Using OpenAI API key from environment variable")
if not openai_api_key:
print("OpenAI API key not found")
await websocket.send_json(
{
"type": "error",
"value": "No OpenAI API key found. Please add your API key in the settings dialog or add it to backend/.env file. If you add it to .env, make sure to restart the backend server.",
}
)
return
# Validate the generated code config
if not generated_code_config in get_args(Stack):
await throw_error(f"Invalid generated code config: {generated_code_config}")
return
# Cast the variable to the Stack type
valid_stack = cast(Stack, generated_code_config)
# Get the OpenAI Base URL from the request. Fall back to environment variable if not provided.
openai_base_url = None
# Disable user-specified OpenAI Base URL in prod
if not os.environ.get("IS_PROD"):
if "openAiBaseURL" in params and params["openAiBaseURL"]:
openai_base_url = params["openAiBaseURL"]
print("Using OpenAI Base URL from client-side settings dialog")
else:
openai_base_url = os.environ.get("OPENAI_BASE_URL")
if openai_base_url:
print("Using OpenAI Base URL from environment variable")
if not openai_base_url:
print("Using official OpenAI URL")
# Get the image generation flag from the request. Fall back to True if not provided.
should_generate_images = (
params["isImageGenerationEnabled"]
if "isImageGenerationEnabled" in params
else True
)
print("generating code...")
await websocket.send_json({"type": "status", "value": "Generating code..."})
async def process_chunk(content: str):
await websocket.send_json({"type": "chunk", "value": content})
# Image cache for updates so that we don't have to regenerate images
image_cache: Dict[str, str] = {}
# If this generation started off with imported code, we need to assemble the prompt differently
if params.get("isImportedFromCode") and params["isImportedFromCode"]:
original_imported_code = params["history"][0]
prompt_messages = assemble_imported_code_prompt(
original_imported_code, valid_stack
)
for index, text in enumerate(params["history"][1:]):
if index % 2 == 0:
message: ChatCompletionMessageParam = {
"role": "user",
"content": text,
}
else:
message: ChatCompletionMessageParam = {
"role": "assistant",
"content": text,
}
prompt_messages.append(message)
else:
# Assemble the prompt
try:
if params.get("resultImage") and params["resultImage"]:
prompt_messages = assemble_prompt(
params["image"], valid_stack, params["resultImage"]
)
else:
prompt_messages = assemble_prompt(params["image"], valid_stack)
except:
await websocket.send_json(
{
"type": "error",
"value": "Error assembling prompt. Contact support at support@picoapps.xyz",
}
)
await websocket.close()
return
if params["generationType"] == "update":
# Transform the history tree into message format
# TODO: Move this to frontend
for index, text in enumerate(params["history"]):
if index % 2 == 0:
message: ChatCompletionMessageParam = {
"role": "assistant",
"content": text,
}
else:
message: ChatCompletionMessageParam = {
"role": "user",
"content": text,
}
prompt_messages.append(message)
image_cache = create_alt_url_mapping(params["history"][-2])
pprint_prompt(prompt_messages)
if SHOULD_MOCK_AI_RESPONSE:
completion = await mock_completion(process_chunk)
else:
try:
completion = await stream_openai_response(
prompt_messages,
api_key=openai_api_key,
base_url=openai_base_url,
callback=lambda x: process_chunk(x),
)
except openai.AuthenticationError as e:
print("[GENERATE_CODE] Authentication failed", e)
error_message = (
"Incorrect OpenAI key. Please make sure your OpenAI API key is correct, or create a new OpenAI API key on your OpenAI dashboard."
+ (
" Alternatively, you can purchase code generation credits directly on this website."
if IS_PROD
else ""
)
)
return await throw_error(error_message)
except openai.NotFoundError as e:
print("[GENERATE_CODE] Model not found", e)
error_message = (
e.message
+ ". Please make sure you have followed the instructions correctly to obtain an OpenAI key with GPT vision access: https://github.com/abi/screenshot-to-code/blob/main/Troubleshooting.md"
+ (
" Alternatively, you can purchase code generation credits directly on this website."
if IS_PROD
else ""
)
)
return await throw_error(error_message)
except openai.RateLimitError as e:
print("[GENERATE_CODE] Rate limit exceeded", e)
error_message = (
"OpenAI error - 'You exceeded your current quota, please check your plan and billing details.'"
+ (
" Alternatively, you can purchase code generation credits directly on this website."
if IS_PROD
else ""
)
)
return await throw_error(error_message)
# Write the messages dict into a log so that we can debug later
write_logs(prompt_messages, completion)
try:
if should_generate_images:
await websocket.send_json(
{"type": "status", "value": "Generating images..."}
)
updated_html = await generate_images(
completion,
api_key=openai_api_key,
base_url=openai_base_url,
image_cache=image_cache,
)
else:
updated_html = completion
await websocket.send_json({"type": "setCode", "value": updated_html})
await websocket.send_json(
{"type": "status", "value": "Code generation complete."}
)
except Exception as e:
traceback.print_exc()
print("Image generation failed", e)
# Send set code even if image generation fails since that triggers
# the frontend to update history
await websocket.send_json({"type": "setCode", "value": completion})
await websocket.send_json(
{"type": "status", "value": "Image generation failed but code is complete."}
)
await websocket.close()

12
backend/routes/home.py Normal file
View File

@ -0,0 +1,12 @@
from fastapi import APIRouter
from fastapi.responses import HTMLResponse
router = APIRouter()
@router.get("/")
async def get_status():
return HTMLResponse(
content="<h3>Your backend is running correctly. Please open the front-end URL (default is http://localhost:5173) to use screenshot-to-code.</h3>"
)

4
backend/routes/screenshot.py
View File

@ -11,7 +11,9 @@ def bytes_to_data_url(image_bytes: bytes, mime_type: str) -> str:
return f"data:{mime_type};base64,{base64_image}"
async def capture_screenshot(target_url, api_key, device="desktop") -> bytes:
async def capture_screenshot(
target_url: str, api_key: str, device: str = "desktop"
) -> bytes:
api_base_url = "https://api.screenshotone.com/take"
params = {

41
backend/run_evals.py Normal file
View File

@ -0,0 +1,41 @@
# Load environment variables first
from dotenv import load_dotenv
load_dotenv()
import os
from typing import Any, Coroutine
import asyncio
from evals.config import EVALS_DIR
from evals.core import generate_code_core
from evals.utils import image_to_data_url
async def main():
INPUT_DIR = EVALS_DIR + "/inputs"
OUTPUT_DIR = EVALS_DIR + "/outputs"
# Get all the files in the directory (only grab pngs)
evals = [f for f in os.listdir(INPUT_DIR) if f.endswith(".png")]
tasks: list[Coroutine[Any, Any, str]] = []
for filename in evals:
filepath = os.path.join(INPUT_DIR, filename)
data_url = await image_to_data_url(filepath)
task = generate_code_core(data_url, "vue_tailwind")
tasks.append(task)
results = await asyncio.gather(*tasks)
os.makedirs(OUTPUT_DIR, exist_ok=True)
for filename, content in zip(evals, results):
# File name is derived from the original filename in evals
output_filename = f"{os.path.splitext(filename)[0]}.html"
output_filepath = os.path.join(OUTPUT_DIR, output_filename)
with open(output_filepath, "w") as file:
file.write(content)
asyncio.run(main())

4
backend/start.py Normal file
View File

@ -0,0 +1,4 @@
import uvicorn
if __name__ == "__main__":
uvicorn.run("main:app", port=7001, reload=True)

20
backend/utils.py
View File

@ -1,28 +1,30 @@
import copy
import json
from typing import List
from openai.types.chat import ChatCompletionMessageParam
def pprint_prompt(prompt_messages):
def pprint_prompt(prompt_messages: List[ChatCompletionMessageParam]):
print(json.dumps(truncate_data_strings(prompt_messages), indent=4))
def truncate_data_strings(data):
def truncate_data_strings(data: List[ChatCompletionMessageParam]): # type: ignore
# Deep clone the data to avoid modifying the original object
cloned_data = copy.deepcopy(data)
if isinstance(cloned_data, dict):
for key, value in cloned_data.items():
for key, value in cloned_data.items(): # type: ignore
# Recursively call the function if the value is a dictionary or a list
if isinstance(value, (dict, list)):
cloned_data[key] = truncate_data_strings(value)
cloned_data[key] = truncate_data_strings(value) # type: ignore
# Truncate the string if it it's long and add ellipsis and length
elif isinstance(value, str):
cloned_data[key] = value[:40]
cloned_data[key] = value[:40] # type: ignore
if len(value) > 40:
cloned_data[key] += "..." + f" ({len(value)} chars)"
cloned_data[key] += "..." + f" ({len(value)} chars)" # type: ignore
elif isinstance(cloned_data, list):
elif isinstance(cloned_data, list): # type: ignore
# Process each item in the list
cloned_data = [truncate_data_strings(item) for item in cloned_data]
cloned_data = [truncate_data_strings(item) for item in cloned_data] # type: ignore
return cloned_data
return cloned_data # type: ignore

5
design-docs.md Normal file
View File

@ -0,0 +1,5 @@
## Version History
Version history is stored as a tree on the client-side.
![Screenshot to Code](https://github.com/abi/screenshot-to-code/assets/23818/e35644aa-b90a-4aa7-8027-b8732796fd7c)

2
frontend/package.json
View File

@ -17,6 +17,7 @@
"@radix-ui/react-accordion": "^1.1.2",
"@radix-ui/react-alert-dialog": "^1.0.5",
"@radix-ui/react-checkbox": "^1.0.4",
"@radix-ui/react-collapsible": "^1.0.3",
"@radix-ui/react-dialog": "^1.0.5",
"@radix-ui/react-hover-card": "^1.0.7",
"@radix-ui/react-icons": "^1.3.0",
@ -40,6 +41,7 @@
"react-dropzone": "^14.2.3",
"react-hot-toast": "^2.4.1",
"react-icons": "^4.12.0",
"react-router-dom": "^6.20.1",
"tailwind-merge": "^2.0.0",
"tailwindcss-animate": "^1.0.7",
"thememirror": "^2.0.1",

95
frontend/src/App.tsx
View File

@ -2,7 +2,7 @@ import { useEffect, useRef, useState } from "react";
import ImageUpload from "./components/ImageUpload";
import CodePreview from "./components/CodePreview";
import Preview from "./components/Preview";
import { CodeGenerationParams, generateCode } from "./generateCode";
import { generateCode } from "./generateCode";
import Spinner from "./components/Spinner";
import classNames from "classnames";
import {
@ -18,7 +18,7 @@ import { Button } from "@/components/ui/button";
import { Textarea } from "@/components/ui/textarea";
import { Tabs, TabsContent, TabsList, TabsTrigger } from "./components/ui/tabs";
import SettingsDialog from "./components/SettingsDialog";
import { Settings, EditorTheme, AppState, GeneratedCodeConfig } from "./types";
import { AppState, CodeGenerationParams, EditorTheme, Settings } from "./types";
import { IS_RUNNING_ON_CLOUD } from "./config";
import { PicoBadge } from "./components/PicoBadge";
import { OnboardingNote } from "./components/OnboardingNote";
@ -33,6 +33,8 @@ import { History } from "./components/history/history_types";
import HistoryDisplay from "./components/history/HistoryDisplay";
import { extractHistoryTree } from "./components/history/utils";
import toast from "react-hot-toast";
import ImportCodeSection from "./components/ImportCodeSection";
import { Stack } from "./lib/stacks";
const IS_OPENAI_DOWN = false;
@ -43,6 +45,7 @@ function App() {
const [referenceImages, setReferenceImages] = useState<string[]>([]);
const [executionConsole, setExecutionConsole] = useState<string[]>([]);
const [updateInstruction, setUpdateInstruction] = useState("");
const [isImportedFromCode, setIsImportedFromCode] = useState<boolean>(false);
// Settings
const [settings, setSettings] = usePersistedState<Settings>(
@ -52,7 +55,7 @@ function App() {
screenshotOneApiKey: null,
isImageGenerationEnabled: true,
editorTheme: EditorTheme.COBALT,
generatedCodeConfig: GeneratedCodeConfig.HTML_TAILWIND,
generatedCodeConfig: Stack.HTML_TAILWIND,
// Only relevant for hosted version
isTermOfServiceAccepted: false,
accessCode: null,
@ -77,7 +80,7 @@ function App() {
if (!settings.generatedCodeConfig) {
setSettings((prev) => ({
...prev,
generatedCodeConfig: GeneratedCodeConfig.HTML_TAILWIND,
generatedCodeConfig: Stack.HTML_TAILWIND,
}));
}
}, [settings.generatedCodeConfig, setSettings]);
@ -117,13 +120,28 @@ function App() {
setGeneratedCode("");
setReferenceImages([]);
setExecutionConsole([]);
setUpdateInstruction("");
setIsImportedFromCode(false);
setAppHistory([]);
setCurrentVersion(null);
setShouldIncludeResultImage(false);
};
const stop = () => {
const cancelCodeGeneration = () => {
wsRef.current?.close?.(USER_CLOSE_WEB_SOCKET_CODE);
// make sure stop can correct the state even if the websocket is already closed
cancelCodeGenerationAndReset();
};
const cancelCodeGenerationAndReset = () => {
// When this is the first version, reset the entire app state
if (currentVersion === null) {
reset();
} else {
// Otherwise, revert to the last version
setGeneratedCode(appHistory[currentVersion].code);
setAppState(AppState.CODE_READY);
}
};
function doGenerateCode(
@ -139,7 +157,9 @@ function App() {
generateCode(
wsRef,
updatedParams,
// On change
(token) => setGeneratedCode((prev) => prev + token),
// On set code
(code) => {
setGeneratedCode(code);
if (params.generationType === "create") {
@ -178,7 +198,13 @@ function App() {
});
}
},
// On status update
(line) => setExecutionConsole((prev) => [...prev, line]),
// On cancel
() => {
cancelCodeGenerationAndReset();
},
// On complete
() => {
setAppState(AppState.CODE_READY);
}
@ -211,10 +237,17 @@ function App() {
return;
}
const updatedHistory = [
...extractHistoryTree(appHistory, currentVersion),
updateInstruction,
];
let historyTree;
try {
historyTree = extractHistoryTree(appHistory, currentVersion);
} catch {
toast.error(
"Version history is invalid. This shouldn't happen. Please contact support or open a Github issue."
);
return;
}
const updatedHistory = [...historyTree, updateInstruction];
if (shouldIncludeResultImage) {
const resultImage = await takeScreenshot();
@ -224,6 +257,7 @@ function App() {
image: referenceImages[0],
resultImage: resultImage,
history: updatedHistory,
isImportedFromCode,
},
currentVersion
);
@ -233,6 +267,7 @@ function App() {
generationType: "update",
image: referenceImages[0],
history: updatedHistory,
isImportedFromCode,
},
currentVersion
);
@ -249,6 +284,32 @@ function App() {
}));
};
function setStack(stack: Stack) {
setSettings((prev) => ({
...prev,
generatedCodeConfig: stack,
}));
}
function importFromCode(code: string, stack: Stack) {
setIsImportedFromCode(true);
// Set up this project
setGeneratedCode(code);
setStack(stack);
setAppHistory([
{
type: "code_create",
parentIndex: null,
code,
inputs: { code },
},
]);
setCurrentVersion(0);
setAppState(AppState.CODE_READY);
}
return (
<div className="mt-2 dark:bg-black dark:text-white">
{IS_RUNNING_ON_CLOUD && <PicoBadge settings={settings} />}
@ -266,13 +327,8 @@ function App() {
</div>
<OutputSettingsSection
generatedCodeConfig={settings.generatedCodeConfig}
setGeneratedCodeConfig={(config: GeneratedCodeConfig) =>
setSettings((prev) => ({
...prev,
generatedCodeConfig: config,
}))
}
stack={settings.generatedCodeConfig}
setStack={(config) => setStack(config)}
shouldDisableUpdates={
appState === AppState.CODING || appState === AppState.CODE_READY
}
@ -302,10 +358,10 @@ function App() {
</div>
<div className="flex mt-4 w-full">
<Button
onClick={stop}
onClick={cancelCodeGeneration}
className="w-full dark:text-white dark:bg-gray-700"
>
Stop
Cancel
</Button>
</div>
<CodePreview code={generatedCode} />
@ -357,6 +413,7 @@ function App() {
{/* Reference image display */}
<div className="flex gap-x-2 mt-2">
{referenceImages.length > 0 && (
<div className="flex flex-col">
<div
className={classNames({
@ -373,6 +430,7 @@ function App() {
Original Screenshot
</div>
</div>
)}
<div className="bg-gray-400 px-4 py-2 rounded text-sm hidden">
<h2 className="text-lg mb-4 border-b border-gray-800">
Console
@ -417,6 +475,7 @@ function App() {
doCreate={doCreate}
screenshotOneApiKey={settings.screenshotOneApiKey}
/>
<ImportCodeSection importFromCode={importFromCode} />
</div>
)}

61
frontend/src/components/ImageUpload.tsx
View File

@ -1,4 +1,5 @@
import { useState, useEffect, useMemo, useCallback } from "react";
import { useState, useEffect, useMemo } from "react";
// useCallback
import { useDropzone } from "react-dropzone";
// import { PromptImage } from "../../../types";
import { toast } from "react-hot-toast";
@ -89,39 +90,39 @@ function ImageUpload({ setReferenceImages }: Props) {
},
});
const pasteEvent = useCallback(
(event: ClipboardEvent) => {
const clipboardData = event.clipboardData;
if (!clipboardData) return;
// const pasteEvent = useCallback(
// (event: ClipboardEvent) => {
// const clipboardData = event.clipboardData;
// if (!clipboardData) return;
const items = clipboardData.items;
const files = [];
for (let i = 0; i < items.length; i++) {
const file = items[i].getAsFile();
if (file && file.type.startsWith("image/")) {
files.push(file);
}
}
// const items = clipboardData.items;
// const files = [];
// for (let i = 0; i < items.length; i++) {
// const file = items[i].getAsFile();
// if (file && file.type.startsWith("image/")) {
// files.push(file);
// }
// }
// Convert images to data URLs and set the prompt images state
Promise.all(files.map((file) => fileToDataURL(file)))
.then((dataUrls) => {
if (dataUrls.length > 0) {
setReferenceImages(dataUrls.map((dataUrl) => dataUrl as string));
}
})
.catch((error) => {
// TODO: Display error to user
console.error("Error reading files:", error);
});
},
[setReferenceImages]
);
// // Convert images to data URLs and set the prompt images state
// Promise.all(files.map((file) => fileToDataURL(file)))
// .then((dataUrls) => {
// if (dataUrls.length > 0) {
// setReferenceImages(dataUrls.map((dataUrl) => dataUrl as string));
// }
// })
// .catch((error) => {
// // TODO: Display error to user
// console.error("Error reading files:", error);
// });
// },
// [setReferenceImages]
// );
// TODO: Make sure we don't listen to paste events in text input components
useEffect(() => {
window.addEventListener("paste", pasteEvent);
}, [pasteEvent]);
// useEffect(() => {
// window.addEventListener("paste", pasteEvent);
// }, [pasteEvent]);
useEffect(() => {
return () => files.forEach((file) => URL.revokeObjectURL(file.preview));

74
frontend/src/components/ImportCodeSection.tsx Normal file
View File

@ -0,0 +1,74 @@
import { useState } from "react";
import { Button } from "./ui/button";
import {
Dialog,
DialogContent,
DialogDescription,
DialogFooter,
DialogHeader,
DialogTitle,
DialogTrigger,
} from "./ui/dialog";
import { Textarea } from "./ui/textarea";
import OutputSettingsSection from "./OutputSettingsSection";
import toast from "react-hot-toast";
import { Stack } from "../lib/stacks";
interface Props {
importFromCode: (code: string, stack: Stack) => void;
}
function ImportCodeSection({ importFromCode }: Props) {
const [code, setCode] = useState("");
const [stack, setStack] = useState<Stack | undefined>(undefined);
const doImport = () => {
if (code === "") {
toast.error("Please paste in some code");
return;
}
if (stack === undefined) {
toast.error("Please select your stack");
return;
}
importFromCode(code, stack);
};
return (
<Dialog>
<DialogTrigger asChild>
<Button variant="secondary">Import from Code</Button>
</DialogTrigger>
<DialogContent className="sm:max-w-[425px]">
<DialogHeader>
<DialogTitle>Paste in your HTML code</DialogTitle>
<DialogDescription>
Make sure that the code you're importing is valid HTML.
</DialogDescription>
</DialogHeader>
<Textarea
value={code}
onChange={(e) => setCode(e.target.value)}
className="w-full h-64"
/>
<OutputSettingsSection
stack={stack}
setStack={(config: Stack) => setStack(config)}
label="Stack:"
shouldDisableUpdates={false}
/>
<DialogFooter>
<Button type="submit" onClick={doImport}>
Import
</Button>
</DialogFooter>
</DialogContent>
</Dialog>
);
}
export default ImportCodeSection;

84
frontend/src/components/OutputSettingsSection.tsx
View File

@ -1,3 +1,4 @@
import React from "react";
import {
Select,
SelectContent,
@ -5,82 +6,63 @@ import {
SelectItem,
SelectTrigger,
} from "./ui/select";
import { GeneratedCodeConfig } from "../types";
import { Badge } from "./ui/badge";
import { Stack, STACK_DESCRIPTIONS } from "../lib/stacks";
function generateDisplayComponent(stack: Stack) {
const stackComponents = STACK_DESCRIPTIONS[stack].components;
function generateDisplayComponent(config: GeneratedCodeConfig) {
switch (config) {
case GeneratedCodeConfig.HTML_TAILWIND:
return (
<div>
<span className="font-semibold">HTML</span> +{" "}
<span className="font-semibold">Tailwind</span>
{stackComponents.map((component, index) => (
<React.Fragment key={index}>
<span className="font-semibold">{component}</span>
{index < stackComponents.length - 1 && " + "}
</React.Fragment>
))}
</div>
);
case GeneratedCodeConfig.REACT_TAILWIND:
return (
<div>
<span className="font-semibold">React</span> +{" "}
<span className="font-semibold">Tailwind</span>
</div>
);
case GeneratedCodeConfig.BOOTSTRAP:
return (
<div>
<span className="font-semibold">Bootstrap</span>
</div>
);
case GeneratedCodeConfig.IONIC_TAILWIND:
return (
<div>
<span className="font-semibold">Ionic</span> +{" "}
<span className="font-semibold">Tailwind</span>
</div>
);
default:
// TODO: Should never reach this out. Error out
return config;
}
}
interface Props {
generatedCodeConfig: GeneratedCodeConfig;
setGeneratedCodeConfig: (config: GeneratedCodeConfig) => void;
stack: Stack | undefined;
setStack: (config: Stack) => void;
label?: string;
shouldDisableUpdates?: boolean;
}
function OutputSettingsSection({
generatedCodeConfig,
setGeneratedCodeConfig,
stack,
setStack,
label = "Generating:",
shouldDisableUpdates = false,
}: Props) {
return (
<div className="flex flex-col gap-y-2 justify-between text-sm">
<div className="grid grid-cols-3 items-center gap-4">
<span>Generating:</span>
<span>{label}</span>
<Select
value={generatedCodeConfig}
onValueChange={(value: string) =>
setGeneratedCodeConfig(value as GeneratedCodeConfig)
}
value={stack}
onValueChange={(value: string) => setStack(value as Stack)}
disabled={shouldDisableUpdates}
>
<SelectTrigger className="col-span-2" id="output-settings-js">
{generateDisplayComponent(generatedCodeConfig)}
{stack ? generateDisplayComponent(stack) : "Select a stack"}
</SelectTrigger>
<SelectContent>
<SelectGroup>
<SelectItem value={GeneratedCodeConfig.HTML_TAILWIND}>
{generateDisplayComponent(GeneratedCodeConfig.HTML_TAILWIND)}
</SelectItem>
<SelectItem value={GeneratedCodeConfig.REACT_TAILWIND}>
{generateDisplayComponent(GeneratedCodeConfig.REACT_TAILWIND)}
</SelectItem>
<SelectItem value={GeneratedCodeConfig.BOOTSTRAP}>
{generateDisplayComponent(GeneratedCodeConfig.BOOTSTRAP)}
</SelectItem>
<SelectItem value={GeneratedCodeConfig.IONIC_TAILWIND}>
{generateDisplayComponent(GeneratedCodeConfig.IONIC_TAILWIND)}
{Object.values(Stack).map((stack) => (
<SelectItem key={stack} value={stack}>
<div className="flex items-center">
{generateDisplayComponent(stack)}
{STACK_DESCRIPTIONS[stack].inBeta && (
<Badge className="ml-2" variant="secondary">
Beta
</Badge>
)}
</div>
</SelectItem>
))}
</SelectGroup>
</SelectContent>
</Select>

8
frontend/src/components/Preview.tsx
View File

@ -1,6 +1,6 @@
import { useEffect, useRef } from 'react';
import { useEffect, useRef } from "react";
import classNames from "classnames";
import useThrottle from "../hooks/useThrottle";
// import useThrottle from "../hooks/useThrottle";
interface Props {
code: string;
@ -8,7 +8,9 @@ interface Props {
}
function Preview({ code, device }: Props) {
const throttledCode = useThrottle(code, 200);
const throttledCode = code;
// Temporary disable throttling for the preview not updating when the code changes
// useThrottle(code, 200);
const iframeRef = useRef<HTMLIFrameElement | null>(null);
useEffect(() => {

70
frontend/src/components/evals/EvalsPage.tsx Normal file
View File

@ -0,0 +1,70 @@
import React, { useEffect } from "react";
import { HTTP_BACKEND_URL } from "../../config";
import RatingPicker from "./RatingPicker";
interface Eval {
input: string;
output: string;
}
function EvalsPage() {
const [evals, setEvals] = React.useState<Eval[]>([]);
const [ratings, setRatings] = React.useState<number[]>([]);
const total = ratings.reduce((a, b) => a + b, 0);
const max = ratings.length * 4;
const score = ((total / max) * 100 || 0).toFixed(2);
useEffect(() => {
if (evals.length > 0) return;
fetch(`${HTTP_BACKEND_URL}/evals`)
.then((res) => res.json())
.then((data) => {
setEvals(data);
setRatings(new Array(data.length).fill(0));
});
}, [evals]);
return (
<div className="mx-auto">
{/* Display total */}
<div className="flex items-center justify-center w-full h-12 bg-zinc-950">
<span className="text-2xl font-semibold text-white">
Total: {total} out of {max} ({score}%)
</span>
</div>
<div className="flex flex-col gap-y-4 mt-4 mx-auto justify-center">
{evals.map((e, index) => (
<div className="flex flex-col justify-center" key={index}>
<div className="flex gap-x-2 justify-center">
<div className="w-1/2 p-1 border">
<img src={e.input} />
</div>
<div className="w-1/2 p-1 border">
{/* Put output into an iframe */}
<iframe
srcDoc={e.output}
className="w-[1200px] h-[800px] transform scale-[0.60]"
style={{ transformOrigin: "top left" }}
></iframe>
</div>
</div>
<div className="ml-8 mt-4 flex justify-center">
<RatingPicker
onSelect={(rating) => {
const newRatings = [...ratings];
newRatings[index] = rating;
setRatings(newRatings);
}}
/>
</div>
</div>
))}
</div>
</div>
);
}
export default EvalsPage;

38
frontend/src/components/evals/RatingPicker.tsx Normal file
View File

@ -0,0 +1,38 @@
import React from "react";
interface Props {
onSelect: (rating: number) => void;
}
function RatingPicker({ onSelect }: Props) {
const [selected, setSelected] = React.useState<number | null>(null);
const renderCircle = (number: number) => {
const isSelected = selected === number;
const bgColor = isSelected ? "bg-black" : "bg-gray-300";
const textColor = isSelected ? "text-white" : "text-black";
return (
<div
className={`flex items-center justify-center w-8 h-8 ${bgColor} rounded-full cursor-pointer`}
onClick={() => {
setSelected(number);
onSelect(number);
}}
>
<span className={`text-lg font-semibold ${textColor}`}>{number}</span>
</div>
);
};
return (
<div className="flex space-x-4">
{renderCircle(1)}
{renderCircle(2)}
{renderCircle(3)}
{renderCircle(4)}
</div>
);
}
export default RatingPicker;

83
frontend/src/components/history/HistoryDisplay.tsx
View File

@ -1,12 +1,16 @@
import { History, HistoryItemType } from "./history_types";
import { History } from "./history_types";
import toast from "react-hot-toast";
import classNames from "classnames";
import {
HoverCard,
HoverCardTrigger,
HoverCardContent,
} from "../ui/hover-card";
import { Badge } from "../ui/badge";
import { renderHistory } from "./utils";
import {
Collapsible,
CollapsibleContent,
CollapsibleTrigger,
} from "../ui/collapsible";
import { Button } from "../ui/button";
import { CaretSortIcon } from "@radix-ui/react-icons";
interface Props {
history: History;
@ -15,42 +19,33 @@ interface Props {
shouldDisableReverts: boolean;
}
function displayHistoryItemType(itemType: HistoryItemType) {
switch (itemType) {
case "ai_create":
return "Create";
case "ai_edit":
return "Edit";
default: {
const exhaustiveCheck: never = itemType;
throw new Error(`Unhandled case: ${exhaustiveCheck}`);
}
}
}
export default function HistoryDisplay({
history,
currentVersion,
revertToVersion,
shouldDisableReverts,
}: Props) {
return history.length === 0 ? null : (
const renderedHistory = renderHistory(history, currentVersion);
return renderedHistory.length === 0 ? null : (
<div className="flex flex-col h-screen">
<h1 className="font-bold mb-2">Versions</h1>
<ul className="space-y-0 flex flex-col-reverse">
{history.map((item, index) => (
{renderedHistory.map((item, index) => (
<li key={index}>
<HoverCard>
<HoverCardTrigger
<Collapsible>
<div
className={classNames(
"flex items-center justify-between space-x-2 p-2",
"flex items-center justify-between space-x-2 w-full pr-2",
"border-b cursor-pointer",
{
" hover:bg-black hover:text-white":
index !== currentVersion,
"bg-slate-500 text-white": index === currentVersion,
" hover:bg-black hover:text-white": !item.isActive,
"bg-slate-500 text-white": item.isActive,
}
)}
>
<div
className="flex justify-between truncate flex-1 p-2"
onClick={() =>
shouldDisableReverts
? toast.error(
@ -59,28 +54,30 @@ export default function HistoryDisplay({
: revertToVersion(index)
}
>
{" "}
<div className="flex gap-x-1 truncate">
<h2 className="text-sm truncate">
{item.type === "ai_edit" ? item.inputs.prompt : "Create"}
</h2>
{/* <h2 className="text-sm">{displayHistoryItemType(item.type)}</h2> */}
{item.parentIndex !== null &&
item.parentIndex !== index - 1 ? (
<h2 className="text-sm truncate">{item.summary}</h2>
{item.parentVersion !== null && (
<h2 className="text-sm">
(parent: v{(item.parentIndex || 0) + 1})
(parent: {item.parentVersion})
</h2>
) : null}
)}
</div>
<h2 className="text-sm">v{index + 1}</h2>
</HoverCardTrigger>
<HoverCardContent>
<div>
{item.type === "ai_edit" ? item.inputs.prompt : "Create"}
</div>
<Badge>{displayHistoryItemType(item.type)}</Badge>
</HoverCardContent>
</HoverCard>
<CollapsibleTrigger asChild>
<Button variant="ghost" size="sm" className="h-6">
<CaretSortIcon className="h-4 w-4" />
<span className="sr-only">Toggle</span>
</Button>
</CollapsibleTrigger>
</div>
<CollapsibleContent className="w-full bg-slate-300 p-2">
<div>Full prompt: {item.summary}</div>
<div className="flex justify-end">
<Badge>{item.type}</Badge>
</div>
</CollapsibleContent>
</Collapsible>
</li>
))}
</ul>

17
frontend/src/components/history/history_types.ts
View File

@ -1,4 +1,4 @@
export type HistoryItemType = "ai_create" | "ai_edit";
export type HistoryItemType = "ai_create" | "ai_edit" | "code_create";
type CommonHistoryItem = {
parentIndex: null | number;
@ -13,6 +13,10 @@ export type HistoryItem =
| ({
type: "ai_edit";
inputs: AiEditInputs;
} & CommonHistoryItem)
| ({
type: "code_create";
inputs: CodeCreateInputs;
} & CommonHistoryItem);
export type AiCreateInputs = {
@ -23,4 +27,15 @@ export type AiEditInputs = {
prompt: string;
};
export type CodeCreateInputs = {
code: string;
};
export type History = HistoryItem[];
export type RenderedHistoryItem = {
type: string;
summary: string;
parentVersion: string | null;
isActive: boolean;
};

139
frontend/src/components/history/utils.test.ts
View File

@ -1,5 +1,5 @@
import { expect, test } from "vitest";
import { extractHistoryTree } from "./utils";
import { extractHistoryTree, renderHistory } from "./utils";
import type { History } from "./history_types";
const basicLinearHistory: History = [
@ -29,6 +29,18 @@ const basicLinearHistory: History = [
},
];
const basicLinearHistoryWithCode: History = [
{
type: "code_create",
parentIndex: null,
code: "<html>1. create</html>",
inputs: {
code: "<html>1. create</html>",
},
},
...basicLinearHistory.slice(1),
];
const basicBranchingHistory: History = [
...basicLinearHistory,
{
@ -53,7 +65,26 @@ const longerBranchingHistory: History = [
},
];
test("should only include history from this point onward", () => {
const basicBadHistory: History = [
{
type: "ai_create",
parentIndex: null,
code: "<html>1. create</html>",
inputs: {
image_url: "",
},
},
{
type: "ai_edit",
parentIndex: 2, // <- Bad parent index
code: "<html>2. edit with better icons</html>",
inputs: {
prompt: "use better icons",
},
},
];
test("should correctly extract the history tree", () => {
expect(extractHistoryTree(basicLinearHistory, 2)).toEqual([
"<html>1. create</html>",
"use better icons",
@ -93,11 +124,107 @@ test("should only include history from this point onward", () => {
"<html>3. edit with better icons and red text</html>",
]);
// Errors - TODO: Handle these
// Errors
// Bad index
// TODO: Throw an exception instead?
expect(extractHistoryTree(basicLinearHistory, 100)).toEqual([]);
expect(extractHistoryTree(basicLinearHistory, -2)).toEqual([]);
expect(() => extractHistoryTree(basicLinearHistory, 100)).toThrow();
expect(() => extractHistoryTree(basicLinearHistory, -2)).toThrow();
// Bad tree
expect(() => extractHistoryTree(basicBadHistory, 1)).toThrow();
});
test("should correctly render the history tree", () => {
expect(renderHistory(basicLinearHistory, 2)).toEqual([
{
isActive: false,
parentVersion: null,
summary: "Create",
type: "Create",
},
{
isActive: false,
parentVersion: null,
summary: "use better icons",
type: "Edit",
},
{
isActive: true,
parentVersion: null,
summary: "make text red",
type: "Edit",
},
]);
// Current version is the first version
expect(renderHistory(basicLinearHistory, 0)).toEqual([
{
isActive: true,
parentVersion: null,
summary: "Create",
type: "Create",
},
{
isActive: false,
parentVersion: null,
summary: "use better icons",
type: "Edit",
},
{
isActive: false,
parentVersion: null,
summary: "make text red",
type: "Edit",
},
]);
// Render a history with code
expect(renderHistory(basicLinearHistoryWithCode, 0)).toEqual([
{
isActive: true,
parentVersion: null,
summary: "Imported from code",
type: "Imported from code",
},
{
isActive: false,
parentVersion: null,
summary: "use better icons",
type: "Edit",
},
{
isActive: false,
parentVersion: null,
summary: "make text red",
type: "Edit",
},
]);
// Render a non-linear history
expect(renderHistory(basicBranchingHistory, 3)).toEqual([
{
isActive: false,
parentVersion: null,
summary: "Create",
type: "Create",
},
{
isActive: false,
parentVersion: null,
summary: "use better icons",
type: "Edit",
},
{
isActive: false,
parentVersion: null,
summary: "make text red",
type: "Edit",
},
{
isActive: true,
parentVersion: "v2",
summary: "make text green",
type: "Edit",
},
]);
});

74
frontend/src/components/history/utils.ts
View File

@ -1,4 +1,9 @@
import { History, HistoryItem } from "./history_types";
import {
History,
HistoryItem,
HistoryItemType,
RenderedHistoryItem,
} from "./history_types";
export function extractHistoryTree(
history: History,
@ -14,19 +19,78 @@ export function extractHistoryTree(
if (item.type === "ai_create") {
// Don't include the image for ai_create
flatHistory.unshift(item.code);
} else {
} else if (item.type === "ai_edit") {
flatHistory.unshift(item.code);
flatHistory.unshift(item.inputs.prompt);
} else if (item.type === "code_create") {
flatHistory.unshift(item.code);
}
// Move to the parent of the current item
currentIndex = item.parentIndex;
} else {
// TODO: Throw an exception here?
// Break the loop if the item is not found (should not happen in a well-formed history)
break;
throw new Error("Malformed history: missing parent index");
}
}
return flatHistory;
}
function displayHistoryItemType(itemType: HistoryItemType) {
switch (itemType) {
case "ai_create":
return "Create";
case "ai_edit":
return "Edit";
case "code_create":
return "Imported from code";
default: {
const exhaustiveCheck: never = itemType;
throw new Error(`Unhandled case: ${exhaustiveCheck}`);
}
}
}
function summarizeHistoryItem(item: HistoryItem) {
const itemType = item.type;
switch (itemType) {
case "ai_create":
return "Create";
case "ai_edit":
return item.inputs.prompt;
case "code_create":
return "Imported from code";
default: {
const exhaustiveCheck: never = itemType;
throw new Error(`Unhandled case: ${exhaustiveCheck}`);
}
}
}
export const renderHistory = (
history: History,
currentVersion: number | null
) => {
const renderedHistory: RenderedHistoryItem[] = [];
for (let i = 0; i < history.length; i++) {
const item = history[i];
// Only show the parent version if it's not the previous version
// (i.e. it's the branching point) and if it's not the first version
const parentVersion =
item.parentIndex !== null && item.parentIndex !== i - 1
? `v${(item.parentIndex || 0) + 1}`
: null;
const type = displayHistoryItemType(item.type);
const isActive = i === currentVersion;
const summary = summarizeHistoryItem(item);
renderedHistory.push({
isActive,
summary: summary,
parentVersion,
type,
});
}
return renderedHistory;
};

9
frontend/src/components/ui/collapsible.tsx Normal file
View File

@ -0,0 +1,9 @@
import * as CollapsiblePrimitive from "@radix-ui/react-collapsible"
const Collapsible = CollapsiblePrimitive.Root
const CollapsibleTrigger = CollapsiblePrimitive.CollapsibleTrigger
const CollapsibleContent = CollapsiblePrimitive.CollapsibleContent
export { Collapsible, CollapsibleTrigger, CollapsibleContent }

21
frontend/src/generateCode.ts
View File

@ -1,26 +1,20 @@
import toast from "react-hot-toast";
import { WS_BACKEND_URL } from "./config";
import { USER_CLOSE_WEB_SOCKET_CODE } from "./constants";
import { FullGenerationSettings } from "./types";
const ERROR_MESSAGE =
"Error generating code. Check the Developer Console AND the backend logs for details. Feel free to open a Github issue.";
const STOP_MESSAGE = "Code generation stopped";
export interface CodeGenerationParams {
generationType: "create" | "update";
image: string;
resultImage?: string;
history?: string[];
// isImageGenerationEnabled: boolean; // TODO: Merge with Settings type in types.ts
}
const CANCEL_MESSAGE = "Code generation cancelled";
export function generateCode(
wsRef: React.MutableRefObject<WebSocket | null>,
params: CodeGenerationParams,
params: FullGenerationSettings,
onChange: (chunk: string) => void,
onSetCode: (code: string) => void,
onStatusUpdate: (status: string) => void,
onCancel: () => void,
onComplete: () => void
) {
const wsUrl = `${WS_BACKEND_URL}/generate-code`;
@ -46,15 +40,18 @@ export function generateCode(
toast.error(response.value);
}
});
ws.addEventListener("close", (event) => {
console.log("Connection closed", event.code, event.reason);
if (event.code === USER_CLOSE_WEB_SOCKET_CODE) {
toast.success(STOP_MESSAGE);
toast.success(CANCEL_MESSAGE);
onCancel();
} else if (event.code !== 1000) {
console.error("WebSocket error code", event);
toast.error(ERROR_MESSAGE);
}
} else {
onComplete();
}
});
ws.addEventListener("error", (error) => {

20
frontend/src/lib/stacks.ts Normal file
View File

@ -0,0 +1,20 @@
// Keep in sync with backend (prompts/types.py)
export enum Stack {
HTML_TAILWIND = "html_tailwind",
REACT_TAILWIND = "react_tailwind",
BOOTSTRAP = "bootstrap",
VUE_TAILWIND = "vue_tailwind",
IONIC_TAILWIND = "ionic_tailwind",
SVG = "svg",
}
export const STACK_DESCRIPTIONS: {
[key in Stack]: { components: string[]; inBeta: boolean };
} = {
html_tailwind: { components: ["HTML", "Tailwind"], inBeta: false },
react_tailwind: { components: ["React", "Tailwind"], inBeta: false },
bootstrap: { components: ["Bootstrap"], inBeta: false },
vue_tailwind: { components: ["Vue", "Tailwind"], inBeta: true },
ionic_tailwind: { components: ["Ionic", "Tailwind"], inBeta: true },
svg: { components: ["SVG"], inBeta: true },
};

11
frontend/src/main.tsx
View File

@ -3,10 +3,17 @@ import ReactDOM from "react-dom/client";
import App from "./App.tsx";
import "./index.css";
import { Toaster } from "react-hot-toast";
import EvalsPage from "./components/evals/EvalsPage.tsx";
import { BrowserRouter as Router, Routes, Route } from "react-router-dom";
ReactDOM.createRoot(document.getElementById("root")!).render(
<React.StrictMode>
<App />
<Toaster toastOptions={{ className:"dark:bg-zinc-950 dark:text-white" }}/>
<Router>
<Routes>
<Route path="/" element={<App />} />
<Route path="/evals" element={<EvalsPage />} />
</Routes>
</Router>
<Toaster toastOptions={{ className: "dark:bg-zinc-950 dark:text-white" }} />
</React.StrictMode>
);

22
frontend/src/types.ts
View File

@ -1,23 +1,17 @@
import { Stack } from "./lib/stacks";
export enum EditorTheme {
ESPRESSO = "espresso",
COBALT = "cobalt",
}
// Keep in sync with backend (prompts.py)
export enum GeneratedCodeConfig {
HTML_TAILWIND = "html_tailwind",
REACT_TAILWIND = "react_tailwind",
BOOTSTRAP = "bootstrap",
IONIC_TAILWIND = "ionic_tailwind",
}
export interface Settings {
openAiApiKey: string | null;
openAiBaseURL: string | null;
screenshotOneApiKey: string | null;
isImageGenerationEnabled: boolean;
editorTheme: EditorTheme;
generatedCodeConfig: GeneratedCodeConfig;
generatedCodeConfig: Stack;
// Only relevant for hosted version
isTermOfServiceAccepted: boolean;
accessCode: string | null;
@ -28,3 +22,13 @@ export enum AppState {
CODING = "CODING",
CODE_READY = "CODE_READY",
}
export interface CodeGenerationParams {
generationType: "create" | "update";
image: string;
resultImage?: string;
history?: string[];
isImportedFromCode?: boolean;
}
export type FullGenerationSettings = CodeGenerationParams & Settings;

22
frontend/yarn.lock
View File

@ -804,7 +804,7 @@
"@radix-ui/react-use-previous" "1.0.1"
"@radix-ui/react-use-size" "1.0.1"
"@radix-ui/react-collapsible@1.0.3":
"@radix-ui/react-collapsible@1.0.3", "@radix-ui/react-collapsible@^1.0.3":
version "1.0.3"
resolved "https://registry.yarnpkg.com/@radix-ui/react-collapsible/-/react-collapsible-1.0.3.tgz#df0e22e7a025439f13f62d4e4a9e92c4a0df5b81"
integrity sha512-UBmVDkmR6IvDsloHVN+3rtx4Mi5TFvylYXpluuv0f37dtaz3H99bp8No0LGXRigVpl3UAT4l9j6bIchh42S/Gg==
@ -1184,6 +1184,11 @@
dependencies:
"@babel/runtime" "^7.13.10"
"@remix-run/router@1.13.1":
version "1.13.1"
resolved "https://registry.yarnpkg.com/@remix-run/router/-/router-1.13.1.tgz#07e2a8006f23a3bc898b3f317e0a58cc8076b86e"
integrity sha512-so+DHzZKsoOcoXrILB4rqDkMDy7NLMErRdOxvzvOKb507YINKUP4Di+shbTZDhSE/pBZ+vr7XGIpcOO0VLSA+Q==
"@rollup/pluginutils@^4.2.0":
version "4.2.1"
resolved "https://registry.yarnpkg.com/@rollup/pluginutils/-/pluginutils-4.2.1.tgz#e6c6c3aba0744edce3fb2074922d3776c0af2a6d"
@ -3144,6 +3149,21 @@ react-remove-scroll@2.5.5:
use-callback-ref "^1.3.0"
use-sidecar "^1.1.2"
react-router-dom@^6.20.1:
version "6.20.1"
resolved "https://registry.yarnpkg.com/react-router-dom/-/react-router-dom-6.20.1.tgz#e34f8075b9304221420de3609e072bb349824984"
integrity sha512-npzfPWcxfQN35psS7rJgi/EW0Gx6EsNjfdJSAk73U/HqMEJZ2k/8puxfwHFgDQhBGmS3+sjnGbMdMSV45axPQw==
dependencies:
"@remix-run/router" "1.13.1"
react-router "6.20.1"
react-router@6.20.1:
version "6.20.1"
resolved "https://registry.yarnpkg.com/react-router/-/react-router-6.20.1.tgz#e8cc326031d235aaeec405bb234af77cf0fe75ef"
integrity sha512-ccvLrB4QeT5DlaxSFFYi/KR8UMQ4fcD8zBcR71Zp1kaYTC5oJKYAp1cbavzGrogwxca+ubjkd7XjFZKBW8CxPA==
dependencies:
"@remix-run/router" "1.13.1"
react-style-singleton@^2.2.1:
version "2.2.1"
resolved "https://registry.yarnpkg.com/react-style-singleton/-/react-style-singleton-2.2.1.tgz#f99e420492b2d8f34d38308ff660b60d0b1205b4"