Github/screenshot-to-code
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'main' into hosted

Browse Source
This commit is contained in:
Abi Raja 2023-12-14 10:39:41 -05:00
commit 732ffc33e1
10 changed files with 141 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
backend/eval.py
View File

@ -49,7 +49,7 @@ async def main():
for filename in evals: for filename in evals:
filepath = os.path.join(INPUT_DIR, filename) filepath = os.path.join(INPUT_DIR, filename)
data_url = await image_to_data_url(filepath) data_url = await image_to_data_url(filepath)
task = generate_code_core(data_url, "html_tailwind") task = generate_code_core(data_url, "svg")
tasks.append(task) tasks.append(task)
results = await asyncio.gather(*tasks) results = await asyncio.gather(*tasks)

12
backend/imported_code_prompts.py
View File

@ -78,3 +78,15 @@ In terms of libraries,
Return only the full code in <html></html> tags. Return only the full code in <html></html> tags.
Do not include markdown "```" or "```html" at the start or end. Do not include markdown "```" or "```html" at the start or end.
""" """
IMPORTED_CODE_SVG_SYSTEM_PROMPT = """
You are an expert at building SVGs.
- Do not add comments in the code such as "<!-- Add other navigation links as needed -->" and "<!-- ... other news items ... -->" in place of writing the full code. WRITE THE FULL CODE.
- Repeat elements as needed to match the screenshot. For example, if there are 15 items, the code should have 15 items. DO NOT LEAVE comments like "<!-- Repeat for each news item -->" or bad things will happen.
- For images, use placeholder images from https://placehold.co and include a detailed description of the image in the alt text so that an image generation AI can generate the image later.
- You can use Google Fonts
Return only the full code in <svg></svg> tags.
Do not include markdown "```" or "```svg" at the start or end.
"""

21
backend/prompts.py
View File

@ -7,12 +7,14 @@ from imported_code_prompts import (
IMPORTED_CODE_IONIC_TAILWIND_SYSTEM_PROMPT, IMPORTED_CODE_IONIC_TAILWIND_SYSTEM_PROMPT,
IMPORTED_CODE_REACT_TAILWIND_SYSTEM_PROMPT, IMPORTED_CODE_REACT_TAILWIND_SYSTEM_PROMPT,
IMPORTED_CODE_TAILWIND_SYSTEM_PROMPT, IMPORTED_CODE_TAILWIND_SYSTEM_PROMPT,
IMPORTED_CODE_SVG_SYSTEM_PROMPT,
) )
from screenshot_system_prompts import ( from screenshot_system_prompts import (
BOOTSTRAP_SYSTEM_PROMPT, BOOTSTRAP_SYSTEM_PROMPT,
IONIC_TAILWIND_SYSTEM_PROMPT, IONIC_TAILWIND_SYSTEM_PROMPT,
REACT_TAILWIND_SYSTEM_PROMPT, REACT_TAILWIND_SYSTEM_PROMPT,
TAILWIND_SYSTEM_PROMPT, TAILWIND_SYSTEM_PROMPT,
SVG_SYSTEM_PROMPT,
) )
@ -20,6 +22,10 @@ USER_PROMPT = """
Generate code for a web page that looks exactly like this. Generate code for a web page that looks exactly like this.
""" """
SVG_USER_PROMPT = """
Generate code for a SVG that looks exactly like this.
"""
def assemble_imported_code_prompt( def assemble_imported_code_prompt(
code: str, stack: str, result_image_data_url: Union[str, None] = None code: str, stack: str, result_image_data_url: Union[str, None] = None
@ -33,9 +39,16 @@ def assemble_imported_code_prompt(
system_content = IMPORTED_CODE_BOOTSTRAP_SYSTEM_PROMPT system_content = IMPORTED_CODE_BOOTSTRAP_SYSTEM_PROMPT
elif stack == "ionic_tailwind": elif stack == "ionic_tailwind":
system_content = IMPORTED_CODE_IONIC_TAILWIND_SYSTEM_PROMPT system_content = IMPORTED_CODE_IONIC_TAILWIND_SYSTEM_PROMPT
elif stack == "svg":
system_content = IMPORTED_CODE_SVG_SYSTEM_PROMPT
else: else:
raise Exception("Code config is not one of available options") raise Exception("Code config is not one of available options")
user_content = (
"Here is the code of the app: " + code
if stack != "svg"
else "Here is the code of the SVG: " + code
)
return [ return [
{ {
"role": "system", "role": "system",
@ -43,7 +56,7 @@ def assemble_imported_code_prompt(
}, },
{ {
"role": "user", "role": "user",
"content": "Here is the code of the app: " + code, "content": user_content,
}, },
] ]
# TODO: Use result_image_data_url # TODO: Use result_image_data_url
@ -64,9 +77,13 @@ def assemble_prompt(
system_content = BOOTSTRAP_SYSTEM_PROMPT system_content = BOOTSTRAP_SYSTEM_PROMPT
elif generated_code_config == "ionic_tailwind": elif generated_code_config == "ionic_tailwind":
system_content = IONIC_TAILWIND_SYSTEM_PROMPT system_content = IONIC_TAILWIND_SYSTEM_PROMPT
elif generated_code_config == "svg":
system_content = SVG_SYSTEM_PROMPT
else: else:
raise Exception("Code config is not one of available options") raise Exception("Code config is not one of available options")
user_prompt = USER_PROMPT if generated_code_config != "svg" else SVG_USER_PROMPT
user_content: List[ChatCompletionContentPartParam] = [ user_content: List[ChatCompletionContentPartParam] = [
{ {
"type": "image_url", "type": "image_url",
@ -74,7 +91,7 @@ def assemble_prompt(
}, },
{ {
"type": "text", "type": "text",
"text": USER_PROMPT, "text": user_prompt,
}, },
] ]

3
backend/routes/generate_code.py
View File

@ -254,6 +254,9 @@ async def stream_code(websocket: WebSocket):
except Exception as e: except Exception as e:
traceback.print_exc() traceback.print_exc()
print("Image generation failed", e) print("Image generation failed", e)
# Send set code even if image generation fails since that triggers
# the frontend to update history
await websocket.send_json({"type": "setCode", "value": completion})
await websocket.send_json( await websocket.send_json(
{"type": "status", "value": "Image generation failed but code is complete."} {"type": "status", "value": "Image generation failed but code is complete."}
) )

18
backend/screenshot_system_prompts.py
View File

@ -110,3 +110,21 @@ In terms of libraries,
Return only the full code in <html></html> tags. Return only the full code in <html></html> tags.
Do not include markdown "```" or "```html" at the start or end. Do not include markdown "```" or "```html" at the start or end.
""" """
SVG_SYSTEM_PROMPT = """
You are an expert at building SVGs.
You take screenshots of a reference web page from the user, and then build a SVG that looks exactly like the screenshot.
- Make sure the SVG looks exactly like the screenshot.
- Pay close attention to background color, text color, font size, font family,
padding, margin, border, etc. Match the colors and sizes exactly.
- Use the exact text from the screenshot.
- Do not add comments in the code such as "<!-- Add other navigation links as needed -->" and "<!-- ... other news items ... -->" in place of writing the full code. WRITE THE FULL CODE.
- Repeat elements as needed to match the screenshot. For example, if there are 15 items, the code should have 15 items. DO NOT LEAVE comments like "<!-- Repeat for each news item -->" or bad things will happen.
- For images, use placeholder images from https://placehold.co and include a detailed description of the image in the alt text so that an image generation AI can generate the image later.
- You can use Google Fonts
Return only the full code in <svg></svg> tags.
Do not include markdown "```" or "```svg" at the start or end.
"""

52
backend/test_prompts.py
View File

@ -113,6 +113,23 @@ Return only the full code in <html></html> tags.
Do not include markdown "```" or "```html" at the start or end. Do not include markdown "```" or "```html" at the start or end.
""" """
SVG_SYSTEM_PROMPT = """
You are an expert at building SVGs.
You take screenshots of a reference web page from the user, and then build a SVG that looks exactly like the screenshot.
- Make sure the SVG looks exactly like the screenshot.
- Pay close attention to background color, text color, font size, font family,
padding, margin, border, etc. Match the colors and sizes exactly.
- Use the exact text from the screenshot.
- Do not add comments in the code such as "<!-- Add other navigation links as needed -->" and "<!-- ... other news items ... -->" in place of writing the full code. WRITE THE FULL CODE.
- Repeat elements as needed to match the screenshot. For example, if there are 15 items, the code should have 15 items. DO NOT LEAVE comments like "<!-- Repeat for each news item -->" or bad things will happen.
- For images, use placeholder images from https://placehold.co and include a detailed description of the image in the alt text so that an image generation AI can generate the image later.
- You can use Google Fonts
Return only the full code in <svg></svg> tags.
Do not include markdown "```" or "```svg" at the start or end.
"""
IMPORTED_CODE_TAILWIND_SYSTEM_PROMPT = """ IMPORTED_CODE_TAILWIND_SYSTEM_PROMPT = """
You are an expert Tailwind developer. You are an expert Tailwind developer.
@ -194,27 +211,55 @@ Return only the full code in <html></html> tags.
Do not include markdown "```" or "```html" at the start or end. Do not include markdown "```" or "```html" at the start or end.
""" """
IMPORTED_CODE_SVG_SYSTEM_PROMPT = """
You are an expert at building SVGs.
- Do not add comments in the code such as "<!-- Add other navigation links as needed -->" and "<!-- ... other news items ... -->" in place of writing the full code. WRITE THE FULL CODE.
- Repeat elements as needed to match the screenshot. For example, if there are 15 items, the code should have 15 items. DO NOT LEAVE comments like "<!-- Repeat for each news item -->" or bad things will happen.
- For images, use placeholder images from https://placehold.co and include a detailed description of the image in the alt text so that an image generation AI can generate the image later.
- You can use Google Fonts
Return only the full code in <svg></svg> tags.
Do not include markdown "```" or "```svg" at the start or end.
"""
USER_PROMPT = """
Generate code for a web page that looks exactly like this.
"""
SVG_USER_PROMPT = """
Generate code for a SVG that looks exactly like this.
"""
def test_prompts(): def test_prompts():
tailwind_prompt = assemble_prompt( tailwind_prompt = assemble_prompt(
"image_data_url", "html_tailwind", "result_image_data_url" "image_data_url", "html_tailwind", "result_image_data_url"
) )
assert tailwind_prompt[0]["content"] == TAILWIND_SYSTEM_PROMPT assert tailwind_prompt[0]["content"] == TAILWIND_SYSTEM_PROMPT
assert tailwind_prompt[1]["content"][2]["text"] == USER_PROMPT # type: ignore
react_tailwind_prompt = assemble_prompt( react_tailwind_prompt = assemble_prompt(
"image_data_url", "react_tailwind", "result_image_data_url" "image_data_url", "react_tailwind", "result_image_data_url"
) )
assert react_tailwind_prompt[0]["content"] == REACT_TAILWIND_SYSTEM_PROMPT assert react_tailwind_prompt[0]["content"] == REACT_TAILWIND_SYSTEM_PROMPT
assert react_tailwind_prompt[1]["content"][2]["text"] == USER_PROMPT # type: ignore
bootstrap_prompt = assemble_prompt( bootstrap_prompt = assemble_prompt(
"image_data_url", "bootstrap", "result_image_data_url" "image_data_url", "bootstrap", "result_image_data_url"
) )
assert bootstrap_prompt[0]["content"] == BOOTSTRAP_SYSTEM_PROMPT assert bootstrap_prompt[0]["content"] == BOOTSTRAP_SYSTEM_PROMPT
assert bootstrap_prompt[1]["content"][2]["text"] == USER_PROMPT # type: ignore
ionic_tailwind = assemble_prompt( ionic_tailwind = assemble_prompt(
"image_data_url", "ionic_tailwind", "result_image_data_url" "image_data_url", "ionic_tailwind", "result_image_data_url"
) )
assert ionic_tailwind[0]["content"] == IONIC_TAILWIND_SYSTEM_PROMPT assert ionic_tailwind[0]["content"] == IONIC_TAILWIND_SYSTEM_PROMPT
assert ionic_tailwind[1]["content"][2]["text"] == USER_PROMPT # type: ignore
svg_prompt = assemble_prompt("image_data_url", "svg", "result_image_data_url")
assert svg_prompt[0]["content"] == SVG_SYSTEM_PROMPT
assert svg_prompt[1]["content"][2]["text"] == SVG_USER_PROMPT # type: ignore
def test_imported_code_prompts(): def test_imported_code_prompts():
@ -253,3 +298,10 @@ def test_imported_code_prompts():
{"role": "user", "content": "Here is the code of the app: code"}, {"role": "user", "content": "Here is the code of the app: code"},
] ]
assert ionic_tailwind == expected_ionic_tailwind assert ionic_tailwind == expected_ionic_tailwind
svg = assemble_imported_code_prompt("code", "svg", "result_image_data_url")
expected_svg = [
{"role": "system", "content": IMPORTED_CODE_SVG_SYSTEM_PROMPT},
{"role": "user", "content": "Here is the code of the SVG: code"},
]
assert svg == expected_svg

5
frontend/src/App.tsx
View File

@ -132,9 +132,11 @@ function App({ navbarComponent }: Props) {
setGeneratedCode(""); setGeneratedCode("");
setReferenceImages([]); setReferenceImages([]);
setExecutionConsole([]); setExecutionConsole([]);
setUpdateInstruction("");
setIsImportedFromCode(false);
setAppHistory([]); setAppHistory([]);
setCurrentVersion(null); setCurrentVersion(null);
setIsImportedFromCode(false); setShouldIncludeResultImage(false);
}; };
const cancelCodeGeneration = () => { const cancelCodeGeneration = () => {
@ -207,6 +209,7 @@ function App({ navbarComponent }: Props) {
}); });
} }
}, },
// On status update
(line) => setExecutionConsole((prev) => [...prev, line]), (line) => setExecutionConsole((prev) => [...prev, line]),
// On cancel // On cancel
() => { () => {

27
frontend/src/components/OutputSettingsSection.tsx
View File

@ -7,6 +7,7 @@ import {
} from "./ui/select"; } from "./ui/select";
import { GeneratedCodeConfig } from "../types"; import { GeneratedCodeConfig } from "../types";
import { addEvent } from "../lib/analytics"; import { addEvent } from "../lib/analytics";
import { Badge } from "./ui/badge";
function generateDisplayComponent(config: GeneratedCodeConfig) { function generateDisplayComponent(config: GeneratedCodeConfig) {
switch (config) { switch (config) {
@ -37,9 +38,16 @@ function generateDisplayComponent(config: GeneratedCodeConfig) {
<span className="font-semibold">Tailwind</span> <span className="font-semibold">Tailwind</span>
</div> </div>
); );
default: case GeneratedCodeConfig.SVG:
// TODO: Should never reach this out. Error out return (
return config; <div>
<span className="font-semibold">SVG</span>
</div>
);
default: {
const exhaustiveCheck: never = config;
throw new Error(`Unhandled case: ${exhaustiveCheck}`);
}
} }
} }
@ -85,7 +93,20 @@ function OutputSettingsSection({
{generateDisplayComponent(GeneratedCodeConfig.BOOTSTRAP)} {generateDisplayComponent(GeneratedCodeConfig.BOOTSTRAP)}
</SelectItem> </SelectItem>
<SelectItem value={GeneratedCodeConfig.IONIC_TAILWIND}> <SelectItem value={GeneratedCodeConfig.IONIC_TAILWIND}>
<div className="flex items-center">
{generateDisplayComponent(GeneratedCodeConfig.IONIC_TAILWIND)} {generateDisplayComponent(GeneratedCodeConfig.IONIC_TAILWIND)}
<Badge className="ml-2" variant="secondary">
Beta
</Badge>
</div>
</SelectItem>
<SelectItem value={GeneratedCodeConfig.SVG}>
<div className="flex items-center">
{generateDisplayComponent(GeneratedCodeConfig.SVG)}
<Badge className="ml-2" variant="secondary">
Beta
</Badge>
</div>
</SelectItem> </SelectItem>
</SelectGroup> </SelectGroup>
</SelectContent> </SelectContent>

8
frontend/src/components/Preview.tsx
View File

@ -1,6 +1,6 @@
import { useEffect, useRef } from 'react'; import { useEffect, useRef } from "react";
import classNames from "classnames"; import classNames from "classnames";
import useThrottle from "../hooks/useThrottle"; // import useThrottle from "../hooks/useThrottle";
interface Props { interface Props {
code: string; code: string;
@ -8,7 +8,9 @@ interface Props {
} }
function Preview({ code, device }: Props) { function Preview({ code, device }: Props) {
const throttledCode = useThrottle(code, 200); const throttledCode = code;
// Temporary disable throttling for the preview not updating when the code changes
// useThrottle(code, 200);
const iframeRef = useRef<HTMLIFrameElement | null>(null); const iframeRef = useRef<HTMLIFrameElement | null>(null);
useEffect(() => { useEffect(() => {

1
frontend/src/types.ts
View File

@ -9,6 +9,7 @@ export enum GeneratedCodeConfig {
REACT_TAILWIND = "react_tailwind", REACT_TAILWIND = "react_tailwind",
BOOTSTRAP = "bootstrap", BOOTSTRAP = "bootstrap",
IONIC_TAILWIND = "ionic_tailwind", IONIC_TAILWIND = "ionic_tailwind",
SVG = "svg",
} }
export interface Settings { export interface Settings {