Github/screenshot-to-code
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge main into sweep/add-reset-button-settings-dialog

Browse Source
This commit is contained in:
sweep-ai[bot] 2023-11-30 21:09:31 +00:00 committed by GitHub
commit 6dbe544532
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 211 additions and 232 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
backend/main.py
View File

@ -69,13 +69,11 @@ async def stream_code(websocket: WebSocket):
print("Received params") print("Received params")
# Read the output settings from the request. Fall back to default if not provided. # Read the code config settings from the request. Fall back to default if not provided.
output_settings = {"css": "tailwind", "js": "vanilla"} generated_code_config = ""
if params["outputSettings"] and params["outputSettings"]["css"]: if "generatedCodeConfig" in params and params["generatedCodeConfig"]:
output_settings["css"] = params["outputSettings"]["css"] generated_code_config = params["generatedCodeConfig"]
if params["outputSettings"] and params["outputSettings"]["js"]: print(f"Generating {generated_code_config} code")
output_settings["js"] = params["outputSettings"]["js"]
print("Using output settings:", output_settings)
# Get the OpenAI API key from the request. Fall back to environment variable if not provided. # Get the OpenAI API key from the request. Fall back to environment variable if not provided.
# If neither is provided, we throw an error. # If neither is provided, we throw an error.
@ -139,12 +137,23 @@ async def stream_code(websocket: WebSocket):
async def process_chunk(content): async def process_chunk(content):
await websocket.send_json({"type": "chunk", "value": content}) await websocket.send_json({"type": "chunk", "value": content})
if params.get("resultImage") and params["resultImage"]: # Assemble the prompt
prompt_messages = assemble_prompt( try:
params["image"], output_settings, params["resultImage"] if params.get("resultImage") and params["resultImage"]:
prompt_messages = assemble_prompt(
params["image"], generated_code_config, params["resultImage"]
)
else:
prompt_messages = assemble_prompt(params["image"], generated_code_config)
except:
await websocket.send_json(
{
"type": "error",
"value": "Error assembling prompt. Contact support at support@picoapps.xyz",
}
) )
else: await websocket.close()
prompt_messages = assemble_prompt(params["image"], output_settings) return
# Image cache for updates so that we don't have to regenerate images # Image cache for updates so that we don't have to regenerate images
image_cache = {} image_cache = {}

55
backend/prompts.py
View File

@ -77,23 +77,60 @@ Return only the full code in <html></html> tags.
Do not include markdown "```" or "```html" at the start or end. Do not include markdown "```" or "```html" at the start or end.
""" """
IONIC_TAILWIND_SYSTEM_PROMPT = """
You are an expert Ionic/Tailwind developer
You take screenshots of a reference web page from the user, and then build single page apps
using Ionic and Tailwind CSS.
You might also be given a screenshot(The second image) of a web page that you have already built, and asked to
update it to look more like the reference image(The first image).
- Make sure the app looks exactly like the screenshot.
- Pay close attention to background color, text color, font size, font family,
padding, margin, border, etc. Match the colors and sizes exactly.
- Use the exact text from the screenshot.
- Do not add comments in the code such as "<!-- Add other navigation links as needed -->" and "<!-- ... other news items ... -->" in place of writing the full code. WRITE THE FULL CODE.
- Repeat elements as needed to match the screenshot. For example, if there are 15 items, the code should have 15 items. DO NOT LEAVE comments like "<!-- Repeat for each news item -->" or bad things will happen.
- For images, use placeholder images from https://placehold.co and include a detailed description of the image in the alt text so that an image generation AI can generate the image later.
In terms of libraries,
- Use these script to include Ionic so that it can run on a standalone page:
<script type="module" src="https://cdn.jsdelivr.net/npm/@ionic/core/dist/ionic/ionic.esm.js"></script>
<script nomodule src="https://cdn.jsdelivr.net/npm/@ionic/core/dist/ionic/ionic.js"></script>
<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/@ionic/core/css/ionic.bundle.css" />
- Use this script to include Tailwind: <script src="https://cdn.tailwindcss.com"></script>
- You can use Google Fonts
- ionicons for icons, add the following <script > tags near the end of the page, right before the closing </body> tag:
<script type="module">
import ionicons from 'https://cdn.jsdelivr.net/npm/ionicons/+esm'
</script>
<script nomodule src="https://cdn.jsdelivr.net/npm/ionicons/dist/esm/ionicons.min.js"></script>
<link href="https://cdn.jsdelivr.net/npm/ionicons/dist/collection/components/icon/icon.min.css" rel="stylesheet">
Return only the full code in <html></html> tags.
Do not include markdown "```" or "```html" at the start or end.
"""
USER_PROMPT = """ USER_PROMPT = """
Generate code for a web page that looks exactly like this. Generate code for a web page that looks exactly like this.
""" """
def assemble_prompt(image_data_url, output_settings: dict, result_image_data_url=None): def assemble_prompt(
image_data_url, generated_code_config: str, result_image_data_url=None
):
# Set the system prompt based on the output settings # Set the system prompt based on the output settings
chosen_prompt_name = "tailwind"
system_content = TAILWIND_SYSTEM_PROMPT system_content = TAILWIND_SYSTEM_PROMPT
if output_settings["css"] == "bootstrap": if generated_code_config == "html_tailwind":
chosen_prompt_name = "bootstrap" system_content = TAILWIND_SYSTEM_PROMPT
system_content = BOOTSTRAP_SYSTEM_PROMPT elif generated_code_config == "react_tailwind":
if output_settings["js"] == "react":
chosen_prompt_name = "react-tailwind"
system_content = REACT_TAILWIND_SYSTEM_PROMPT system_content = REACT_TAILWIND_SYSTEM_PROMPT
elif generated_code_config == "bootstrap":
print("Using system prompt:", chosen_prompt_name) system_content = BOOTSTRAP_SYSTEM_PROMPT
elif generated_code_config == "ionic_tailwind":
system_content = IONIC_TAILWIND_SYSTEM_PROMPT
else:
raise Exception("Code config is not one of available options")
user_content = [ user_content = [
{ {

49
backend/test_prompts.py
View File

@ -79,19 +79,58 @@ Return only the full code in <html></html> tags.
Do not include markdown "```" or "```html" at the start or end. Do not include markdown "```" or "```html" at the start or end.
""" """
IONIC_TAILWIND_SYSTEM_PROMPT = """
You are an expert Ionic/Tailwind developer
You take screenshots of a reference web page from the user, and then build single page apps
using Ionic and Tailwind CSS.
You might also be given a screenshot(The second image) of a web page that you have already built, and asked to
update it to look more like the reference image(The first image).
- Make sure the app looks exactly like the screenshot.
- Pay close attention to background color, text color, font size, font family,
padding, margin, border, etc. Match the colors and sizes exactly.
- Use the exact text from the screenshot.
- Do not add comments in the code such as "<!-- Add other navigation links as needed -->" and "<!-- ... other news items ... -->" in place of writing the full code. WRITE THE FULL CODE.
- Repeat elements as needed to match the screenshot. For example, if there are 15 items, the code should have 15 items. DO NOT LEAVE comments like "<!-- Repeat for each news item -->" or bad things will happen.
- For images, use placeholder images from https://placehold.co and include a detailed description of the image in the alt text so that an image generation AI can generate the image later.
In terms of libraries,
- Use these script to include Ionic so that it can run on a standalone page:
<script type="module" src="https://cdn.jsdelivr.net/npm/@ionic/core/dist/ionic/ionic.esm.js"></script>
<script nomodule src="https://cdn.jsdelivr.net/npm/@ionic/core/dist/ionic/ionic.js"></script>
<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/@ionic/core/css/ionic.bundle.css" />
- Use this script to include Tailwind: <script src="https://cdn.tailwindcss.com"></script>
- You can use Google Fonts
- ionicons for icons, add the following <script > tags near the end of the page, right before the closing </body> tag:
<script type="module">
import ionicons from 'https://cdn.jsdelivr.net/npm/ionicons/+esm'
</script>
<script nomodule src="https://cdn.jsdelivr.net/npm/ionicons/dist/esm/ionicons.min.js"></script>
<link href="https://cdn.jsdelivr.net/npm/ionicons/dist/collection/components/icon/icon.min.css" rel="stylesheet">
Return only the full code in <html></html> tags.
Do not include markdown "```" or "```html" at the start or end.
"""
def test_prompts(): def test_prompts():
tailwind_prompt = assemble_prompt( tailwind_prompt = assemble_prompt(
"image_data_url", {"css": "tailwind", "js": "vanilla"}, "result_image_data_url" "image_data_url", "html_tailwind", "result_image_data_url"
) )
assert tailwind_prompt[0]["content"] == TAILWIND_SYSTEM_PROMPT assert tailwind_prompt[0]["content"] == TAILWIND_SYSTEM_PROMPT
react_tailwind_prompt = assemble_prompt(
"image_data_url", "react_tailwind", "result_image_data_url"
)
assert react_tailwind_prompt[0]["content"] == REACT_TAILWIND_SYSTEM_PROMPT
bootstrap_prompt = assemble_prompt( bootstrap_prompt = assemble_prompt(
"image_data_url", {"css": "bootstrap", "js": "vanilla"}, "result_image_data_url" "image_data_url", "bootstrap", "result_image_data_url"
) )
assert bootstrap_prompt[0]["content"] == BOOTSTRAP_SYSTEM_PROMPT assert bootstrap_prompt[0]["content"] == BOOTSTRAP_SYSTEM_PROMPT
react_tailwind_prompt = assemble_prompt( ionic_tailwind = assemble_prompt(
"image_data_url", {"css": "tailwind", "js": "react"}, "result_image_data_url" "image_data_url", "ionic_tailwind", "result_image_data_url"
) )
assert react_tailwind_prompt[0]["content"] == REACT_TAILWIND_SYSTEM_PROMPT assert ionic_tailwind[0]["content"] == IONIC_TAILWIND_SYSTEM_PROMPT

41
frontend/src/App.tsx
View File

@ -1,4 +1,4 @@
import { useRef, useState } from "react"; import { useEffect, useRef, useState } from "react";
import ImageUpload from "./components/ImageUpload"; import ImageUpload from "./components/ImageUpload";
import CodePreview from "./components/CodePreview"; import CodePreview from "./components/CodePreview";
import Preview from "./components/Preview"; import Preview from "./components/Preview";
@ -18,14 +18,7 @@ import { Button } from "@/components/ui/button";
import { Textarea } from "@/components/ui/textarea"; import { Textarea } from "@/components/ui/textarea";
import { Tabs, TabsContent, TabsList, TabsTrigger } from "./components/ui/tabs"; import { Tabs, TabsContent, TabsList, TabsTrigger } from "./components/ui/tabs";
import SettingsDialog from "./components/SettingsDialog"; import SettingsDialog from "./components/SettingsDialog";
import { import { Settings, EditorTheme, AppState, GeneratedCodeConfig } from "./types";
Settings,
EditorTheme,
AppState,
CSSOption,
OutputSettings,
JSFrameworkOption,
} from "./types";
import { IS_RUNNING_ON_CLOUD } from "./config"; import { IS_RUNNING_ON_CLOUD } from "./config";
import { PicoBadge } from "./components/PicoBadge"; import { PicoBadge } from "./components/PicoBadge";
import { OnboardingNote } from "./components/OnboardingNote"; import { OnboardingNote } from "./components/OnboardingNote";
@ -51,20 +44,31 @@ function App() {
screenshotOneApiKey: null, screenshotOneApiKey: null,
isImageGenerationEnabled: true, isImageGenerationEnabled: true,
editorTheme: EditorTheme.COBALT, editorTheme: EditorTheme.COBALT,
generatedCodeConfig: GeneratedCodeConfig.HTML_TAILWIND,
// Only relevant for hosted version
isTermOfServiceAccepted: false, isTermOfServiceAccepted: false,
accessCode: null, accessCode: null,
}, },
"setting" "setting"
); );
const [outputSettings, setOutputSettings] = useState<OutputSettings>({
css: CSSOption.TAILWIND,
js: JSFrameworkOption.NO_FRAMEWORK,
});
const [shouldIncludeResultImage, setShouldIncludeResultImage] = const [shouldIncludeResultImage, setShouldIncludeResultImage] =
useState<boolean>(false); useState<boolean>(false);
const wsRef = useRef<WebSocket>(null); const wsRef = useRef<WebSocket>(null);
// When the user already has the settings in local storage, newly added keys
// do not get added to the settings so if it's falsy, we populate it with the default
// value
useEffect(() => {
if (!settings.generatedCodeConfig) {
setSettings((prev) => ({
...prev,
generatedCodeConfig: GeneratedCodeConfig.HTML_TAILWIND,
}));
}
}, [settings.generatedCodeConfig, setSettings]);
const takeScreenshot = async (): Promise<string> => { const takeScreenshot = async (): Promise<string> => {
const iframeElement = document.querySelector( const iframeElement = document.querySelector(
"#preview-desktop" "#preview-desktop"
@ -114,7 +118,7 @@ function App() {
setAppState(AppState.CODING); setAppState(AppState.CODING);
// Merge settings with params // Merge settings with params
const updatedParams = { ...params, ...settings, outputSettings }; const updatedParams = { ...params, ...settings };
generateCode( generateCode(
wsRef, wsRef,
@ -188,8 +192,13 @@ function App() {
</div> </div>
<OutputSettingsSection <OutputSettingsSection
outputSettings={outputSettings} generatedCodeConfig={settings.generatedCodeConfig}
setOutputSettings={setOutputSettings} setGeneratedCodeConfig={(config: GeneratedCodeConfig) =>
setSettings((prev) => ({
...prev,
generatedCodeConfig: config,
}))
}
shouldDisableUpdates={ shouldDisableUpdates={
appState === AppState.CODING || appState === AppState.CODE_READY appState === AppState.CODING || appState === AppState.CODE_READY
} }

241
frontend/src/components/OutputSettingsSection.tsx
View File

@ -5,195 +5,86 @@ import {
SelectItem, SelectItem,
SelectTrigger, SelectTrigger,
} from "./ui/select"; } from "./ui/select";
import { CSSOption, JSFrameworkOption, OutputSettings } from "../types"; import { GeneratedCodeConfig } from "../types";
import toast from "react-hot-toast";
import { Label } from "@radix-ui/react-label";
import { Button } from "./ui/button";
import { Popover, PopoverTrigger, PopoverContent } from "./ui/popover";
function displayCSSOption(option: CSSOption) { function generateDisplayComponent(config: GeneratedCodeConfig) {
switch (option) { switch (config) {
case CSSOption.TAILWIND: case GeneratedCodeConfig.HTML_TAILWIND:
return "Tailwind"; return (
case CSSOption.BOOTSTRAP: <div>
return "Bootstrap"; <span className="font-semibold">HTML</span> +{" "}
<span className="font-semibold">Tailwind</span>
</div>
);
case GeneratedCodeConfig.REACT_TAILWIND:
return (
<div>
<span className="font-semibold">React</span> +{" "}
<span className="font-semibold">Tailwind</span>
</div>
);
case GeneratedCodeConfig.BOOTSTRAP:
return (
<div>
<span className="font-semibold">Bootstrap</span>
</div>
);
case GeneratedCodeConfig.IONIC_TAILWIND:
return (
<div>
<span className="font-semibold">Ionic</span> +{" "}
<span className="font-semibold">Tailwind</span>
</div>
);
default: default:
return option; // TODO: Should never reach this out. Error out
} return config;
}
function displayJSOption(option: JSFrameworkOption) {
switch (option) {
case JSFrameworkOption.REACT:
return "React";
case JSFrameworkOption.NO_FRAMEWORK:
return "No Framework";
default:
return option;
}
}
function convertStringToCSSOption(option: string) {
switch (option) {
case "tailwind":
return CSSOption.TAILWIND;
case "bootstrap":
return CSSOption.BOOTSTRAP;
default:
throw new Error(`Unknown CSS option: ${option}`);
}
}
function generateDisplayString(settings: OutputSettings) {
if (
settings.js === JSFrameworkOption.REACT &&
settings.css === CSSOption.TAILWIND
) {
return (
<div className="text-gray-800 dark:text-white">
Generating <span className="font-bold">React</span> +{" "}
<span className="font-bold">Tailwind</span> code
</div>
);
} else if (
settings.js === JSFrameworkOption.NO_FRAMEWORK &&
settings.css === CSSOption.TAILWIND
) {
return (
<div className="text-gray-800 dark:text-white">
Generating <span className="font-bold">HTML</span> +{" "}
<span className="font-bold">Tailwind</span> code
</div>
);
} else if (
settings.js === JSFrameworkOption.NO_FRAMEWORK &&
settings.css === CSSOption.BOOTSTRAP
) {
return (
<div className="text-gray-800 dark:text-white">
Generating <span className="font-bold">HTML</span> +{" "}
<span className="font-bold">Bootstrap</span> code
</div>
);
} }
} }
interface Props { interface Props {
outputSettings: OutputSettings; generatedCodeConfig: GeneratedCodeConfig;
setOutputSettings: React.Dispatch<React.SetStateAction<OutputSettings>>; setGeneratedCodeConfig: (config: GeneratedCodeConfig) => void;
shouldDisableUpdates?: boolean; shouldDisableUpdates?: boolean;
} }
function OutputSettingsSection({ function OutputSettingsSection({
outputSettings, generatedCodeConfig,
setOutputSettings, setGeneratedCodeConfig,
shouldDisableUpdates = false, shouldDisableUpdates = false,
}: Props) { }: Props) {
const onCSSValueChange = (value: string) => {
setOutputSettings((prev) => {
if (prev.js === JSFrameworkOption.REACT) {
if (value !== CSSOption.TAILWIND) {
toast.error(
'React only supports Tailwind CSS. Change JS framework to "No Framework" to use Bootstrap.'
);
}
return {
css: CSSOption.TAILWIND,
js: JSFrameworkOption.REACT,
};
} else {
return {
...prev,
css: convertStringToCSSOption(value),
};
}
});
};
const onJsFrameworkChange = (value: string) => {
if (value === JSFrameworkOption.REACT) {
setOutputSettings(() => ({
css: CSSOption.TAILWIND,
js: value as JSFrameworkOption,
}));
} else {
setOutputSettings((prev) => ({
...prev,
js: value as JSFrameworkOption,
}));
}
};
return ( return (
<div className="flex flex-col gap-y-2 justify-between text-sm"> <div className="flex flex-col gap-y-2 justify-between text-sm">
{generateDisplayString(outputSettings)}{" "} <div className="grid grid-cols-3 items-center gap-4">
{!shouldDisableUpdates && ( <span>Generating:</span>
<Popover> <Select
<PopoverTrigger asChild> value={generatedCodeConfig}
<Button variant="outline">Customize</Button> onValueChange={(value: string) =>
</PopoverTrigger> setGeneratedCodeConfig(value as GeneratedCodeConfig)
<PopoverContent className="w-80 text-sm"> }
<div className="grid gap-4"> disabled={shouldDisableUpdates}
<div className="space-y-2"> >
<h4 className="font-medium leading-none">Code Settings</h4> <SelectTrigger className="col-span-2" id="output-settings-js">
<p className="text-muted-foreground"> {generateDisplayComponent(generatedCodeConfig)}
Customize your code output </SelectTrigger>
</p> <SelectContent>
</div> <SelectGroup>
<div className="grid gap-2"> <SelectItem value={GeneratedCodeConfig.HTML_TAILWIND}>
<div className="grid grid-cols-3 items-center gap-4"> {generateDisplayComponent(GeneratedCodeConfig.HTML_TAILWIND)}
<Label htmlFor="output-settings-js">JS</Label> </SelectItem>
<Select <SelectItem value={GeneratedCodeConfig.REACT_TAILWIND}>
value={outputSettings.js} {generateDisplayComponent(GeneratedCodeConfig.REACT_TAILWIND)}
onValueChange={onJsFrameworkChange} </SelectItem>
> <SelectItem value={GeneratedCodeConfig.BOOTSTRAP}>
<SelectTrigger {generateDisplayComponent(GeneratedCodeConfig.BOOTSTRAP)}
className="col-span-2 h-8" </SelectItem>
id="output-settings-js" <SelectItem value={GeneratedCodeConfig.IONIC_TAILWIND}>
> {generateDisplayComponent(GeneratedCodeConfig.IONIC_TAILWIND)}
{displayJSOption(outputSettings.js)} </SelectItem>
</SelectTrigger> </SelectGroup>
<SelectContent> </SelectContent>
<SelectGroup> </Select>
<SelectItem value={JSFrameworkOption.NO_FRAMEWORK}> </div>
No Framework
</SelectItem>
<SelectItem value={JSFrameworkOption.REACT}>
React
</SelectItem>
</SelectGroup>
</SelectContent>
</Select>
</div>
<div className="grid grid-cols-3 items-center gap-4">
<Label htmlFor="output-settings-css">CSS</Label>
<Select
value={outputSettings.css}
onValueChange={onCSSValueChange}
>
<SelectTrigger
className="col-span-2 h-8"
id="output-settings-css"
>
{displayCSSOption(outputSettings.css)}
</SelectTrigger>
<SelectContent>
<SelectGroup>
<SelectItem value={CSSOption.TAILWIND}>
Tailwind
</SelectItem>
<SelectItem value={CSSOption.BOOTSTRAP}>
Bootstrap
</SelectItem>
</SelectGroup>
</SelectContent>
</Select>
</div>
</div>
</div>
</PopoverContent>
</Popover>
)}
</div> </div>
); );
} }

24
frontend/src/types.ts
View File

@ -3,20 +3,12 @@ export enum EditorTheme {
COBALT = "cobalt", COBALT = "cobalt",
} }
export enum CSSOption { // Keep in sync with backend (prompts.py)
TAILWIND = "tailwind", export enum GeneratedCodeConfig {
HTML_TAILWIND = "html_tailwind",
REACT_TAILWIND = "react_tailwind",
BOOTSTRAP = "bootstrap", BOOTSTRAP = "bootstrap",
} IONIC_TAILWIND = "ionic_tailwind",
export enum JSFrameworkOption {
NO_FRAMEWORK = "vanilla",
REACT = "react",
VUE = "vue",
}
export interface OutputSettings {
css: CSSOption;
js: JSFrameworkOption;
} }
export interface Settings { export interface Settings {
@ -25,8 +17,10 @@ export interface Settings {
screenshotOneApiKey: string | null; screenshotOneApiKey: string | null;
isImageGenerationEnabled: boolean; isImageGenerationEnabled: boolean;
editorTheme: EditorTheme; editorTheme: EditorTheme;
isTermOfServiceAccepted: boolean; // Only relevant for hosted version generatedCodeConfig: GeneratedCodeConfig;
accessCode: string | null; // Only relevant for hosted version // Only relevant for hosted version
isTermOfServiceAccepted: boolean;
accessCode: string | null;
} }
export enum AppState { export enum AppState {