diff --git a/README.md b/README.md
index 1365a69..eb9725b 100644
--- a/README.md
+++ b/README.md
@@ -4,10 +4,13 @@ This is a simple app that converts a screenshot to HTML/Tailwind CSS. It uses GP
https://github.com/abi/screenshot-to-code/assets/23818/6cebadae-2fe3-4986-ac6a-8fb9db030045
-See Examples section below for more demos.
+See [Examples](#examples) section below for more demos.
+
+🆕 [Try it here](https://picoapps.xyz/free-tools/screenshot-to-code) (bring your own OpenAI key - **your key must have access to GPT-4 Vision. See [FAQ](#faqs) section below for details**). Or see [Getting Started](#getting-started) below for local install instructions.
## Updates
+- Nov 16 - Added a setting to disable DALL-E image generation if you don't need that
- Nov 16 - View code directly within the app
- Nov 15 - 🔥 You can now instruct the AI to update the code as you wish. Useful if the AI messed up some styles or missed a section.
@@ -50,16 +53,28 @@ Application will be up and running at http://localhost:5173
Note that you can't develop the application with this setup as the file changes won't trigger a rebuild.
-## Feedback
+## FAQs
-If you have feature requests, bug reports or other feedback, open an issue or ping me on [Twitter](https://twitter.com/_abi_).
+- **I'm running into an error when setting up the backend. How can I fix it?** [Try this](https://github.com/abi/screenshot-to-code/issues/3#issuecomment-1814777959). If that still doesn't work, open an issue.
+- **How do I get an OpenAI API key that has the GPT4 Vision model available?** Create an OpenAI account. And then, you need to buy at least $1 worth of credit on the [Billing dashboard](https://platform.openai.com/account/billing/overview).
+- **How can I provide feedback?** For feedback, feature requests and bug reports, open an issue or ping me on [Twitter](https://twitter.com/_abi_).
## Examples
-Hacker News but it gets the colors wrong at first so we nudge it
+**NYTimes**
+
+| Original | Replica |
+| -------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------- |
+|  |  |
+
+**Instagram page (with not Taylor Swift pics)**
+
+https://github.com/abi/screenshot-to-code/assets/23818/503eb86a-356e-4dfc-926a-dabdb1ac7ba1
+
+**Hacker News** but it gets the colors wrong at first so we nudge it
https://github.com/abi/screenshot-to-code/assets/23818/3fec0f77-44e8-4fb3-a769-ac7410315e5d
## Hosted Version
-Hosted version coming soon on [Pico](https://picoapps.xyz?ref=github).
+🆕 [Try it here](https://picoapps.xyz/free-tools/screenshot-to-code) (bring your own OpenAI key - **your key must have access to GPT-4 Vision. See [FAQ](#faqs) section for details**). Or see [Getting Started](#getting-started) for local install instructions.
diff --git a/backend/build.sh b/backend/build.sh
new file mode 100644
index 0000000..baab923
--- /dev/null
+++ b/backend/build.sh
@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+# exit on error
+set -o errexit
+
+echo "Installing the latest version of poetry..."
+pip install --upgrade pip
+pip install poetry==1.4.1
+
+rm poetry.lock
+poetry lock
+python -m poetry install
diff --git a/backend/image_generation.py b/backend/image_generation.py
index 5d4b81c..080334f 100644
--- a/backend/image_generation.py
+++ b/backend/image_generation.py
@@ -5,8 +5,8 @@ from openai import AsyncOpenAI
from bs4 import BeautifulSoup
-async def process_tasks(prompts):
- tasks = [generate_image(prompt) for prompt in prompts]
+async def process_tasks(prompts, api_key):
+ tasks = [generate_image(prompt, api_key) for prompt in prompts]
results = await asyncio.gather(*tasks, return_exceptions=True)
processed_results = []
@@ -20,8 +20,8 @@ async def process_tasks(prompts):
return processed_results
-async def generate_image(prompt):
- client = AsyncOpenAI(api_key=os.environ.get("OPENAI_API_KEY"))
+async def generate_image(prompt, api_key):
+ client = AsyncOpenAI(api_key=api_key)
image_params = {
"model": "dall-e-3",
"quality": "standard",
@@ -60,7 +60,7 @@ def create_alt_url_mapping(code):
return mapping
-async def generate_images(code, image_cache):
+async def generate_images(code, api_key, image_cache):
# Find all images
soup = BeautifulSoup(code, "html.parser")
images = soup.find_all("img")
@@ -82,8 +82,12 @@ async def generate_images(code, image_cache):
# Remove duplicates
prompts = list(set(alts))
+ # Return early if there are no images to replace
+ if len(prompts) == 0:
+ return code
+
# Generate images
- results = await process_tasks(prompts)
+ results = await process_tasks(prompts, api_key)
# Create a dict mapping alt text to image URL
mapped_image_urls = dict(zip(prompts, results))
@@ -110,4 +114,5 @@ async def generate_images(code, image_cache):
print("Image generation failed for alt text:" + img.get("alt"))
# Return the modified HTML
- return str(soup)
+ # (need to prettify it because BeautifulSoup messes up the formatting)
+ return soup.prettify()
diff --git a/backend/llm.py b/backend/llm.py
index 686b008..b52c3c9 100644
--- a/backend/llm.py
+++ b/backend/llm.py
@@ -4,10 +4,12 @@ from openai import AsyncOpenAI
MODEL_GPT_4_VISION = "gpt-4-vision-preview"
-client = AsyncOpenAI(api_key=os.environ.get("OPENAI_API_KEY"))
+async def stream_openai_response(
+ messages, api_key: str, callback: Callable[[str], Awaitable[None]]
+):
+ client = AsyncOpenAI(api_key=api_key)
-async def stream_openai_response(messages, callback: Callable[[str], Awaitable[None]]):
model = MODEL_GPT_4_VISION
# Base parameters
diff --git a/backend/main.py b/backend/main.py
index 5abd333..c6e5556 100644
--- a/backend/main.py
+++ b/backend/main.py
@@ -11,7 +11,7 @@ from datetime import datetime
from fastapi import FastAPI, WebSocket
from llm import stream_openai_response
-from mock import MOCK_HTML, mock_completion
+from mock import mock_completion
from image_generation import create_alt_url_mapping, generate_images
from prompts import assemble_prompt
@@ -23,12 +23,18 @@ SHOULD_MOCK_AI_RESPONSE = False
def write_logs(prompt_messages, completion):
- # Create run_logs directory if it doesn't exist
- if not os.path.exists("run_logs"):
- os.makedirs("run_logs")
+ # Get the logs path from environment, default to the current working directory
+ logs_path = os.environ.get("LOGS_PATH", os.getcwd())
- # Generate a unique filename using the current timestamp
- filename = datetime.now().strftime("run_logs/messages_%Y%m%d_%H%M%S.json")
+ # Create run_logs directory if it doesn't exist within the specified logs path
+ logs_directory = os.path.join(logs_path, "run_logs")
+ if not os.path.exists(logs_directory):
+ os.makedirs(logs_directory)
+
+ print("Writing to logs directory:", logs_directory)
+
+ # Generate a unique filename using the current timestamp within the logs directory
+ filename = datetime.now().strftime(f"{logs_directory}/messages_%Y%m%d_%H%M%S.json")
# Write the messages dict into a new file for each run
with open(filename, "w") as f:
@@ -41,6 +47,33 @@ async def stream_code_test(websocket: WebSocket):
params = await websocket.receive_json()
+ # Get the OpenAI API key from the request. Fall back to environment variable if not provided.
+ # If neither is provided, we throw an error.
+ if params["openAiApiKey"]:
+ openai_api_key = params["openAiApiKey"]
+ print("Using OpenAI API key from client-side settings dialog")
+ else:
+ openai_api_key = os.environ.get("OPENAI_API_KEY")
+ if openai_api_key:
+ print("Using OpenAI API key from environment variable")
+
+ if not openai_api_key:
+ print("OpenAI API key not found")
+ await websocket.send_json(
+ {
+ "type": "error",
+ "value": "No OpenAI API key found. Please add your API key in the settings dialog or add it to backend/.env file.",
+ }
+ )
+ return
+
+ should_generate_images = (
+ params["isImageGenerationEnabled"]
+ if "isImageGenerationEnabled" in params
+ else True
+ )
+
+ print("generating code...")
await websocket.send_json({"type": "status", "value": "Generating code..."})
async def process_chunk(content):
@@ -66,17 +99,23 @@ async def stream_code_test(websocket: WebSocket):
else:
completion = await stream_openai_response(
prompt_messages,
- lambda x: process_chunk(x),
+ api_key=openai_api_key,
+ callback=lambda x: process_chunk(x),
)
# Write the messages dict into a log so that we can debug later
write_logs(prompt_messages, completion)
- # Generate images
- await websocket.send_json({"type": "status", "value": "Generating images..."})
-
try:
- updated_html = await generate_images(completion, image_cache=image_cache)
+ if should_generate_images:
+ await websocket.send_json(
+ {"type": "status", "value": "Generating images..."}
+ )
+ updated_html = await generate_images(
+ completion, api_key=openai_api_key, image_cache=image_cache
+ )
+ else:
+ updated_html = completion
await websocket.send_json({"type": "setCode", "value": updated_html})
await websocket.send_json(
{"type": "status", "value": "Code generation complete."}
diff --git a/backend/mock.py b/backend/mock.py
index ec26339..90dc7d3 100644
--- a/backend/mock.py
+++ b/backend/mock.py
@@ -2,7 +2,7 @@ import asyncio
async def mock_completion(process_chunk):
- code_to_return = MOCK_HTML_2
+ code_to_return = NO_IMAGES_NYTIMES_MOCK_CODE
for i in range(0, len(code_to_return), 10):
await process_chunk(code_to_return[i : i + 10])
@@ -11,7 +11,7 @@ async def mock_completion(process_chunk):
return code_to_return
-MOCK_HTML = """
+APPLE_MOCK_CODE = """
@@ -68,7 +68,7 @@ MOCK_HTML = """
"""
-MOCK_HTML_2 = """
+NYTIMES_MOCK_CODE = """
@@ -138,3 +138,70 @@ MOCK_HTML_2 = """
"""
+
+NO_IMAGES_NYTIMES_MOCK_CODE = """
+
+
+
+
+ The New York Times - News
+
+
+
+
+
+
+
+
+
+
+
+
+
Tuesday, November 14, 2023 Today's Paper
+
+
+
+
Account
+
+
+
+
+
+
+
+
+
+
Israeli Military Raids Gaza’s Largest Hospital
+
Israeli troops have entered the Al-Shifa Hospital complex, where conditions have grown dire and Israel says Hamas fighters are embedded.
-
- Preview
+
+ Desktop
+
+
+ Mobile
@@ -193,8 +224,11 @@ function App() {
-
-
+
+
+
+
+
diff --git a/frontend/src/components/CodeMirror.tsx b/frontend/src/components/CodeMirror.tsx
index 7c6d9df..3ba1c51 100644
--- a/frontend/src/components/CodeMirror.tsx
+++ b/frontend/src/components/CodeMirror.tsx
@@ -60,6 +60,6 @@ function CodeMirror({ code }: Props) {
}
}, [code]);
- return ;
+ return ;
}
export default CodeMirror;
diff --git a/frontend/src/components/ImageUpload.tsx b/frontend/src/components/ImageUpload.tsx
index 16b5373..4a46d99 100644
--- a/frontend/src/components/ImageUpload.tsx
+++ b/frontend/src/components/ImageUpload.tsx
@@ -1,4 +1,4 @@
-import { useState, useEffect, useMemo } from "react";
+import { useState, useEffect, useMemo, useCallback } from "react";
import { useDropzone } from "react-dropzone";
// import { PromptImage } from "../../../types";
import { toast } from "react-hot-toast";
@@ -35,7 +35,7 @@ const rejectStyle = {
borderColor: "#ff1744",
};
-// TODO: Move to a seperate file
+// TODO: Move to a separate file
function fileToDataURL(file: File) {
return new Promise((resolve, reject) => {
const reader = new FileReader();
@@ -80,7 +80,7 @@ function ImageUpload({ setReferenceImages }: Props) {
setReferenceImages(dataUrls.map((dataUrl) => dataUrl as string));
})
.catch((error) => {
- // TODO: Display error to user
+ toast.error("Error reading files" + error);
console.error("Error reading files:", error);
});
},
@@ -89,6 +89,38 @@ function ImageUpload({ setReferenceImages }: Props) {
},
});
+ const pasteEvent = useCallback(
+ (event: ClipboardEvent) => {
+ const clipboardData = event.clipboardData;
+ if (!clipboardData) return;
+
+ const items = clipboardData.items;
+ const files = [];
+ for (let i = 0; i < items.length; i++) {
+ const file = items[i].getAsFile();
+ if (file && file.type.startsWith("image/")) {
+ files.push(file);
+ }
+ }
+
+ // Convert images to data URLs and set the prompt images state
+ Promise.all(files.map((file) => fileToDataURL(file)))
+ .then((dataUrls) => {
+ setReferenceImages(dataUrls.map((dataUrl) => dataUrl as string));
+ })
+ .catch((error) => {
+ // TODO: Display error to user
+ console.error("Error reading files:", error);
+ });
+ },
+ [setReferenceImages]
+ );
+
+ // TODO: Make sure we don't listen to paste events in text input components
+ useEffect(() => {
+ window.addEventListener("paste", pasteEvent);
+ }, [pasteEvent]);
+
useEffect(() => {
return () => files.forEach((file) => URL.revokeObjectURL(file.preview));
}, [files]); // Added files as a dependency
@@ -108,7 +140,7 @@ function ImageUpload({ setReferenceImages }: Props) {
{/* eslint-disable-next-line @typescript-eslint/no-explicit-any */}
-
Drop a screenshot here, or click to select
+
Drop a screenshot here, paste from clipboard, or click to select
);
diff --git a/frontend/src/components/OnboardingNote.tsx b/frontend/src/components/OnboardingNote.tsx
new file mode 100644
index 0000000..437bc1e
--- /dev/null
+++ b/frontend/src/components/OnboardingNote.tsx
@@ -0,0 +1,20 @@
+export function OnboardingNote() {
+ return (
+
+ Please add your OpenAI API key (must have GPT4 vision access) in the
+ settings dialog (gear icon above).
+
+
+ How do you get an OpenAI API key that has the GPT4 Vision model available?
+ Create an OpenAI account. And then, you need to buy at least $1 worth of
+ credit on the Billing dashboard.
+
+
+ This key is never stored. This app is open source. You can{" "}
+
+ check the code to confirm.
+
+
+
+ );
+}
diff --git a/frontend/src/components/PicoBadge.tsx b/frontend/src/components/PicoBadge.tsx
new file mode 100644
index 0000000..077a99d
--- /dev/null
+++ b/frontend/src/components/PicoBadge.tsx
@@ -0,0 +1,12 @@
+export function PicoBadge() {
+ return (
+
+