diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml
new file mode 100644
index 0000000..f3c0ced
--- /dev/null
+++ b/.github/FUNDING.yml
@@ -0,0 +1 @@
+github: [abi]
diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md
new file mode 100644
index 0000000..386e7e3
--- /dev/null
+++ b/.github/ISSUE_TEMPLATE/bug_report.md
@@ -0,0 +1,21 @@
+---
+name: Bug report
+about: Create a report to help us improve
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**To Reproduce**
+Steps to reproduce the behavior:
+1. Go to '...'
+2. Click on '....'
+3. Scroll down to '....'
+4. See error
+
+**Screenshots of backend AND frontend terminal logs**
+If applicable, add screenshots to help explain your problem.
diff --git a/.github/ISSUE_TEMPLATE/custom.md b/.github/ISSUE_TEMPLATE/custom.md
new file mode 100644
index 0000000..48d5f81
--- /dev/null
+++ b/.github/ISSUE_TEMPLATE/custom.md
@@ -0,0 +1,10 @@
+---
+name: Custom issue template
+about: Describe this issue template's purpose here.
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+
diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md
new file mode 100644
index 0000000..bbcbbe7
--- /dev/null
+++ b/.github/ISSUE_TEMPLATE/feature_request.md
@@ -0,0 +1,20 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+**Is your feature request related to a problem? Please describe.**
+A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+
+**Describe the solution you'd like**
+A clear and concise description of what you want to happen.
+
+**Describe alternatives you've considered**
+A clear and concise description of any alternative solutions or features you've considered.
+
+**Additional context**
+Add any other context or screenshots about the feature request here.
diff --git a/.vscode/settings.json b/.vscode/settings.json
index d6e2638..bfbce46 100644
--- a/.vscode/settings.json
+++ b/.vscode/settings.json
@@ -1,3 +1,5 @@
{
- "python.analysis.typeCheckingMode": "strict"
+ "python.analysis.typeCheckingMode": "strict",
+ "python.analysis.extraPaths": ["./backend"],
+ "python.autoComplete.extraPaths": ["./backend"]
}
diff --git a/Evaluation.md b/Evaluation.md
new file mode 100644
index 0000000..e937b78
--- /dev/null
+++ b/Evaluation.md
@@ -0,0 +1,19 @@
+## Evaluating models and prompts
+
+Evaluation dataset consists of 16 screenshots. A Python script for running screenshot-to-code on the dataset and a UI for rating outputs is included. With this set up, we can compare and evaluate various models and prompts.
+
+### Running evals
+
+- Input screenshots should be located at `backend/evals_data/inputs` and the outputs will be `backend/evals_data/outputs`. If you want to modify this, modify `EVALS_DIR` in `backend/evals/config.py`. You can download the input screenshot dataset here: TODO.
+- Set a stack and model (`STACK` var, `MODEL` var) in `backend/run_evals.py`
+- Run `OPENAI_API_KEY=sk-... python run_evals.py` - this runs the screenshot-to-code on the input dataset in parallel but it will still take a few minutes to complete.
+- Once the script is done, you can find the outputs in `backend/evals_data/outputs`.
+
+### Rating evals
+
+In order to view and rate the outputs, visit your front-end at `/evals`.
+
+- Rate each output on a scale of 1-4
+- You can also print the page as PDF to share your results with others.
+
+Generally, I run three tests for each model/prompt + stack combo and take the average score out of those tests to evaluate.
diff --git a/README.md b/README.md
index bc82497..7a7531a 100644
--- a/README.md
+++ b/README.md
@@ -1,32 +1,48 @@
# screenshot-to-code
-This simple app converts a screenshot to code (HTML/Tailwind CSS, or React or Bootstrap or Vue). It uses GPT-4 Vision to generate the code and DALL-E 3 to generate similar-looking images. You can now also enter a URL to clone a live website!
+A simple tool to convert screenshots, mockups and Figma designs into clean, functional code using AI. **Now supporting GPT-4O!**
https://github.com/abi/screenshot-to-code/assets/23818/6cebadae-2fe3-4986-ac6a-8fb9db030045
+Supported stacks:
+
+- HTML + Tailwind
+- React + Tailwind
+- Vue + Tailwind
+- Bootstrap
+- Ionic + Tailwind
+- SVG
+
+Supported AI models:
+
+- GPT-4O - Best model!
+- GPT-4 Turbo (Apr 2024)
+- GPT-4 Vision (Nov 2023)
+- Claude 3 Sonnet
+- DALL-E 3 for image generation
+
See the [Examples](#-examples) section below for more demos.
-## ๐ Try It Out!
+We also just added experimental support for taking a video/screen recording of a website in action and turning that into a functional prototype.
-๐ [Try it here](https://screenshottocode.com) (bring your own OpenAI key - **your key must have access to GPT-4 Vision. See [FAQ](#%EF%B8%8F-faqs) section below for details**). Or see [Getting Started](#-getting-started) below for local install instructions.
+
-## ๐ Recent Updates
+[Learn more about video here](https://github.com/abi/screenshot-to-code/wiki/Screen-Recording-to-Code).
-- Dec 11 - Start a new project from existing code (allows you to come back to an older project)
-- Dec 7 - ๐ฅ ๐ฅ ๐ฅ View a history of your edits, and branch off them
-- Nov 30 - Dark mode, output code in Ionic (thanks [@dialmedu](https://github.com/dialmedu)), set OpenAI base URL
-- Nov 28 - ๐ฅ ๐ฅ ๐ฅ Customize your stack: React or Bootstrap or TailwindCSS
-- Nov 23 - Send in a screenshot of the current replicated version (sometimes improves quality of subsequent generations)
-- Nov 21 - Edit code in the code editor and preview changes live thanks to [@clean99](https://github.com/clean99)
-- Nov 20 - Paste in a URL to screenshot and clone (requires [ScreenshotOne free API key](https://screenshotone.com?via=screenshot-to-code))
-- Nov 19 - Support for dark/light code editor theme - thanks [@kachbit](https://github.com/kachbit)
-- Nov 16 - Added a setting to disable DALL-E image generation if you don't need that
-- Nov 16 - View code directly within the app
-- Nov 15 - You can now instruct the AI to update the code as you wish. It is helpful if the AI messed up some styles or missed a section.
+[Follow me on Twitter for updates](https://twitter.com/_abi_).
+
+## Sponsors
+
+
+
+
+## ๐ Try It Out without no install
+
+[Try it live on the hosted version (paid)](https://screenshottocode.com).
## ๐ Getting Started
-The app has a React/Vite frontend and a FastAPI backend. You will need an OpenAI API key with access to the GPT-4 Vision API.
+The app has a React/Vite frontend and a FastAPI backend. You will need an OpenAI API key with access to the GPT-4 Vision API or an Anthropic key if you want to use Claude Sonnet, or for experimental video support.
Run the backend (I use Poetry for package management - `pip install poetry` if you don't have it):
@@ -38,11 +54,7 @@ poetry shell
poetry run uvicorn main:app --reload --port 7001
```
-You can also run the backend (when you're in `backend`):
-
-```bash
-poetry run pyright
-```
+If you want to use Anthropic, add the `ANTHROPIC_API_KEY` to `backend/.env` with your API key from Anthropic.
Run the frontend:
@@ -62,10 +74,6 @@ For debugging purposes, if you don't want to waste GPT4-Vision credits, you can
MOCK=true poetry run uvicorn main:app --reload --port 7001
```
-## Configuration
-
-- You can configure the OpenAI base URL if you need to use a proxy: Set OPENAI_BASE_URL in the `backend/.env` or directly in the UI in the settings dialog
-
## Docker
If you have Docker installed on your system, in the root directory, run:
@@ -81,6 +89,9 @@ The app will be up and running at http://localhost:5173. Note that you can't dev
- **I'm running into an error when setting up the backend. How can I fix it?** [Try this](https://github.com/abi/screenshot-to-code/issues/3#issuecomment-1814777959). If that still doesn't work, open an issue.
- **How do I get an OpenAI API key?** See https://github.com/abi/screenshot-to-code/blob/main/Troubleshooting.md
+- **How can I configure an OpenAI proxy?** - If you're not able to access the OpenAI API directly (due to e.g. country restrictions), you can try a VPN or you can configure the OpenAI base URL to use a proxy: Set OPENAI_BASE_URL in the `backend/.env` or directly in the UI in the settings dialog. Make sure the URL has "v1" in the path so it should look like this: `https://xxx.xxxxx.xxx/v1`
+- **How can I update the backend host that my front-end connects to?** - Configure VITE_HTTP_BACKEND_URL and VITE_WS_BACKEND_URL in front/.env.local For example, set VITE_HTTP_BACKEND_URL=http://124.10.20.1:7001
+- **Seeing UTF-8 errors when running the backend?** - On windows, open the .env file with notepad++, then go to Encoding and select UTF-8.
- **How can I provide feedback?** For feedback, feature requests and bug reports, open an issue or ping me on [Twitter](https://twitter.com/_abi_).
## ๐ Examples
@@ -101,6 +112,6 @@ https://github.com/abi/screenshot-to-code/assets/23818/3fec0f77-44e8-4fb3-a769-a
## ๐ Hosted Version
-๐ [Try it here](https://screenshottocode.com) (bring your own OpenAI key - **your key must have access to GPT-4 Vision. See [FAQ](#%EF%B8%8F-faqs) section for details**). Or see [Getting Started](#-getting-started) for local install instructions.
+๐ [Try it here (paid)](https://screenshottocode.com). Or see [Getting Started](#-getting-started) for local install instructions to use with your own API keys.
[](https://www.buymeacoffee.com/abiraja)
diff --git a/Troubleshooting.md b/Troubleshooting.md
index 20fa815..89aa3ba 100644
--- a/Troubleshooting.md
+++ b/Troubleshooting.md
@@ -5,13 +5,18 @@ You don't need a ChatGPT Pro account. Screenshot to code uses API keys from your
1. Open [OpenAI Dashboard](https://platform.openai.com/)
1. Go to Settings > Billing
1. Click at the Add payment details
-
+
+
4. You have to buy some credits. The minimum is $5.
-
5. Go to Settings > Limits and check at the bottom of the page, your current tier has to be "Tier 1" to have GPT4 access
-
-6. Go to Screenshot to code and paste it in the Settings dialog under OpenAI key (gear icon). Your key is only stored in your browser. Never stored on our servers.
+
-Some users have also reported that it can take upto 30 minutes after your credit purchase for the GPT4 vision model to be activated.
+6. Navigate to OpenAI [api keys](https://platform.openai.com/api-keys) page and create and copy a new secret key.
+7. Go to Screenshot to code and paste it in the Settings dialog under OpenAI key (gear icon). Your key is only stored in your browser. Never stored on our servers.
-If you've followed these steps, and it still doesn't work, feel free to open a Github issue.
+## Still not working?
+
+- Some users have also reported that it can take upto 30 minutes after your credit purchase for the GPT4 vision model to be activated.
+- You need to add credits to your account AND set it to renew when credits run out in order to be upgraded to Tier 1. Make sure your "Settings > Limits" page shows that you are at Tier 1.
+
+If you've followed these steps, and it still doesn't work, feel free to open a Github issue. We only provide support for the open source version since we don't have debugging logs on the hosted version. If you're looking to use the hosted version, we recommend getting a paid subscription on screenshottocode.com
diff --git a/backend/.gitignore b/backend/.gitignore
index a42aad3..5d03006 100644
--- a/backend/.gitignore
+++ b/backend/.gitignore
@@ -154,3 +154,7 @@ cython_debug/
# Temporary eval output
evals_data
+
+
+# Temporary video evals (Remove before merge)
+video_evals
diff --git a/backend/.pre-commit-config.yaml b/backend/.pre-commit-config.yaml
index 9a495ad..b54da93 100644
--- a/backend/.pre-commit-config.yaml
+++ b/backend/.pre-commit-config.yaml
@@ -16,3 +16,10 @@ repos:
pass_filenames: false
always_run: true
files: ^backend/
+ # - id: poetry-pyright
+ # name: Run pyright with Poetry
+ # entry: poetry run --directory backend pyright
+ # language: system
+ # pass_filenames: false
+ # always_run: true
+ # files: ^backend/
diff --git a/backend/access_token.py b/backend/access_token.py
deleted file mode 100644
index 35fa657..0000000
--- a/backend/access_token.py
+++ /dev/null
@@ -1,21 +0,0 @@
-import json
-import os
-import httpx
-
-
-async def validate_access_token(access_code: str):
- async with httpx.AsyncClient() as client:
- url = (
- "https://backend.buildpicoapps.com/screenshot_to_code/validate_access_token"
- )
- data = json.dumps(
- {
- "access_code": access_code,
- "secret": os.environ.get("PICO_BACKEND_SECRET"),
- }
- )
- headers = {"Content-Type": "application/json"}
-
- response = await client.post(url, content=data, headers=headers)
- response_data = response.json()
- return response_data
diff --git a/backend/config.py b/backend/config.py
index 7dd21f9..05592b0 100644
--- a/backend/config.py
+++ b/backend/config.py
@@ -3,8 +3,13 @@
# TODO: Should only be set to true when value is 'True', not any abitrary truthy value
import os
+ANTHROPIC_API_KEY = os.environ.get("ANTHROPIC_API_KEY", None)
+
+# Debugging-related
SHOULD_MOCK_AI_RESPONSE = bool(os.environ.get("MOCK", False))
+IS_DEBUG_ENABLED = bool(os.environ.get("IS_DEBUG_ENABLED", False))
+DEBUG_DIR = os.environ.get("DEBUG_DIR", "")
# Set to True when running in production (on the hosted version)
# Used as a feature flag to enable or disable certain features
diff --git a/backend/custom_types.py b/backend/custom_types.py
new file mode 100644
index 0000000..b6c9fee
--- /dev/null
+++ b/backend/custom_types.py
@@ -0,0 +1,7 @@
+from typing import Literal
+
+
+InputMode = Literal[
+ "image",
+ "video",
+]
diff --git a/backend/debug/DebugFileWriter.py b/backend/debug/DebugFileWriter.py
new file mode 100644
index 0000000..bbcd77b
--- /dev/null
+++ b/backend/debug/DebugFileWriter.py
@@ -0,0 +1,30 @@
+import os
+import logging
+import uuid
+
+from config import DEBUG_DIR, IS_DEBUG_ENABLED
+
+
+class DebugFileWriter:
+ def __init__(self):
+ if not IS_DEBUG_ENABLED:
+ return
+
+ try:
+ self.debug_artifacts_path = os.path.expanduser(
+ f"{DEBUG_DIR}/{str(uuid.uuid4())}"
+ )
+ os.makedirs(self.debug_artifacts_path, exist_ok=True)
+ print(f"Debugging artifacts will be stored in: {self.debug_artifacts_path}")
+ except:
+ logging.error("Failed to create debug directory")
+
+ def write_to_file(self, filename: str, content: str) -> None:
+ try:
+ with open(os.path.join(self.debug_artifacts_path, filename), "w") as file:
+ file.write(content)
+ except Exception as e:
+ logging.error(f"Failed to write to file: {e}")
+
+ def extract_html_content(self, text: str) -> str:
+ return str(text.split("")[-1].rsplit("", 1)[0] + "