diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 0000000..f3c0ced --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1 @@ +github: [abi] diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 0000000..386e7e3 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,21 @@ +--- +name: Bug report +about: Create a report to help us improve +title: '' +labels: '' +assignees: '' + +--- + +**Describe the bug** +A clear and concise description of what the bug is. + +**To Reproduce** +Steps to reproduce the behavior: +1. Go to '...' +2. Click on '....' +3. Scroll down to '....' +4. See error + +**Screenshots of backend AND frontend terminal logs** +If applicable, add screenshots to help explain your problem. diff --git a/.github/ISSUE_TEMPLATE/custom.md b/.github/ISSUE_TEMPLATE/custom.md new file mode 100644 index 0000000..48d5f81 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/custom.md @@ -0,0 +1,10 @@ +--- +name: Custom issue template +about: Describe this issue template's purpose here. +title: '' +labels: '' +assignees: '' + +--- + + diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 0000000..bbcbbe7 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,20 @@ +--- +name: Feature request +about: Suggest an idea for this project +title: '' +labels: '' +assignees: '' + +--- + +**Is your feature request related to a problem? Please describe.** +A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] + +**Describe the solution you'd like** +A clear and concise description of what you want to happen. + +**Describe alternatives you've considered** +A clear and concise description of any alternative solutions or features you've considered. + +**Additional context** +Add any other context or screenshots about the feature request here. diff --git a/README.md b/README.md index a017ed4..8f01c7f 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # screenshot-to-code -A simple tool to convert screenshots, mockups and Figma designs into clean, functional code using AI. +A simple tool to convert screenshots, mockups and Figma designs into clean, functional code using AI. **Now supporting GPT-4O!** https://github.com/abi/screenshot-to-code/assets/23818/6cebadae-2fe3-4986-ac6a-8fb9db030045 @@ -15,9 +15,10 @@ Supported stacks: Supported AI models: -- GPT-4 Turbo (Apr 2024) - Best model -- GPT-4 Vision (Nov 2023) - Good model that's better than GPT-4 Turbo on some inputs -- Claude 3 Sonnet - Faster, and on par or better than GPT-4 vision for many inputs +- GPT-4O - Best model! +- GPT-4 Turbo (Apr 2024) +- GPT-4 Vision (Nov 2023) +- Claude 3 Sonnet - DALL-E 3 for image generation See the [Examples](#-examples) section below for more demos. @@ -30,13 +31,22 @@ We also just added experimental support for taking a video/screen recording of a [Follow me on Twitter for updates](https://twitter.com/_abi_). -## πŸš€ Try It Out without no install +## Sponsors + + + +## πŸš€ Hosted Version [Try it live on the hosted version (paid)](https://screenshottocode.com). ## πŸ›  Getting Started -The app has a React/Vite frontend and a FastAPI backend. You will need an OpenAI API key with access to the GPT-4 Vision API or an Anthropic key if you want to use Claude Sonnet, or for experimental video support. +The app has a React/Vite frontend and a FastAPI backend. + +Keys needed: + +* [OpenAI API key with access to GPT-4](https://github.com/abi/screenshot-to-code/blob/main/Troubleshooting.md) +* Anthropic key (optional) - only if you want to use Claude Sonnet, or for experimental video support. Run the backend (I use Poetry for package management - `pip install poetry` if you don't have it): @@ -48,7 +58,7 @@ poetry shell poetry run uvicorn main:app --reload --port 7001 ``` -If you want to use Anthropic, add the `ANTHROPIC_API_KEY` to `backend/.env` with your API key from Anthropic. +If you want to use Anthropic, add `ANTHROPIC_API_KEY` to `backend/.env`. You can also set up the keys using the settings dialog on the front-end (click the gear icon after loading the frontend). Run the frontend: @@ -107,5 +117,3 @@ https://github.com/abi/screenshot-to-code/assets/23818/3fec0f77-44e8-4fb3-a769-a ## 🌍 Hosted Version πŸ†• [Try it here (paid)](https://screenshottocode.com). Or see [Getting Started](#-getting-started) for local install instructions to use with your own API keys. - -[!["Buy Me A Coffee"](https://www.buymeacoffee.com/assets/img/custom_images/orange_img.png)](https://www.buymeacoffee.com/abiraja) diff --git a/Troubleshooting.md b/Troubleshooting.md index 89aa3ba..dbb1186 100644 --- a/Troubleshooting.md +++ b/Troubleshooting.md @@ -1,4 +1,4 @@ -### Getting an OpenAI API key with GPT4-Vision model access +### Getting an OpenAI API key with GPT-4 model access You don't need a ChatGPT Pro account. Screenshot to code uses API keys from your OpenAI developer account. In order to get access to the GPT4 Vision model, log into your OpenAI account and then, follow these instructions: diff --git a/backend/image_generation.py b/backend/image_generation.py index b93792c..e3f609f 100644 --- a/backend/image_generation.py +++ b/backend/image_generation.py @@ -5,7 +5,7 @@ from openai import AsyncOpenAI from bs4 import BeautifulSoup -async def process_tasks(prompts: List[str], api_key: str, base_url: str): +async def process_tasks(prompts: List[str], api_key: str, base_url: str | None): tasks = [generate_image(prompt, api_key, base_url) for prompt in prompts] results = await asyncio.gather(*tasks, return_exceptions=True) @@ -15,22 +15,23 @@ async def process_tasks(prompts: List[str], api_key: str, base_url: str): print(f"An exception occurred: {result}") processed_results.append(None) else: - processed_results.append(result) # type: ignore + processed_results.append(result) return processed_results -async def generate_image(prompt: str, api_key: str, base_url: str): +async def generate_image( + prompt: str, api_key: str, base_url: str | None +) -> Union[str, None]: client = AsyncOpenAI(api_key=api_key, base_url=base_url) - image_params: Dict[str, Union[str, int]] = { - "model": "dall-e-3", - "quality": "standard", - "style": "natural", - "n": 1, - "size": "1024x1024", - "prompt": prompt, - } - res = await client.images.generate(**image_params) # type: ignore + res = await client.images.generate( + model="dall-e-3", + quality="standard", + style="natural", + n=1, + size="1024x1024", + prompt=prompt, + ) await client.close() return res.data[0].url @@ -63,13 +64,13 @@ def create_alt_url_mapping(code: str) -> Dict[str, str]: async def generate_images( code: str, api_key: str, base_url: Union[str, None], image_cache: Dict[str, str] -): +) -> str: # Find all images soup = BeautifulSoup(code, "html.parser") images = soup.find_all("img") # Extract alt texts as image prompts - alts = [] + alts: List[str | None] = [] for img in images: # Only include URL if the image starts with https://placehold.co # and it's not already in the image_cache @@ -77,26 +78,26 @@ async def generate_images( img["src"].startswith("https://placehold.co") and image_cache.get(img.get("alt")) is None ): - alts.append(img.get("alt", None)) # type: ignore + alts.append(img.get("alt", None)) # Exclude images with no alt text - alts = [alt for alt in alts if alt is not None] # type: ignore + filtered_alts: List[str] = [alt for alt in alts if alt is not None] # Remove duplicates - prompts = list(set(alts)) # type: ignore + prompts = list(set(filtered_alts)) # Return early if there are no images to replace - if len(prompts) == 0: # type: ignore + if len(prompts) == 0: return code # Generate images - results = await process_tasks(prompts, api_key, base_url) # type: ignore + results = await process_tasks(prompts, api_key, base_url) # Create a dict mapping alt text to image URL - mapped_image_urls = dict(zip(prompts, results)) # type: ignore + mapped_image_urls = dict(zip(prompts, results)) # Merge with image_cache - mapped_image_urls = {**mapped_image_urls, **image_cache} # type: ignore + mapped_image_urls = {**mapped_image_urls, **image_cache} # Replace old image URLs with the generated URLs for img in images: diff --git a/backend/llm.py b/backend/llm.py index ce01f15..53bf5f9 100644 --- a/backend/llm.py +++ b/backend/llm.py @@ -16,6 +16,7 @@ from utils import pprint_prompt class Llm(Enum): GPT_4_VISION = "gpt-4-vision-preview" GPT_4_TURBO_2024_04_09 = "gpt-4-turbo-2024-04-09" + GPT_4O_2024_05_13 = "gpt-4o-2024-05-13" CLAUDE_3_SONNET = "claude-3-sonnet-20240229" CLAUDE_3_SONNET_BEDROCK = "anthropic.claude-3-sonnet-20240229-v1:0" CLAUDE_3_OPUS = "claude-3-opus-20240229" @@ -51,16 +52,21 @@ async def stream_openai_response( } # Add 'max_tokens' only if the model is a GPT4 vision or Turbo model - if model == Llm.GPT_4_VISION or model == Llm.GPT_4_TURBO_2024_04_09: + if ( + model == Llm.GPT_4_VISION + or model == Llm.GPT_4_TURBO_2024_04_09 + or model == Llm.GPT_4O_2024_05_13 + ): params["max_tokens"] = 4096 stream = await client.chat.completions.create(**params) # type: ignore full_response = "" async for chunk in stream: # type: ignore assert isinstance(chunk, ChatCompletionChunk) - content = chunk.choices[0].delta.content or "" - full_response += content - await callback(content) + if chunk.choices and len(chunk.choices) > 0 and chunk.choices[0].delta and chunk.choices[0].delta.content: + content = chunk.choices[0].delta.content or "" + full_response += content + await callback(content) await client.close() diff --git a/backend/routes/evals.py b/backend/routes/evals.py index 798a9d8..22262cd 100644 --- a/backend/routes/evals.py +++ b/backend/routes/evals.py @@ -7,10 +7,13 @@ from evals.config import EVALS_DIR router = APIRouter() +# Update this if the number of outputs generated per input changes +N = 1 + class Eval(BaseModel): input: str - output: str + outputs: list[str] @router.get("/evals") @@ -25,21 +28,27 @@ async def get_evals(): input_file_path = os.path.join(input_dir, file) input_file = await image_to_data_url(input_file_path) - # Construct the corresponding output file name - output_file_name = file.replace(".png", ".html") - output_file_path = os.path.join(output_dir, output_file_name) + # Construct the corresponding output file names + output_file_names = [ + file.replace(".png", f"_{i}.html") for i in range(0, N) + ] # Assuming 3 outputs for each input - # Check if the output file exists - if os.path.exists(output_file_path): - with open(output_file_path, "r") as f: - output_file_data = f.read() - else: - output_file_data = "Output file not found." + output_files_data: list[str] = [] + for output_file_name in output_file_names: + output_file_path = os.path.join(output_dir, output_file_name) + # Check if the output file exists + if os.path.exists(output_file_path): + with open(output_file_path, "r") as f: + output_files_data.append(f.read()) + else: + output_files_data.append( + "

Output file not found.

" + ) evals.append( Eval( input=input_file, - output=output_file_data, + outputs=output_files_data, ) ) diff --git a/backend/routes/generate_code.py b/backend/routes/generate_code.py index e294bde..ff924a1 100644 --- a/backend/routes/generate_code.py +++ b/backend/routes/generate_code.py @@ -13,7 +13,7 @@ from llm import ( ) from openai.types.chat import ChatCompletionMessageParam from mock_llm import mock_completion -from typing import Dict, List, cast, get_args +from typing import Dict, List, Union, cast, get_args from image_generation import create_alt_url_mapping, generate_images from prompts import assemble_imported_code_prompt, assemble_prompt from datetime import datetime @@ -84,7 +84,7 @@ async def stream_code(websocket: WebSocket): # Read the model from the request. Fall back to default if not provided. code_generation_model_str = params.get( - "codeGenerationModel", Llm.GPT_4_VISION.value + "codeGenerationModel", Llm.GPT_4O_2024_05_13.value ) try: code_generation_model = convert_frontend_str_to_llm(code_generation_model_str) @@ -111,6 +111,7 @@ async def stream_code(websocket: WebSocket): if not openai_api_key and ( code_generation_model == Llm.GPT_4_VISION or code_generation_model == Llm.GPT_4_TURBO_2024_04_09 + or code_generation_model == Llm.GPT_4O_2024_05_13 ): print("OpenAI API key not found") await throw_error( @@ -118,8 +119,19 @@ async def stream_code(websocket: WebSocket): ) return + # Get the Anthropic API key from the request. Fall back to environment variable if not provided. + # If neither is provided, we throw an error later only if Claude is used. + anthropic_api_key = None + if "anthropicApiKey" in params and params["anthropicApiKey"]: + anthropic_api_key = params["anthropicApiKey"] + print("Using Anthropic API key from client-side settings dialog") + else: + anthropic_api_key = ANTHROPIC_API_KEY + if anthropic_api_key: + print("Using Anthropic API key from environment variable") + # Get the OpenAI Base URL from the request. Fall back to environment variable if not provided. - openai_base_url = None + openai_base_url: Union[str, None] = None # Disable user-specified OpenAI Base URL in prod if not os.environ.get("IS_PROD"): if "openAiBaseURL" in params and params["openAiBaseURL"]: @@ -217,53 +229,53 @@ async def stream_code(websocket: WebSocket): else: try: if validated_input_mode == "video": - if not ANTHROPIC_API_KEY and not AWS_ACCESS_KEY and not AWS_SECRET_ACCESS_KEY: - await throw_error( - "Video only works with Anthropic models. Neither Anthropic API key or AWS Access Key found. Please add the environment variable ANTHROPIC_API_KEY or AWS_ACCESS_KEY/AWS_SECRET_ACCESS_KEY to backend/.env" - ) - raise Exception("No Anthropic key") + if not anthropic_api_key and not AWS_ACCESS_KEY and not AWS_SECRET_ACCESS_KEY: + await throw_error( + "Video only works with Anthropic models. Neither Anthropic API key or AWS Access Key found. Please add the environment variable ANTHROPIC_API_KEY or AWS_ACCESS_KEY/AWS_SECRET_ACCESS_KEY to backend/.env or in the settings dialog" + ) + raise Exception("No Anthropic key") - if ANTHROPIC_API_KEY: - completion = await stream_claude_response_native( - system_prompt=VIDEO_PROMPT, - messages=prompt_messages, # type: ignore - api_key=ANTHROPIC_API_KEY, - callback=lambda x: process_chunk(x), - model=Llm.CLAUDE_3_OPUS, - include_thinking=True, - ) - else: - completion = await stream_claude_response_native_aws_bedrock( - system_prompt=VIDEO_PROMPT, - messages=prompt_messages, # type: ignore - access_key=AWS_ACCESS_KEY, - secret_access_key=AWS_SECRET_ACCESS_KEY, - aws_region_name=AWS_REGION_NAME, - callback=lambda x: process_chunk(x), - model=Llm.CLAUDE_3_OPUS, - include_thinking=True, - ) - exact_llm_version = Llm.CLAUDE_3_OPUS - elif code_generation_model == Llm.CLAUDE_3_SONNET: - if not ANTHROPIC_API_KEY and not AWS_ACCESS_KEY and not AWS_SECRET_ACCESS_KEY: - await throw_error( - "No Anthropic API key or AWS Access Key found. Please add the environment variable ANTHROPIC_API_KEY or AWS_ACCESS_KEY/AWS_SECRET_ACCESS_KEY to backend/.env" - ) - raise Exception("No Anthropic key") - if ANTHROPIC_API_KEY: - completion = await stream_claude_response( - prompt_messages, # type: ignore - api_key=ANTHROPIC_API_KEY, - callback=lambda x: process_chunk(x), - ) - else: - completion = await stream_claude_response_aws_bedrock( - prompt_messages, # type: ignore - access_key=AWS_ACCESS_KEY, - secret_access_key=AWS_SECRET_ACCESS_KEY, - aws_region_name=AWS_REGION_NAME, - callback=lambda x: process_chunk(x), - ) + if anthropic_api_key: + completion = await stream_claude_response_native( + system_prompt=VIDEO_PROMPT, + messages=prompt_messages, # type: ignore + api_key=anthropic_api_key, + callback=lambda x: process_chunk(x), + model=Llm.CLAUDE_3_OPUS, + include_thinking=True, + ) + else: + completion = await stream_claude_response_native_aws_bedrock( + system_prompt=VIDEO_PROMPT, + messages=prompt_messages, # type: ignore + access_key=AWS_ACCESS_KEY, + secret_access_key=AWS_SECRET_ACCESS_KEY, + aws_region_name=AWS_REGION_NAME, + callback=lambda x: process_chunk(x), + model=Llm.CLAUDE_3_OPUS, + include_thinking=True, + ) + exact_llm_version = Llm.CLAUDE_3_OPUS + elif code_generation_model == Llm.CLAUDE_3_SONNET: + if not anthropic_api_key and not AWS_ACCESS_KEY and not AWS_SECRET_ACCESS_KEY: + await throw_error( + "No Anthropic API key or AWS Access Key found. Please add the environment variable ANTHROPIC_API_KEY or AWS_ACCESS_KEY/AWS_SECRET_ACCESS_KEY to backend/.env or in the settings dialog" + ) + raise Exception("No Anthropic key") + if anthropic_api_key: + completion = await stream_claude_response( + prompt_messages, # type: ignore + api_key=anthropic_api_key, + callback=lambda x: process_chunk(x), + ) + else: + completion = await stream_claude_response_aws_bedrock( + prompt_messages, # type: ignore + access_key=AWS_ACCESS_KEY, + secret_access_key=AWS_SECRET_ACCESS_KEY, + aws_region_name=AWS_REGION_NAME, + callback=lambda x: process_chunk(x), + ) exact_llm_version = code_generation_model else: completion = await stream_openai_response( diff --git a/backend/run_evals.py b/backend/run_evals.py index f26c708..bbf355a 100644 --- a/backend/run_evals.py +++ b/backend/run_evals.py @@ -13,8 +13,9 @@ from evals.config import EVALS_DIR from evals.core import generate_code_core from evals.utils import image_to_data_url -STACK = "html_tailwind" -MODEL = Llm.CLAUDE_3_SONNET +STACK = "ionic_tailwind" +MODEL = Llm.GPT_4O_2024_05_13 +N = 1 # Number of outputs to generate async def main(): @@ -28,16 +29,21 @@ async def main(): for filename in evals: filepath = os.path.join(INPUT_DIR, filename) data_url = await image_to_data_url(filepath) - task = generate_code_core(image_url=data_url, stack=STACK, model=MODEL) - tasks.append(task) + for _ in range(N): # Generate N tasks for each input + task = generate_code_core(image_url=data_url, stack=STACK, model=MODEL) + tasks.append(task) results = await asyncio.gather(*tasks) os.makedirs(OUTPUT_DIR, exist_ok=True) - for filename, content in zip(evals, results): - # File name is derived from the original filename in evals - output_filename = f"{os.path.splitext(filename)[0]}.html" + for i, content in enumerate(results): + # Calculate index for filename and output number + eval_index = i // N + output_number = i % N + filename = evals[eval_index] + # File name is derived from the original filename in evals with an added output number + output_filename = f"{os.path.splitext(filename)[0]}_{output_number}.html" output_filepath = os.path.join(OUTPUT_DIR, output_filename) with open(output_filepath, "w") as file: file.write(content) diff --git a/backend/test_llm.py b/backend/test_llm.py index ec005a3..aeb02ab 100644 --- a/backend/test_llm.py +++ b/backend/test_llm.py @@ -24,6 +24,11 @@ class TestConvertFrontendStrToLlm(unittest.TestCase): Llm.GPT_4_TURBO_2024_04_09, "Should convert 'gpt-4-turbo-2024-04-09' to Llm.GPT_4_TURBO_2024_04_09", ) + self.assertEqual( + convert_frontend_str_to_llm("gpt-4o-2024-05-13"), + Llm.GPT_4O_2024_05_13, + "Should convert 'gpt-4o-2024-05-13' to Llm.GPT_4O_2024_05_13", + ) def test_convert_invalid_string_raises_exception(self): with self.assertRaises(ValueError): diff --git a/frontend/.gitignore b/frontend/.gitignore index 17ceca3..a0d3702 100644 --- a/frontend/.gitignore +++ b/frontend/.gitignore @@ -25,3 +25,6 @@ dist-ssr # Env files .env* + +# Test files +src/tests/results/ diff --git a/frontend/Dockerfile b/frontend/Dockerfile index b176926..8579e17 100644 --- a/frontend/Dockerfile +++ b/frontend/Dockerfile @@ -1,4 +1,4 @@ -FROM node:20.9-bullseye-slim +FROM node:22-bullseye-slim # Set the working directory in the container WORKDIR /app @@ -6,6 +6,9 @@ WORKDIR /app # Copy package.json and yarn.lock COPY package.json yarn.lock /app/ +# Set the environment variable to skip Puppeteer download +ENV PUPPETEER_SKIP_DOWNLOAD=true + # Install dependencies RUN yarn install diff --git a/frontend/jest.config.js b/frontend/jest.config.js new file mode 100644 index 0000000..310efb5 --- /dev/null +++ b/frontend/jest.config.js @@ -0,0 +1,9 @@ +export default { + preset: "ts-jest", + testEnvironment: "node", + setupFiles: ["/src/setupTests.ts"], + transform: { + "^.+\\.tsx?$": "ts-jest", + }, + testTimeout: 30000, +}; diff --git a/frontend/package.json b/frontend/package.json index 7109443..4652dc7 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -10,7 +10,7 @@ "build-hosted": "tsc && vite build --mode prod", "lint": "eslint . --ext ts,tsx --report-unused-disable-directives --max-warnings 0", "preview": "vite preview", - "test": "vitest" + "test": "jest" }, "dependencies": { "@codemirror/lang-html": "^6.4.6", @@ -46,21 +46,28 @@ "tailwindcss-animate": "^1.0.7", "thememirror": "^2.0.1", "vite-plugin-checker": "^0.6.2", - "webm-duration-fix": "^1.0.4" + "webm-duration-fix": "^1.0.4", + "zustand": "^4.5.2" }, "devDependencies": { + "@types/jest": "^29.5.12", "@types/node": "^20.9.0", + "@types/puppeteer": "^7.0.4", "@types/react": "^18.2.15", "@types/react-dom": "^18.2.7", "@typescript-eslint/eslint-plugin": "^6.0.0", "@typescript-eslint/parser": "^6.0.0", "@vitejs/plugin-react": "^4.0.3", "autoprefixer": "^10.4.16", + "dotenv": "^16.4.5", "eslint": "^8.45.0", "eslint-plugin-react-hooks": "^4.6.0", "eslint-plugin-react-refresh": "^0.4.3", + "jest": "^29.7.0", "postcss": "^8.4.31", + "puppeteer": "^22.6.4", "tailwindcss": "^3.3.5", + "ts-jest": "^29.1.2", "typescript": "^5.0.2", "vite": "^4.4.5", "vite-plugin-html": "^3.2.0", diff --git a/frontend/src/.env.jest.example b/frontend/src/.env.jest.example new file mode 100644 index 0000000..59bc657 --- /dev/null +++ b/frontend/src/.env.jest.example @@ -0,0 +1,2 @@ +TEST_SCREENSHOTONE_API_KEY= +TEST_ROOT_PATH= diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index eef3a11..5156a0f 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -40,6 +40,8 @@ import ModelSettingsSection from "./components/ModelSettingsSection"; import { extractHtml } from "./components/preview/extractHtml"; import useBrowserTabIndicator from "./hooks/useBrowserTabIndicator"; import TipLink from "./components/core/TipLink"; +import SelectAndEditModeToggleButton from "./components/select-and-edit/SelectAndEditModeToggleButton"; +import { useAppStore } from "./store/app-store"; const IS_OPENAI_DOWN = false; @@ -54,16 +56,19 @@ function App() { const [updateInstruction, setUpdateInstruction] = useState(""); const [isImportedFromCode, setIsImportedFromCode] = useState(false); + const { disableInSelectAndEditMode } = useAppStore(); + // Settings const [settings, setSettings] = usePersistedState( { openAiApiKey: null, openAiBaseURL: null, + anthropicApiKey: null, screenshotOneApiKey: null, isImageGenerationEnabled: true, editorTheme: EditorTheme.COBALT, generatedCodeConfig: Stack.HTML_TAILWIND, - codeGenerationModel: CodeGenerationModel.GPT_4_TURBO_2024_04_09, + codeGenerationModel: CodeGenerationModel.GPT_4O_2024_05_13, // Only relevant for hosted version isTermOfServiceAccepted: false, }, @@ -89,6 +94,14 @@ function App() { CodeGenerationModel.GPT_4_TURBO_2024_04_09 && settings.generatedCodeConfig === Stack.REACT_TAILWIND; + const showGpt4OMessage = + selectedCodeGenerationModel !== CodeGenerationModel.GPT_4O_2024_05_13 && + appState === AppState.INITIAL; + + const showSelectAndEditFeature = + selectedCodeGenerationModel === CodeGenerationModel.GPT_4O_2024_05_13 && + settings.generatedCodeConfig === Stack.HTML_TAILWIND; + // Indicate coding state using the browser tab's favicon and title useBrowserTabIndicator(appState === AppState.CODING); @@ -144,6 +157,7 @@ function App() { setAppHistory([]); setCurrentVersion(null); setShouldIncludeResultImage(false); + disableInSelectAndEditMode(); }; const regenerate = () => { @@ -232,7 +246,9 @@ function App() { parentIndex: parentVersion, code, inputs: { - prompt: updateInstruction, + prompt: params.history + ? params.history[params.history.length - 1] + : updateInstruction, }, }, ]; @@ -274,7 +290,10 @@ function App() { } // Subsequent updates - async function doUpdate() { + async function doUpdate( + updateInstruction: string, + selectedElement?: HTMLElement + ) { if (currentVersion === null) { toast.error( "No current version set. Contact support or open a Github issue." @@ -292,7 +311,17 @@ function App() { return; } - const updatedHistory = [...historyTree, updateInstruction]; + let modifiedUpdateInstruction = updateInstruction; + + // Send in a reference to the selected element if it exists + if (selectedElement) { + modifiedUpdateInstruction = + updateInstruction + + " referring to this element specifically: " + + selectedElement.outerHTML; + } + + const updatedHistory = [...historyTree, modifiedUpdateInstruction]; if (shouldIncludeResultImage) { const resultImage = await takeScreenshot(); @@ -403,6 +432,15 @@ function App() { )} + {showGpt4OMessage && ( +
+

+ Now supporting GPT-4o. Higher quality and 2x faster. Give it a + try! +

+
+ )} + {appState !== AppState.CODE_READY && } {IS_RUNNING_ON_CLOUD && !settings.openAiApiKey && } @@ -468,8 +506,8 @@ function App() { /> @@ -477,10 +515,13 @@ function App() {
+ {showSelectAndEditFeature && ( + + )}
@@ -586,7 +627,7 @@ function App() { @@ -609,10 +650,18 @@ function App() {
- + - + - +

Drag & drop a screenshot here,
or click to upload diff --git a/frontend/src/components/ImportCodeSection.tsx b/frontend/src/components/ImportCodeSection.tsx index b320a97..c31e753 100644 --- a/frontend/src/components/ImportCodeSection.tsx +++ b/frontend/src/components/ImportCodeSection.tsx @@ -38,7 +38,9 @@ function ImportCodeSection({ importFromCode }: Props) { return (

- + @@ -62,7 +64,7 @@ function ImportCodeSection({ importFromCode }: Props) { /> - diff --git a/frontend/src/components/Preview.tsx b/frontend/src/components/Preview.tsx index eb9ea6d..d601f78 100644 --- a/frontend/src/components/Preview.tsx +++ b/frontend/src/components/Preview.tsx @@ -1,21 +1,35 @@ -import { useEffect, useRef } from "react"; +import { useEffect, useRef, useState } from "react"; import classNames from "classnames"; import useThrottle from "../hooks/useThrottle"; +import EditPopup from "./select-and-edit/EditPopup"; interface Props { code: string; device: "mobile" | "desktop"; + doUpdate: (updateInstruction: string, selectedElement?: HTMLElement) => void; } -function Preview({ code, device }: Props) { +function Preview({ code, device, doUpdate }: Props) { const iframeRef = useRef(null); // Don't update code more often than every 200ms. const throttledCode = useThrottle(code, 200); + // Select and edit functionality + const [clickEvent, setClickEvent] = useState(null); + useEffect(() => { - if (iframeRef.current) { - iframeRef.current.srcdoc = throttledCode; + const iframe = iframeRef.current; + if (iframe) { + iframe.srcdoc = throttledCode; + + // Set up click handler for select and edit funtionality + iframe.addEventListener("load", function () { + iframe.contentWindow?.document.body.addEventListener( + "click", + setClickEvent + ); + }); } }, [throttledCode]); @@ -34,6 +48,7 @@ function Preview({ code, device }: Props) { } )} > + ); } diff --git a/frontend/src/components/SettingsDialog.tsx b/frontend/src/components/SettingsDialog.tsx index 2e7814b..97d8f38 100644 --- a/frontend/src/components/SettingsDialog.tsx +++ b/frontend/src/components/SettingsDialog.tsx @@ -49,7 +49,7 @@ function SettingsDialog({ settings, setSettings }: Props) {