From 06a022a8c013f2826b6d134ce8bd0ed8941495bd Mon Sep 17 00:00:00 2001
From: Leo <leowang657@gmail.com>
Date: Tue, 30 Apr 2024 16:32:38 -0700
Subject: [PATCH 01/19] changed node version from 20.9 to 22

---
 frontend/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/frontend/Dockerfile b/frontend/Dockerfile
index b176926..b779dd2 100644
--- a/frontend/Dockerfile
+++ b/frontend/Dockerfile
@@ -1,4 +1,4 @@
-FROM node:20.9-bullseye-slim
+FROM node:22-bullseye-slim
 
 # Set the working directory in the container
 WORKDIR /app

From ac8198e5ba9c29b7ec15e7f612623e9aa8f343ef Mon Sep 17 00:00:00 2001
From: Abi Raja <abimanyuraja@gmail.com>
Date: Fri, 17 May 2024 12:30:26 -0400
Subject: [PATCH 02/19] Add field for Anthropic API key in settings

---
 frontend/src/components/SettingsDialog.tsx | 20 ++++++++++++++++++++
 frontend/src/types.ts                      |  1 +
 2 files changed, 21 insertions(+)

diff --git a/frontend/src/components/SettingsDialog.tsx b/frontend/src/components/SettingsDialog.tsx
index 2e7814b..1a9c5a6 100644
--- a/frontend/src/components/SettingsDialog.tsx
+++ b/frontend/src/components/SettingsDialog.tsx
@@ -108,6 +108,26 @@ function SettingsDialog({ settings, setSettings }: Props) {
             </>
           )}
 
+          <Label htmlFor="anthropic-api-key">
+            <div>Anthropic API key</div>
+            <div className="font-light mt-2 leading-relaxed">
+              Only stored in your browser. Never stored on servers. Overrides
+              your .env config.
+            </div>
+          </Label>
+
+          <Input
+            id="anthropic-api-key"
+            placeholder="Anthropic API key"
+            value={settings.anthropicApiKey || ""}
+            onChange={(e) =>
+              setSettings((s) => ({
+                ...s,
+                anthropicApiKey: e.target.value,
+              }))
+            }
+          />
+
           <Accordion type="single" collapsible className="w-full">
             <AccordionItem value="item-1">
               <AccordionTrigger>Screenshot by URL Config</AccordionTrigger>
diff --git a/frontend/src/types.ts b/frontend/src/types.ts
index bbabeff..dbe10ad 100644
--- a/frontend/src/types.ts
+++ b/frontend/src/types.ts
@@ -16,6 +16,7 @@ export interface Settings {
   codeGenerationModel: CodeGenerationModel;
   // Only relevant for hosted version
   isTermOfServiceAccepted: boolean;
+  anthropicApiKey: string | null; // Added property for anthropic API key
 }
 
 export enum AppState {

From 1f9c5b2c80a52c285323350581d4cd605bda3097 Mon Sep 17 00:00:00 2001
From: Abi Raja <abimanyuraja@gmail.com>
Date: Fri, 17 May 2024 14:47:54 -0400
Subject: [PATCH 03/19] set initial value

---
 frontend/src/App.tsx | 1 +
 1 file changed, 1 insertion(+)

diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx
index fd1576e..c05c44f 100644
--- a/frontend/src/App.tsx
+++ b/frontend/src/App.tsx
@@ -59,6 +59,7 @@ function App() {
     {
       openAiApiKey: null,
       openAiBaseURL: null,
+      anthropicApiKey: null,
       screenshotOneApiKey: null,
       isImageGenerationEnabled: true,
       editorTheme: EditorTheme.COBALT,

From d31ebcaa27eaf882bc57e84bba6ce301d042638e Mon Sep 17 00:00:00 2001
From: Naman <1608naman@gmail.com>
Date: Wed, 29 May 2024 07:11:03 +0530
Subject: [PATCH 04/19] types fix in image generation file

---
 backend/image_generation.py     | 26 ++++++++++++++------------
 backend/routes/generate_code.py |  4 ++--
 2 files changed, 16 insertions(+), 14 deletions(-)

diff --git a/backend/image_generation.py b/backend/image_generation.py
index b93792c..a1c4629 100644
--- a/backend/image_generation.py
+++ b/backend/image_generation.py
@@ -11,16 +11,16 @@ async def process_tasks(prompts: List[str], api_key: str, base_url: str):
 
     processed_results: List[Union[str, None]] = []
     for result in results:
-        if isinstance(result, Exception):
+        if isinstance(result, BaseException):
             print(f"An exception occurred: {result}")
             processed_results.append(None)
         else:
-            processed_results.append(result)  # type: ignore
+            processed_results.append(result)
 
     return processed_results
 
 
-async def generate_image(prompt: str, api_key: str, base_url: str):
+async def generate_image(prompt: str, api_key: str, base_url: str) -> Union[str, None]:
     client = AsyncOpenAI(api_key=api_key, base_url=base_url)
     image_params: Dict[str, Union[str, int]] = {
         "model": "dall-e-3",
@@ -63,13 +63,15 @@ def create_alt_url_mapping(code: str) -> Dict[str, str]:
 
 async def generate_images(
     code: str, api_key: str, base_url: Union[str, None], image_cache: Dict[str, str]
-):
+) -> Union[str, None]:
+    if base_url is None:
+        return code
     # Find all images
     soup = BeautifulSoup(code, "html.parser")
     images = soup.find_all("img")
 
     # Extract alt texts as image prompts
-    alts = []
+    alts: List[str | None] = []
     for img in images:
         # Only include URL if the image starts with https://placehold.co
         # and it's not already in the image_cache
@@ -77,26 +79,26 @@ async def generate_images(
             img["src"].startswith("https://placehold.co")
             and image_cache.get(img.get("alt")) is None
         ):
-            alts.append(img.get("alt", None))  # type: ignore
+            alts.append(img.get("alt", None))
 
     # Exclude images with no alt text
-    alts = [alt for alt in alts if alt is not None]  # type: ignore
+    filtered_alts: List[str] = [alt for alt in alts if alt is not None]
 
     # Remove duplicates
-    prompts = list(set(alts))  # type: ignore
+    prompts = list(set(filtered_alts))
 
     # Return early if there are no images to replace
-    if len(prompts) == 0:  # type: ignore
+    if len(prompts) == 0:
         return code
 
     # Generate images
-    results = await process_tasks(prompts, api_key, base_url)  # type: ignore
+    results = await process_tasks(prompts, api_key, base_url)
 
     # Create a dict mapping alt text to image URL
-    mapped_image_urls = dict(zip(prompts, results))  # type: ignore
+    mapped_image_urls = dict(zip(prompts, results))
 
     # Merge with image_cache
-    mapped_image_urls = {**mapped_image_urls, **image_cache}  # type: ignore
+    mapped_image_urls = {**mapped_image_urls, **image_cache}
 
     # Replace old image URLs with the generated URLs
     for img in images:
diff --git a/backend/routes/generate_code.py b/backend/routes/generate_code.py
index e7186fc..fc7330e 100644
--- a/backend/routes/generate_code.py
+++ b/backend/routes/generate_code.py
@@ -13,7 +13,7 @@ from llm import (
 )
 from openai.types.chat import ChatCompletionMessageParam
 from mock_llm import mock_completion
-from typing import Dict, List, cast, get_args
+from typing import Dict, List, Union, cast, get_args
 from image_generation import create_alt_url_mapping, generate_images
 from prompts import assemble_imported_code_prompt, assemble_prompt
 from datetime import datetime
@@ -121,7 +121,7 @@ async def stream_code(websocket: WebSocket):
         return
 
     # Get the OpenAI Base URL from the request. Fall back to environment variable if not provided.
-    openai_base_url = None
+    openai_base_url: Union[str, None] = None
     # Disable user-specified OpenAI Base URL in prod
     if not os.environ.get("IS_PROD"):
         if "openAiBaseURL" in params and params["openAiBaseURL"]:

From c5695975c710a6dd5d3f8c311c53857ca7f73d4a Mon Sep 17 00:00:00 2001
From: Naman <1608naman@gmail.com>
Date: Sat, 25 May 2024 00:32:13 +0530
Subject: [PATCH 05/19] handle none type in llm.py

---
 backend/llm.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/backend/llm.py b/backend/llm.py
index e541046..44f97e1 100644
--- a/backend/llm.py
+++ b/backend/llm.py
@@ -59,9 +59,10 @@ async def stream_openai_response(
     full_response = ""
     async for chunk in stream:  # type: ignore
         assert isinstance(chunk, ChatCompletionChunk)
-        content = chunk.choices[0].delta.content or ""
-        full_response += content
-        await callback(content)
+        if chunk.choices and chunk.choices[0].delta.content:
+            content = chunk.choices[0].delta.content or ""
+            full_response += content
+            await callback(content)
 
     await client.close()
 

From f0ee68666092b2579147aabfc5846f2568c983a3 Mon Sep 17 00:00:00 2001
From: Naman <1608naman@gmail.com>
Date: Sat, 25 May 2024 07:02:02 +0530
Subject: [PATCH 06/19] null check for chunk.choices

---
 backend/llm.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/backend/llm.py b/backend/llm.py
index 44f97e1..e32051c 100644
--- a/backend/llm.py
+++ b/backend/llm.py
@@ -59,7 +59,7 @@ async def stream_openai_response(
     full_response = ""
     async for chunk in stream:  # type: ignore
         assert isinstance(chunk, ChatCompletionChunk)
-        if chunk.choices and chunk.choices[0].delta.content:
+        if chunk.choices and len(chunk.choices) > 0 and chunk.choices[0].delta and chunk.choices[0].delta.content:
             content = chunk.choices[0].delta.content or ""
             full_response += content
             await callback(content)

From 8e579e425ee0bb12baf19186b71ee8a7b840d266 Mon Sep 17 00:00:00 2001
From: Abi Raja <abimanyuraja@gmail.com>
Date: Fri, 31 May 2024 13:51:43 -0400
Subject: [PATCH 07/19] clean up settings dialog look

---
 frontend/src/components/SettingsDialog.tsx | 84 +++++++++++-----------
 1 file changed, 44 insertions(+), 40 deletions(-)

diff --git a/frontend/src/components/SettingsDialog.tsx b/frontend/src/components/SettingsDialog.tsx
index 1a9c5a6..97d8f38 100644
--- a/frontend/src/components/SettingsDialog.tsx
+++ b/frontend/src/components/SettingsDialog.tsx
@@ -49,7 +49,7 @@ function SettingsDialog({ settings, setSettings }: Props) {
         <div className="flex items-center space-x-2">
           <Label htmlFor="image-generation">
             <div>DALL-E Placeholder Image Generation</div>
-            <div className="font-light mt-2">
+            <div className="font-light mt-2 text-xs">
               More fun with it but if you want to save money, turn it off.
             </div>
           </Label>
@@ -64,29 +64,31 @@ function SettingsDialog({ settings, setSettings }: Props) {
             }
           />
         </div>
-        <div className="flex flex-col space-y-4">
-          <Label htmlFor="openai-api-key">
-            <div>OpenAI API key</div>
-            <div className="font-light mt-2 leading-relaxed">
-              Only stored in your browser. Never stored on servers. Overrides
-              your .env config.
-            </div>
-          </Label>
+        <div className="flex flex-col space-y-6">
+          <div>
+            <Label htmlFor="openai-api-key">
+              <div>OpenAI API key</div>
+              <div className="font-light mt-1 mb-2 text-xs leading-relaxed">
+                Only stored in your browser. Never stored on servers. Overrides
+                your .env config.
+              </div>
+            </Label>
 
-          <Input
-            id="openai-api-key"
-            placeholder="OpenAI API key"
-            value={settings.openAiApiKey || ""}
-            onChange={(e) =>
-              setSettings((s) => ({
-                ...s,
-                openAiApiKey: e.target.value,
-              }))
-            }
-          />
+            <Input
+              id="openai-api-key"
+              placeholder="OpenAI API key"
+              value={settings.openAiApiKey || ""}
+              onChange={(e) =>
+                setSettings((s) => ({
+                  ...s,
+                  openAiApiKey: e.target.value,
+                }))
+              }
+            />
+          </div>
 
           {!IS_RUNNING_ON_CLOUD && (
-            <>
+            <div>
               <Label htmlFor="openai-api-key">
                 <div>OpenAI Base URL (optional)</div>
                 <div className="font-light mt-2 leading-relaxed">
@@ -105,28 +107,30 @@ function SettingsDialog({ settings, setSettings }: Props) {
                   }))
                 }
               />
-            </>
+            </div>
           )}
 
-          <Label htmlFor="anthropic-api-key">
-            <div>Anthropic API key</div>
-            <div className="font-light mt-2 leading-relaxed">
-              Only stored in your browser. Never stored on servers. Overrides
-              your .env config.
-            </div>
-          </Label>
+          <div>
+            <Label htmlFor="anthropic-api-key">
+              <div>Anthropic API key</div>
+              <div className="font-light mt-1 text-xs leading-relaxed">
+                Only stored in your browser. Never stored on servers. Overrides
+                your .env config.
+              </div>
+            </Label>
 
-          <Input
-            id="anthropic-api-key"
-            placeholder="Anthropic API key"
-            value={settings.anthropicApiKey || ""}
-            onChange={(e) =>
-              setSettings((s) => ({
-                ...s,
-                anthropicApiKey: e.target.value,
-              }))
-            }
-          />
+            <Input
+              id="anthropic-api-key"
+              placeholder="Anthropic API key"
+              value={settings.anthropicApiKey || ""}
+              onChange={(e) =>
+                setSettings((s) => ({
+                  ...s,
+                  anthropicApiKey: e.target.value,
+                }))
+              }
+            />
+          </div>
 
           <Accordion type="single" collapsible className="w-full">
             <AccordionItem value="item-1">

From c64968b8f09e534224f69b31463bd197e9a8808e Mon Sep 17 00:00:00 2001
From: Abi Raja <abimanyuraja@gmail.com>
Date: Fri, 31 May 2024 14:06:08 -0400
Subject: [PATCH 08/19] read anthropic api key from front-end

---
 backend/routes/generate_code.py | 23 +++++++++++++++++------
 1 file changed, 17 insertions(+), 6 deletions(-)

diff --git a/backend/routes/generate_code.py b/backend/routes/generate_code.py
index e7186fc..61ce9c9 100644
--- a/backend/routes/generate_code.py
+++ b/backend/routes/generate_code.py
@@ -120,6 +120,17 @@ async def stream_code(websocket: WebSocket):
         )
         return
 
+    # Get the Anthropic API key from the request. Fall back to environment variable if not provided.
+    # If neither is provided, we throw an error later only if Claude is used.
+    anthropic_api_key = None
+    if params["anthropicApiKey"]:
+        anthropic_api_key = params["anthropicApiKey"]
+        print("Using Anthropic API key from client-side settings dialog")
+    else:
+        anthropic_api_key = ANTHROPIC_API_KEY
+        if anthropic_api_key:
+            print("Using Anthropic API key from environment variable")
+
     # Get the OpenAI Base URL from the request. Fall back to environment variable if not provided.
     openai_base_url = None
     # Disable user-specified OpenAI Base URL in prod
@@ -219,31 +230,31 @@ async def stream_code(websocket: WebSocket):
     else:
         try:
             if validated_input_mode == "video":
-                if not ANTHROPIC_API_KEY:
+                if not anthropic_api_key:
                     await throw_error(
-                        "Video only works with Anthropic models. No Anthropic API key found. Please add the environment variable ANTHROPIC_API_KEY to backend/.env"
+                        "Video only works with Anthropic models. No Anthropic API key found. Please add the environment variable ANTHROPIC_API_KEY to backend/.env or in the settings dialog"
                     )
                     raise Exception("No Anthropic key")
 
                 completion = await stream_claude_response_native(
                     system_prompt=VIDEO_PROMPT,
                     messages=prompt_messages,  # type: ignore
-                    api_key=ANTHROPIC_API_KEY,
+                    api_key=anthropic_api_key,
                     callback=lambda x: process_chunk(x),
                     model=Llm.CLAUDE_3_OPUS,
                     include_thinking=True,
                 )
                 exact_llm_version = Llm.CLAUDE_3_OPUS
             elif code_generation_model == Llm.CLAUDE_3_SONNET:
-                if not ANTHROPIC_API_KEY:
+                if not anthropic_api_key:
                     await throw_error(
-                        "No Anthropic API key found. Please add the environment variable ANTHROPIC_API_KEY to backend/.env"
+                        "No Anthropic API key found. Please add the environment variable ANTHROPIC_API_KEY to backend/.env or in the settings dialog"
                     )
                     raise Exception("No Anthropic key")
 
                 completion = await stream_claude_response(
                     prompt_messages,  # type: ignore
-                    api_key=ANTHROPIC_API_KEY,
+                    api_key=anthropic_api_key,
                     callback=lambda x: process_chunk(x),
                 )
                 exact_llm_version = code_generation_model

From f6f8b2f7075cdb97915f491c351167e6572d2e8e Mon Sep 17 00:00:00 2001
From: Abi Raja <abimanyuraja@gmail.com>
Date: Fri, 31 May 2024 14:24:01 -0400
Subject: [PATCH 09/19] fix bug

---
 backend/routes/generate_code.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/backend/routes/generate_code.py b/backend/routes/generate_code.py
index 61ce9c9..3fb92f1 100644
--- a/backend/routes/generate_code.py
+++ b/backend/routes/generate_code.py
@@ -123,7 +123,7 @@ async def stream_code(websocket: WebSocket):
     # Get the Anthropic API key from the request. Fall back to environment variable if not provided.
     # If neither is provided, we throw an error later only if Claude is used.
     anthropic_api_key = None
-    if params["anthropicApiKey"]:
+    if "anthropicApiKey" in params and params["anthropicApiKey"]:
         anthropic_api_key = params["anthropicApiKey"]
         print("Using Anthropic API key from client-side settings dialog")
     else:

From 2204896e3162574bdc608532b8aad8da6abc2529 Mon Sep 17 00:00:00 2001
From: Abi Raja <abimanyuraja@gmail.com>
Date: Fri, 31 May 2024 14:40:25 -0400
Subject: [PATCH 10/19] Update Troubleshooting.md

---
 Troubleshooting.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Troubleshooting.md b/Troubleshooting.md
index 89aa3ba..dbb1186 100644
--- a/Troubleshooting.md
+++ b/Troubleshooting.md
@@ -1,4 +1,4 @@
-### Getting an OpenAI API key with GPT4-Vision model access
+### Getting an OpenAI API key with GPT-4 model access
 
 You don't need a ChatGPT Pro account. Screenshot to code uses API keys from your OpenAI developer account. In order to get access to the GPT4 Vision model, log into your OpenAI account and then, follow these instructions:
 

From 328dc7931dda61320a1d45334828dfbe723c0c9b Mon Sep 17 00:00:00 2001
From: Abi Raja <abimanyuraja@gmail.com>
Date: Fri, 31 May 2024 14:44:11 -0400
Subject: [PATCH 11/19] Update README.md

---
 README.md | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/README.md b/README.md
index 1762f91..8f01c7f 100644
--- a/README.md
+++ b/README.md
@@ -35,14 +35,18 @@ We also just added experimental support for taking a video/screen recording of a
 
 <a href="https://konghq.com/products/kong-konnect/register?utm_medium=referral&utm_source=github&utm_campaign=platform&utm_content=screenshot-to-code" target="_blank" title="Kong - powering the API world"><img src="https://picoapps.xyz/s2c-sponsors/Kong-GitHub-240x100.png"></a>
 
-
-## 🚀 Try It Out without no install
+## 🚀 Hosted Version
 
 [Try it live on the hosted version (paid)](https://screenshottocode.com).
 
 ## 🛠 Getting Started
 
-The app has a React/Vite frontend and a FastAPI backend. You will need an OpenAI API key with access to the GPT-4 Vision API or an Anthropic key if you want to use Claude Sonnet, or for experimental video support.
+The app has a React/Vite frontend and a FastAPI backend. 
+
+Keys needed:
+
+* [OpenAI API key with access to GPT-4](https://github.com/abi/screenshot-to-code/blob/main/Troubleshooting.md)
+* Anthropic key (optional) - only if you want to use Claude Sonnet, or for experimental video support.
 
 Run the backend (I use Poetry for package management - `pip install poetry` if you don't have it):
 
@@ -54,7 +58,7 @@ poetry shell
 poetry run uvicorn main:app --reload --port 7001
 ```
 
-If you want to use Anthropic, add the `ANTHROPIC_API_KEY` to `backend/.env` with your API key from Anthropic.
+If you want to use Anthropic, add `ANTHROPIC_API_KEY` to `backend/.env`. You can also set up the keys using the settings dialog on the front-end (click the gear icon after loading the frontend).
 
 Run the frontend:
 
@@ -113,5 +117,3 @@ https://github.com/abi/screenshot-to-code/assets/23818/3fec0f77-44e8-4fb3-a769-a
 ## 🌍 Hosted Version
 
 🆕 [Try it here (paid)](https://screenshottocode.com). Or see [Getting Started](#-getting-started) for local install instructions to use with your own API keys.
-
-[!["Buy Me A Coffee"](https://www.buymeacoffee.com/assets/img/custom_images/orange_img.png)](https://www.buymeacoffee.com/abiraja)

From a73b1f6dc300f0a558ba47ad03aaef62a3c28887 Mon Sep 17 00:00:00 2001
From: Naman Dhingra <1608naman@gmail.com>
Date: Wed, 5 Jun 2024 00:31:21 +0530
Subject: [PATCH 12/19] modify frontend dockerfile to skip puppeteer download

---
 frontend/Dockerfile | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/frontend/Dockerfile b/frontend/Dockerfile
index b176926..6e2e3f3 100644
--- a/frontend/Dockerfile
+++ b/frontend/Dockerfile
@@ -6,6 +6,9 @@ WORKDIR /app
 # Copy package.json and yarn.lock
 COPY package.json yarn.lock /app/
 
+# Set the environment variable to skip Puppeteer download
+ENV PUPPETEER_SKIP_DOWNLOAD=true
+
 # Install dependencies
 RUN yarn install
 

From 8de0e95c54ce15a1259eabc2246e8926928ef2c7 Mon Sep 17 00:00:00 2001
From: Abi Raja <abimanyuraja@gmail.com>
Date: Wed, 5 Jun 2024 15:12:47 -0400
Subject: [PATCH 13/19] fix up types

---
 backend/image_generation.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/backend/image_generation.py b/backend/image_generation.py
index a1c4629..c99dd40 100644
--- a/backend/image_generation.py
+++ b/backend/image_generation.py
@@ -5,13 +5,13 @@ from openai import AsyncOpenAI
 from bs4 import BeautifulSoup
 
 
-async def process_tasks(prompts: List[str], api_key: str, base_url: str):
+async def process_tasks(prompts: List[str], api_key: str, base_url: str | None):
     tasks = [generate_image(prompt, api_key, base_url) for prompt in prompts]
     results = await asyncio.gather(*tasks, return_exceptions=True)
 
     processed_results: List[Union[str, None]] = []
     for result in results:
-        if isinstance(result, BaseException):
+        if isinstance(result, Exception):
             print(f"An exception occurred: {result}")
             processed_results.append(None)
         else:
@@ -20,7 +20,9 @@ async def process_tasks(prompts: List[str], api_key: str, base_url: str):
     return processed_results
 
 
-async def generate_image(prompt: str, api_key: str, base_url: str) -> Union[str, None]:
+async def generate_image(
+    prompt: str, api_key: str, base_url: str | None
+) -> Union[str, None]:
     client = AsyncOpenAI(api_key=api_key, base_url=base_url)
     image_params: Dict[str, Union[str, int]] = {
         "model": "dall-e-3",
@@ -63,9 +65,7 @@ def create_alt_url_mapping(code: str) -> Dict[str, str]:
 
 async def generate_images(
     code: str, api_key: str, base_url: Union[str, None], image_cache: Dict[str, str]
-) -> Union[str, None]:
-    if base_url is None:
-        return code
+) -> str:
     # Find all images
     soup = BeautifulSoup(code, "html.parser")
     images = soup.find_all("img")

From 7cd9b294c350e67092ebb5ff7c2d57f953463ea4 Mon Sep 17 00:00:00 2001
From: Abi Raja <abimanyuraja@gmail.com>
Date: Wed, 5 Jun 2024 15:18:54 -0400
Subject: [PATCH 14/19] fix another type ignore

---
 backend/image_generation.py | 17 ++++++++---------
 1 file changed, 8 insertions(+), 9 deletions(-)

diff --git a/backend/image_generation.py b/backend/image_generation.py
index c99dd40..e3f609f 100644
--- a/backend/image_generation.py
+++ b/backend/image_generation.py
@@ -24,15 +24,14 @@ async def generate_image(
     prompt: str, api_key: str, base_url: str | None
 ) -> Union[str, None]:
     client = AsyncOpenAI(api_key=api_key, base_url=base_url)
-    image_params: Dict[str, Union[str, int]] = {
-        "model": "dall-e-3",
-        "quality": "standard",
-        "style": "natural",
-        "n": 1,
-        "size": "1024x1024",
-        "prompt": prompt,
-    }
-    res = await client.images.generate(**image_params)  # type: ignore
+    res = await client.images.generate(
+        model="dall-e-3",
+        quality="standard",
+        style="natural",
+        n=1,
+        size="1024x1024",
+        prompt=prompt,
+    )
     await client.close()
     return res.data[0].url
 

From 402a279eaf700a0ae1477f465100e9f837504bde Mon Sep 17 00:00:00 2001
From: Abi Raja <abimanyuraja@gmail.com>
Date: Wed, 5 Jun 2024 20:41:35 -0400
Subject: [PATCH 15/19] revert debugging statements

---
 backend/routes/generate_code.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/backend/routes/generate_code.py b/backend/routes/generate_code.py
index 6780dbe..e7186fc 100644
--- a/backend/routes/generate_code.py
+++ b/backend/routes/generate_code.py
@@ -21,7 +21,7 @@ import json
 from prompts.claude_prompts import VIDEO_PROMPT
 from prompts.types import Stack
 
-from utils import pprint_prompt
+# from utils import pprint_prompt
 from video.utils import extract_tag_content, assemble_claude_prompt_video
 from ws.constants import APP_ERROR_WEB_SOCKET_CODE  # type: ignore
 
@@ -210,7 +210,7 @@ async def stream_code(websocket: WebSocket):
         video_data_url = params["image"]
         prompt_messages = await assemble_claude_prompt_video(video_data_url)
 
-    pprint_prompt(prompt_messages)  # type: ignore
+    # pprint_prompt(prompt_messages)  # type: ignore
 
     if SHOULD_MOCK_AI_RESPONSE:
         completion = await mock_completion(

From 1c13897f450ea7fb31540bded1c9a65337053a15 Mon Sep 17 00:00:00 2001
From: Abi Raja <abimanyuraja@gmail.com>
Date: Wed, 5 Jun 2024 20:46:52 -0400
Subject: [PATCH 16/19] remove dummy state

---
 frontend/src/store/app-store.ts | 2 --
 1 file changed, 2 deletions(-)

diff --git a/frontend/src/store/app-store.ts b/frontend/src/store/app-store.ts
index 2b52513..834c47a 100644
--- a/frontend/src/store/app-store.ts
+++ b/frontend/src/store/app-store.ts
@@ -3,13 +3,11 @@ import { create } from "zustand";
 // Store for app-wide state
 interface AppStore {
   inSelectAndEditMode: boolean;
-  inputMode: "image" | "video";
   toggleInSelectAndEditMode: () => void;
   disableInSelectAndEditMode: () => void;
 }
 
 export const useAppStore = create<AppStore>((set) => ({
-  inputMode: "image",
   inSelectAndEditMode: false,
   toggleInSelectAndEditMode: () =>
     set((state) => ({ inSelectAndEditMode: !state.inSelectAndEditMode })),

From f7c9209d1803eff2568b1380c28f32041b47eef6 Mon Sep 17 00:00:00 2001
From: Abi Raja <abimanyuraja@gmail.com>
Date: Wed, 5 Jun 2024 20:49:31 -0400
Subject: [PATCH 17/19] remove unused code

---
 frontend/src/components/select-and-edit/utils.ts | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/frontend/src/components/select-and-edit/utils.ts b/frontend/src/components/select-and-edit/utils.ts
index f3180fd..9462778 100644
--- a/frontend/src/components/select-and-edit/utils.ts
+++ b/frontend/src/components/select-and-edit/utils.ts
@@ -18,7 +18,5 @@ export function getAdjustedCoordinates(
   const offsetX = rect ? rect.left : 0;
   const offsetY = rect ? rect.top : 0;
 
-  // Adjust for scale
-  const scale = 1; // the scale factor applied to the iframe
-  return { x: x / scale + offsetX, y: y / scale + offsetY };
+  return { x: x + offsetX, y: y + offsetY };
 }

From 49a24d8b0c139e4f334d82528c09be43f4519fca Mon Sep 17 00:00:00 2001
From: Abi Raja <abimanyuraja@gmail.com>
Date: Sat, 22 Jun 2024 19:39:49 +0800
Subject: [PATCH 18/19] support Claude Sonnet 3.5

---
 backend/evals/core.py           |  3 ++-
 backend/llm.py                  | 10 ++++++++--
 backend/routes/generate_code.py |  6 +++++-
 frontend/src/lib/models.ts      |  2 ++
 4 files changed, 17 insertions(+), 4 deletions(-)

diff --git a/backend/evals/core.py b/backend/evals/core.py
index 5e05362..2fc0352 100644
--- a/backend/evals/core.py
+++ b/backend/evals/core.py
@@ -15,7 +15,7 @@ async def generate_code_core(image_url: str, stack: Stack, model: Llm) -> str:
     async def process_chunk(content: str):
         pass
 
-    if model == Llm.CLAUDE_3_SONNET:
+    if model == Llm.CLAUDE_3_SONNET or model == Llm.CLAUDE_3_5_SONNET_2024_06_20:
         if not anthropic_api_key:
             raise Exception("Anthropic API key not found")
 
@@ -23,6 +23,7 @@ async def generate_code_core(image_url: str, stack: Stack, model: Llm) -> str:
             prompt_messages,
             api_key=anthropic_api_key,
             callback=lambda x: process_chunk(x),
+            model=model,
         )
     else:
         if not openai_api_key:
diff --git a/backend/llm.py b/backend/llm.py
index e32051c..51b7efe 100644
--- a/backend/llm.py
+++ b/backend/llm.py
@@ -17,6 +17,7 @@ class Llm(Enum):
     CLAUDE_3_SONNET = "claude-3-sonnet-20240229"
     CLAUDE_3_OPUS = "claude-3-opus-20240229"
     CLAUDE_3_HAIKU = "claude-3-haiku-20240307"
+    CLAUDE_3_5_SONNET_2024_06_20 = "claude-3-5-sonnet-20240620"
 
 
 # Will throw errors if you send a garbage string
@@ -59,7 +60,12 @@ async def stream_openai_response(
     full_response = ""
     async for chunk in stream:  # type: ignore
         assert isinstance(chunk, ChatCompletionChunk)
-        if chunk.choices and len(chunk.choices) > 0 and chunk.choices[0].delta and chunk.choices[0].delta.content:
+        if (
+            chunk.choices
+            and len(chunk.choices) > 0
+            and chunk.choices[0].delta
+            and chunk.choices[0].delta.content
+        ):
             content = chunk.choices[0].delta.content or ""
             full_response += content
             await callback(content)
@@ -74,12 +80,12 @@ async def stream_claude_response(
     messages: List[ChatCompletionMessageParam],
     api_key: str,
     callback: Callable[[str], Awaitable[None]],
+    model: Llm,
 ) -> str:
 
     client = AsyncAnthropic(api_key=api_key)
 
     # Base parameters
-    model = Llm.CLAUDE_3_SONNET
     max_tokens = 4096
     temperature = 0.0
 
diff --git a/backend/routes/generate_code.py b/backend/routes/generate_code.py
index 379042e..362f742 100644
--- a/backend/routes/generate_code.py
+++ b/backend/routes/generate_code.py
@@ -245,7 +245,10 @@ async def stream_code(websocket: WebSocket):
                     include_thinking=True,
                 )
                 exact_llm_version = Llm.CLAUDE_3_OPUS
-            elif code_generation_model == Llm.CLAUDE_3_SONNET:
+            elif (
+                code_generation_model == Llm.CLAUDE_3_SONNET
+                or code_generation_model == Llm.CLAUDE_3_5_SONNET_2024_06_20
+            ):
                 if not anthropic_api_key:
                     await throw_error(
                         "No Anthropic API key found. Please add the environment variable ANTHROPIC_API_KEY to backend/.env or in the settings dialog"
@@ -256,6 +259,7 @@ async def stream_code(websocket: WebSocket):
                     prompt_messages,  # type: ignore
                     api_key=anthropic_api_key,
                     callback=lambda x: process_chunk(x),
+                    model=code_generation_model,
                 )
                 exact_llm_version = code_generation_model
             else:
diff --git a/frontend/src/lib/models.ts b/frontend/src/lib/models.ts
index d6fd09d..250953b 100644
--- a/frontend/src/lib/models.ts
+++ b/frontend/src/lib/models.ts
@@ -2,6 +2,7 @@
 // Order here matches dropdown order
 export enum CodeGenerationModel {
   GPT_4O_2024_05_13 = "gpt-4o-2024-05-13",
+  CLAUDE_3_5_SONNET_2024_06_20 = "claude-3-5-sonnet-20240620",
   GPT_4_TURBO_2024_04_09 = "gpt-4-turbo-2024-04-09",
   GPT_4_VISION = "gpt_4_vision",
   CLAUDE_3_SONNET = "claude_3_sonnet",
@@ -12,6 +13,7 @@ export const CODE_GENERATION_MODEL_DESCRIPTIONS: {
   [key in CodeGenerationModel]: { name: string; inBeta: boolean };
 } = {
   "gpt-4o-2024-05-13": { name: "GPT-4o 🌟", inBeta: false },
+  "claude-3-5-sonnet-20240620": { name: "Claude 3.5 Sonnet 🌟", inBeta: false },
   "gpt-4-turbo-2024-04-09": { name: "GPT-4 Turbo (Apr 2024)", inBeta: false },
   gpt_4_vision: { name: "GPT-4 Vision (Nov 2023)", inBeta: false },
   claude_3_sonnet: { name: "Claude 3 Sonnet", inBeta: false },

From f75294ca648235b1c40a217a1eb6ede3d5f8ab39 Mon Sep 17 00:00:00 2001
From: Abi Raja <abimanyuraja@gmail.com>
Date: Sat, 22 Jun 2024 19:41:56 +0800
Subject: [PATCH 19/19] update better model banner to include sonnet 3.5

---
 frontend/src/App.tsx | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx
index 5156a0f..82bd0a0 100644
--- a/frontend/src/App.tsx
+++ b/frontend/src/App.tsx
@@ -94,8 +94,10 @@ function App() {
       CodeGenerationModel.GPT_4_TURBO_2024_04_09 &&
     settings.generatedCodeConfig === Stack.REACT_TAILWIND;
 
-  const showGpt4OMessage =
+  const showBetterModelMessage =
     selectedCodeGenerationModel !== CodeGenerationModel.GPT_4O_2024_05_13 &&
+    selectedCodeGenerationModel !==
+      CodeGenerationModel.CLAUDE_3_5_SONNET_2024_06_20 &&
     appState === AppState.INITIAL;
 
   const showSelectAndEditFeature =
@@ -432,11 +434,11 @@ function App() {
             </div>
           )}
 
-          {showGpt4OMessage && (
+          {showBetterModelMessage && (
             <div className="rounded-lg p-2 bg-fuchsia-200">
               <p className="text-gray-800 text-sm">
-                Now supporting GPT-4o. Higher quality and 2x faster. Give it a
-                try!
+                Now supporting GPT-4o and Claude Sonnet 3.5. Higher quality and
+                2x faster. Give it a try!
               </p>
             </div>
           )}