diff --git a/README.md b/README.md index a6dfabc..7a56758 100644 --- a/README.md +++ b/README.md @@ -45,6 +45,12 @@ Open http://localhost:5173 to use the app. If you prefer to run the backend on a different port, update VITE_WS_BACKEND_URL in `frontend/.env.local` +For debugging purposes, if you don't want to waste GPT4-Vision credits, you can run the backend in mock mode (which streams a pre-recorded response): + +```bash +MOCK=true poetry run uvicorn main:app --reload --port 7001 +``` + ## Docker If you have Docker installed on your system, in the root directory, run: @@ -59,19 +65,17 @@ The app will be up and running at http://localhost:5173. Note that you can't dev ## 🙋‍♂️ FAQs - **I'm running into an error when setting up the backend. How can I fix it?** [Try this](https://github.com/abi/screenshot-to-code/issues/3#issuecomment-1814777959). If that still doesn't work, open an issue. -- **How do I get an OpenAI API key that has the GPT4 Vision model available?** Create an OpenAI account. And then, you need to buy at least $1 worth of credit on the [Billing dashboard](https://platform.openai.com/account/billing/overview). +- **How do I get an OpenAI API key that has the GPT4 Vision model available?** Create an OpenAI account. And then, you need to buy at least $1 worth of credit on the [Billing dashboard](https://platform.openai.com/account/billing/overview). Also, see [OpenAI docs](https://help.openai.com/en/articles/7102672-how-can-i-access-gpt-4). - **How can I provide feedback?** For feedback, feature requests and bug reports, open an issue or ping me on [Twitter](https://twitter.com/_abi_). ## 📚 Examples **NYTimes** -| Original | Replica | -| -------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------- | +| Original | Replica | +| --------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------- | | Screenshot 2023-11-20 at 12 54 03 PM | Screenshot 2023-11-20 at 12 59 56 PM | - - **Instagram page (with not Taylor Swift pics)** https://github.com/abi/screenshot-to-code/assets/23818/503eb86a-356e-4dfc-926a-dabdb1ac7ba1 diff --git a/backend/main.py b/backend/main.py index 72b4dd7..93d2e9a 100644 --- a/backend/main.py +++ b/backend/main.py @@ -33,7 +33,8 @@ app.add_middleware( # Useful for debugging purposes when you don't want to waste GPT4-Vision credits # Setting to True will stream a mock response instead of calling the OpenAI API -SHOULD_MOCK_AI_RESPONSE = False +# TODO: Should only be set to true when value is 'True', not any abitrary truthy value +SHOULD_MOCK_AI_RESPONSE = bool(os.environ.get("MOCK", False)) app.include_router(screenshot.router) @@ -96,7 +97,10 @@ async def stream_code(websocket: WebSocket): async def process_chunk(content): await websocket.send_json({"type": "chunk", "value": content}) - prompt_messages = assemble_prompt(params["image"]) + if params.get("resultImage") and params["resultImage"]: + prompt_messages = assemble_prompt(params["image"], params["resultImage"]) + else: + prompt_messages = assemble_prompt(params["image"]) # Image cache for updates so that we don't have to regenerate images image_cache = {} diff --git a/backend/prompts.py b/backend/prompts.py index 3761404..1ba254d 100644 --- a/backend/prompts.py +++ b/backend/prompts.py @@ -2,8 +2,8 @@ SYSTEM_PROMPT = """ You are an expert Tailwind developer You take screenshots of a reference web page from the user, and then build single page apps using Tailwind, HTML and JS. -You might also be given a screenshot of a web page that you have already built, and asked to -update it to look more like the reference image. +You might also be given a screenshot(The second image) of a web page that you have already built, and asked to +update it to look more like the reference image(The first image). - Make sure the app looks exactly like the screenshot. - Pay close attention to background color, text color, font size, font family, @@ -27,21 +27,26 @@ USER_PROMPT = """ Generate code for a web page that looks exactly like this. """ - -def assemble_prompt(image_data_url): +def assemble_prompt(image_data_url, result_image_data_url=None): + content = [ + { + "type": "image_url", + "image_url": {"url": image_data_url, "detail": "high"}, + }, + { + "type": "text", + "text": USER_PROMPT, + }, + ] + if result_image_data_url: + content.insert(1, { + "type": "image_url", + "image_url": {"url": result_image_data_url, "detail": "high"}, + }) return [ {"role": "system", "content": SYSTEM_PROMPT}, { "role": "user", - "content": [ - { - "type": "image_url", - "image_url": {"url": image_data_url, "detail": "high"}, - }, - { - "type": "text", - "text": USER_PROMPT, - }, - ], + "content": content, }, ] diff --git a/frontend/package.json b/frontend/package.json index 912d0b7..ef840e7 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -22,6 +22,7 @@ "classnames": "^2.3.2", "clsx": "^2.0.0", "codemirror": "^6.0.1", + "html2canvas": "^1.4.1", "copy-to-clipboard": "^3.3.3", "react": "^18.2.0", "react-dom": "^18.2.0", diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 5c946dc..4c843d6 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -13,6 +13,8 @@ import { FaMobile, FaUndo, } from "react-icons/fa"; + +import { Switch } from "./components/ui/switch"; import copy from "copy-to-clipboard"; import toast from "react-hot-toast"; import { Button } from "@/components/ui/button"; @@ -27,6 +29,7 @@ import { OnboardingNote } from "./components/OnboardingNote"; import { usePersistedState } from "./hooks/usePersistedState"; import { UrlInputSection } from "./components/UrlInputSection"; import TermsOfServiceDialog from "./components/TermsOfServiceDialog"; +import html2canvas from "html2canvas"; import { USER_CLOSE_WEB_SOCKET_CODE } from "./constants"; function App() { @@ -46,8 +49,23 @@ function App() { }, "setting" ); + const [shouldIncludeResultImage, setShouldIncludeResultImage] = + useState(false); const wsRef = useRef(null); + const takeScreenshot = async (): Promise => { + const iframeElement = document.querySelector( + "#preview-desktop" + ) as HTMLIFrameElement; + if (!iframeElement?.contentWindow?.document.body) { + return ""; + } + + const canvas = await html2canvas(iframeElement.contentWindow.document.body); + const png = canvas.toDataURL("image/png"); + return png; + }; + const downloadCode = () => { // Create a blob from the generated code const blob = new Blob([generatedCode], { type: "text/html" }); @@ -75,6 +93,8 @@ function App() { const stop = () => { wsRef.current?.close?.(USER_CLOSE_WEB_SOCKET_CODE); + // make sure stop can correct the state even if the websocket is already closed + setAppState(AppState.CODE_READY); }; function doGenerateCode(params: CodeGenerationParams) { @@ -106,14 +126,23 @@ function App() { } // Subsequent updates - function doUpdate() { + async function doUpdate() { const updatedHistory = [...history, generatedCode, updateInstruction]; - - doGenerateCode({ - generationType: "update", - image: referenceImages[0], - history: updatedHistory, - }); + if (shouldIncludeResultImage) { + const resultImage = await takeScreenshot(); + doGenerateCode({ + generationType: "update", + image: referenceImages[0], + resultImage: resultImage, + history: updatedHistory, + }); + } else { + doGenerateCode({ + generationType: "update", + image: referenceImages[0], + history: updatedHistory, + }); + } setHistory(updatedHistory); setGeneratedCode(""); @@ -183,6 +212,15 @@ function App() { onChange={(e) => setUpdateInstruction(e.target.value)} value={updateInstruction} /> +
+
+ Include screenshot of current version? +
+ +
diff --git a/frontend/src/components/Preview.tsx b/frontend/src/components/Preview.tsx index 3a603ff..24f33c0 100644 --- a/frontend/src/components/Preview.tsx +++ b/frontend/src/components/Preview.tsx @@ -23,6 +23,7 @@ function Preview({ code, device }: Props) { return (