Github/screenshot-to-code
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

support screen recording

Browse Source
This commit is contained in:
Abi Raja 2024-03-14 15:57:44 -04:00
parent b69edb7948
commit c08cf0ae57
7 changed files with 247 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
backend/poetry.lock generated
View File

@ -592,13 +592,13 @@ files = [
[[package]]
name = "openai"
version = "1.13.3"
version = "1.14.0"
description = "The official Python library for the openai API"
optional = false
python-versions = ">=3.7.1"
files = [
{file = "openai-1.13.3-py3-none-any.whl", hash = "sha256:5769b62abd02f350a8dd1a3a242d8972c947860654466171d60fb0972ae0a41c"},
{file = "openai-1.13.3.tar.gz", hash = "sha256:ff6c6b3bc7327e715e4b3592a923a5a1c7519ff5dd764a83d69f633d49e77a7b"},
{file = "openai-1.14.0-py3-none-any.whl", hash = "sha256:5c9fd3a59f5cbdb4020733ddf79a22f6b7a36d561968cb3f3dd255cdd263d9fe"},
{file = "openai-1.14.0.tar.gz", hash = "sha256:e287057adf0ec3315abc32ddcc968d095879abd9b68bf51c0402dab13ab5ae9b"},
]
[package.dependencies]
@ -615,13 +615,13 @@ datalib = ["numpy (>=1)", "pandas (>=1.2.3)", "pandas-stubs (>=1.1.0.11)"]
[[package]]
name = "packaging"
version = "23.2"
version = "24.0"
description = "Core utilities for Python packages"
optional = false
python-versions = ">=3.7"
files = [
{file = "packaging-23.2-py3-none-any.whl", hash = "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7"},
{file = "packaging-23.2.tar.gz", hash = "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5"},
{file = "packaging-24.0-py3-none-any.whl", hash = "sha256:2ddfb553fdf02fb784c234c7ba6ccc288296ceabec964ad2eae3777778130bc5"},
{file = "packaging-24.0.tar.gz", hash = "sha256:eb82c5e3e56209074766e6885bb04b8c38a0c015d0a30036ebe7ece34c9989e9"},
]
[[package]]
@ -825,13 +825,13 @@ email = ["email-validator (>=1.0.3)"]
[[package]]
name = "pyright"
version = "1.1.352"
version = "1.1.354"
description = "Command line wrapper for pyright"
optional = false
python-versions = ">=3.7"
files = [
{file = "pyright-1.1.352-py3-none-any.whl", hash = "sha256:0040cf173c6a60704e553bfd129dfe54de59cc76d0b2b80f77cfab4f50701d64"},
{file = "pyright-1.1.352.tar.gz", hash = "sha256:a621c0dfbcf1291b3610641a07380fefaa1d0e182890a1b2a7f13b446e8109a9"},
{file = "pyright-1.1.354-py3-none-any.whl", hash = "sha256:f28d61ae8ae035fc52ded1070e8d9e786051a26a4127bbd7a4ba0399b81b37b5"},
{file = "pyright-1.1.354.tar.gz", hash = "sha256:b1070dc774ff2e79eb0523fe87f4ba9a90550de7e4b030a2bc9e031864029a1f"},
]
[package.dependencies]
@ -959,18 +959,18 @@ use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"]
[[package]]
name = "setuptools"
version = "69.1.1"
version = "69.2.0"
description = "Easily download, build, install, upgrade, and uninstall Python packages"
optional = false
python-versions = ">=3.8"
files = [
{file = "setuptools-69.1.1-py3-none-any.whl", hash = "sha256:02fa291a0471b3a18b2b2481ed902af520c69e8ae0919c13da936542754b4c56"},
{file = "setuptools-69.1.1.tar.gz", hash = "sha256:5c0806c7d9af348e6dd3777b4f4dbb42c7ad85b190104837488eab9a7c945cf8"},
{file = "setuptools-69.2.0-py3-none-any.whl", hash = "sha256:c21c49fb1042386df081cb5d86759792ab89efca84cf114889191cd09aacc80c"},
{file = "setuptools-69.2.0.tar.gz", hash = "sha256:0ff4183f8f42cd8fa3acea16c45205521a4ef28f73c6391d8a25e92893134f2e"},
]
[package.extras]
docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"]
testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.2)", "pip (>=19.1)", "pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff (>=0.2.1)", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"]
testing = ["build[virtualenv]", "filelock (>=3.4.0)", "importlib-metadata", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "mypy (==1.9)", "packaging (>=23.2)", "pip (>=19.1)", "pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff (>=0.2.1)", "pytest-timeout", "pytest-xdist (>=3)", "tomli", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"]
testing-integration = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.2)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"]
[[package]]

3
frontend/package.json
View File

@ -45,7 +45,8 @@
"tailwind-merge": "^2.0.0",
"tailwindcss-animate": "^1.0.7",
"thememirror": "^2.0.1",
"vite-plugin-checker": "^0.6.2"
"vite-plugin-checker": "^0.6.2",
"webm-duration-fix": "^1.0.4"
},
"devDependencies": {
"@types/node": "^20.9.0",

52
frontend/src/components/ImageUpload.tsx
View File

@ -5,6 +5,8 @@ import { useDropzone } from "react-dropzone";
import { toast } from "react-hot-toast";
import { URLS } from "../urls";
import { Badge } from "./ui/badge";
import ScreenRecorder from "./recording/ScreenRecorder";
import { ScreenRecorderState } from "../types";
const baseStyle = {
flex: 1,
@ -61,16 +63,23 @@ interface Props {
function ImageUpload({ setReferenceImages }: Props) {
const [files, setFiles] = useState<FileWithPreview[]>([]);
// TODO: Switch to Zustand
const [screenRecorderState, setScreenRecorderState] =
useState<ScreenRecorderState>(ScreenRecorderState.INITIAL);
const { getRootProps, getInputProps, isFocused, isDragAccept, isDragReject } =
useDropzone({
maxFiles: 1,
maxSize: 1024 * 1024 * 20, // 20 MB
accept: {
// Image formats
"image/png": [".png"],
"image/jpeg": [".jpeg"],
"image/jpg": [".jpg"],
// Video formats
"video/quicktime": [".mov"],
"video/mp4": [".mp4"],
"video/webm": [".webm"],
},
onDrop: (acceptedFiles) => {
// Set up the preview thumbnail images
@ -154,21 +163,34 @@ function ImageUpload({ setReferenceImages }: Props) {
return (
<section className="container">
{/* eslint-disable-next-line @typescript-eslint/no-explicit-any */}
<div {...getRootProps({ style: style as any })}>
<input {...getInputProps()} />
<p className="text-slate-700 text-lg">
Drag & drop a screenshot here, <br />
or click to upload
</p>
</div>
<div className="text-center text-sm text-slate-800 mt-4">
<Badge>New!</Badge> Upload a screen recording in .mp4 or .mov format to
clone a whole app (experimental).{" "}
<a className="underline" href={URLS["intro-to-video"]} target="_blank">
Learn more.
</a>
</div>
{screenRecorderState === ScreenRecorderState.INITIAL && (
/* eslint-disable-next-line @typescript-eslint/no-explicit-any */
<div {...getRootProps({ style: style as any })}>
<input {...getInputProps()} />
<p className="text-slate-700 text-lg">
Drag & drop a screenshot here, <br />
or click to upload
</p>
</div>
)}
{screenRecorderState === ScreenRecorderState.INITIAL && (
<div className="text-center text-sm text-slate-800 mt-4">
<Badge>New!</Badge> Upload a screen recording (.mp4, .mov) or record
your screen to clone a whole app (experimental).{" "}
<a
className="underline"
href={URLS["intro-to-video"]}
target="_blank"
>
Learn more.
</a>
</div>
)}
<ScreenRecorder
screenRecorderState={screenRecorderState}
setScreenRecorderState={setScreenRecorderState}
generateCode={setReferenceImages}
/>
</section>
);
}

115
frontend/src/components/recording/ScreenRecorder.tsx Normal file
View File

@ -0,0 +1,115 @@
import { useState } from "react";
import { Button } from "../ui/button";
import { ScreenRecorderState } from "../../types";
import { blobToBase64DataUrl } from "./utils";
import fixWebmDuration from "webm-duration-fix";
import toast from "react-hot-toast";
interface Props {
screenRecorderState: ScreenRecorderState;
setScreenRecorderState: (state: ScreenRecorderState) => void;
generateCode: (
referenceImages: string[],
inputMode: "image" | "video"
) => void;
}
function ScreenRecorder({
screenRecorderState,
setScreenRecorderState,
generateCode,
}: Props) {
const [mediaRecorder, setMediaRecorder] = useState<MediaRecorder | null>(
null
);
const [screenRecordingDataUrl, setScreenRecordingDataUrl] = useState<
string | null
>(null);
const startScreenRecording = async () => {
try {
// Get the screen recording stream
const stream = await navigator.mediaDevices.getDisplayMedia({
video: true,
audio: { echoCancellation: true },
});
// TODO: Test across different browsers
// Create the media recorder
const options = { mimeType: "video/webm" };
const mediaRecorder = new MediaRecorder(stream, options);
setMediaRecorder(mediaRecorder);
const chunks: BlobPart[] = [];
// Accumalate chunks as data is available
mediaRecorder.ondataavailable = (e: BlobEvent) => chunks.push(e.data);
// When media recorder is stopped, create a data URL
mediaRecorder.onstop = async () => {
// TODO: Do I need to fix duration if it's not a webm?
const completeBlob = await fixWebmDuration(
new Blob(chunks, {
type: options.mimeType,
})
);
const dataUrl = await blobToBase64DataUrl(completeBlob);
setScreenRecordingDataUrl(dataUrl);
setScreenRecorderState(ScreenRecorderState.FINISHED);
};
// Start recording
mediaRecorder.start();
setScreenRecorderState(ScreenRecorderState.RECORDING);
} catch (error) {
toast.error("Could not start screen recording");
throw error;
}
};
const stopScreenRecording = () => {
if (mediaRecorder) {
mediaRecorder.stop();
setMediaRecorder(null);
}
};
const kickoffGeneration = () => {
if (screenRecordingDataUrl) {
generateCode([screenRecordingDataUrl], "video");
} else {
toast.error("Screen recording does not exist. Please try again.");
throw new Error("No screen recording data url");
}
};
return (
<div className="flex items-center justify-center my-3">
{screenRecorderState === ScreenRecorderState.INITIAL && (
<Button onClick={startScreenRecording}>Record Screen</Button>
)}
{screenRecorderState === ScreenRecorderState.RECORDING && (
<div className="flex items-center flex-col gap-y-4">
<div className="flex items-center mr-2 text-xl gap-x-1">
<span className="block h-10 w-10 bg-red-600 rounded-full mr-1 animate-pulse"></span>
<span>Recording...</span>
</div>
<Button onClick={stopScreenRecording}>Finish Recording</Button>
</div>
)}
{screenRecorderState === ScreenRecorderState.FINISHED && (
<div className="flex items-center flex-col gap-y-4">
<div className="flex items-center mr-2 text-xl gap-x-1">
<span>Screen Recording Captured.</span>
</div>
<Button onClick={kickoffGeneration}>Generate</Button>
</div>
)}
</div>
);
}
export default ScreenRecorder;

31
frontend/src/components/recording/utils.ts Normal file
View File

@ -0,0 +1,31 @@
export function downloadBlob(blob: Blob) {
// Create a URL for the blob object
const videoURL = URL.createObjectURL(blob);
// Create a temporary anchor element and trigger the download
const a = document.createElement("a");
a.href = videoURL;
a.download = "recording.webm";
document.body.appendChild(a);
a.click();
document.body.removeChild(a);
// Clear object URL
URL.revokeObjectURL(videoURL);
}
export function blobToBase64DataUrl(blob: Blob): Promise<string> {
return new Promise((resolve, reject) => {
const reader = new FileReader();
reader.onloadend = () => {
if (reader.result) {
resolve(reader.result as string);
} else {
reject(new Error("FileReader did not return a result."));
}
};
reader.onerror = () =>
reject(new Error("FileReader encountered an error."));
reader.readAsDataURL(blob);
});
}

6
frontend/src/types.ts
View File

@ -25,6 +25,12 @@ export enum AppState {
CODE_READY = "CODE_READY",
}
export enum ScreenRecorderState {
INITIAL = "initial",
RECORDING = "recording",
FINISHED = "finished",
}
export interface CodeGenerationParams {
generationType: "create" | "update";
inputMode: "image" | "video";

43
frontend/yarn.lock
View File

@ -1610,6 +1610,11 @@ base64-arraybuffer@^1.0.2:
resolved "https://registry.npmjs.org/base64-arraybuffer/-/base64-arraybuffer-1.0.2.tgz#1c37589a7c4b0746e34bd1feb951da2df01c1bdc"
integrity sha512-I3yl4r9QB5ZRY3XuJVEPfc2XhZO6YweFPI+UovAzn+8/hb3oJ6lnysaFcjVpkCPfVWFUDvoZ8kmVDP7WyRtYtQ==
base64-js@^1.3.1:
version "1.5.1"
resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-1.5.1.tgz#1b1b440160a5bf7ad40b650f095963481903930a"
integrity sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==
binary-extensions@^2.0.0:
version "2.2.0"
resolved "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.2.0.tgz"
@ -1657,6 +1662,14 @@ buffer-from@^1.0.0:
resolved "https://registry.yarnpkg.com/buffer-from/-/buffer-from-1.1.2.tgz#2b146a6fd72e80b4f55d255f35ed59a3a9a41bd5"
integrity sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==
buffer@^6.0.3:
version "6.0.3"
resolved "https://registry.yarnpkg.com/buffer/-/buffer-6.0.3.tgz#2ace578459cc8fbe2a70aaa8f52ee63b6a74c6c6"
integrity sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==
dependencies:
base64-js "^1.3.1"
ieee754 "^1.2.1"
cac@^6.7.14:
version "6.7.14"
resolved "https://registry.yarnpkg.com/cac/-/cac-6.7.14.tgz#804e1e6f506ee363cb0e3ccbb09cad5dd9870959"
@ -1993,6 +2006,11 @@ dotenv@^16.0.0:
resolved "https://registry.yarnpkg.com/dotenv/-/dotenv-16.3.1.tgz#369034de7d7e5b120972693352a3bf112172cc3e"
integrity sha512-IPzF4w4/Rd94bA9imS68tZBaYyBWSCE47V1RGuMrB94iyTOIEwRmVL2x/4An+6mETpLrKJ5hQkB8W4kFAadeIQ==
ebml-block@^1.1.2:
version "1.1.2"
resolved "https://registry.yarnpkg.com/ebml-block/-/ebml-block-1.1.2.tgz#fd49951b0faf5a3049bdd61c851a76b5e679c290"
integrity sha512-HgNlIsRFP6D9VKU5atCeHRJY7XkJP8bOe8yEhd8NB7B3b4++VWTyauz6g650iiPmLfPLGlVpoJmGSgMfXDYusg==
ejs@^3.1.6:
version "3.1.9"
resolved "https://registry.yarnpkg.com/ejs/-/ejs-3.1.9.tgz#03c9e8777fe12686a9effcef22303ca3d8eeb361"
@ -2186,6 +2204,11 @@ esutils@^2.0.2:
resolved "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz"
integrity sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==
events@^3.3.0:
version "3.3.0"
resolved "https://registry.yarnpkg.com/events/-/events-3.3.0.tgz#31a95ad0a924e2d2c419a813aeb2c4e878ea7400"
integrity sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==
execa@^8.0.1:
version "8.0.1"
resolved "https://registry.yarnpkg.com/execa/-/execa-8.0.1.tgz#51f6a5943b580f963c3ca9c6321796db8cc39b8c"
@ -2467,6 +2490,11 @@ human-signals@^5.0.0:
resolved "https://registry.yarnpkg.com/human-signals/-/human-signals-5.0.0.tgz#42665a284f9ae0dade3ba41ebc37eb4b852f3a28"
integrity sha512-AXcZb6vzzrFAUE61HnN4mpLqd/cSIwNQjtNWR0euPm6y0iqx3G4gOXaIDdtdDwZmhwe82LA6+zinmW4UBWVePQ==
ieee754@^1.2.1:
version "1.2.1"
resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.2.1.tgz#8eb7a10a63fff25d15a57b001586d177d1b0d352"
integrity sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==
ignore@^5.2.0, ignore@^5.2.4:
version "5.2.4"
resolved "https://registry.npmjs.org/ignore/-/ignore-5.2.4.tgz"
@ -2498,6 +2526,11 @@ inherits@2:
resolved "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz"
integrity sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==
int64-buffer@^1.0.1:
version "1.0.1"
resolved "https://registry.yarnpkg.com/int64-buffer/-/int64-buffer-1.0.1.tgz#c78d841b444cadf036cd04f8683696c740f15dca"
integrity sha512-+3azY4pXrjAupJHU1V9uGERWlhoqNswJNji6aD/02xac7oxol508AsMC5lxKhEqyZeDFy3enq5OGWXF4u75hiw==
invariant@^2.2.4:
version "2.2.4"
resolved "https://registry.yarnpkg.com/invariant/-/invariant-2.2.4.tgz#610f3c92c9359ce1db616e538008d23ff35158e6"
@ -3765,6 +3798,16 @@ w3c-keyname@^2.2.4:
resolved "https://registry.yarnpkg.com/w3c-keyname/-/w3c-keyname-2.2.8.tgz#7b17c8c6883d4e8b86ac8aba79d39e880f8869c5"
integrity sha512-dpojBhNsCNN7T82Tm7k26A6G9ML3NkhDsnw9n/eoxSRlVBB4CEtIQ/KTCLI2Fwf3ataSXRhYFkQi3SlnFwPvPQ==
webm-duration-fix@^1.0.4:
version "1.0.4"
resolved "https://registry.yarnpkg.com/webm-duration-fix/-/webm-duration-fix-1.0.4.tgz#fef235cb3d3ed3363507f705a7577dbb9fdedae6"
integrity sha512-kvhmSmEnuohtK+j+mJswqCCM2ViKb9W8Ch0oAxcaeUvpok5CsMORQLnea+CYKDXPG6JH12H0CbRK85qhfeZLew==
dependencies:
buffer "^6.0.3"
ebml-block "^1.1.2"
events "^3.3.0"
int64-buffer "^1.0.1"
which@^2.0.1:
version "2.0.2"
resolved "https://registry.npmjs.org/which/-/which-2.0.2.tgz"