From c08cf0ae571b1cd93d70f04a8f17b68c6452b35b Mon Sep 17 00:00:00 2001 From: Abi Raja Date: Thu, 14 Mar 2024 15:57:44 -0400 Subject: [PATCH] support screen recording --- backend/poetry.lock | 26 ++-- frontend/package.json | 3 +- frontend/src/components/ImageUpload.tsx | 52 +++++--- .../components/recording/ScreenRecorder.tsx | 115 ++++++++++++++++++ frontend/src/components/recording/utils.ts | 31 +++++ frontend/src/types.ts | 6 + frontend/yarn.lock | 43 +++++++ 7 files changed, 247 insertions(+), 29 deletions(-) create mode 100644 frontend/src/components/recording/ScreenRecorder.tsx create mode 100644 frontend/src/components/recording/utils.ts diff --git a/backend/poetry.lock b/backend/poetry.lock index 61ff096..d6b5d02 100644 --- a/backend/poetry.lock +++ b/backend/poetry.lock @@ -592,13 +592,13 @@ files = [ [[package]] name = "openai" -version = "1.13.3" +version = "1.14.0" description = "The official Python library for the openai API" optional = false python-versions = ">=3.7.1" files = [ - {file = "openai-1.13.3-py3-none-any.whl", hash = "sha256:5769b62abd02f350a8dd1a3a242d8972c947860654466171d60fb0972ae0a41c"}, - {file = "openai-1.13.3.tar.gz", hash = "sha256:ff6c6b3bc7327e715e4b3592a923a5a1c7519ff5dd764a83d69f633d49e77a7b"}, + {file = "openai-1.14.0-py3-none-any.whl", hash = "sha256:5c9fd3a59f5cbdb4020733ddf79a22f6b7a36d561968cb3f3dd255cdd263d9fe"}, + {file = "openai-1.14.0.tar.gz", hash = "sha256:e287057adf0ec3315abc32ddcc968d095879abd9b68bf51c0402dab13ab5ae9b"}, ] [package.dependencies] @@ -615,13 +615,13 @@ datalib = ["numpy (>=1)", "pandas (>=1.2.3)", "pandas-stubs (>=1.1.0.11)"] [[package]] name = "packaging" -version = "23.2" +version = "24.0" description = "Core utilities for Python packages" optional = false python-versions = ">=3.7" files = [ - {file = "packaging-23.2-py3-none-any.whl", hash = "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7"}, - {file = "packaging-23.2.tar.gz", hash = "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5"}, + {file = "packaging-24.0-py3-none-any.whl", hash = "sha256:2ddfb553fdf02fb784c234c7ba6ccc288296ceabec964ad2eae3777778130bc5"}, + {file = "packaging-24.0.tar.gz", hash = "sha256:eb82c5e3e56209074766e6885bb04b8c38a0c015d0a30036ebe7ece34c9989e9"}, ] [[package]] @@ -825,13 +825,13 @@ email = ["email-validator (>=1.0.3)"] [[package]] name = "pyright" -version = "1.1.352" +version = "1.1.354" description = "Command line wrapper for pyright" optional = false python-versions = ">=3.7" files = [ - {file = "pyright-1.1.352-py3-none-any.whl", hash = "sha256:0040cf173c6a60704e553bfd129dfe54de59cc76d0b2b80f77cfab4f50701d64"}, - {file = "pyright-1.1.352.tar.gz", hash = "sha256:a621c0dfbcf1291b3610641a07380fefaa1d0e182890a1b2a7f13b446e8109a9"}, + {file = "pyright-1.1.354-py3-none-any.whl", hash = "sha256:f28d61ae8ae035fc52ded1070e8d9e786051a26a4127bbd7a4ba0399b81b37b5"}, + {file = "pyright-1.1.354.tar.gz", hash = "sha256:b1070dc774ff2e79eb0523fe87f4ba9a90550de7e4b030a2bc9e031864029a1f"}, ] [package.dependencies] @@ -959,18 +959,18 @@ use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] [[package]] name = "setuptools" -version = "69.1.1" +version = "69.2.0" description = "Easily download, build, install, upgrade, and uninstall Python packages" optional = false python-versions = ">=3.8" files = [ - {file = "setuptools-69.1.1-py3-none-any.whl", hash = "sha256:02fa291a0471b3a18b2b2481ed902af520c69e8ae0919c13da936542754b4c56"}, - {file = "setuptools-69.1.1.tar.gz", hash = "sha256:5c0806c7d9af348e6dd3777b4f4dbb42c7ad85b190104837488eab9a7c945cf8"}, + {file = "setuptools-69.2.0-py3-none-any.whl", hash = "sha256:c21c49fb1042386df081cb5d86759792ab89efca84cf114889191cd09aacc80c"}, + {file = "setuptools-69.2.0.tar.gz", hash = "sha256:0ff4183f8f42cd8fa3acea16c45205521a4ef28f73c6391d8a25e92893134f2e"}, ] [package.extras] docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] -testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.2)", "pip (>=19.1)", "pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff (>=0.2.1)", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "importlib-metadata", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "mypy (==1.9)", "packaging (>=23.2)", "pip (>=19.1)", "pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff (>=0.2.1)", "pytest-timeout", "pytest-xdist (>=3)", "tomli", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] testing-integration = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.2)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] [[package]] diff --git a/frontend/package.json b/frontend/package.json index 8b4f0ee..7109443 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -45,7 +45,8 @@ "tailwind-merge": "^2.0.0", "tailwindcss-animate": "^1.0.7", "thememirror": "^2.0.1", - "vite-plugin-checker": "^0.6.2" + "vite-plugin-checker": "^0.6.2", + "webm-duration-fix": "^1.0.4" }, "devDependencies": { "@types/node": "^20.9.0", diff --git a/frontend/src/components/ImageUpload.tsx b/frontend/src/components/ImageUpload.tsx index f5f5fc7..366f104 100644 --- a/frontend/src/components/ImageUpload.tsx +++ b/frontend/src/components/ImageUpload.tsx @@ -5,6 +5,8 @@ import { useDropzone } from "react-dropzone"; import { toast } from "react-hot-toast"; import { URLS } from "../urls"; import { Badge } from "./ui/badge"; +import ScreenRecorder from "./recording/ScreenRecorder"; +import { ScreenRecorderState } from "../types"; const baseStyle = { flex: 1, @@ -61,16 +63,23 @@ interface Props { function ImageUpload({ setReferenceImages }: Props) { const [files, setFiles] = useState([]); + // TODO: Switch to Zustand + const [screenRecorderState, setScreenRecorderState] = + useState(ScreenRecorderState.INITIAL); + const { getRootProps, getInputProps, isFocused, isDragAccept, isDragReject } = useDropzone({ maxFiles: 1, maxSize: 1024 * 1024 * 20, // 20 MB accept: { + // Image formats "image/png": [".png"], "image/jpeg": [".jpeg"], "image/jpg": [".jpg"], + // Video formats "video/quicktime": [".mov"], "video/mp4": [".mp4"], + "video/webm": [".webm"], }, onDrop: (acceptedFiles) => { // Set up the preview thumbnail images @@ -154,21 +163,34 @@ function ImageUpload({ setReferenceImages }: Props) { return (
- {/* eslint-disable-next-line @typescript-eslint/no-explicit-any */} -
- -

- Drag & drop a screenshot here,
- or click to upload -

-
-
- New! Upload a screen recording in .mp4 or .mov format to - clone a whole app (experimental).{" "} - - Learn more. - -
+ {screenRecorderState === ScreenRecorderState.INITIAL && ( + /* eslint-disable-next-line @typescript-eslint/no-explicit-any */ +
+ +

+ Drag & drop a screenshot here,
+ or click to upload +

+
+ )} + {screenRecorderState === ScreenRecorderState.INITIAL && ( +
+ New! Upload a screen recording (.mp4, .mov) or record + your screen to clone a whole app (experimental).{" "} + + Learn more. + +
+ )} +
); } diff --git a/frontend/src/components/recording/ScreenRecorder.tsx b/frontend/src/components/recording/ScreenRecorder.tsx new file mode 100644 index 0000000..c2cc06d --- /dev/null +++ b/frontend/src/components/recording/ScreenRecorder.tsx @@ -0,0 +1,115 @@ +import { useState } from "react"; +import { Button } from "../ui/button"; +import { ScreenRecorderState } from "../../types"; +import { blobToBase64DataUrl } from "./utils"; +import fixWebmDuration from "webm-duration-fix"; +import toast from "react-hot-toast"; + +interface Props { + screenRecorderState: ScreenRecorderState; + setScreenRecorderState: (state: ScreenRecorderState) => void; + generateCode: ( + referenceImages: string[], + inputMode: "image" | "video" + ) => void; +} + +function ScreenRecorder({ + screenRecorderState, + setScreenRecorderState, + generateCode, +}: Props) { + const [mediaRecorder, setMediaRecorder] = useState( + null + ); + const [screenRecordingDataUrl, setScreenRecordingDataUrl] = useState< + string | null + >(null); + + const startScreenRecording = async () => { + try { + // Get the screen recording stream + const stream = await navigator.mediaDevices.getDisplayMedia({ + video: true, + audio: { echoCancellation: true }, + }); + + // TODO: Test across different browsers + // Create the media recorder + const options = { mimeType: "video/webm" }; + const mediaRecorder = new MediaRecorder(stream, options); + setMediaRecorder(mediaRecorder); + + const chunks: BlobPart[] = []; + + // Accumalate chunks as data is available + mediaRecorder.ondataavailable = (e: BlobEvent) => chunks.push(e.data); + + // When media recorder is stopped, create a data URL + mediaRecorder.onstop = async () => { + // TODO: Do I need to fix duration if it's not a webm? + const completeBlob = await fixWebmDuration( + new Blob(chunks, { + type: options.mimeType, + }) + ); + + const dataUrl = await blobToBase64DataUrl(completeBlob); + setScreenRecordingDataUrl(dataUrl); + setScreenRecorderState(ScreenRecorderState.FINISHED); + }; + + // Start recording + mediaRecorder.start(); + setScreenRecorderState(ScreenRecorderState.RECORDING); + } catch (error) { + toast.error("Could not start screen recording"); + throw error; + } + }; + + const stopScreenRecording = () => { + if (mediaRecorder) { + mediaRecorder.stop(); + setMediaRecorder(null); + } + }; + + const kickoffGeneration = () => { + if (screenRecordingDataUrl) { + generateCode([screenRecordingDataUrl], "video"); + } else { + toast.error("Screen recording does not exist. Please try again."); + throw new Error("No screen recording data url"); + } + }; + + return ( +
+ {screenRecorderState === ScreenRecorderState.INITIAL && ( + + )} + + {screenRecorderState === ScreenRecorderState.RECORDING && ( +
+
+ + Recording... +
+ +
+ )} + + {screenRecorderState === ScreenRecorderState.FINISHED && ( +
+
+ Screen Recording Captured. +
+ +
+ )} +
+ ); +} + +export default ScreenRecorder; diff --git a/frontend/src/components/recording/utils.ts b/frontend/src/components/recording/utils.ts new file mode 100644 index 0000000..b482ecf --- /dev/null +++ b/frontend/src/components/recording/utils.ts @@ -0,0 +1,31 @@ +export function downloadBlob(blob: Blob) { + // Create a URL for the blob object + const videoURL = URL.createObjectURL(blob); + + // Create a temporary anchor element and trigger the download + const a = document.createElement("a"); + a.href = videoURL; + a.download = "recording.webm"; + document.body.appendChild(a); + a.click(); + document.body.removeChild(a); + + // Clear object URL + URL.revokeObjectURL(videoURL); +} + +export function blobToBase64DataUrl(blob: Blob): Promise { + return new Promise((resolve, reject) => { + const reader = new FileReader(); + reader.onloadend = () => { + if (reader.result) { + resolve(reader.result as string); + } else { + reject(new Error("FileReader did not return a result.")); + } + }; + reader.onerror = () => + reject(new Error("FileReader encountered an error.")); + reader.readAsDataURL(blob); + }); +} diff --git a/frontend/src/types.ts b/frontend/src/types.ts index 89440fe..fd46082 100644 --- a/frontend/src/types.ts +++ b/frontend/src/types.ts @@ -25,6 +25,12 @@ export enum AppState { CODE_READY = "CODE_READY", } +export enum ScreenRecorderState { + INITIAL = "initial", + RECORDING = "recording", + FINISHED = "finished", +} + export interface CodeGenerationParams { generationType: "create" | "update"; inputMode: "image" | "video"; diff --git a/frontend/yarn.lock b/frontend/yarn.lock index 5f09885..1edf79e 100644 --- a/frontend/yarn.lock +++ b/frontend/yarn.lock @@ -1610,6 +1610,11 @@ base64-arraybuffer@^1.0.2: resolved "https://registry.npmjs.org/base64-arraybuffer/-/base64-arraybuffer-1.0.2.tgz#1c37589a7c4b0746e34bd1feb951da2df01c1bdc" integrity sha512-I3yl4r9QB5ZRY3XuJVEPfc2XhZO6YweFPI+UovAzn+8/hb3oJ6lnysaFcjVpkCPfVWFUDvoZ8kmVDP7WyRtYtQ== +base64-js@^1.3.1: + version "1.5.1" + resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-1.5.1.tgz#1b1b440160a5bf7ad40b650f095963481903930a" + integrity sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA== + binary-extensions@^2.0.0: version "2.2.0" resolved "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.2.0.tgz" @@ -1657,6 +1662,14 @@ buffer-from@^1.0.0: resolved "https://registry.yarnpkg.com/buffer-from/-/buffer-from-1.1.2.tgz#2b146a6fd72e80b4f55d255f35ed59a3a9a41bd5" integrity sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ== +buffer@^6.0.3: + version "6.0.3" + resolved "https://registry.yarnpkg.com/buffer/-/buffer-6.0.3.tgz#2ace578459cc8fbe2a70aaa8f52ee63b6a74c6c6" + integrity sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA== + dependencies: + base64-js "^1.3.1" + ieee754 "^1.2.1" + cac@^6.7.14: version "6.7.14" resolved "https://registry.yarnpkg.com/cac/-/cac-6.7.14.tgz#804e1e6f506ee363cb0e3ccbb09cad5dd9870959" @@ -1993,6 +2006,11 @@ dotenv@^16.0.0: resolved "https://registry.yarnpkg.com/dotenv/-/dotenv-16.3.1.tgz#369034de7d7e5b120972693352a3bf112172cc3e" integrity sha512-IPzF4w4/Rd94bA9imS68tZBaYyBWSCE47V1RGuMrB94iyTOIEwRmVL2x/4An+6mETpLrKJ5hQkB8W4kFAadeIQ== +ebml-block@^1.1.2: + version "1.1.2" + resolved "https://registry.yarnpkg.com/ebml-block/-/ebml-block-1.1.2.tgz#fd49951b0faf5a3049bdd61c851a76b5e679c290" + integrity sha512-HgNlIsRFP6D9VKU5atCeHRJY7XkJP8bOe8yEhd8NB7B3b4++VWTyauz6g650iiPmLfPLGlVpoJmGSgMfXDYusg== + ejs@^3.1.6: version "3.1.9" resolved "https://registry.yarnpkg.com/ejs/-/ejs-3.1.9.tgz#03c9e8777fe12686a9effcef22303ca3d8eeb361" @@ -2186,6 +2204,11 @@ esutils@^2.0.2: resolved "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz" integrity sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g== +events@^3.3.0: + version "3.3.0" + resolved "https://registry.yarnpkg.com/events/-/events-3.3.0.tgz#31a95ad0a924e2d2c419a813aeb2c4e878ea7400" + integrity sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q== + execa@^8.0.1: version "8.0.1" resolved "https://registry.yarnpkg.com/execa/-/execa-8.0.1.tgz#51f6a5943b580f963c3ca9c6321796db8cc39b8c" @@ -2467,6 +2490,11 @@ human-signals@^5.0.0: resolved "https://registry.yarnpkg.com/human-signals/-/human-signals-5.0.0.tgz#42665a284f9ae0dade3ba41ebc37eb4b852f3a28" integrity sha512-AXcZb6vzzrFAUE61HnN4mpLqd/cSIwNQjtNWR0euPm6y0iqx3G4gOXaIDdtdDwZmhwe82LA6+zinmW4UBWVePQ== +ieee754@^1.2.1: + version "1.2.1" + resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.2.1.tgz#8eb7a10a63fff25d15a57b001586d177d1b0d352" + integrity sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA== + ignore@^5.2.0, ignore@^5.2.4: version "5.2.4" resolved "https://registry.npmjs.org/ignore/-/ignore-5.2.4.tgz" @@ -2498,6 +2526,11 @@ inherits@2: resolved "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz" integrity sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ== +int64-buffer@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/int64-buffer/-/int64-buffer-1.0.1.tgz#c78d841b444cadf036cd04f8683696c740f15dca" + integrity sha512-+3azY4pXrjAupJHU1V9uGERWlhoqNswJNji6aD/02xac7oxol508AsMC5lxKhEqyZeDFy3enq5OGWXF4u75hiw== + invariant@^2.2.4: version "2.2.4" resolved "https://registry.yarnpkg.com/invariant/-/invariant-2.2.4.tgz#610f3c92c9359ce1db616e538008d23ff35158e6" @@ -3765,6 +3798,16 @@ w3c-keyname@^2.2.4: resolved "https://registry.yarnpkg.com/w3c-keyname/-/w3c-keyname-2.2.8.tgz#7b17c8c6883d4e8b86ac8aba79d39e880f8869c5" integrity sha512-dpojBhNsCNN7T82Tm7k26A6G9ML3NkhDsnw9n/eoxSRlVBB4CEtIQ/KTCLI2Fwf3ataSXRhYFkQi3SlnFwPvPQ== +webm-duration-fix@^1.0.4: + version "1.0.4" + resolved "https://registry.yarnpkg.com/webm-duration-fix/-/webm-duration-fix-1.0.4.tgz#fef235cb3d3ed3363507f705a7577dbb9fdedae6" + integrity sha512-kvhmSmEnuohtK+j+mJswqCCM2ViKb9W8Ch0oAxcaeUvpok5CsMORQLnea+CYKDXPG6JH12H0CbRK85qhfeZLew== + dependencies: + buffer "^6.0.3" + ebml-block "^1.1.2" + events "^3.3.0" + int64-buffer "^1.0.1" + which@^2.0.1: version "2.0.2" resolved "https://registry.npmjs.org/which/-/which-2.0.2.tgz"