Github/screenshot-to-code
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'main' into hosted

Browse Source
This commit is contained in:
Abi Raja 2024-05-13 16:06:10 -04:00
commit 9a13fcc3d0
8 changed files with 65 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
backend/llm.py
View File

@ -14,6 +14,7 @@ from utils import pprint_prompt
class Llm(Enum): class Llm(Enum):
GPT_4_VISION = "gpt-4-vision-preview" GPT_4_VISION = "gpt-4-vision-preview"
GPT_4_TURBO_2024_04_09 = "gpt-4-turbo-2024-04-09" GPT_4_TURBO_2024_04_09 = "gpt-4-turbo-2024-04-09"
GPT_4O_2024_05_13 = "gpt-4o-2024-05-13"
CLAUDE_3_SONNET = "claude-3-sonnet-20240229" CLAUDE_3_SONNET = "claude-3-sonnet-20240229"
CLAUDE_3_OPUS = "claude-3-opus-20240229" CLAUDE_3_OPUS = "claude-3-opus-20240229"
CLAUDE_3_HAIKU = "claude-3-haiku-20240307" CLAUDE_3_HAIKU = "claude-3-haiku-20240307"
@ -48,7 +49,11 @@ async def stream_openai_response(
} }
# Add 'max_tokens' only if the model is a GPT4 vision or Turbo model # Add 'max_tokens' only if the model is a GPT4 vision or Turbo model
if model == Llm.GPT_4_VISION or model == Llm.GPT_4_TURBO_2024_04_09: if (
model == Llm.GPT_4_VISION
or model == Llm.GPT_4_TURBO_2024_04_09
or model == Llm.GPT_4O_2024_05_13
):
params["max_tokens"] = 4096 params["max_tokens"] = 4096
stream = await client.chat.completions.create(**params) # type: ignore stream = await client.chat.completions.create(**params) # type: ignore

31
backend/routes/evals.py
View File

@ -7,10 +7,13 @@ from evals.config import EVALS_DIR
router = APIRouter() router = APIRouter()
# Update this if the number of outputs generated per input changes
N = 1
class Eval(BaseModel): class Eval(BaseModel):
input: str input: str
output: str outputs: list[str]
@router.get("/evals") @router.get("/evals")
@ -25,21 +28,27 @@ async def get_evals():
input_file_path = os.path.join(input_dir, file) input_file_path = os.path.join(input_dir, file)
input_file = await image_to_data_url(input_file_path) input_file = await image_to_data_url(input_file_path)
# Construct the corresponding output file name # Construct the corresponding output file names
output_file_name = file.replace(".png", ".html") output_file_names = [
output_file_path = os.path.join(output_dir, output_file_name) file.replace(".png", f"_{i}.html") for i in range(0, N)
] # Assuming 3 outputs for each input
# Check if the output file exists output_files_data: list[str] = []
if os.path.exists(output_file_path): for output_file_name in output_file_names:
with open(output_file_path, "r") as f: output_file_path = os.path.join(output_dir, output_file_name)
output_file_data = f.read() # Check if the output file exists
else: if os.path.exists(output_file_path):
output_file_data = "Output file not found." with open(output_file_path, "r") as f:
output_files_data.append(f.read())
else:
output_files_data.append(
"<html><h1>Output file not found.</h1></html>"
)
evals.append( evals.append(
Eval( Eval(
input=input_file, input=input_file,
output=output_file_data, outputs=output_files_data,
) )
) )

3
backend/routes/generate_code.py
View File

@ -85,7 +85,7 @@ async def stream_code(websocket: WebSocket):
# Read the model from the request. Fall back to default if not provided. # Read the model from the request. Fall back to default if not provided.
code_generation_model_str = params.get( code_generation_model_str = params.get(
"codeGenerationModel", Llm.GPT_4_VISION.value "codeGenerationModel", Llm.GPT_4O_2024_05_13.value
) )
try: try:
code_generation_model = convert_frontend_str_to_llm(code_generation_model_str) code_generation_model = convert_frontend_str_to_llm(code_generation_model_str)
@ -146,6 +146,7 @@ async def stream_code(websocket: WebSocket):
if not openai_api_key and ( if not openai_api_key and (
code_generation_model == Llm.GPT_4_VISION code_generation_model == Llm.GPT_4_VISION
or code_generation_model == Llm.GPT_4_TURBO_2024_04_09 or code_generation_model == Llm.GPT_4_TURBO_2024_04_09
or code_generation_model == Llm.GPT_4O_2024_05_13
): ):
print("OpenAI API key not found") print("OpenAI API key not found")
await throw_error( await throw_error(

20
backend/run_evals.py
View File

@ -13,8 +13,9 @@ from evals.config import EVALS_DIR
from evals.core import generate_code_core from evals.core import generate_code_core
from evals.utils import image_to_data_url from evals.utils import image_to_data_url
STACK = "html_tailwind" STACK = "ionic_tailwind"
MODEL = Llm.CLAUDE_3_SONNET MODEL = Llm.GPT_4O_2024_05_13
N = 1 # Number of outputs to generate
async def main(): async def main():
@ -28,16 +29,21 @@ async def main():
for filename in evals: for filename in evals:
filepath = os.path.join(INPUT_DIR, filename) filepath = os.path.join(INPUT_DIR, filename)
data_url = await image_to_data_url(filepath) data_url = await image_to_data_url(filepath)
task = generate_code_core(image_url=data_url, stack=STACK, model=MODEL) for _ in range(N): # Generate N tasks for each input
tasks.append(task) task = generate_code_core(image_url=data_url, stack=STACK, model=MODEL)
tasks.append(task)
results = await asyncio.gather(*tasks) results = await asyncio.gather(*tasks)
os.makedirs(OUTPUT_DIR, exist_ok=True) os.makedirs(OUTPUT_DIR, exist_ok=True)
for filename, content in zip(evals, results): for i, content in enumerate(results):
# File name is derived from the original filename in evals # Calculate index for filename and output number
output_filename = f"{os.path.splitext(filename)[0]}.html" eval_index = i // N
output_number = i % N
filename = evals[eval_index]
# File name is derived from the original filename in evals with an added output number
output_filename = f"{os.path.splitext(filename)[0]}_{output_number}.html"
output_filepath = os.path.join(OUTPUT_DIR, output_filename) output_filepath = os.path.join(OUTPUT_DIR, output_filename)
with open(output_filepath, "w") as file: with open(output_filepath, "w") as file:
file.write(content) file.write(content)

5
backend/test_llm.py
View File

@ -24,6 +24,11 @@ class TestConvertFrontendStrToLlm(unittest.TestCase):
Llm.GPT_4_TURBO_2024_04_09, Llm.GPT_4_TURBO_2024_04_09,
"Should convert 'gpt-4-turbo-2024-04-09' to Llm.GPT_4_TURBO_2024_04_09", "Should convert 'gpt-4-turbo-2024-04-09' to Llm.GPT_4_TURBO_2024_04_09",
) )
self.assertEqual(
convert_frontend_str_to_llm("gpt-4o-2024-05-13"),
Llm.GPT_4O_2024_05_13,
"Should convert 'gpt-4o-2024-05-13' to Llm.GPT_4O_2024_05_13",
)
def test_convert_invalid_string_raises_exception(self): def test_convert_invalid_string_raises_exception(self):
with self.assertRaises(ValueError): with self.assertRaises(ValueError):

2
frontend/src/App.tsx
View File

@ -74,7 +74,7 @@ function App({ navbarComponent }: Props) {
isImageGenerationEnabled: true, isImageGenerationEnabled: true,
editorTheme: EditorTheme.COBALT, editorTheme: EditorTheme.COBALT,
generatedCodeConfig: Stack.HTML_TAILWIND, generatedCodeConfig: Stack.HTML_TAILWIND,
codeGenerationModel: CodeGenerationModel.GPT_4_TURBO_2024_04_09, codeGenerationModel: CodeGenerationModel.GPT_4O_2024_05_13,
// Only relevant for hosted version // Only relevant for hosted version
isTermOfServiceAccepted: false, isTermOfServiceAccepted: false,
}, },

26
frontend/src/components/evals/EvalsPage.tsx
View File

@ -4,7 +4,7 @@ import RatingPicker from "./RatingPicker";
interface Eval { interface Eval {
input: string; input: string;
output: string; outputs: string[];
} }
function EvalsPage() { function EvalsPage() {
@ -38,18 +38,22 @@ function EvalsPage() {
<div className="flex flex-col gap-y-4 mt-4 mx-auto justify-center"> <div className="flex flex-col gap-y-4 mt-4 mx-auto justify-center">
{evals.map((e, index) => ( {evals.map((e, index) => (
<div className="flex flex-col justify-center" key={index}> <div className="flex flex-col justify-center" key={index}>
<div className="flex gap-x-2 justify-center"> <h2 className="font-bold text-lg ml-4">{index}</h2>
<div className="flex gap-x-2 justify-center ml-4">
{/* Update w if N changes to a fixed number like w-[600px] */}
<div className="w-1/2 p-1 border"> <div className="w-1/2 p-1 border">
<img src={e.input} /> <img src={e.input} alt={`Input for eval ${index}`} />
</div>
<div className="w-1/2 p-1 border">
{/* Put output into an iframe */}
<iframe
srcDoc={e.output}
className="w-[1200px] h-[800px] transform scale-[0.60]"
style={{ transformOrigin: "top left" }}
></iframe>
</div> </div>
{e.outputs.map((output, outputIndex) => (
<div className="w-1/2 p-1 border" key={outputIndex}>
{/* Put output into an iframe */}
<iframe
srcDoc={output}
className="w-[1200px] h-[800px] transform scale-[0.60]"
style={{ transformOrigin: "top left" }}
></iframe>
</div>
))}
</div> </div>
<div className="ml-8 mt-4 flex justify-center"> <div className="ml-8 mt-4 flex justify-center">
<RatingPicker <RatingPicker

3
frontend/src/lib/models.ts
View File

@ -1,5 +1,7 @@
// Keep in sync with backend (llm.py) // Keep in sync with backend (llm.py)
// Order here matches dropdown order
export enum CodeGenerationModel { export enum CodeGenerationModel {
GPT_4O_2024_05_13 = "gpt-4o-2024-05-13",
GPT_4_TURBO_2024_04_09 = "gpt-4-turbo-2024-04-09", GPT_4_TURBO_2024_04_09 = "gpt-4-turbo-2024-04-09",
GPT_4_VISION = "gpt_4_vision", GPT_4_VISION = "gpt_4_vision",
CLAUDE_3_SONNET = "claude_3_sonnet", CLAUDE_3_SONNET = "claude_3_sonnet",
@ -13,6 +15,7 @@ export const CODE_GENERATION_MODEL_DESCRIPTIONS: {
isPaid: boolean; isPaid: boolean;
}; };
} = { } = {
"gpt-4o-2024-05-13": { name: "GPT-4O 🌟", inBeta: false, isPaid: false },
"gpt-4-turbo-2024-04-09": { "gpt-4-turbo-2024-04-09": {
name: "GPT-4 Turbo (Apr 2024)", name: "GPT-4 Turbo (Apr 2024)",
inBeta: false, inBeta: false,