Merge branch 'main' into hosted
This commit is contained in:
Abi Raja
commit
9a13fcc3d0
@ -14,6 +14,7 @@ from utils import pprint_prompt
|
||||
class Llm(Enum):
|
||||
GPT_4_VISION = "gpt-4-vision-preview"
|
||||
GPT_4_TURBO_2024_04_09 = "gpt-4-turbo-2024-04-09"
|
||||
GPT_4O_2024_05_13 = "gpt-4o-2024-05-13"
|
||||
CLAUDE_3_SONNET = "claude-3-sonnet-20240229"
|
||||
CLAUDE_3_OPUS = "claude-3-opus-20240229"
|
||||
CLAUDE_3_HAIKU = "claude-3-haiku-20240307"
|
||||
@ -48,7 +49,11 @@ async def stream_openai_response(
|
||||
}
|
||||
|
||||
# Add 'max_tokens' only if the model is a GPT4 vision or Turbo model
|
||||
if model == Llm.GPT_4_VISION or model == Llm.GPT_4_TURBO_2024_04_09:
|
||||
if (
|
||||
model == Llm.GPT_4_VISION
|
||||
or model == Llm.GPT_4_TURBO_2024_04_09
|
||||
or model == Llm.GPT_4O_2024_05_13
|
||||
):
|
||||
params["max_tokens"] = 4096
|
||||
|
||||
stream = await client.chat.completions.create(**params) # type: ignore
|
||||
|
||||
@ -7,10 +7,13 @@ from evals.config import EVALS_DIR
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
# Update this if the number of outputs generated per input changes
|
||||
N = 1
|
||||
|
||||
|
||||
class Eval(BaseModel):
|
||||
input: str
|
||||
output: str
|
||||
outputs: list[str]
|
||||
|
||||
|
||||
@router.get("/evals")
|
||||
@ -25,21 +28,27 @@ async def get_evals():
|
||||
input_file_path = os.path.join(input_dir, file)
|
||||
input_file = await image_to_data_url(input_file_path)
|
||||
|
||||
# Construct the corresponding output file name
|
||||
output_file_name = file.replace(".png", ".html")
|
||||
output_file_path = os.path.join(output_dir, output_file_name)
|
||||
# Construct the corresponding output file names
|
||||
output_file_names = [
|
||||
file.replace(".png", f"_{i}.html") for i in range(0, N)
|
||||
] # Assuming 3 outputs for each input
|
||||
|
||||
output_files_data: list[str] = []
|
||||
for output_file_name in output_file_names:
|
||||
output_file_path = os.path.join(output_dir, output_file_name)
|
||||
# Check if the output file exists
|
||||
if os.path.exists(output_file_path):
|
||||
with open(output_file_path, "r") as f:
|
||||
output_file_data = f.read()
|
||||
output_files_data.append(f.read())
|
||||
else:
|
||||
output_file_data = "Output file not found."
|
||||
output_files_data.append(
|
||||
"<html><h1>Output file not found.</h1></html>"
|
||||
)
|
||||
|
||||
evals.append(
|
||||
Eval(
|
||||
input=input_file,
|
||||
output=output_file_data,
|
||||
outputs=output_files_data,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@ -85,7 +85,7 @@ async def stream_code(websocket: WebSocket):
|
||||
|
||||
# Read the model from the request. Fall back to default if not provided.
|
||||
code_generation_model_str = params.get(
|
||||
"codeGenerationModel", Llm.GPT_4_VISION.value
|
||||
"codeGenerationModel", Llm.GPT_4O_2024_05_13.value
|
||||
)
|
||||
try:
|
||||
code_generation_model = convert_frontend_str_to_llm(code_generation_model_str)
|
||||
@ -146,6 +146,7 @@ async def stream_code(websocket: WebSocket):
|
||||
if not openai_api_key and (
|
||||
code_generation_model == Llm.GPT_4_VISION
|
||||
or code_generation_model == Llm.GPT_4_TURBO_2024_04_09
|
||||
or code_generation_model == Llm.GPT_4O_2024_05_13
|
||||
):
|
||||
print("OpenAI API key not found")
|
||||
await throw_error(
|
||||
|
||||
@ -13,8 +13,9 @@ from evals.config import EVALS_DIR
|
||||
from evals.core import generate_code_core
|
||||
from evals.utils import image_to_data_url
|
||||
|
||||
STACK = "html_tailwind"
|
||||
MODEL = Llm.CLAUDE_3_SONNET
|
||||
STACK = "ionic_tailwind"
|
||||
MODEL = Llm.GPT_4O_2024_05_13
|
||||
N = 1 # Number of outputs to generate
|
||||
|
||||
|
||||
async def main():
|
||||
@ -28,6 +29,7 @@ async def main():
|
||||
for filename in evals:
|
||||
filepath = os.path.join(INPUT_DIR, filename)
|
||||
data_url = await image_to_data_url(filepath)
|
||||
for _ in range(N): # Generate N tasks for each input
|
||||
task = generate_code_core(image_url=data_url, stack=STACK, model=MODEL)
|
||||
tasks.append(task)
|
||||
|
||||
@ -35,9 +37,13 @@ async def main():
|
||||
|
||||
os.makedirs(OUTPUT_DIR, exist_ok=True)
|
||||
|
||||
for filename, content in zip(evals, results):
|
||||
# File name is derived from the original filename in evals
|
||||
output_filename = f"{os.path.splitext(filename)[0]}.html"
|
||||
for i, content in enumerate(results):
|
||||
# Calculate index for filename and output number
|
||||
eval_index = i // N
|
||||
output_number = i % N
|
||||
filename = evals[eval_index]
|
||||
# File name is derived from the original filename in evals with an added output number
|
||||
output_filename = f"{os.path.splitext(filename)[0]}_{output_number}.html"
|
||||
output_filepath = os.path.join(OUTPUT_DIR, output_filename)
|
||||
with open(output_filepath, "w") as file:
|
||||
file.write(content)
|
||||
|
||||
@ -24,6 +24,11 @@ class TestConvertFrontendStrToLlm(unittest.TestCase):
|
||||
Llm.GPT_4_TURBO_2024_04_09,
|
||||
"Should convert 'gpt-4-turbo-2024-04-09' to Llm.GPT_4_TURBO_2024_04_09",
|
||||
)
|
||||
self.assertEqual(
|
||||
convert_frontend_str_to_llm("gpt-4o-2024-05-13"),
|
||||
Llm.GPT_4O_2024_05_13,
|
||||
"Should convert 'gpt-4o-2024-05-13' to Llm.GPT_4O_2024_05_13",
|
||||
)
|
||||
|
||||
def test_convert_invalid_string_raises_exception(self):
|
||||
with self.assertRaises(ValueError):
|
||||
|
||||
@ -74,7 +74,7 @@ function App({ navbarComponent }: Props) {
|
||||
isImageGenerationEnabled: true,
|
||||
editorTheme: EditorTheme.COBALT,
|
||||
generatedCodeConfig: Stack.HTML_TAILWIND,
|
||||
codeGenerationModel: CodeGenerationModel.GPT_4_TURBO_2024_04_09,
|
||||
codeGenerationModel: CodeGenerationModel.GPT_4O_2024_05_13,
|
||||
// Only relevant for hosted version
|
||||
isTermOfServiceAccepted: false,
|
||||
},
|
||||
|
||||
@ -4,7 +4,7 @@ import RatingPicker from "./RatingPicker";
|
||||
|
||||
interface Eval {
|
||||
input: string;
|
||||
output: string;
|
||||
outputs: string[];
|
||||
}
|
||||
|
||||
function EvalsPage() {
|
||||
@ -38,18 +38,22 @@ function EvalsPage() {
|
||||
<div className="flex flex-col gap-y-4 mt-4 mx-auto justify-center">
|
||||
{evals.map((e, index) => (
|
||||
<div className="flex flex-col justify-center" key={index}>
|
||||
<div className="flex gap-x-2 justify-center">
|
||||
<h2 className="font-bold text-lg ml-4">{index}</h2>
|
||||
<div className="flex gap-x-2 justify-center ml-4">
|
||||
{/* Update w if N changes to a fixed number like w-[600px] */}
|
||||
<div className="w-1/2 p-1 border">
|
||||
<img src={e.input} />
|
||||
<img src={e.input} alt={`Input for eval ${index}`} />
|
||||
</div>
|
||||
<div className="w-1/2 p-1 border">
|
||||
{e.outputs.map((output, outputIndex) => (
|
||||
<div className="w-1/2 p-1 border" key={outputIndex}>
|
||||
{/* Put output into an iframe */}
|
||||
<iframe
|
||||
srcDoc={e.output}
|
||||
srcDoc={output}
|
||||
className="w-[1200px] h-[800px] transform scale-[0.60]"
|
||||
style={{ transformOrigin: "top left" }}
|
||||
></iframe>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
<div className="ml-8 mt-4 flex justify-center">
|
||||
<RatingPicker
|
||||
|
||||
@ -1,5 +1,7 @@
|
||||
// Keep in sync with backend (llm.py)
|
||||
// Order here matches dropdown order
|
||||
export enum CodeGenerationModel {
|
||||
GPT_4O_2024_05_13 = "gpt-4o-2024-05-13",
|
||||
GPT_4_TURBO_2024_04_09 = "gpt-4-turbo-2024-04-09",
|
||||
GPT_4_VISION = "gpt_4_vision",
|
||||
CLAUDE_3_SONNET = "claude_3_sonnet",
|
||||
@ -13,6 +15,7 @@ export const CODE_GENERATION_MODEL_DESCRIPTIONS: {
|
||||
isPaid: boolean;
|
||||
};
|
||||
} = {
|
||||
"gpt-4o-2024-05-13": { name: "GPT-4O 🌟", inBeta: false, isPaid: false },
|
||||
"gpt-4-turbo-2024-04-09": {
|
||||
name: "GPT-4 Turbo (Apr 2024)",
|
||||
inBeta: false,
|
||||
|
||||
Loading…
Reference in New Issue
Block a user