diff --git a/backend/eval_config.py b/backend/eval_config.py deleted file mode 100644 index 62a3b8f..0000000 --- a/backend/eval_config.py +++ /dev/null @@ -1 +0,0 @@ -EVALS_DIR = "./evals" diff --git a/backend/evals/__init__.py b/backend/evals/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/evals/config.py b/backend/evals/config.py new file mode 100644 index 0000000..7643027 --- /dev/null +++ b/backend/evals/config.py @@ -0,0 +1 @@ +EVALS_DIR = "./evals_data" diff --git a/backend/evals/core.py b/backend/evals/core.py new file mode 100644 index 0000000..61db1a3 --- /dev/null +++ b/backend/evals/core.py @@ -0,0 +1,29 @@ +import os + +from llm import stream_openai_response +from prompts import assemble_prompt +from prompts.types import Stack +from utils import pprint_prompt + + +async def generate_code_core(image_url: str, stack: Stack) -> str: + prompt_messages = assemble_prompt(image_url, stack) + openai_api_key = os.environ.get("OPENAI_API_KEY") + openai_base_url = None + + pprint_prompt(prompt_messages) + + async def process_chunk(content: str): + pass + + if not openai_api_key: + raise Exception("OpenAI API key not found") + + completion = await stream_openai_response( + prompt_messages, + api_key=openai_api_key, + base_url=openai_base_url, + callback=lambda x: process_chunk(x), + ) + + return completion diff --git a/backend/eval_utils.py b/backend/evals/utils.py similarity index 100% rename from backend/eval_utils.py rename to backend/evals/utils.py diff --git a/backend/routes/evals.py b/backend/routes/evals.py index 48a3a95..798a9d8 100644 --- a/backend/routes/evals.py +++ b/backend/routes/evals.py @@ -1,8 +1,8 @@ import os from fastapi import APIRouter from pydantic import BaseModel -from eval_utils import image_to_data_url -from eval_config import EVALS_DIR +from evals.utils import image_to_data_url +from evals.config import EVALS_DIR router = APIRouter() diff --git a/backend/eval.py b/backend/run_evals.py similarity index 57% rename from backend/eval.py rename to backend/run_evals.py index 75641a3..ddb7eaa 100644 --- a/backend/eval.py +++ b/backend/run_evals.py @@ -1,42 +1,15 @@ # Load environment variables first -from typing import Any, Coroutine from dotenv import load_dotenv -from eval_config import EVALS_DIR -from eval_utils import image_to_data_url -from prompts.types import Stack - load_dotenv() import os -from llm import stream_openai_response -from prompts import assemble_prompt +from typing import Any, Coroutine import asyncio -from utils import pprint_prompt - - -async def generate_code_core(image_url: str, stack: Stack) -> str: - prompt_messages = assemble_prompt(image_url, stack) - openai_api_key = os.environ.get("OPENAI_API_KEY") - openai_base_url = None - - pprint_prompt(prompt_messages) - - async def process_chunk(content: str): - pass - - if not openai_api_key: - raise Exception("OpenAI API key not found") - - completion = await stream_openai_response( - prompt_messages, - api_key=openai_api_key, - base_url=openai_base_url, - callback=lambda x: process_chunk(x), - ) - - return completion +from evals.config import EVALS_DIR +from evals.core import generate_code_core +from evals.utils import image_to_data_url async def main():