add bedrock claude 3.5
This commit is contained in:
xie river
parent
d285e0e94e
commit
1f65c29c4d
@ -37,9 +37,9 @@ We also just added experimental support for taking a video/screen recording of a
|
||||
|
||||
## 🛠 Getting Started
|
||||
|
||||
### 使用AWS Bedrock Claude 3 sonnet注意事项
|
||||
- 如果使用Bedrock Claude 3需要在运行机器上安装 https://aws.amazon.com/cn/cli/， 并配置aws iam 账号的ak sk，另外还需要开通该账号Bedrock Claude 3 访问的权限。
|
||||
- 如果使用Bedrock Claude 3，则无须配置OPENAI_API_KEY 或者 ANTHROPIC_API_KEY 到.env中
|
||||
### 使用AWS Bedrock Claude 3/3.5 sonnet注意事项
|
||||
- 如果使用Bedrock Claude 3/3.5需要在运行机器上安装 https://aws.amazon.com/cn/cli/， 并配置aws iam 账号的ak sk，另外还需要开通该账号Bedrock Claude 3 访问的权限。
|
||||
- 如果使用Bedrock Claude 3/3.5，则无须配置OPENAI_API_KEY 或者 ANTHROPIC_API_KEY 到.env中
|
||||
|
||||
|
||||
The app has a React/Vite frontend and a FastAPI backend. You will need an OpenAI API key with access to the GPT-4 Vision API or an Anthropic key if you want to use Claude Sonnet, or for experimental video support.
|
||||
|
||||
@ -19,29 +19,36 @@ from datetime import datetime
|
||||
from botocore.config import Config
|
||||
|
||||
#get modelARN
|
||||
region = 'us-west-2'
|
||||
REGION = os.environ.get('region','us-east-1')
|
||||
PROFILE = os.environ.get('profile', 'default')
|
||||
session = boto3.Session(profile_name=PROFILE,region_name=REGION)
|
||||
bedrock_runtime = session.client(
|
||||
service_name="bedrock-runtime",
|
||||
region_name=REGION
|
||||
)
|
||||
|
||||
config = Config(read_timeout=1000) #timeout
|
||||
|
||||
boto3_bedrock = boto3.client('bedrock',region)
|
||||
bedrock_runtime = boto3.client('bedrock-runtime',config=config)
|
||||
modelId = "anthropic.claude-3-sonnet-20240229-v1:0"
|
||||
# Actual model versions that are passed to the LLMs and stored in our logs
|
||||
class Llm(Enum):
|
||||
GPT_4_VISION = "gpt-4-vision-preview"
|
||||
GPT_4_TURBO_2024_04_09 = "gpt-4-turbo-2024-04-09"
|
||||
GPT_4O_2024_05_13 = "gpt-4o-2024-05-13"
|
||||
CLAUDE_3_SONNET = "claude-3-sonnet-20240229"
|
||||
CLAUDE_3_5_SONNET = "claude-3-5-sonnet-20240620"
|
||||
CLAUDE_3_OPUS = "claude-3-opus-20240229"
|
||||
CLAUDE_3_HAIKU = "claude-3-haiku-20240307"
|
||||
|
||||
|
||||
BEDROCK_LLM_MODELID_LIST = {Llm.CLAUDE_3_5_SONNET: 'anthropic.claude-3-sonnet-20240229-v1:0',
|
||||
Llm.CLAUDE_3_SONNET: 'anthropic.claude-3-5-sonnet-20240620-v1:0',}
|
||||
|
||||
# Will throw errors if you send a garbage string
|
||||
def convert_frontend_str_to_llm(frontend_str: str) -> Llm:
|
||||
if frontend_str == "gpt_4_vision":
|
||||
return Llm.GPT_4_VISION
|
||||
elif frontend_str == "claude_3_sonnet":
|
||||
return Llm.CLAUDE_3_SONNET
|
||||
elif frontend_str == "claude_3_5_sonnet":
|
||||
return Llm.CLAUDE_3_5_SONNET
|
||||
else:
|
||||
return Llm(frontend_str)
|
||||
|
||||
@ -85,65 +92,6 @@ async def stream_openai_response(
|
||||
return full_response
|
||||
|
||||
|
||||
# TODO: Have a seperate function that translates OpenAI messages to Claude messages
|
||||
async def stream_claude_response_bak(
|
||||
messages: List[ChatCompletionMessageParam],
|
||||
api_key: str,
|
||||
callback: Callable[[str], Awaitable[None]],
|
||||
) -> str:
|
||||
|
||||
client = AsyncAnthropic(api_key=api_key)
|
||||
|
||||
# Base parameters
|
||||
model = Llm.CLAUDE_3_SONNET
|
||||
max_tokens = 4096
|
||||
temperature = 0.0
|
||||
|
||||
# Translate OpenAI messages to Claude messages
|
||||
system_prompt = cast(str, messages[0].get("content"))
|
||||
claude_messages = [dict(message) for message in messages[1:]]
|
||||
for message in claude_messages:
|
||||
if not isinstance(message["content"], list):
|
||||
continue
|
||||
|
||||
for content in message["content"]: # type: ignore
|
||||
if content["type"] == "image_url":
|
||||
content["type"] = "image"
|
||||
|
||||
# Extract base64 data and media type from data URL
|
||||
# Example base64 data URL: data:image/png;base64,iVBOR...
|
||||
image_data_url = cast(str, content["image_url"]["url"])
|
||||
media_type = image_data_url.split(";")[0].split(":")[1]
|
||||
base64_data = image_data_url.split(",")[1]
|
||||
|
||||
# Remove OpenAI parameter
|
||||
del content["image_url"]
|
||||
|
||||
content["source"] = {
|
||||
"type": "base64",
|
||||
"media_type": media_type,
|
||||
"data": base64_data,
|
||||
}
|
||||
|
||||
# Stream Claude response
|
||||
async with client.messages.stream(
|
||||
model=model.value,
|
||||
max_tokens=max_tokens,
|
||||
temperature=temperature,
|
||||
system=system_prompt,
|
||||
messages=claude_messages, # type: ignore
|
||||
) as stream:
|
||||
async for text in stream.text_stream:
|
||||
await callback(text)
|
||||
|
||||
# Return final message
|
||||
response = await stream.get_final_message()
|
||||
|
||||
# Close the Anthropic client
|
||||
await client.close()
|
||||
|
||||
return response.content[0].text
|
||||
|
||||
async def stream_claude_response(
|
||||
messages: List[ChatCompletionMessageParam],
|
||||
api_key: str,
|
||||
@ -153,7 +101,6 @@ async def stream_claude_response(
|
||||
# client = AsyncAnthropic(api_key=api_key)
|
||||
|
||||
# Base parameters
|
||||
model = Llm.CLAUDE_3_SONNET
|
||||
max_tokens = 4096
|
||||
temperature = 0.0
|
||||
|
||||
@ -190,7 +137,8 @@ async def stream_claude_response(
|
||||
"anthropic_version": "bedrock-2023-05-31",
|
||||
"max_tokens": max_tokens,
|
||||
"messages": claude_messages,
|
||||
"temperature":temperature
|
||||
"temperature":temperature,
|
||||
"system":system_prompt,
|
||||
}
|
||||
}
|
||||
|
||||
@ -224,10 +172,11 @@ async def stream_claude_response_native(
|
||||
api_key: str,
|
||||
callback: Callable[[str], Awaitable[None]],
|
||||
include_thinking: bool = False,
|
||||
model: Llm = Llm.CLAUDE_3_OPUS,
|
||||
model: Llm = Llm.CLAUDE_3_5_SONNET,
|
||||
) -> str:
|
||||
|
||||
# client = AsyncAnthropic(api_key=api_key)
|
||||
modelId = BEDROCK_LLM_MODELID_LIST[model]
|
||||
|
||||
# Base model parameters
|
||||
max_tokens = 4096
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
// Keep in sync with backend (llm.py)
|
||||
// Order here matches dropdown order
|
||||
export enum CodeGenerationModel {
|
||||
CLAUDE_3_5_SONNET = "claude_3_5_sonnet",
|
||||
CLAUDE_3_SONNET = "claude_3_sonnet",
|
||||
GPT_4O_2024_05_13 = "gpt-4o-2024-05-13",
|
||||
GPT_4_TURBO_2024_04_09 = "gpt-4-turbo-2024-04-09",
|
||||
@ -15,4 +16,5 @@ export const CODE_GENERATION_MODEL_DESCRIPTIONS: {
|
||||
"gpt-4-turbo-2024-04-09": { name: "GPT-4 Turbo (Apr 2024)", inBeta: false },
|
||||
gpt_4_vision: { name: "GPT-4 Vision (Nov 2023)", inBeta: false },
|
||||
claude_3_sonnet: { name: "Claude 3 Sonnet", inBeta: false },
|
||||
claude_3_5_sonnet: { name: "Claude 3.5 Sonnet", inBeta: false },
|
||||
};
|
||||
|
||||
Loading…
Reference in New Issue
Block a user