github/autogen
1
1
Fork 0
mirror of https://github.com/microsoft/autogen.git synced 2026-04-20 03:02:16 -04:00
Code Issues Packages Projects Releases Wiki Activity

Fix tests for GPT assistant (#1505)

Browse Source 
* fixed tests for gpt assistant

* fixed tests for gpt assistant

* api_type openai

* tests updated

---------

Co-authored-by: Chi Wang <wang.chi@microsoft.com>
This commit is contained in:
Davor Runje
2024-02-02 17:27:13 +01:00
committed by GitHub
parent 015070b604
commit b0f42a1bd4
4 changed files with 234 additions and 184 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
autogen/oai/client.py
View File

@@ -90,13 +90,12 @@ class OpenAIWrapper:
{
"model": "gpt-3.5-turbo",
"api_key": os.environ.get("OPENAI_API_KEY"),
"api_type": "open_ai",
"api_type": "openai",
"base_url": "https://api.openai.com/v1",
},
{
"model": "llama-7B",
"base_url": "http://127.0.0.1:8080",
"api_type": "open_ai",
}
]
```

4
autogen/oai/completion.py
View File

@@ -744,13 +744,13 @@ class Completion(openai_Completion):
{
"model": "gpt-3.5-turbo",
"api_key": os.environ.get("OPENAI_API_KEY"),
"api_type": "open_ai",
"api_type": "openai",
"base_url": "https://api.openai.com/v1",
},
{
"model": "llama-7B",
"base_url": "http://127.0.0.1:8080",
"api_type": "open_ai",
"api_type": "openai",
}
],
prompt="Hi",

403
test/agentchat/contrib/test_gpt_assistant.py
View File

@@ -1,3 +1,5 @@
from unittest.mock import MagicMock
import uuid
import pytest
import os
import sys
@@ -26,15 +28,19 @@ if not skip:
)
def ask_ossinsight(question):
return f"That is a good question, but I don't know the answer yet. Please ask your human developer friend to help you. \n\n{question}"
@pytest.mark.skipif(
sys.platform in ["darwin", "win32"] or skip,
reason="do not run on MacOS or windows OR dependency is not installed OR requested to skip",
)
def test_config_list() -> None:
assert len(config_list) > 0
@pytest.mark.skipif(
sys.platform in ["darwin", "win32"] or skip,
reason="do not run on MacOS or windows OR dependency is not installed OR requested to skip",
)
def test_gpt_assistant_chat():
def test_gpt_assistant_chat() -> None:
ossinsight_api_schema = {
"name": "ossinsight_data_api",
"parameters": {
@@ -49,30 +55,50 @@ def test_gpt_assistant_chat():
},
"description": "This is an API endpoint allowing users (analysts) to input question about GitHub in text format to retrieve the related and structured data.",
}
ask_ossinsight_mock = MagicMock()
name = "For test_gpt_assistant_chat"
def ask_ossinsight(question: str) -> str:
ask_ossinsight_mock(question)
return "The repository microsoft/autogen has 123,456 stars on GitHub."
name = f"For test_gpt_assistant_chat {uuid.uuid4()}"
analyst = GPTAssistantAgent(
name=name,
llm_config={"tools": [{"type": "function", "function": ossinsight_api_schema}], "config_list": config_list},
instructions="Hello, Open Source Project Analyst. You'll conduct comprehensive evaluations of open source projects or organizations on the GitHub platform",
)
analyst.register_function(
function_map={
"ossinsight_data_api": ask_ossinsight,
}
)
try:
analyst.register_function(
function_map={
"ossinsight_data_api": ask_ossinsight,
}
)
ok, response = analyst._invoke_assistant(
[{"role": "user", "content": "What is the most popular open source project on GitHub?"}]
)
executable = analyst.can_execute_function("ossinsight_data_api")
analyst.reset()
threads_count = len(analyst._openai_threads)
analyst.delete_assistant()
ok, response = analyst._invoke_assistant(
[{"role": "user", "content": "How many stars microsoft/autogen has on GitHub?"}]
)
executable = analyst.can_execute_function("ossinsight_data_api")
analyst.reset()
threads_count = len(analyst._openai_threads)
finally:
analyst.delete_assistant()
# check response
assert ok is True
assert response.get("role", "") == "assistant"
assert len(response.get("content", "")) > 0
# check the question asked
ask_ossinsight_mock.assert_called_once()
question_asked = ask_ossinsight_mock.call_args[0][0].lower()
for word in "microsoft autogen stars github".split(" "):
assert word in question_asked
# check the answer
response_content = response.get("content", "").lower()
assert len(response_content) > 0
for word in "microsoft autogen 123 456".split(" "):
assert word in response_content
assert executable is False
assert threads_count == 0
@@ -81,12 +107,12 @@ def test_gpt_assistant_chat():
sys.platform in ["darwin", "win32"] or skip,
reason="do not run on MacOS or windows OR dependency is not installed OR requested to skip",
)
def test_get_assistant_instructions():
def test_get_assistant_instructions() -> None:
"""
Test function to create a new GPTAssistantAgent, set its instructions, retrieve the instructions,
and assert that the retrieved instructions match the set instructions.
"""
name = "For test_get_assistant_instructions"
name = f"For test_get_assistant_instructions {uuid.uuid4()}"
assistant = GPTAssistantAgent(
name,
instructions="This is a test",
@@ -105,7 +131,7 @@ def test_get_assistant_instructions():
sys.platform in ["darwin", "win32"] or skip,
reason="do not run on MacOS or windows OR dependency is not installed OR requested to skip",
)
def test_gpt_assistant_instructions_overwrite():
def test_gpt_assistant_instructions_overwrite() -> None:
"""
Test that the instructions of a GPTAssistantAgent can be overwritten or not depending on the value of the
`overwrite_instructions` parameter when creating a new assistant with the same ID.
@@ -117,7 +143,7 @@ def test_gpt_assistant_instructions_overwrite():
4. Check that the instructions of the assistant have been overwritten with the new ones.
"""
name = "For test_gpt_assistant_instructions_overwrite"
name = f"For test_gpt_assistant_instructions_overwrite {uuid.uuid4()}"
instructions1 = "This is a test #1"
instructions2 = "This is a test #2"
@@ -129,19 +155,22 @@ def test_gpt_assistant_instructions_overwrite():
},
)
assistant_id = assistant.assistant_id
assistant = GPTAssistantAgent(
name,
instructions=instructions2,
llm_config={
"config_list": config_list,
"assistant_id": assistant_id,
},
overwrite_instructions=True,
)
try:
assistant_id = assistant.assistant_id
assistant = GPTAssistantAgent(
name,
instructions=instructions2,
llm_config={
"config_list": config_list,
"assistant_id": assistant_id,
},
overwrite_instructions=True,
)
instruction_match = assistant.get_assistant_instructions() == instructions2
assistant.delete_assistant()
instruction_match = assistant.get_assistant_instructions() == instructions2
finally:
assistant.delete_assistant()
assert instruction_match is True
@@ -150,12 +179,12 @@ def test_gpt_assistant_instructions_overwrite():
sys.platform in ["darwin", "win32"] or skip,
reason="do not run on MacOS or windows OR dependency is not installed OR requested to skip",
)
def test_gpt_assistant_existing_no_instructions():
def test_gpt_assistant_existing_no_instructions() -> None:
"""
Test function to check if the GPTAssistantAgent can retrieve instructions for an existing assistant
even if the assistant was created with no instructions initially.
"""
name = "For test_gpt_assistant_existing_no_instructions"
name = f"For test_gpt_assistant_existing_no_instructions {uuid.uuid4()}"
instructions = "This is a test #1"
assistant = GPTAssistantAgent(
@@ -166,19 +195,23 @@ def test_gpt_assistant_existing_no_instructions():
},
)
assistant_id = assistant.assistant_id
try:
assistant_id = assistant.assistant_id
# create a new assistant with the same ID but no instructions
assistant = GPTAssistantAgent(
name,
llm_config={
"config_list": config_list,
"assistant_id": assistant_id,
},
)
# create a new assistant with the same ID but no instructions
assistant = GPTAssistantAgent(
name,
llm_config={
"config_list": config_list,
"assistant_id": assistant_id,
},
)
instruction_match = assistant.get_assistant_instructions() == instructions
finally:
assistant.delete_assistant()
instruction_match = assistant.get_assistant_instructions() == instructions
assistant.delete_assistant()
assert instruction_match is True
@@ -186,7 +219,7 @@ def test_gpt_assistant_existing_no_instructions():
sys.platform in ["darwin", "win32"] or skip,
reason="do not run on MacOS or windows OR dependency is not installed OR requested to skip",
)
def test_get_assistant_files():
def test_get_assistant_files() -> None:
"""
Test function to create a new GPTAssistantAgent, set its instructions, retrieve the instructions,
and assert that the retrieved instructions match the set instructions.
@@ -194,7 +227,7 @@ def test_get_assistant_files():
current_file_path = os.path.abspath(__file__)
openai_client = OpenAIWrapper(config_list=config_list)._clients[0]
file = openai_client.files.create(file=open(current_file_path, "rb"), purpose="assistants")
name = "For test_get_assistant_files"
name = f"For test_get_assistant_files {uuid.uuid4()}"
assistant = GPTAssistantAgent(
name,
@@ -206,11 +239,14 @@ def test_get_assistant_files():
},
)
files = assistant.openai_client.beta.assistants.files.list(assistant_id=assistant.assistant_id)
retrieved_file_ids = [fild.id for fild in files]
expected_file_id = file.id
try:
files = assistant.openai_client.beta.assistants.files.list(assistant_id=assistant.assistant_id)
retrieved_file_ids = [fild.id for fild in files]
expected_file_id = file.id
finally:
assistant.delete_assistant()
assistant.delete_assistant()
openai_client.files.delete(file.id)
assert expected_file_id in retrieved_file_ids
@@ -220,12 +256,12 @@ def test_get_assistant_files():
sys.platform in ["darwin", "win32"] or skip,
reason="do not run on MacOS or windows OR dependency is not installed OR requested to skip",
)
def test_assistant_retrieval():
def test_assistant_retrieval() -> None:
"""
Test function to check if the GPTAssistantAgent can retrieve the same assistant
"""
name = "For test_assistant_retrieval"
name = f"For test_assistant_retrieval {uuid.uuid4()}"
function_1_schema = {
"name": "call_function_1",
@@ -240,45 +276,47 @@ def test_assistant_retrieval():
openai_client = OpenAIWrapper(config_list=config_list)._clients[0]
current_file_path = os.path.abspath(__file__)
file_1 = openai_client.files.create(file=open(current_file_path, "rb"), purpose="assistants")
file_2 = openai_client.files.create(file=open(current_file_path, "rb"), purpose="assistants")
all_llm_config = {
"tools": [
{"type": "function", "function": function_1_schema},
{"type": "function", "function": function_2_schema},
{"type": "retrieval"},
{"type": "code_interpreter"},
],
"file_ids": [file_1.id, file_2.id],
"config_list": config_list,
}
name = "For test_gpt_assistant_chat"
assistant_first = GPTAssistantAgent(
name,
instructions="This is a test",
llm_config=all_llm_config,
)
candidate_first = retrieve_assistants_by_name(assistant_first.openai_client, name)
assistant_second = GPTAssistantAgent(
name,
instructions="This is a test",
llm_config=all_llm_config,
)
candidate_second = retrieve_assistants_by_name(assistant_second.openai_client, name)
try:
assistant_first.delete_assistant()
assistant_second.delete_assistant()
except openai.NotFoundError:
# Not found error is expected because the same assistant can not be deleted twice
pass
all_llm_config = {
"tools": [
{"type": "function", "function": function_1_schema},
{"type": "function", "function": function_2_schema},
{"type": "retrieval"},
{"type": "code_interpreter"},
],
"file_ids": [file_1.id, file_2.id],
"config_list": config_list,
}
openai_client.files.delete(file_1.id)
openai_client.files.delete(file_2.id)
name = f"For test_assistant_retrieval {uuid.uuid4()}"
assistant_first = GPTAssistantAgent(
name,
instructions="This is a test",
llm_config=all_llm_config,
)
candidate_first = retrieve_assistants_by_name(assistant_first.openai_client, name)
try:
assistant_second = GPTAssistantAgent(
name,
instructions="This is a test",
llm_config=all_llm_config,
)
candidate_second = retrieve_assistants_by_name(assistant_second.openai_client, name)
finally:
assistant_first.delete_assistant()
with pytest.raises(openai.NotFoundError):
assistant_second.delete_assistant()
finally:
openai_client.files.delete(file_1.id)
openai_client.files.delete(file_2.id)
assert candidate_first == candidate_second
assert len(candidate_first) == 1
@@ -291,10 +329,10 @@ def test_assistant_retrieval():
sys.platform in ["darwin", "win32"] or skip,
reason="do not run on MacOS or windows OR dependency is not installed OR requested to skip",
)
def test_assistant_mismatch_retrieval():
def test_assistant_mismatch_retrieval() -> None:
"""Test function to check if the GPTAssistantAgent can filter out the mismatch assistant"""
name = "For test_assistant_retrieval"
name = f"For test_assistant_retrieval {uuid.uuid4()}"
function_1_schema = {
"name": "call_function",
@@ -317,82 +355,92 @@ def test_assistant_mismatch_retrieval():
file_1 = openai_client.files.create(file=open(current_file_path, "rb"), purpose="assistants")
file_2 = openai_client.files.create(file=open(current_file_path, "rb"), purpose="assistants")
all_llm_config = {
"tools": [
{"type": "function", "function": function_1_schema},
{"type": "function", "function": function_2_schema},
{"type": "retrieval"},
{"type": "code_interpreter"},
],
"file_ids": [file_1.id, file_2.id],
"config_list": config_list,
}
try:
all_llm_config = {
"tools": [
{"type": "function", "function": function_1_schema},
{"type": "function", "function": function_2_schema},
{"type": "retrieval"},
{"type": "code_interpreter"},
],
"file_ids": [file_1.id, file_2.id],
"config_list": config_list,
}
name = "For test_gpt_assistant_chat"
name = f"For test_assistant_retrieval {uuid.uuid4()}"
assistant_first = GPTAssistantAgent(
name,
instructions="This is a test",
llm_config=all_llm_config,
)
candidate_first = retrieve_assistants_by_name(assistant_first.openai_client, name)
assert len(candidate_first) == 1
assistant_first, assistant_instructions_mistaching = None, None
assistant_file_ids_mismatch, assistant_tools_mistaching = None, None
try:
assistant_first = GPTAssistantAgent(
name,
instructions="This is a test",
llm_config=all_llm_config,
)
candidate_first = retrieve_assistants_by_name(assistant_first.openai_client, name)
assert len(candidate_first) == 1
# test instructions mismatch
assistant_instructions_mistaching = GPTAssistantAgent(
name,
instructions="This is a test for mismatch instructions",
llm_config=all_llm_config,
)
candidate_instructions_mistaching = retrieve_assistants_by_name(
assistant_instructions_mistaching.openai_client, name
)
assert len(candidate_instructions_mistaching) == 2
# test instructions mismatch
assistant_instructions_mistaching = GPTAssistantAgent(
name,
instructions="This is a test for mismatch instructions",
llm_config=all_llm_config,
)
candidate_instructions_mistaching = retrieve_assistants_by_name(
assistant_instructions_mistaching.openai_client, name
)
assert len(candidate_instructions_mistaching) == 2
# test mismatch fild ids
file_ids_mismatch_llm_config = {
"tools": [
{"type": "code_interpreter"},
{"type": "retrieval"},
{"type": "function", "function": function_2_schema},
{"type": "function", "function": function_1_schema},
],
"file_ids": [file_2.id],
"config_list": config_list,
}
assistant_file_ids_mismatch = GPTAssistantAgent(
name,
instructions="This is a test",
llm_config=file_ids_mismatch_llm_config,
)
candidate_file_ids_mismatch = retrieve_assistants_by_name(assistant_file_ids_mismatch.openai_client, name)
assert len(candidate_file_ids_mismatch) == 3
# test mismatch fild ids
file_ids_mismatch_llm_config = {
"tools": [
{"type": "code_interpreter"},
{"type": "retrieval"},
{"type": "function", "function": function_2_schema},
{"type": "function", "function": function_1_schema},
],
"file_ids": [file_2.id],
"config_list": config_list,
}
assistant_file_ids_mismatch = GPTAssistantAgent(
name,
instructions="This is a test",
llm_config=file_ids_mismatch_llm_config,
)
candidate_file_ids_mismatch = retrieve_assistants_by_name(assistant_file_ids_mismatch.openai_client, name)
assert len(candidate_file_ids_mismatch) == 3
# test tools mismatch
tools_mismatch_llm_config = {
"tools": [
{"type": "code_interpreter"},
{"type": "retrieval"},
{"type": "function", "function": function_3_schema},
],
"file_ids": [file_2.id, file_1.id],
"config_list": config_list,
}
assistant_tools_mistaching = GPTAssistantAgent(
name,
instructions="This is a test",
llm_config=tools_mismatch_llm_config,
)
candidate_tools_mismatch = retrieve_assistants_by_name(assistant_tools_mistaching.openai_client, name)
assert len(candidate_tools_mismatch) == 4
# test tools mismatch
tools_mismatch_llm_config = {
"tools": [
{"type": "code_interpreter"},
{"type": "retrieval"},
{"type": "function", "function": function_3_schema},
],
"file_ids": [file_2.id, file_1.id],
"config_list": config_list,
}
assistant_tools_mistaching = GPTAssistantAgent(
name,
instructions="This is a test",
llm_config=tools_mismatch_llm_config,
)
candidate_tools_mismatch = retrieve_assistants_by_name(assistant_tools_mistaching.openai_client, name)
assert len(candidate_tools_mismatch) == 4
openai_client.files.delete(file_1.id)
openai_client.files.delete(file_2.id)
finally:
if assistant_first:
assistant_first.delete_assistant()
if assistant_instructions_mistaching:
assistant_instructions_mistaching.delete_assistant()
if assistant_file_ids_mismatch:
assistant_file_ids_mismatch.delete_assistant()
if assistant_tools_mistaching:
assistant_tools_mistaching.delete_assistant()
assistant_first.delete_assistant()
assistant_instructions_mistaching.delete_assistant()
assistant_file_ids_mismatch.delete_assistant()
assistant_tools_mistaching.delete_assistant()
finally:
openai_client.files.delete(file_1.id)
openai_client.files.delete(file_2.id)
candidates = retrieve_assistants_by_name(openai_client, name)
assert len(candidates) == 0
@@ -402,7 +450,7 @@ def test_assistant_mismatch_retrieval():
sys.platform in ["darwin", "win32"] or skip,
reason="do not run on MacOS or windows OR dependency is not installed OR requested to skip",
)
def test_gpt_assistant_tools_overwrite():
def test_gpt_assistant_tools_overwrite() -> None:
"""
Test that the tools of a GPTAssistantAgent can be overwritten or not depending on the value of the
`overwrite_tools` parameter when creating a new assistant with the same ID.
@@ -414,7 +462,6 @@ def test_gpt_assistant_tools_overwrite():
4. Check that the tools of the assistant have been overwritten with the new ones.
"""
name = "For test_gpt_assistant_tools_overwrite"
original_tools = [
{
"type": "function",
@@ -483,8 +530,10 @@ def test_gpt_assistant_tools_overwrite():
},
]
name = f"For test_gpt_assistant_tools_overwrite {uuid.uuid4()}"
# Create an assistant with original tools
assistant = GPTAssistantAgent(
assistant_org = GPTAssistantAgent(
name,
llm_config={
"config_list": config_list,
@@ -492,22 +541,24 @@ def test_gpt_assistant_tools_overwrite():
},
)
assistant_id = assistant.assistant_id
assistant_id = assistant_org.assistant_id
# Create a new assistant with new tools and overwrite_tools set to True
assistant = GPTAssistantAgent(
name,
llm_config={
"config_list": config_list,
"assistant_id": assistant_id,
"tools": new_tools,
},
overwrite_tools=True,
)
try:
# Create a new assistant with new tools and overwrite_tools set to True
assistant = GPTAssistantAgent(
name,
llm_config={
"config_list": config_list,
"assistant_id": assistant_id,
"tools": new_tools,
},
overwrite_tools=True,
)
# Add logic to retrieve the tools from the assistant and assert
retrieved_tools = assistant.llm_config.get("tools", [])
assistant.delete_assistant()
# Add logic to retrieve the tools from the assistant and assert
retrieved_tools = assistant.llm_config.get("tools", [])
finally:
assistant_org.delete_assistant()
assert retrieved_tools == new_tools

8
website/blog/2023-07-14-Local-LLMs/index.md
View File

@@ -78,7 +78,7 @@ response = oai.Completion.create(
{
"model": "chatglm2-6b",
"base_url": "http://localhost:8000/v1",
"api_type": "open_ai",
"api_type": "openai",
"api_key": "NULL", # just a placeholder
}
],
@@ -92,7 +92,7 @@ response = oai.ChatCompletion.create(
{
"model": "chatglm2-6b",
"base_url": "http://localhost:8000/v1",
"api_type": "open_ai",
"api_type": "openai",
"api_key": "NULL",
}
],
@@ -126,13 +126,13 @@ response = oai.ChatCompletion.create(
{
"model": "chatglm2-6b",
"base_url": "http://localhost:8000/v1",
"api_type": "open_ai",
"api_type": "openai",
"api_key": "NULL",
},
{
"model": "vicuna-7b-v1.3",
"base_url": "http://localhost:8000/v1",
"api_type": "open_ai",
"api_type": "openai",
"api_key": "NULL",
}
],