From 142d46f4ce810707f6e65895b1ae14322d780477 Mon Sep 17 00:00:00 2001 From: Jack Collins <6640905+jackmpcollins@users.noreply.github.com> Date: Fri, 29 Nov 2024 13:32:49 -0800 Subject: [PATCH] Test Ollama via `OpenaiChatModel` (#281) * Add test_openai_chat_model_complete_ollama * Add async test * Add openai_ollama marker to pyproject and vcr markers * Add vcr cassettes for ollama tests * Update version in uv.lock --- pyproject.toml | 3 +- ...sync[Return True.-output_types1-bool].yaml | 64 ++++++++ ...n [1, 2, 3, 4, 5]-output_types2-list].yaml | 66 ++++++++ ...e_async[Say hello!-output_types0-str].yaml | 129 +++++++++++++++ ...lama[Return True.-output_types1-bool].yaml | 64 ++++++++ ...n [1, 2, 3, 4, 5]-output_types2-list].yaml | 66 ++++++++ ..._ollama[Say hello!-output_types0-str].yaml | 148 ++++++++++++++++++ .../test_openai_chat_model_ollama.py | 40 +++++ tests/conftest.py | 1 + uv.lock | 2 +- 10 files changed, 581 insertions(+), 2 deletions(-) create mode 100644 tests/chat_model/cassettes/test_openai_chat_model_ollama/test_openai_chat_model_complete_async[Return True.-output_types1-bool].yaml create mode 100644 tests/chat_model/cassettes/test_openai_chat_model_ollama/test_openai_chat_model_complete_async[Return [1, 2, 3, 4, 5]-output_types2-list].yaml create mode 100644 tests/chat_model/cassettes/test_openai_chat_model_ollama/test_openai_chat_model_complete_async[Say hello!-output_types0-str].yaml create mode 100644 tests/chat_model/cassettes/test_openai_chat_model_ollama/test_openai_chat_model_complete_ollama[Return True.-output_types1-bool].yaml create mode 100644 tests/chat_model/cassettes/test_openai_chat_model_ollama/test_openai_chat_model_complete_ollama[Return [1, 2, 3, 4, 5]-output_types2-list].yaml create mode 100644 tests/chat_model/cassettes/test_openai_chat_model_ollama/test_openai_chat_model_complete_ollama[Say hello!-output_types0-str].yaml create mode 100644 tests/chat_model/test_openai_chat_model_ollama.py diff --git a/pyproject.toml b/pyproject.toml index 741150a..3d79c8d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -78,10 +78,11 @@ asyncio_mode = "auto" markers = [ "anthropic: Tests that query the Anthropic API. Requires the ANTHROPIC_API_KEY environment variable to be set.", "litellm_anthropic: Tests that query the Anthropic API via litellm. Requires the ANTHROPIC_API_KEY environment variable to be set.", - "litellm_ollama: Tests that query Ollama. Requires ollama to be installed and running on localhost:11434.", + "litellm_ollama: Tests that query Ollama via Litellm. Requires ollama to be installed and running on localhost:11434.", "litellm_openai: Tests that query the OpenAI API via litellm. Requires the OPENAI_API_KEY environment variable to be set.", "mistral: Tests that query the Mistral API (via openai). Requires the MISTRAL_API_KEY environment variable to be set.", "openai: Tests that query the OpenAI API. Requires the OPENAI_API_KEY environment variable to be set.", + "openai_ollama: Tests that query Ollama via OpenAI. Requires ollama to be installed and running on localhost:11434.", ] [tool.ruff] diff --git a/tests/chat_model/cassettes/test_openai_chat_model_ollama/test_openai_chat_model_complete_async[Return True.-output_types1-bool].yaml b/tests/chat_model/cassettes/test_openai_chat_model_ollama/test_openai_chat_model_complete_async[Return True.-output_types1-bool].yaml new file mode 100644 index 0000000..77a69d2 --- /dev/null +++ b/tests/chat_model/cassettes/test_openai_chat_model_ollama/test_openai_chat_model_complete_async[Return True.-output_types1-bool].yaml @@ -0,0 +1,64 @@ +interactions: +- request: + body: '{"messages": [{"role": "user", "content": "Return True."}], "model": "llama3.1", + "parallel_tool_calls": false, "stream": true, "stream_options": {"include_usage": + true}, "tool_choice": {"type": "function", "function": {"name": "return_bool"}}, + "tools": [{"type": "function", "function": {"name": "return_bool", "parameters": + {"properties": {"value": {"title": "Value", "type": "boolean"}}, "required": + ["value"], "type": "object"}}}]}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '434' + content-type: + - application/json + host: + - localhost:11434 + user-agent: + - AsyncOpenAI/Python 1.54.4 + x-stainless-arch: + - arm64 + x-stainless-async: + - async:asyncio + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.54.4 + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.10.15 + method: POST + uri: http://localhost:11434/v1/chat/completions + response: + body: + string: 'data: {"id":"chatcmpl-573","object":"chat.completion.chunk","created":1732915688,"model":"llama3.1","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":"","tool_calls":[{"id":"call_wqha42ih","type":"function","function":{"name":"return_bool","arguments":"{\"value\":true}"}}]},"finish_reason":null}]} + + + data: {"id":"chatcmpl-573","object":"chat.completion.chunk","created":1732915688,"model":"llama3.1","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":""},"finish_reason":"stop"}]} + + + data: [DONE] + + + ' + headers: + Content-Type: + - text/event-stream + Date: + - Fri, 29 Nov 2024 21:28:08 GMT + Transfer-Encoding: + - chunked + status: + code: 200 + message: OK +version: 1 diff --git a/tests/chat_model/cassettes/test_openai_chat_model_ollama/test_openai_chat_model_complete_async[Return [1, 2, 3, 4, 5]-output_types2-list].yaml b/tests/chat_model/cassettes/test_openai_chat_model_ollama/test_openai_chat_model_complete_async[Return [1, 2, 3, 4, 5]-output_types2-list].yaml new file mode 100644 index 0000000..e2fc055 --- /dev/null +++ b/tests/chat_model/cassettes/test_openai_chat_model_ollama/test_openai_chat_model_complete_async[Return [1, 2, 3, 4, 5]-output_types2-list].yaml @@ -0,0 +1,66 @@ +interactions: +- request: + body: '{"messages": [{"role": "user", "content": "Return [1, 2, 3, 4, 5]"}], "model": + "llama3.1", "parallel_tool_calls": false, "stream": true, "stream_options": + {"include_usage": true}, "tool_choice": {"type": "function", "function": {"name": + "return_list_of_int"}}, "tools": [{"type": "function", "function": {"name": + "return_list_of_int", "parameters": {"properties": {"value": {"items": {"type": + "integer"}, "title": "Value", "type": "array"}}, "required": ["value"], "type": + "object"}}}]}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '486' + content-type: + - application/json + host: + - localhost:11434 + user-agent: + - AsyncOpenAI/Python 1.54.4 + x-stainless-arch: + - arm64 + x-stainless-async: + - async:asyncio + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.54.4 + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.10.15 + method: POST + uri: http://localhost:11434/v1/chat/completions + response: + body: + string: 'data: {"id":"chatcmpl-877","object":"chat.completion.chunk","created":1732915691,"model":"llama3.1","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":"","tool_calls":[{"id":"call_kichi5ua","type":"function","function":{"name":"return_list_of_int","arguments":"{\"value\":\"[1, + 2, 3, 4, 5]\"}"}}]},"finish_reason":null}]} + + + data: {"id":"chatcmpl-877","object":"chat.completion.chunk","created":1732915691,"model":"llama3.1","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":""},"finish_reason":"stop"}]} + + + data: [DONE] + + + ' + headers: + Content-Type: + - text/event-stream + Date: + - Fri, 29 Nov 2024 21:28:11 GMT + Transfer-Encoding: + - chunked + status: + code: 200 + message: OK +version: 1 diff --git a/tests/chat_model/cassettes/test_openai_chat_model_ollama/test_openai_chat_model_complete_async[Say hello!-output_types0-str].yaml b/tests/chat_model/cassettes/test_openai_chat_model_ollama/test_openai_chat_model_complete_async[Say hello!-output_types0-str].yaml new file mode 100644 index 0000000..bbbe710 --- /dev/null +++ b/tests/chat_model/cassettes/test_openai_chat_model_ollama/test_openai_chat_model_complete_async[Say hello!-output_types0-str].yaml @@ -0,0 +1,129 @@ +interactions: +- request: + body: '{"messages": [{"role": "user", "content": "Say hello!"}], "model": "llama3.1", + "stream": true, "stream_options": {"include_usage": true}}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '137' + content-type: + - application/json + host: + - localhost:11434 + user-agent: + - AsyncOpenAI/Python 1.54.4 + x-stainless-arch: + - arm64 + x-stainless-async: + - async:asyncio + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.54.4 + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.10.15 + method: POST + uri: http://localhost:11434/v1/chat/completions + response: + body: + string: 'data: {"id":"chatcmpl-402","object":"chat.completion.chunk","created":1732915683,"model":"llama3.1","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":"Hello"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-402","object":"chat.completion.chunk","created":1732915684,"model":"llama3.1","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":"!"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-402","object":"chat.completion.chunk","created":1732915684,"model":"llama3.1","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":" + How"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-402","object":"chat.completion.chunk","created":1732915684,"model":"llama3.1","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":" + are"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-402","object":"chat.completion.chunk","created":1732915684,"model":"llama3.1","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":" + you"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-402","object":"chat.completion.chunk","created":1732915684,"model":"llama3.1","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":" + doing"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-402","object":"chat.completion.chunk","created":1732915684,"model":"llama3.1","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":" + today"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-402","object":"chat.completion.chunk","created":1732915684,"model":"llama3.1","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":"?"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-402","object":"chat.completion.chunk","created":1732915684,"model":"llama3.1","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":" + Is"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-402","object":"chat.completion.chunk","created":1732915684,"model":"llama3.1","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":" + there"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-402","object":"chat.completion.chunk","created":1732915684,"model":"llama3.1","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":" + anything"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-402","object":"chat.completion.chunk","created":1732915684,"model":"llama3.1","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":" + I"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-402","object":"chat.completion.chunk","created":1732915684,"model":"llama3.1","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":" + can"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-402","object":"chat.completion.chunk","created":1732915685,"model":"llama3.1","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":" + help"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-402","object":"chat.completion.chunk","created":1732915685,"model":"llama3.1","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":" + with"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-402","object":"chat.completion.chunk","created":1732915685,"model":"llama3.1","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":" + or"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-402","object":"chat.completion.chunk","created":1732915685,"model":"llama3.1","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":" + chat"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-402","object":"chat.completion.chunk","created":1732915685,"model":"llama3.1","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":" + about"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-402","object":"chat.completion.chunk","created":1732915685,"model":"llama3.1","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":"?"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-402","object":"chat.completion.chunk","created":1732915685,"model":"llama3.1","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":""},"finish_reason":"stop"}]} + + + data: [DONE] + + + ' + headers: + Content-Type: + - text/event-stream + Date: + - Fri, 29 Nov 2024 21:28:03 GMT + Transfer-Encoding: + - chunked + status: + code: 200 + message: OK +version: 1 diff --git a/tests/chat_model/cassettes/test_openai_chat_model_ollama/test_openai_chat_model_complete_ollama[Return True.-output_types1-bool].yaml b/tests/chat_model/cassettes/test_openai_chat_model_ollama/test_openai_chat_model_complete_ollama[Return True.-output_types1-bool].yaml new file mode 100644 index 0000000..7b3ab70 --- /dev/null +++ b/tests/chat_model/cassettes/test_openai_chat_model_ollama/test_openai_chat_model_complete_ollama[Return True.-output_types1-bool].yaml @@ -0,0 +1,64 @@ +interactions: +- request: + body: '{"messages": [{"role": "user", "content": "Return True."}], "model": "llama3.1", + "parallel_tool_calls": false, "stream": true, "stream_options": {"include_usage": + true}, "tool_choice": {"type": "function", "function": {"name": "return_bool"}}, + "tools": [{"type": "function", "function": {"name": "return_bool", "parameters": + {"properties": {"value": {"title": "Value", "type": "boolean"}}, "required": + ["value"], "type": "object"}}}]}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '434' + content-type: + - application/json + host: + - localhost:11434 + user-agent: + - OpenAI/Python 1.54.4 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.54.4 + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.10.15 + method: POST + uri: http://localhost:11434/v1/chat/completions + response: + body: + string: 'data: {"id":"chatcmpl-341","object":"chat.completion.chunk","created":1732915676,"model":"llama3.1","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":"","tool_calls":[{"id":"call_dxlghlch","type":"function","function":{"name":"return_bool","arguments":"{\"value\":true}"}}]},"finish_reason":null}]} + + + data: {"id":"chatcmpl-341","object":"chat.completion.chunk","created":1732915676,"model":"llama3.1","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":""},"finish_reason":"stop"}]} + + + data: [DONE] + + + ' + headers: + Content-Type: + - text/event-stream + Date: + - Fri, 29 Nov 2024 21:27:56 GMT + Transfer-Encoding: + - chunked + status: + code: 200 + message: OK +version: 1 diff --git a/tests/chat_model/cassettes/test_openai_chat_model_ollama/test_openai_chat_model_complete_ollama[Return [1, 2, 3, 4, 5]-output_types2-list].yaml b/tests/chat_model/cassettes/test_openai_chat_model_ollama/test_openai_chat_model_complete_ollama[Return [1, 2, 3, 4, 5]-output_types2-list].yaml new file mode 100644 index 0000000..4b8fe6c --- /dev/null +++ b/tests/chat_model/cassettes/test_openai_chat_model_ollama/test_openai_chat_model_complete_ollama[Return [1, 2, 3, 4, 5]-output_types2-list].yaml @@ -0,0 +1,66 @@ +interactions: +- request: + body: '{"messages": [{"role": "user", "content": "Return [1, 2, 3, 4, 5]"}], "model": + "llama3.1", "parallel_tool_calls": false, "stream": true, "stream_options": + {"include_usage": true}, "tool_choice": {"type": "function", "function": {"name": + "return_list_of_int"}}, "tools": [{"type": "function", "function": {"name": + "return_list_of_int", "parameters": {"properties": {"value": {"items": {"type": + "integer"}, "title": "Value", "type": "array"}}, "required": ["value"], "type": + "object"}}}]}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '486' + content-type: + - application/json + host: + - localhost:11434 + user-agent: + - OpenAI/Python 1.54.4 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.54.4 + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.10.15 + method: POST + uri: http://localhost:11434/v1/chat/completions + response: + body: + string: 'data: {"id":"chatcmpl-168","object":"chat.completion.chunk","created":1732915680,"model":"llama3.1","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":"","tool_calls":[{"id":"call_ygc23skr","type":"function","function":{"name":"return_list_of_int","arguments":"{\"value\":\"[1, + 2, 3, 4, 5]\"}"}}]},"finish_reason":null}]} + + + data: {"id":"chatcmpl-168","object":"chat.completion.chunk","created":1732915680,"model":"llama3.1","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":""},"finish_reason":"stop"}]} + + + data: [DONE] + + + ' + headers: + Content-Type: + - text/event-stream + Date: + - Fri, 29 Nov 2024 21:28:00 GMT + Transfer-Encoding: + - chunked + status: + code: 200 + message: OK +version: 1 diff --git a/tests/chat_model/cassettes/test_openai_chat_model_ollama/test_openai_chat_model_complete_ollama[Say hello!-output_types0-str].yaml b/tests/chat_model/cassettes/test_openai_chat_model_ollama/test_openai_chat_model_complete_ollama[Say hello!-output_types0-str].yaml new file mode 100644 index 0000000..7f3a541 --- /dev/null +++ b/tests/chat_model/cassettes/test_openai_chat_model_ollama/test_openai_chat_model_complete_ollama[Say hello!-output_types0-str].yaml @@ -0,0 +1,148 @@ +interactions: +- request: + body: '{"messages": [{"role": "user", "content": "Say hello!"}], "model": "llama3.1", + "stream": true, "stream_options": {"include_usage": true}}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '137' + content-type: + - application/json + host: + - localhost:11434 + user-agent: + - OpenAI/Python 1.54.4 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.54.4 + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.10.15 + method: POST + uri: http://localhost:11434/v1/chat/completions + response: + body: + string: 'data: {"id":"chatcmpl-725","object":"chat.completion.chunk","created":1732915672,"model":"llama3.1","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":"Hello"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-725","object":"chat.completion.chunk","created":1732915672,"model":"llama3.1","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":"!"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-725","object":"chat.completion.chunk","created":1732915672,"model":"llama3.1","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":" + It"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-725","object":"chat.completion.chunk","created":1732915672,"model":"llama3.1","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":"''s"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-725","object":"chat.completion.chunk","created":1732915672,"model":"llama3.1","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":" + nice"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-725","object":"chat.completion.chunk","created":1732915672,"model":"llama3.1","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":" + to"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-725","object":"chat.completion.chunk","created":1732915672,"model":"llama3.1","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":" + meet"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-725","object":"chat.completion.chunk","created":1732915672,"model":"llama3.1","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":" + you"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-725","object":"chat.completion.chunk","created":1732915672,"model":"llama3.1","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":"."},"finish_reason":null}]} + + + data: {"id":"chatcmpl-725","object":"chat.completion.chunk","created":1732915672,"model":"llama3.1","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":" + Is"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-725","object":"chat.completion.chunk","created":1732915673,"model":"llama3.1","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":" + there"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-725","object":"chat.completion.chunk","created":1732915673,"model":"llama3.1","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":" + something"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-725","object":"chat.completion.chunk","created":1732915673,"model":"llama3.1","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":" + I"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-725","object":"chat.completion.chunk","created":1732915673,"model":"llama3.1","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":" + can"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-725","object":"chat.completion.chunk","created":1732915673,"model":"llama3.1","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":" + help"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-725","object":"chat.completion.chunk","created":1732915673,"model":"llama3.1","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":" + you"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-725","object":"chat.completion.chunk","created":1732915673,"model":"llama3.1","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":" + with"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-725","object":"chat.completion.chunk","created":1732915673,"model":"llama3.1","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":" + or"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-725","object":"chat.completion.chunk","created":1732915673,"model":"llama3.1","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":" + would"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-725","object":"chat.completion.chunk","created":1732915673,"model":"llama3.1","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":" + you"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-725","object":"chat.completion.chunk","created":1732915673,"model":"llama3.1","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":" + like"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-725","object":"chat.completion.chunk","created":1732915673,"model":"llama3.1","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":" + to"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-725","object":"chat.completion.chunk","created":1732915674,"model":"llama3.1","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":" + chat"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-725","object":"chat.completion.chunk","created":1732915674,"model":"llama3.1","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":"?"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-725","object":"chat.completion.chunk","created":1732915674,"model":"llama3.1","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":""},"finish_reason":"stop"}]} + + + data: [DONE] + + + ' + headers: + Content-Type: + - text/event-stream + Date: + - Fri, 29 Nov 2024 21:27:52 GMT + Transfer-Encoding: + - chunked + status: + code: 200 + message: OK +version: 1 diff --git a/tests/chat_model/test_openai_chat_model_ollama.py b/tests/chat_model/test_openai_chat_model_ollama.py new file mode 100644 index 0000000..b9d0a66 --- /dev/null +++ b/tests/chat_model/test_openai_chat_model_ollama.py @@ -0,0 +1,40 @@ +import pytest + +from magentic.chat_model.message import UserMessage +from magentic.chat_model.openai_chat_model import OpenaiChatModel + + +@pytest.mark.parametrize( + ("prompt", "output_types", "expected_output_type"), + [ + ("Say hello!", [str], str), + ("Return True.", [bool], bool), + ("Return [1, 2, 3, 4, 5]", [list[int]], list), + ], +) +@pytest.mark.openai_ollama +def test_openai_chat_model_complete_ollama(prompt, output_types, expected_output_type): + chat_model = OpenaiChatModel("llama3.1", base_url="http://localhost:11434/v1/") + message = chat_model.complete( + messages=[UserMessage(prompt)], output_types=output_types + ) + assert isinstance(message.content, expected_output_type) + + +@pytest.mark.parametrize( + ("prompt", "output_types", "expected_output_type"), + [ + ("Say hello!", [str], str), + ("Return True.", [bool], bool), + ("Return [1, 2, 3, 4, 5]", [list[int]], list), + ], +) +@pytest.mark.openai_ollama +async def test_openai_chat_model_complete_async( + prompt, output_types, expected_output_type +): + chat_model = OpenaiChatModel("llama3.1", base_url="http://localhost:11434/v1/") + message = await chat_model.acomplete( + messages=[UserMessage(prompt)], output_types=output_types + ) + assert isinstance(message.content, expected_output_type) diff --git a/tests/conftest.py b/tests/conftest.py index e5480b6..3166c8a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -70,6 +70,7 @@ def pytest_collection_modifyitems( "litellm_openai", "mistral", "openai", + "openai_ollama", ] for item in items: # Apply vcr marker to all LLM tests diff --git a/uv.lock b/uv.lock index 65bd95d..4db97cd 100644 --- a/uv.lock +++ b/uv.lock @@ -1389,7 +1389,7 @@ wheels = [ [[package]] name = "magentic" -version = "0.32.0" +version = "0.33.0" source = { editable = "." } dependencies = [ { name = "filetype" },