Merge pull request #38 from atomiechen/dev

Bump version to 0.9.0

CHANGELOG.md

@@ -5,6 +5,63 @@ All notable changes to HandyLLM will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
 
 
+## [0.9.0] - 2024-07-28
+
+### Added
+
+Check 🌟s for attractive new features!
+
+- `hprompt.py`: 
+  - 🌟 `image_url` in chat hprompt file now supports local path (file://), both absolute and relative
+  - 🌟 add `fetch()`, `afetch()`, `stream()` and `astream()` methods for direct and typed API responses
+  - add `RunConfig.var_map_file_format` for specifying variable map file format, including JSON / YAML; `load_var_map()` supports format param
+- `requestor.py`: 
+  - 🌟 add `fetch()`, `afetch()`, `stream()` and `astream()` methods for typed API responses
+  - use generic and add `DictRequestor`, `BinRequestor`, `ChatRequestor` and `CompletionsRequestor`
+- `OpenAIClient`: 
+  - 🌟 constructor supports `endpoint_manager`, `endpoints` and `load_path` param; supports loading from YAML file and `Mapping` obj
+  - APIs support `endpoints` param
+  - APIs `endpoint` param supports `Mapping` type
+- 🌟 added `cache_manager.py`: `CacheManager` for general purpose caching to text files
+  - add `load_method` and `dump_method` params
+  - infers format from file suffix when convert handler is not provided
+- 🌟 added `response.py`: `DictProxy` for quick access to well-defined response dict
+- `EndpointManager`: supports loading from YAML file using `endpoints` key, or from `Iterable` obj
+- `__init__.py` import everything from hprompt for convenience
+- rename `_types.py` to `types.py` and expose all definitions
+- `prompt_converter.py`:
+  - add generator sink `consume_stream2fd()`
+- `utils.py`:
+  - add generator filter `trans_stream_chat()`, generator sink `echo_consumer()`
+- a lot improved type hints
+- added tests:
+  - load prompt type specification
+  - variable map substitution
+  - ChatPrompt and CompletionsPrompt's API calls, supports for RunConfig.on_chunk, and addition operations
+  - chat hprompt `image_url`
+  - `OpenAIClient` loading, chat `fetch()` & `stream()`
+  - `endpoint_manager.py`
+  - `cache_manager.py`
+  - audio speech
+  - legacy OpenAIAPI
+
+### Changed
+
+- `hprompt.py`: 
+  - add 'endpoints' to default record blacklist
+  - remove the `var_map` related configurations from the evaluated prompt, as it is already applied
+- `EndpointManager`: 
+  - raises ValueError when getting endpoint out of empty
+
+### Fixed
+
+- `hprompt.py`: 'type' object is not subscriptable on python 3.8
+
+### Removed
+
+- `prompt_converter.py`: remove `stream_msgs2raw()` and `astream_msgs2raw()` as no longer needed
+
+
 ## [0.8.2] - 2024-06-30
 
 ### Added

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "HandyLLM"
-version = "0.8.2"
+version = "0.9.0"
 authors = [
   { name="Atomie CHEN", email="[email protected]" },
 ]

src/handyllm/hprompt.py

@@ -23,7 +23,7 @@
 import sys
 from pathlib import Path
 from datetime import datetime
-from typing import IO, AsyncGenerator, Generator, Generic, MutableMapping, Optional, Tuple, Type, Union, TypeVar, cast
+from typing import IO, AsyncGenerator, Generator, Generic, MutableMapping, Optional, Type, Union, TypeVar, cast
 from abc import abstractmethod, ABC
 from contextlib import asynccontextmanager, contextmanager
 
@@ -35,12 +35,12 @@
 from .prompt_converter import PromptConverter
 from .openai_client import ClientMode, OpenAIClient
 from .utils import (
-    astream_chat_all, astream_completions, local_path_to_base64, 
+    astream_chat_all, astream_completions, echo_consumer, trans_stream_chat, local_path_to_base64, 
     stream_chat_all, stream_completions, 
 )
 from .run_config import RunConfig, RecordRequestMode, CredentialType, VarMapFileFormat
-from .types import PathType, SyncHandlerCompletions, VarMapType, SyncHandlerChat
-from .response import ChatResponse, CompletionsResponse, ToolCallDelta
+from .types import PathType, SyncHandlerChat, SyncHandlerCompletions, VarMapType
+from .response import ChatChunk, ChatResponse, CompletionsChunk, CompletionsResponse
 
 
 PromptType = TypeVar('PromptType', bound='HandyPrompt')
@@ -518,7 +518,7 @@ def open_and_dump_frontmatter(cls, run_config: RunConfig, request: MutableMappin
             yield fout
 
 
-class ChatPrompt(HandyPrompt[ChatResponse, Tuple[str, Optional[str], ToolCallDelta]]):
+class ChatPrompt(HandyPrompt[ChatResponse, ChatChunk]):
 
     def __init__(
         self, 
@@ -581,7 +581,7 @@ def _run_with_client(
             role = ""
             content = ""
             tool_calls = []
-            for r, text, tool_call in cls._stream_with_client(client, evaled_prompt):
+            for r, text, tool_call in stream_chat_all(cls._stream_with_client(client, evaled_prompt)):
                 if r != role:
                     role = r
                 if tool_call:
@@ -613,7 +613,7 @@ async def _arun_with_client(
             role = ""
             content = ""
             tool_calls = []
-            async for r, text, tool_call in cls._astream_with_client(client, evaled_prompt):
+            async for r, text, tool_call in astream_chat_all(cls._astream_with_client(client, evaled_prompt)):
                 if r != role:
                     role = r
                 if tool_call:
@@ -638,18 +638,23 @@ def _stream_with_client(
         evaled_prompt: HandyPrompt,
         ):
         run_config = evaled_prompt.run_config
-        print("why it is None", client, client._sync_client, client._async_client)
         requestor = client.chat(
             messages=evaled_prompt.data,
             **evaled_prompt.request
         )
         response = requestor.stream()
         with cls.open_and_dump_frontmatter(run_config, evaled_prompt.request) as fout:
-            for role, content, tool_call in converter.stream_msgs2raw(stream_chat_all(response), fout):
-                if run_config.on_chunk:
+            producer = trans_stream_chat(
+                converter.consume_stream2fd(fout) if fout else echo_consumer()
+            )
+            next(producer)  # "prime" the coroutine
+            for chat_chunk in response:
+                ret = producer.send(chat_chunk)
+                if run_config.on_chunk and ret:
                     run_config.on_chunk = cast(SyncHandlerChat, run_config.on_chunk)
-                    run_config.on_chunk(role, content, tool_call)
-                yield role, content, tool_call
+                    run_config.on_chunk(*ret)
+                yield chat_chunk
+            producer.close()
 
     @classmethod
     async def _astream_with_client(
@@ -664,14 +669,20 @@ async def _astream_with_client(
         )
         response = await requestor.astream()
         with cls.open_and_dump_frontmatter(run_config, evaled_prompt.request) as fout:
-            async for role, content, tool_call in converter.astream_msgs2raw(astream_chat_all(response), fout):
-                if run_config.on_chunk:
+            producer = trans_stream_chat(
+                converter.consume_stream2fd(fout) if fout else echo_consumer()
+            )
+            next(producer)  # "prime" the coroutine
+            async for chat_chunk in response:
+                ret = producer.send(chat_chunk)
+                if run_config.on_chunk and ret:
                     if inspect.iscoroutinefunction(run_config.on_chunk):
-                        await run_config.on_chunk(role, content, tool_call)
+                        await run_config.on_chunk(*ret)
                     else:
                         run_config.on_chunk = cast(SyncHandlerChat, run_config.on_chunk)
-                        run_config.on_chunk(role, content, tool_call)
-                yield role, content, tool_call
+                        run_config.on_chunk(*ret)
+                yield chat_chunk
+            producer.close()
 
     @classmethod
     def _fetch_with_client(
@@ -733,7 +744,7 @@ def add_message(self, role: str = "user", content: Optional[str] = None, tool_ca
         self.messages.append(msg)
 
 
-class CompletionsPrompt(HandyPrompt[CompletionsResponse, str]):
+class CompletionsPrompt(HandyPrompt[CompletionsResponse, CompletionsChunk]):
 
     def __init__(
         self, 
@@ -771,7 +782,7 @@ def _run_with_client(
         response = None
         if stream:
             content = ""
-            for text in cls._stream_with_client(client, evaled_prompt):
+            for text in stream_completions(cls._stream_with_client(client, evaled_prompt)):
                 content += text
         else:
             response = cls._fetch_with_client(client, evaled_prompt)
@@ -792,7 +803,7 @@ async def _arun_with_client(
         response = None
         if stream:
             content = ""
-            async for text in cls._astream_with_client(client, evaled_prompt):
+            async for text in astream_completions(cls._astream_with_client(client, evaled_prompt)):
                 content += text
         else:
             response = await cls._afetch_with_client(client, evaled_prompt)
@@ -814,13 +825,17 @@ def _stream_with_client(
         )
         response = requestor.stream()
         with cls.open_and_dump_frontmatter(run_config, evaled_prompt.request) as fout:
-            for text in stream_completions(response):
-                if fout:
-                    fout.write(text)
-                if run_config.on_chunk:
-                    run_config.on_chunk = cast(SyncHandlerCompletions, run_config.on_chunk)
-                    run_config.on_chunk(text)
-                yield text
+            for chunk in response:
+                try:
+                    text = cast(str, chunk['choices'][0]['text'])
+                    if fout:
+                        fout.write(text)
+                    if run_config.on_chunk:
+                        run_config.on_chunk = cast(SyncHandlerCompletions, run_config.on_chunk)
+                        run_config.on_chunk(text)
+                except (KeyError, IndexError):
+                    pass
+                yield chunk
 
     @classmethod
     async def _astream_with_client(
@@ -835,16 +850,20 @@ async def _astream_with_client(
         )
         response = await requestor.astream()
         with cls.open_and_dump_frontmatter(run_config, evaled_prompt.request) as fout:
-            async for text in astream_completions(response):
-                if fout:
-                    fout.write(text)
-                if run_config.on_chunk:
-                    if inspect.iscoroutinefunction(run_config.on_chunk):
-                        await run_config.on_chunk(text)
-                    else:
-                        run_config.on_chunk = cast(SyncHandlerCompletions, run_config.on_chunk)
-                        run_config.on_chunk(text)
-                yield text
+            async for chunk in response:
+                try:
+                    text = cast(str, chunk['choices'][0]['text'])
+                    if fout:
+                        fout.write(text)
+                    if run_config.on_chunk:
+                        if inspect.iscoroutinefunction(run_config.on_chunk):
+                            await run_config.on_chunk(text)
+                        else:
+                            run_config.on_chunk = cast(SyncHandlerCompletions, run_config.on_chunk)
+                            run_config.on_chunk(text)
+                except (KeyError, IndexError):
+                    pass
+                yield chunk
 
     @classmethod
     def _fetch_with_client(

src/handyllm/openai_client.py

@@ -37,31 +37,31 @@ class OpenAIClient:
     # set this to your API type;
     # or environment variable OPENAI_API_TYPE will be used;
     # can be None (roll back to default).
-    api_type: Union[TYPE_API_TYPES, None]
+    api_type: Optional[TYPE_API_TYPES]
 
     # set this to your API base;
     # or environment variable OPENAI_API_BASE will be used.
     # can be None (roll back to default).
-    api_base: Union[str, None]
+    api_base: Optional[str]
 
     # set this to your API key; 
     # or environment variable OPENAI_API_KEY will be used.
-    api_key: Union[str, None]
+    api_key: Optional[str]
 
     # set this to your organization ID; 
     # or environment variable OPENAI_ORGANIZATION / OPENAI_ORG_ID will be used;
     # can be None.
-    organization: Union[str, None]
+    organization: Optional[str]
 
     # set this to your API version;
     # or environment variable OPENAI_API_VERSION will be used;
     # cannot be None if using Azure API.
-    api_version: Union[str, None]
+    api_version: Optional[str]
 
     # set this to your model-engine map;
     # or environment variable MODEL_ENGINE_MAP will be used;
     # can be None.
-    model_engine_map: Union[dict, None]
+    model_engine_map: Optional[dict[str, str]]
 
     # set this to your endpoint manager
     endpoint_manager: Optional[EndpointManager] = None
@@ -70,12 +70,12 @@ def __init__(
         self, 
         mode: Union[str, ClientMode] = ClientMode.SYNC,
         *,
-        api_base: Union[str, None] = None,
-        api_key: Union[str, None] = None,
-        organization: Union[str, None] = None,
-        api_type: Union[TYPE_API_TYPES, None] = None,
-        api_version: Union[str, None] = None,
-        model_engine_map: Union[dict, None] = None,
+        api_base: Optional[str] = None,
+        api_key: Optional[str] = None,
+        organization: Optional[str] = None,
+        api_type: Optional[TYPE_API_TYPES] = None,
+        api_version: Optional[str] = None,
+        model_engine_map: Optional[dict[str, str]] = None,
         endpoint_manager: Optional[EndpointManager] = None,
         endpoints: Optional[Iterable] = None,
         load_path: Optional[PathType] = None,