diff --git a/docs/concepts/patching.md b/docs/concepts/patching.md index e4f1cd7..d7e2988 100644 --- a/docs/concepts/patching.md +++ b/docs/concepts/patching.md @@ -54,6 +54,7 @@ from openai import OpenAI client = instructor.patch(OpenAI(), mode=Mode.MD_JSON) ``` + ### Schema Integration In JSON Mode, the schema is part of the system message: @@ -82,3 +83,64 @@ user = UserExtract.from_response(response, mode=Mode.JSON) assert user.name.lower() == "jason" assert user.age == 25 ``` + +## Understanding the Chat Completion Parametsrs + +### Mode: FUNCTIONS + +- Adds `functions` and `function_call` keys with OpenAI schema information. +- No direct content change in messages, but function-based response processing is guided. + +```python +if mode == Mode.FUNCTIONS: + chat_completion_parameters["functions"] = [response_model.openai_schema] + chat_completion_parameters["function_call"] = {"name": response_model.openai_schema["name"]} +``` + +### Mode: TOOLS + +- Adds `tools` and `tool_choice` keys with tool details following the OpenAI schema. +- Messages aren't modified in content; tool-based response processing is guided. + +```python +if mode == Mode.TOOLS: + chat_completion_parameters["tools"] = [{"type": "function", "function": response_model.openai_schema}] + chat_completion_parameters["tool_choice"] = {"type": "function", "function": {"name": response_model.openai_schema["name"]}} +``` + +### Mode: JSON + +- Appends `response_format` to indicate a JSON object type. +- Adds or modifies a system message for JSON format adherence and schema instructions. + +```python +if mode == Mode.JSON: + chat_completion_parameters["response_format"] = {"type": "json_object"} + chat_completion_parameters["messages"].append({"role": "system", "content": "Provide response in JSON format adhering to the specified schema."}) +``` + +### Mode: MD_JSON + +- Similar to JSON mode, but with Markdown code block formatting. +- Adds a stop sequence for the Markdown JSON response and system message for instructions. + +````python +if mode == Mode.MD_JSON: + chat_completion_parameters["response_format"] = {"type": "json_object"} + chat_completion_parameters["stop"] = "```" + chat_completion_parameters["messages"].append({"role": "system", "content": "Provide response in Markdown-formatted JSON within the code block."}) +```` + +### Mode: JSON_SCHEMA + +- Appends `response_format` with a JSON object type and specific schema. +- Modifies system messages for JSON schema formatting instructions. +- Only supported by Anyscale! + +```python +if mode == Mode.JSON_SCHEMA: + chat_completion_parameters["response_format"] = {"type": "json_object", "schema": response_model.model_json_schema()} + chat_completion_parameters["messages"].append({"role": "system", "content": "Format the response according to the following JSON schema: " + str(response_model.model_json_schema())}) +``` + +In each mode, the chat completion parameters are adapted to ensure the assistant's response adheres to the specific format required. diff --git a/examples/ollama/run.py b/examples/ollama/run.py new file mode 100644 index 0000000..17697f6 --- /dev/null +++ b/examples/ollama/run.py @@ -0,0 +1,32 @@ +from litellm import completion +from pydantic import BaseModel + +import instructor +from instructor.patch import wrap_chatcompletion + +completion = wrap_chatcompletion(completion, mode=instructor.Mode.MD_JSON) + + +class User(BaseModel): + name: str + age: int + + +response = completion( + model="ollama/mistral", + response_model=User, + messages=[ + { + "role": "system", + "content": "You are a JSON Output system, only return valid JSON. YOU CAN ONLY RETURN WITH JSON NO TALKING", + }, + { + "role": "user", + "content": "Extract `My name is John and I am 25 years old` into JSON", + }, + ], + api_base="http://localhost:11434", +) + +assert response.name == "John", "Name is not John" +assert response.age == 25, "Age is not 25" diff --git a/instructor/patch.py b/instructor/patch.py index 458800e..a2e46cf 100644 --- a/instructor/patch.py +++ b/instructor/patch.py @@ -49,10 +49,10 @@ def dump_message(message: ChatCompletionMessage) -> ChatCompletionMessageParam: "role": message.role, "content": message.content or "", } - if message.tool_calls is not None: + if message.get("tool_calls", None) is not None: ret["tool_calls"] = message.model_dump()["tool_calls"] ret["content"] += json.dumps(message.model_dump()["tool_calls"]) - if message.function_call is not None: + if message.get("function_call", None) is not None: ret["content"] += json.dumps(message.model_dump()["function_call"]) return ret @@ -247,9 +247,9 @@ async def retry_async( "role": "tool", "tool_call_id": response.choices[0].message.tool_calls[0].id, "name": response.choices[0].message.tool_calls[0].function.name, - "content": "failure" + "content": "failure", } - ) + ) kwargs["messages"].append( { "role": "user", @@ -302,7 +302,7 @@ def retry_sync( "role": "tool", "tool_call_id": response.choices[0].message.tool_calls[0].id, "name": response.choices[0].message.tool_calls[0].function.name, - "content": "failure" + "content": "failure", } ) kwargs["messages"].append( @@ -417,4 +417,4 @@ def apatch(client: AsyncOpenAI, mode: Mode = Mode.FUNCTIONS): - `validation_context` parameter to validate the response using the pydantic model - `strict` parameter to use strict json parsing """ - return patch(client, mode=mode) \ No newline at end of file + return patch(client, mode=mode)