diff --git a/docs/blog/posts/together.md b/docs/blog/posts/together.md new file mode 100644 index 0000000..d5f1ed2 --- /dev/null +++ b/docs/blog/posts/together.md @@ -0,0 +1,86 @@ +--- +draft: False +date: 2024-01-27 +slug: together +tags: + - patching + - open source +authors: + - jxnl +--- + +# Structured Outputs with Together AI + +Open-source LLMS are gaining popularity, and with the release of Together's Function calling models, its been easier than ever to get structured outputs. + +By the end of this blog post, you will learn how to effectively utilize instructor with Together AI. But before we proceed, let's first explore the concept of patching. + +!!! note "Other Languages" + + This blog post is written in Python, but the concepts are applicable to other languages as well, as we currently have support for [Javascript]( + https://instructor-ai.github.io/instructor-js) and [Elixir](https://hexdocs.pm/instructor/Instructor.html) + + + +## Patching + +Instructor's patch enhances the openai api it with the following features: + +- `response_model` in `create` calls that returns a pydantic model +- `max_retries` in `create` calls that retries the call if it fails by using a backoff strategy + +!!! note "Learn More" + + To learn more, please refer to the [docs](../../index.md). To understand the benefits of using Pydantic with Instructor, visit the tips and tricks section of the [why use Pydantic](../../why.md) page. + +## Together AI + +The good news is that Together employs the same OpenAI client, and its models support some of these output modes too! + +!!! note "Getting access" + + If you want to try this out for yourself check out the [Together AI](https://www.together.ai/) website. You can get started [here](http://api.together.ai/). + +```python +import os +import openai +from pydantic import BaseModel +import instructor + +client = openai.OpenAI( + base_url="https://api.together.xyz/v1", + api_key=os.environ["TOGETHER_API_KEY"], +) + + +# By default, the patch function will patch the ChatCompletion.create and ChatCompletion.create methods to support the response_model parameter +client = instructor.patch(client, mode=instructor.Mode.TOOLS) + + +# Now, we can use the response_model parameter using only a base model +# rather than having to use the OpenAISchema class +class UserExtract(BaseModel): + name: str + age: int + + +user: UserExtract = client.chat.completions.create( + model="mistralai/Mixtral-8x7B-Instruct-v0.1", + response_model=UserExtract, + messages=[ + {"role": "user", "content": "Extract jason is 25 years old"}, + ], +) + +assert isinstance(user, UserExtract), "Should be instance of UserExtract" +assert user.name.lower() == "jason" +assert user.age == 25 + +print(user.model_dump_json(indent=2)) +{ + "name": "Jason", + "age": 25, +} +``` + +You can find more information about Together's function calling support [here](https://docs.together.ai/docs/function-calling). diff --git a/examples/patching/anyscale.py b/examples/patching/anyscale.py new file mode 100644 index 0000000..d4e7785 --- /dev/null +++ b/examples/patching/anyscale.py @@ -0,0 +1,37 @@ +import os +import instructor + +from openai import OpenAI +from pydantic import BaseModel + + +# By default, the patch function will patch the ChatCompletion.create and ChatCompletion.acreate methods. to support response_model parameter +client = instructor.patch( + OpenAI( + base_url="https://api.endpoints.anyscale.com/v1", + api_key=os.environ["ANYSCALE_API_KEY"], + ), + mode=instructor.Mode.JSON_SCHEMA, +) + + +# Now, we can use the response_model parameter using only a base model +# rather than having to use the OpenAISchema class +class UserExtract(BaseModel): + name: str + age: int + + +user: UserExtract = client.chat.completions.create( + model="mistralai/Mixtral-8x7B-Instruct-v0.1", + response_model=UserExtract, + messages=[ + {"role": "user", "content": "Extract jason is 25 years old"}, + ], +) # type: ignore + +print(user) +{ + "name": "Jason", + "age": 25, +} diff --git a/examples/patching/patching.py b/examples/patching/oai.py similarity index 88% rename from examples/patching/patching.py rename to examples/patching/oai.py index 728b60c..205e9bf 100644 --- a/examples/patching/patching.py +++ b/examples/patching/oai.py @@ -1,3 +1,4 @@ +import os import instructor from openai import OpenAI @@ -5,7 +6,10 @@ from pydantic import BaseModel # By default, the patch function will patch the ChatCompletion.create and ChatCompletion.acreate methods. to support response_model parameter -client = instructor.patch(OpenAI()) +client = instructor.patch( + OpenAI(), + mode=instructor.Mode.TOOLS, +) # Now, we can use the response_model parameter using only a base model diff --git a/examples/patching/together.py b/examples/patching/together.py new file mode 100644 index 0000000..c6bbb07 --- /dev/null +++ b/examples/patching/together.py @@ -0,0 +1,35 @@ +import os +import openai +from pydantic import BaseModel +import instructor + +client = openai.OpenAI( + base_url="https://api.together.xyz/v1", + api_key=os.environ["TOGETHER_API_KEY"], +) + + +# By default, the patch function will patch the ChatCompletion.create and ChatCompletion.acreate methods. to support response_model parameter +client = instructor.patch(client, mode=instructor.Mode.TOOLS) + + +# Now, we can use the response_model parameter using only a base model +# rather than having to use the OpenAISchema class +class UserExtract(BaseModel): + name: str + age: int + + +user: UserExtract = client.chat.completions.create( + model="mistralai/Mixtral-8x7B-Instruct-v0.1", + response_model=UserExtract, + messages=[ + {"role": "user", "content": "Extract jason is 25 years old"}, + ], +) # type: ignore + +print(user.model_dump_json(indent=2)) +{ + "name": "Jason", + "age": 25, +} diff --git a/tests/__init__.py b/tests/__init__.py deleted file mode 100644 index e69de29..0000000