From a5c7965d3aa5a7a0c4a2a48ac6f60c777012d99e Mon Sep 17 00:00:00 2001 From: Jason Liu Date: Wed, 20 Dec 2023 16:21:25 -0500 Subject: [PATCH 1/7] updates --- tutorials/1.introduction.ipynb | 180 +++++++++------------------ tutorials/3.0.applications-rag.ipynb | 166 ++++++++++++------------ 2 files changed, 136 insertions(+), 210 deletions(-) diff --git a/tutorials/1.introduction.ipynb b/tutorials/1.introduction.ipynb index 0d23fca..e702ed6 100644 --- a/tutorials/1.introduction.ipynb +++ b/tutorials/1.introduction.ipynb @@ -24,7 +24,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -40,30 +40,9 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Jason is 10\n", - "None is 10\n", - "Next year Jason will be 11 years old\n" - ] - }, - { - "ename": "TypeError", - "evalue": "can only concatenate str (not \"int\") to str", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mTypeError\u001b[0m Traceback (most recent call last)", - "\u001b[1;32m/Users/jasonliu/dev/instructor/tutorials/1.introduction.ipynb Cell 5\u001b[0m line \u001b[0;36m9\n\u001b[1;32m 7\u001b[0m name \u001b[39m=\u001b[39m obj\u001b[39m.\u001b[39mget(\u001b[39m\"\u001b[39m\u001b[39mfirst_name\u001b[39m\u001b[39m\"\u001b[39m)\n\u001b[1;32m 8\u001b[0m age \u001b[39m=\u001b[39m obj\u001b[39m.\u001b[39mget(\u001b[39m\"\u001b[39m\u001b[39mage\u001b[39m\u001b[39m\"\u001b[39m)\n\u001b[0;32m----> 9\u001b[0m \u001b[39mprint\u001b[39m(\u001b[39mf\u001b[39m\u001b[39m\"\u001b[39m\u001b[39mNext year \u001b[39m\u001b[39m{\u001b[39;00mname\u001b[39m}\u001b[39;00m\u001b[39m will be \u001b[39m\u001b[39m{\u001b[39;00mage\u001b[39m+\u001b[39;49m\u001b[39m1\u001b[39;49m\u001b[39m}\u001b[39;00m\u001b[39m years old\u001b[39m\u001b[39m\"\u001b[39m)\n", - "\u001b[0;31mTypeError\u001b[0m: can only concatenate str (not \"int\") to str" - ] - } - ], + "outputs": [], "source": [ "for obj in data:\n", " name = obj.get(\"first_name\")\n", @@ -94,20 +73,9 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "Person(name='Sam', age=30)" - ] - }, - "execution_count": 3, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "from pydantic import BaseModel, Field\n", "\n", @@ -123,20 +91,9 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "Person(name='Sam', age=30)" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "# Data is correctly casted to the right type\n", "person = Person.model_validate({\"name\": \"Sam\", \"age\": \"30\"})\n", @@ -145,21 +102,9 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "ename": "AssertionError", - "evalue": "", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mAssertionError\u001b[0m Traceback (most recent call last)", - "\u001b[1;32m/Users/jasonliu/dev/instructor/tutorials/1.introduction.ipynb Cell 10\u001b[0m line \u001b[0;36m2\n\u001b[1;32m 1\u001b[0m \u001b[39massert\u001b[39;00m person\u001b[39m.\u001b[39mname \u001b[39m==\u001b[39m \u001b[39m\"\u001b[39m\u001b[39mSam\u001b[39m\u001b[39m\"\u001b[39m\n\u001b[0;32m----> 2\u001b[0m \u001b[39massert\u001b[39;00m person\u001b[39m.\u001b[39mage \u001b[39m==\u001b[39m \u001b[39m20\u001b[39m\n", - "\u001b[0;31mAssertionError\u001b[0m: " - ] - } - ], + "outputs": [], "source": [ "assert person.name == \"Sam\"\n", "assert person.age == 20" @@ -167,25 +112,12 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "ename": "ValidationError", - "evalue": "1 validation error for Person\nage\n Input should be a valid integer, unable to parse string as an integer [type=int_parsing, input_value='30.2', input_type=str]\n For further information visit https://errors.pydantic.dev/2.5/v/int_parsing", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mValidationError\u001b[0m Traceback (most recent call last)", - "\u001b[1;32m/Users/jasonliu/dev/instructor/tutorials/1.introduction.ipynb Cell 11\u001b[0m line \u001b[0;36m2\n\u001b[1;32m 1\u001b[0m \u001b[39m# Data is validated to get better error messages\u001b[39;00m\n\u001b[0;32m----> 2\u001b[0m person \u001b[39m=\u001b[39m Person\u001b[39m.\u001b[39;49mmodel_validate({\u001b[39m\"\u001b[39;49m\u001b[39mname\u001b[39;49m\u001b[39m\"\u001b[39;49m: \u001b[39m\"\u001b[39;49m\u001b[39mSam\u001b[39;49m\u001b[39m\"\u001b[39;49m, \u001b[39m\"\u001b[39;49m\u001b[39mage\u001b[39;49m\u001b[39m\"\u001b[39;49m: \u001b[39m\"\u001b[39;49m\u001b[39m30.2\u001b[39;49m\u001b[39m\"\u001b[39;49m})\n\u001b[1;32m 3\u001b[0m person\n", - "File \u001b[0;32m~/dev/instructor/.venv/lib/python3.11/site-packages/pydantic/main.py:503\u001b[0m, in \u001b[0;36mBaseModel.model_validate\u001b[0;34m(cls, obj, strict, from_attributes, context)\u001b[0m\n\u001b[1;32m 501\u001b[0m \u001b[39m# `__tracebackhide__` tells pytest and some other tools to omit this function from tracebacks\u001b[39;00m\n\u001b[1;32m 502\u001b[0m __tracebackhide__ \u001b[39m=\u001b[39m \u001b[39mTrue\u001b[39;00m\n\u001b[0;32m--> 503\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mcls\u001b[39;49m\u001b[39m.\u001b[39;49m__pydantic_validator__\u001b[39m.\u001b[39;49mvalidate_python(\n\u001b[1;32m 504\u001b[0m obj, strict\u001b[39m=\u001b[39;49mstrict, from_attributes\u001b[39m=\u001b[39;49mfrom_attributes, context\u001b[39m=\u001b[39;49mcontext\n\u001b[1;32m 505\u001b[0m )\n", - "\u001b[0;31mValidationError\u001b[0m: 1 validation error for Person\nage\n Input should be a valid integer, unable to parse string as an integer [type=int_parsing, input_value='30.2', input_type=str]\n For further information visit https://errors.pydantic.dev/2.5/v/int_parsing" - ] - } - ], + "outputs": [], "source": [ "# Data is validated to get better error messages\n", - "person = Person.model_validate({\"name\": \"Sam\", \"age\": \"30.2\"})\n", + "person = Person.model_validate({\"first_name\": \"Sam\", \"age\": \"30.2\"})\n", "person" ] }, @@ -202,41 +134,38 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## Asking for JSON from OpenAI\n" + "## Fundamental problem with asking for JSON from OpenAI\n" ] }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 25, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "{\n", - " \"Jason\": {\n", - " \"age\": 10\n", - " }\n", - "}\n", - "Here is the JSON representation of `jason is 10` as a JSON object:\n", - "\n", - "```\n", - "{\n", - " \"name\": \"Jason\",\n", - " \"age\": 10\n", - "}\n", - "```\n", - "Here is the JSON object representation of \"Jason is 10\":\n", - "\n", - "```json\n", - "{\n", - " \"name\": \"Jason\",\n", - " \"age\": 10\n", - "}\n", - "```\n", - "\n", - "In this JSON object, the key \"name\" corresponds to the value \"Jason\" and the key \"age\" corresponds to the value 10.\n" + "correctly parsed person=Person(name='Jason', age=10)\n", + "correctly parsed person=Person(name='Jason', age=10)\n", + "correctly parsed person=Person(name='Jason', age=10)\n", + "correctly parsed person=Person(name='Jason', age=10)\n", + "correctly parsed person=Person(name='Jason', age=10)\n", + "correctly parsed person=Person(name='Jason', age=10)\n", + "correctly parsed person=Person(name='Jason', age=10)\n", + "correctly parsed person=Person(name='Jason', age=10)\n", + "correctly parsed person=Person(name='Jason', age=10)\n", + "correctly parsed person=Person(name='Jason', age=10)\n", + "correctly parsed person=Person(name='Jason', age=10)\n", + "correctly parsed person=Person(name='Jason', age=10)\n", + "correctly parsed person=Person(name='Jason', age=10)\n", + "correctly parsed person=Person(name='Jason', age=10)\n", + "correctly parsed person=Person(name='Jason', age=10)\n", + "correctly parsed person=Person(name='Jason', age=10)\n", + "correctly parsed person=Person(name='Jason', age=10)\n", + "correctly parsed person=Person(name='Jason', age=10)\n", + "correctly parsed person=Person(name='Jason', age=10)\n", + "correctly parsed person=Person(name='Jason', age=10)\n" ] } ], @@ -248,7 +177,7 @@ "resp = client.chat.completions.create(\n", " model=\"gpt-3.5-turbo\",\n", " messages=[\n", - " {\"role\": \"user\", \"content\": \"Please give me jason is 10 as a json object\"},\n", + " {\"role\": \"user\", \"content\": \"Please give me jason is 10 as a json object ```json\\n\"},\n", " ],\n", " n=20,\n", " temperature=1,\n", @@ -257,8 +186,10 @@ "for choice in resp.choices:\n", " json = choice.message.content\n", " try:\n", - " Person.model_validate_json(json)\n", + " person = Person.model_validate_json(json)\n", + " print(f\"correctly parsed {person=}\")\n", " except Exception as e:\n", + " print(\"error!!\")\n", " print(json)" ] }, @@ -277,16 +208,16 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 26, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "PersonBirthday(name='Jason Liu', age=30, birthday=datetime.date(2023, 11, 30))" + "PersonBirthday(name='Jason Liu', age=30, birthday=datetime.date(2023, 12, 19))" ] }, - "execution_count": 13, + "execution_count": 26, "metadata": {}, "output_type": "execute_result" } @@ -335,7 +266,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 27, "metadata": {}, "outputs": [ { @@ -349,7 +280,7 @@ " 'type': 'object'}" ] }, - "execution_count": 14, + "execution_count": 27, "metadata": {}, "output_type": "execute_result" } @@ -367,7 +298,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 28, "metadata": {}, "outputs": [ { @@ -390,7 +321,7 @@ " 'type': 'object'}" ] }, - "execution_count": 15, + "execution_count": 28, "metadata": {}, "output_type": "execute_result" } @@ -425,14 +356,14 @@ "source": [ "# The core idea around Instructor\n", "\n", - "1. Using function calling allows us to specify the schema we want\n", - "2. Pydantic can be used to define the schema and documentation AND validate the response at runtime\n", + "1. Using function calling allows us use a llm that is finetuned to use json_schema and output json.\n", + "2. Pydantic can be used to define the object, schema, and validation in one single class, allow us to encapsulate everything neatly\n", "3. As a library with 100M downloads, we can leverage pydantic to do all the heavy lifting for us and fit nicely with the python ecosystem\n" ] }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 41, "metadata": {}, "outputs": [ { @@ -441,7 +372,7 @@ "PersonAddress(name='Jason Liu', age=30, address=Address(address='123 Main St', city='San Francisco', state='CA'))" ] }, - "execution_count": 16, + "execution_count": 41, "metadata": {}, "output_type": "execute_result" } @@ -451,10 +382,10 @@ "import datetime\n", "\n", "# patch the client to add `response_model` to the `create` method\n", - "client = instructor.patch(client)\n", + "client = instructor.patch(OpenAI(), mode=instructor.Mode.MD_JSON)\n", "\n", "resp = client.chat.completions.create(\n", - " model=\"gpt-3.5-turbo\",\n", + " model=\"gpt-3.5-turbo-1106\",\n", " messages=[\n", " {\n", " \"role\": \"user\",\n", @@ -484,7 +415,7 @@ "source": [ "## Is instructor the only way to do this?\n", "\n", - "No. Libraries like Marvin, Langchain, and Llamaindex all now leverage the pydantic object in similar ways however they all have different approaches to how they do it. With instructor the goal is to be as light weight as possible, get you as close as possible to the openai api, and then get out of your way.\n", + "No. Libraries like Marvin, Langchain, and Llamaindex all now leverage the Pydantic object in similar ways. The goal is to be as light weight as possible, get you as close as possible to the openai api, and then get out of your way.\n", "\n", "More importantly, we've also added straight forward validation and reasking to the mix.\n", "\n", @@ -496,6 +427,13 @@ "- [Langchain](https://python.langchain.com/docs/modules/model_io/output_parsers/pydantic)\n", "- [LlamaIndex](https://gpt-index.readthedocs.io/en/latest/examples/output_parsing/openai_pydantic_program.html)\n" ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { diff --git a/tutorials/3.0.applications-rag.ipynb b/tutorials/3.0.applications-rag.ipynb index 95dc574..0aadb5b 100644 --- a/tutorials/3.0.applications-rag.ipynb +++ b/tutorials/3.0.applications-rag.ipynb @@ -74,7 +74,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 1, "metadata": {}, "outputs": [], "source": [ @@ -107,7 +107,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ @@ -120,46 +120,39 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 3, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "{'hypothetical_questions': ['What is the basic concept behind Simple RAG?',\n", - " 'Why might Simple RAG not perform well with '\n", - " 'complex queries?',\n", - " \"In what ways does Simple RAG's embedding search \"\n", - " 'fall short?'],\n", - " 'keywords': ['Simple RAG',\n", - " 'Retriever-Augmented Generation',\n", - " 'user queries',\n", - " 'embedding search',\n", - " 'vector database',\n", - " 'query-document mismatch'],\n", - " 'summary': 'Simple RAG is an implementation that embeds user queries for a '\n", - " 'single embedding search in a vector database. Although '\n", - " 'straightforward, it struggles with complex queries and varied '\n", - " 'data sources because of its basic framework and query-document '\n", - " 'mismatch issues.',\n", - " 'topic': 'Simple Retriever-Augmented Generation (RAG)'}\n", - "{'hypothetical_questions': ['What kind of limitations does Simple RAG face?',\n", - " 'How does a monolithic search backend affect '\n", - " \"Simple RAG's performance?\",\n", - " 'Can Simple RAG handle complex, context-specific '\n", - " 'queries effectively?'],\n", - " 'keywords': ['limitations',\n", - " 'Simple RAG',\n", - " 'query-document mismatch',\n", + "{'hypothetical_questions': [],\n", + " 'keywords': ['RAG',\n", + " 'Retrieval-Augmented Generation',\n", + " 'embedding',\n", + " 'query',\n", + " 'vector database'],\n", + " 'summary': 'The simplest implementation of RAG (Retrieval-Augmented '\n", + " 'Generation) involves embedding a user query and conducting a '\n", + " 'single embedding search in a vector database, often containing a '\n", + " 'vector store of Wikipedia articles. This approach is intended to '\n", + " 'retrieve relevant documents to aid in generating responses.',\n", + " 'topic': 'Simple RAG Description'}\n", + "{'hypothetical_questions': ['What are the common challenges encountered with '\n", + " 'the simple RAG approach?',\n", + " 'How does the monolithic search backend limit the '\n", + " 'simple RAG?'],\n", + " 'keywords': ['query-document mismatch',\n", " 'monolithic search backend',\n", " 'text search limitations',\n", " 'limited planning ability'],\n", - " 'summary': 'The limitations of Simple RAG include query-document mismatch, '\n", - " 'reliance on a monolithic search backend, restrictions to simple '\n", - " 'text searches, and limited planning ability, which results in '\n", - " 'suboptimal outcomes when handling nuanced or context-specific '\n", - " 'queries.',\n", + " 'summary': 'Simple RAG has several limitations including query-document '\n", + " 'mismatch where embeddings may not align, reliance on a monolithic '\n", + " 'search backend which reduces flexibility, text search limitations '\n", + " \"that can't grasp nuances of complex queries, and limited planning \"\n", + " 'ability that fails to consider additional contextual information '\n", + " 'for refining search results.',\n", " 'topic': 'Limitations of Simple RAG'}\n" ] } @@ -227,7 +220,7 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 4, "metadata": {}, "outputs": [], "source": [ @@ -260,40 +253,38 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 6, "metadata": {}, "outputs": [ { - "name": "stdout", - "output_type": "stream", - "text": [ - "{\n", - " \"rewritten_query\": \"recent developments in AI\",\n", - " \"published_daterange\": {\n", - " \"start\": \"2023-01-01\",\n", - " \"end\": \"2023-11-30\"\n", - " }\n", - "}\n" - ] + "data": { + "text/plain": [ + "Query(rewritten_query='recent developments in AI', published_daterange=DateRange(start=datetime.date(2023, 1, 1), end=datetime.date(2023, 12, 20)))" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ - "query = client.chat.completions.create(\n", - " model=\"gpt-3.5-turbo\",\n", - " response_model=Query,\n", - " messages=[\n", - " {\n", - " \"role\": \"system\", \n", - " \"content\": f\"You're a query understanding system for the Metafor Systems search engine. Today is {date.today()}. Here are some tips: ...\"\n", - " },\n", - " {\n", - " \"role\": \"user\", \n", - " \"content\": \"query: What are some recent developments in AI?\"\n", - " }\n", - " ],\n", - ")\n", + "def expand_query(q) -> Query:\n", + " return client.chat.completions.create(\n", + " model=\"gpt-3.5-turbo\",\n", + " response_model=Query,\n", + " messages=[\n", + " {\n", + " \"role\": \"system\", \n", + " \"content\": f\"You're a query understanding system for the Metafor Systems search engine. Today is {date.today()}. Here are some tips: ...\"\n", + " },\n", + " {\n", + " \"role\": \"user\", \n", + " \"content\": f\"query: {q}\"\n", + " }\n", + " ],\n", + " )\n", "\n", - "print(query.model_dump_json(indent=4)) # Printing the Json dump of the model" + "expand_query(\"What are some recent developments in AI?\")" ] }, { @@ -305,22 +296,18 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": 9, "metadata": {}, "outputs": [ { - "name": "stdout", - "output_type": "stream", - "text": [ - "{\n", - " \"rewritten_query\": \"recent developments in artificial intelligence\",\n", - " \"published_daterange\": {\n", - " \"chain_of_thought\": \"Given that it's currently late 2023, a recent timeframe would ideally be within the last year to ensure the developments are current. Therefore, a suitable date range for recent AI developments could start from late 2022 to the present date in 2023.\",\n", - " \"start\": \"2022-11-18\",\n", - " \"end\": \"2023-11-18\"\n", - " }\n", - "}\n" - ] + "data": { + "text/plain": [ + "Query(rewritten_query='latest advancements in artificial intelligence', published_daterange=DateRange(chain_of_thought=\"Given that 'recent' suggests developments within the last few months to a year, the best time range to search would be from the beginning of the current year to the current date. This will ensure that the developments are timely and up-to-date.\", start=datetime.date(2023, 1, 1), end=datetime.date(2023, 12, 20)))" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ @@ -336,22 +323,23 @@ " published_daterange: DateRange\n", "\n", "\n", - "query = client.chat.completions.create(\n", - " model=\"gpt-4-1106-preview\",\n", - " response_model=Query,\n", - " messages=[\n", - " {\n", - " \"role\": \"system\", \n", - " \"content\": f\"You're a query understanding system for a search engine. Today is {date.today()}.\"\n", - " },\n", - " {\n", - " \"role\": \"user\", \n", - " \"content\": \"What are some recent developments in AI?\"\n", - " }\n", - " ],\n", - ")\n", + "def expand_query(q) -> Query:\n", + " return client.chat.completions.create(\n", + " model=\"gpt-4-1106-preview\",\n", + " response_model=Query,\n", + " messages=[\n", + " {\n", + " \"role\": \"system\", \n", + " \"content\": f\"You're a query understanding system for the Metafor Systems search engine. Today is {date.today()}. Here are some tips: ...\"\n", + " },\n", + " {\n", + " \"role\": \"user\", \n", + " \"content\": f\"query: {q}\"\n", + " }\n", + " ],\n", + " )\n", "\n", - "print(query.model_dump_json(indent=4)) # Printing the Json dump of the model" + "expand_query(\"What are some recent developments in AI?\")" ] }, { From db9836b9c06de2aa82b4b5617ab4a633a396f795 Mon Sep 17 00:00:00 2001 From: Jason Liu Date: Thu, 21 Dec 2023 15:12:47 -0500 Subject: [PATCH 2/7] bump --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 7b8a52f..987816e 100644 --- a/.gitignore +++ b/.gitignore @@ -161,3 +161,4 @@ cython_debug/ #.idea/ examples/citation_with_extraction/fly.toml my_cache_directory/ +tutorials/wandb/latest-run From b27285a345d608146f49f468a45f6eec573ee3f8 Mon Sep 17 00:00:00 2001 From: Jason Liu Date: Thu, 21 Dec 2023 15:13:08 -0500 Subject: [PATCH 3/7] bump --- .gitignore | 2 +- tutorials/2.tips.ipynb | 354 +++++++++------------------ tutorials/3.0.applications-rag.ipynb | 273 ++++++++++++++++++--- 3 files changed, 357 insertions(+), 272 deletions(-) diff --git a/.gitignore b/.gitignore index 987816e..ea3a62a 100644 --- a/.gitignore +++ b/.gitignore @@ -161,4 +161,4 @@ cython_debug/ #.idea/ examples/citation_with_extraction/fly.toml my_cache_directory/ -tutorials/wandb/latest-run +tutorials/wandb/* diff --git a/tutorials/2.tips.ipynb b/tutorials/2.tips.ipynb index be5d94f..69bc771 100644 --- a/tutorials/2.tips.ipynb +++ b/tutorials/2.tips.ipynb @@ -32,21 +32,10 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": null, "id": "fdf5e1d9-31ad-4e8a-a55e-e2e70fff598d", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'age': 17, 'name': 'Harry Potter', 'house': }" - ] - }, - "execution_count": 10, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "import instructor\n", "from openai import OpenAI\n", @@ -88,39 +77,20 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": null, "id": "c609eb44", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Hello, I'm Harry Potter, I'm 17 years old and I'm from Gryffindor\n" - ] - } - ], + "outputs": [], "source": [ "resp.say_hello()" ] }, { "cell_type": "code", - "execution_count": 12, + "execution_count": null, "id": "03db160c-81e9-4373-bfec-7a107224b6dd", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'age': 17, 'name': 'Harry Potter', 'house': 'Gryffindor'}" - ] - }, - "execution_count": 12, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "class Character(BaseModel):\n", " age: int\n", @@ -148,7 +118,7 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": null, "id": "0e7938b8-4666-4df4-bd80-f53e8baf7550", "metadata": {}, "outputs": [], @@ -193,33 +163,7 @@ "execution_count": null, "id": "69a58d01-ab6f-41b6-bc0c-b0e55fdb6fe4", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'age': 38,\n", - " 'name': 'Severus Snape',\n", - " 'house': 'Slytherin',\n", - " 'properties': [{'index': '1', 'key': 'patronus', 'value': 'Doe'},\n", - " {'index': '2',\n", - " 'key': 'position',\n", - " 'value': 'Potions Master, Defense Against the Dark Arts teacher, Headmaster'},\n", - " {'index': '3',\n", - " 'key': 'loyalty',\n", - " 'value': 'Hogwarts, Albus Dumbledore, Order of the Phoenix, Lily Evans'},\n", - " {'index': '4',\n", - " 'key': 'skills',\n", - " 'value': 'Potions expertise, Occlumency, Legilimency'},\n", - " {'index': '5',\n", - " 'key': 'disguised_loyalty',\n", - " 'value': 'Death Eater (formerly, as a double agent)'}]}" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "class Property(BaseModel):\n", " index: str = Field(..., description=\"Monotonically increasing ID\")\n", @@ -260,16 +204,7 @@ "execution_count": null, "id": "1f2a2b14-a956-4f96-90c9-e11ca04ab7d1", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "age=38 name='Severus Snape' house='Slytherin'\n", - "age=115 name='Albus Dumbledore' house='Gryffindor'\n" - ] - } - ], + "outputs": [], "source": [ "from typing import Iterable\n", "\n", @@ -282,7 +217,7 @@ "\n", "resp = client.chat.completions.create(\n", " model=\"gpt-4-1106-preview\",\n", - " messages=[{\"role\": \"user\", \"content\": \"Snape and Dumbledore from Harry Potter\"}],\n", + " messages=[{\"role\": \"user\", \"content\": \"Five characters from Harry Potter\"}],\n", " response_model=Iterable[Character],\n", ")\n", "\n", @@ -295,16 +230,7 @@ "execution_count": null, "id": "a3091aba", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "age=38 name='Severus Snape' house='Slytherin'\n", - "age=115 name='Albus Dumbledore' house='Gryffindor'\n" - ] - } - ], + "outputs": [], "source": [ "from typing import Iterable\n", "\n", @@ -317,7 +243,7 @@ "\n", "resp = client.chat.completions.create(\n", " model=\"gpt-4-1106-preview\",\n", - " messages=[{\"role\": \"user\", \"content\": \"Snape and Dumbledore from Harry Potter\"}],\n", + " messages=[{\"role\": \"user\", \"content\": \"Five characters from Harry Potter\"}],\n", " stream=True,\n", " response_model=Iterable[Character],\n", ")\n", @@ -341,29 +267,17 @@ "execution_count": null, "id": "6de8768e-b36a-4a51-9cf9-940d178552f6", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "id=1 name='Harry Potter' friends=[2, 3, 4, 5]\n", - "id=2 name='Hermione Granger' friends=[1, 3, 4, 5]\n", - "id=3 name='Ron Weasley' friends=[1, 2, 4, 5]\n", - "id=4 name='Draco Malfoy' friends=[5]\n", - "id=5 name='Neville Longbottom' friends=[1, 2, 3, 4]\n" - ] - } - ], + "outputs": [], "source": [ "class Character(BaseModel):\n", " id: int\n", " name: str\n", - " friends: List[int]\n", + " friends_array: List[int] = Field(description=\"Relationships to their friends using the id\")\n", "\n", "\n", "resp = client.chat.completions.create(\n", " model=\"gpt-4-1106-preview\",\n", - " messages=[{\"role\": \"user\", \"content\": \"The 5 kids from Harry Potter\"}],\n", + " messages=[{\"role\": \"user\", \"content\": \"5 kids from Harry Potter\"}],\n", " stream=True,\n", " response_model=Iterable[Character],\n", ")\n", @@ -372,142 +286,6 @@ " print(character)" ] }, - { - "cell_type": "code", - "execution_count": null, - "id": "b31e10d7-ebd2-49b4-b2c4-20dd67ca135d", - "metadata": {}, - "outputs": [ - { - "data": { - "image/svg+xml": [ - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "1\n", - "\n", - "Harry Potter\n", - "\n", - "\n", - "\n", - "2\n", - "\n", - "Hermione Granger\n", - "\n", - "\n", - "\n", - "1->2\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "3\n", - "\n", - "Ron Weasley\n", - "\n", - "\n", - "\n", - "1->3\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "4\n", - "\n", - "Draco Malfoy\n", - "\n", - "\n", - "\n", - "1->4\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "5\n", - "\n", - "Neville Longbottom\n", - "\n", - "\n", - "\n", - "1->5\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "2->3\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "2->4\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "3->4\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "3->5\n", - "\n", - "\n", - "\n", - "\n", - "\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "from graphviz import Digraph\n", - "from IPython.display import display\n", - "\n", - "dot = Digraph()\n", - "\n", - "resp = client.chat.completions.create(\n", - " model=\"gpt-4-1106-preview\",\n", - " messages=[{\"role\": \"user\", \"content\": \"The 5 kids from Harry Potter\"}],\n", - " response_model=Iterable[Character],\n", - ")\n", - "\n", - "# Create nodes for each user\n", - "for user in resp:\n", - " dot.node(str(user.id), user.name)\n", - "\n", - "# Create edges for friends\n", - "for user in resp:\n", - " for friend_id in user.friends:\n", - " # To avoid duplicating edges, only add an edge if the friend ID is greater than the user ID\n", - " if friend_id > user.id:\n", - " dot.edge(str(user.id), str(friend_id))\n", - "\n", - "\n", - "# Render the graph to a file\n", - "display(dot)" - ] - }, { "cell_type": "markdown", "id": "523b5797-71a5-4a96-a4b7-21280fb73015", @@ -515,6 +293,108 @@ "source": [ "With the tools we've discussed, we can find numerous real-world applications in production settings. These include extracting action items from transcripts, generating fake data, filling out forms, and creating objects that correspond to generative UI. These simple tricks will be highly useful.\n" ] + }, + { + "cell_type": "markdown", + "id": "a9d20fd9-0cd0-4300-a8c1-d16388969e8e", + "metadata": {}, + "source": [ + "# Missing Data\n", + "\n", + "The Maybe pattern is a concept in functional programming used for error handling. Instead of raising exceptions or returning None, you can use a Maybe type to encapsulate both the result and potential errors.\n", + "\n", + "This pattern is particularly useful when making LLM calls, as providing language models with an escape hatch can effectively reduce hallucinations." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c04f44aa-dc4b-4499-a151-e812512e77e6", + "metadata": {}, + "outputs": [], + "source": [ + "from typing import Optional\n", + "\n", + "class Character(BaseModel):\n", + " age: int\n", + " name: str\n", + "\n", + "class MaybeCharacter(BaseModel):\n", + " result: Optional[Character] = Field(default=None)\n", + " error: bool = Field(default=False)\n", + " message: Optional[str]" + ] + }, + { + "cell_type": "code", + "execution_count": 63, + "id": "a2155190-e104-4ed6-a17f-e0732499dd51", + "metadata": {}, + "outputs": [], + "source": [ + "def extract(content: str) -> MaybeCharacter:\n", + " return client.chat.completions.create(\n", + " model=\"gpt-3.5-turbo\",\n", + " response_model=MaybeCharacter,\n", + " messages=[\n", + " {\"role\": \"user\", \"content\": f\"Extract `{content}`\"},\n", + " ],\n", + " )" + ] + }, + { + "cell_type": "code", + "execution_count": 64, + "id": "a7b59afa-9bf0-4dc0-a5ca-de584514f33b", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "MaybeCharacter(result=Character(age=17, name='Harry Potter'), error=False, message=None)" + ] + }, + "execution_count": 64, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "extract(\"Harry Potter\")" + ] + }, + { + "cell_type": "code", + "execution_count": 66, + "id": "b5ddd5c1-ca75-49a9-95ad-181170435291", + "metadata": {}, + "outputs": [ + { + "ename": "ValueError", + "evalue": "404 Error", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mValueError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[66], line 4\u001b[0m\n\u001b[1;32m 1\u001b[0m user \u001b[38;5;241m=\u001b[39m extract(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m404 Error\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m 3\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m user\u001b[38;5;241m.\u001b[39merror:\n\u001b[0;32m----> 4\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(user\u001b[38;5;241m.\u001b[39mmessage)\n", + "\u001b[0;31mValueError\u001b[0m: 404 Error" + ] + } + ], + "source": [ + "user = extract(\"404 Error\")\n", + "\n", + "if user.error:\n", + " raise ValueError(user.message)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1e14f7cb-d99c-4696-a1fa-e08319bf5d68", + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { diff --git a/tutorials/3.0.applications-rag.ipynb b/tutorials/3.0.applications-rag.ipynb index 0aadb5b..62ce23b 100644 --- a/tutorials/3.0.applications-rag.ipynb +++ b/tutorials/3.0.applications-rag.ipynb @@ -127,33 +127,68 @@ "name": "stdout", "output_type": "stream", "text": [ - "{'hypothetical_questions': [],\n", - " 'keywords': ['RAG',\n", - " 'Retrieval-Augmented Generation',\n", - " 'embedding',\n", - " 'query',\n", - " 'vector database'],\n", - " 'summary': 'The simplest implementation of RAG (Retrieval-Augmented '\n", - " 'Generation) involves embedding a user query and conducting a '\n", - " 'single embedding search in a vector database, often containing a '\n", - " 'vector store of Wikipedia articles. This approach is intended to '\n", - " 'retrieve relevant documents to aid in generating responses.',\n", - " 'topic': 'Simple RAG Description'}\n", - "{'hypothetical_questions': ['What are the common challenges encountered with '\n", - " 'the simple RAG approach?',\n", - " 'How does the monolithic search backend limit the '\n", - " 'simple RAG?'],\n", + "{'hypothetical_questions': ['How does a simple RAG Model work?',\n", + " 'What are the key challenges faced by simple RAG '\n", + " 'systems?',\n", + " 'Can a simple RAG model handle complex queries '\n", + " 'effectively?'],\n", + " 'keywords': ['Retrieval-Augmented Generation',\n", + " 'RAG',\n", + " 'user query',\n", + " 'vector database',\n", + " 'embeddings',\n", + " 'search limitations'],\n", + " 'summary': 'The simple Retrieval-Augmented Generation (RAG) model is an '\n", + " 'approach that uses a vector database to embed and search for user '\n", + " 'queries, such as Wikipedia articles. It provides answers by '\n", + " 'aligning query and document embeddings. However, it has several '\n", + " 'limitations, including query-document mismatch, reliance on a '\n", + " 'monolithic search backend, text search limitations, and a limited '\n", + " 'planning ability.',\n", + " 'topic': 'Simple RAG'}\n", + "{'hypothetical_questions': ['What is a query-document mismatch in the context '\n", + " 'of RAG models?',\n", + " 'Why might a simple RAG model struggle with '\n", + " 'specific user queries?'],\n", " 'keywords': ['query-document mismatch',\n", - " 'monolithic search backend',\n", - " 'text search limitations',\n", - " 'limited planning ability'],\n", - " 'summary': 'Simple RAG has several limitations including query-document '\n", - " 'mismatch where embeddings may not align, reliance on a monolithic '\n", - " 'search backend which reduces flexibility, text search limitations '\n", - " \"that can't grasp nuances of complex queries, and limited planning \"\n", - " 'ability that fails to consider additional contextual information '\n", - " 'for refining search results.',\n", - " 'topic': 'Limitations of Simple RAG'}\n" + " 'RAG limitation',\n", + " 'embedding alignment'],\n", + " 'summary': \"A limitation where the simple RAG system's query and document \"\n", + " 'embeddings may not align properly, leading to ineffective '\n", + " \"retrieval of information specific to a user's query about, for \"\n", + " \"example, 'climate change effects on marine life'.\",\n", + " 'topic': 'Query-Document Mismatch'}\n", + "{'hypothetical_questions': ['Why is depending on a monolithic search backend a '\n", + " 'limitation for simple RAG?',\n", + " 'How can a monolithic search backend affect the '\n", + " 'quality of search results?'],\n", + " 'keywords': ['monolithic search backend', 'RAG system', 'data sources'],\n", + " 'summary': 'In simple RAG, the reliance on a single search method and backend '\n", + " \"can limit the system's ability to access diverse or specialized \"\n", + " \"data sources, such as when searching for 'latest research in \"\n", + " \"quantum computing'.\",\n", + " 'topic': 'Monolithic Search Backend'}\n", + "{'hypothetical_questions': ['How do text search limitations impact the '\n", + " 'effectiveness of simple RAG?',\n", + " 'Can simple RAG models understand the context of '\n", + " 'search terms?'],\n", + " 'keywords': ['text search limitations', 'RAG', 'advanced search'],\n", + " 'summary': 'The simple RAG model is limited to straightforward text queries '\n", + " 'without advanced search capabilities, failing to resolve nuanced '\n", + " \"queries like 'what problems did we fix last week' due to the \"\n", + " \"presence of generic terms such as 'problem' and 'last week' \"\n", + " 'throughout documents.',\n", + " 'topic': 'Text Search Limitations'}\n", + "{'hypothetical_questions': ['What does limited planning ability imply for a '\n", + " \"RAG model's search results?\",\n", + " 'Can simple RAG models provide context-specific '\n", + " 'information effectively?'],\n", + " 'keywords': ['limited planning ability', 'contextual information', 'RAG'],\n", + " 'summary': 'Simple RAG models struggle to incorporate additional context in '\n", + " 'their searches, which may result in less relevant or overly '\n", + " 'general responses to queries that require specific insights, like '\n", + " \"'Tips for first-time Europe travelers'.\",\n", + " 'topic': 'Limited Planning Ability'}\n" ] } ], @@ -253,16 +288,16 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 5, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "Query(rewritten_query='recent developments in AI', published_daterange=DateRange(start=datetime.date(2023, 1, 1), end=datetime.date(2023, 12, 20)))" + "Query(rewritten_query='recent developments in AI', published_daterange=DateRange(start=datetime.date(2023, 1, 1), end=datetime.date(2023, 12, 21)))" ] }, - "execution_count": 6, + "execution_count": 5, "metadata": {}, "output_type": "execute_result" } @@ -296,16 +331,16 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 16, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "Query(rewritten_query='latest advancements in artificial intelligence', published_daterange=DateRange(chain_of_thought=\"Given that 'recent' suggests developments within the last few months to a year, the best time range to search would be from the beginning of the current year to the current date. This will ensure that the developments are timely and up-to-date.\", start=datetime.date(2023, 1, 1), end=datetime.date(2023, 12, 20)))" + "Query(rewritten_query='Latest advancements in Artificial Intelligence as of December 2023', published_daterange=DateRange(chain_of_thought=\"To get the most recent developments in AI, the date range should be quite recent. Considering today's date is 2023-12-21, a suitable range might be from the past three months to now.\", start=datetime.date(2023, 9, 21), end=datetime.date(2023, 12, 21)))" ] }, - "execution_count": 9, + "execution_count": 16, "metadata": {}, "output_type": "execute_result" } @@ -319,8 +354,12 @@ " end: date\n", "\n", "class Query(BaseModel):\n", - " rewritten_query: str\n", - " published_daterange: DateRange\n", + " rewritten_query: str = Field(\n", + " description=\"Rewrite the query to make it more specific\"\n", + " )\n", + " published_daterange: DateRange = Field(\n", + " description=\"Effective date range to search in\"\n", + " )\n", "\n", "\n", "def expand_query(q) -> Query:\n", @@ -342,6 +381,172 @@ "expand_query(\"What are some recent developments in AI?\")" ] }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "data": { + "text/html": [ + "wandb version 0.16.1 is available! To upgrade, please run:\n", + " $ pip install wandb --upgrade" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "Tracking run with wandb version 0.16.0" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "Run data is saved locally in /Users/jasonliu/dev/instructor/tutorials/wandb/run-20231221_145615-zv4el5or" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "Syncing run swept-sound-11 to Weights & Biases (docs)
" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + " View project at https://wandb.ai/instructor/query-understanding" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + " View run at https://wandb.ai/instructor/query-understanding/runs/zv4el5or" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "wandb: WARNING Source type is set to 'repo' but some required information is missing from the environment. A job will not be created from this run. See https://docs.wandb.ai/guides/launch/create-job\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "eacf93c56db445ffafcfbf673ee44a91", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "VBox(children=(Label(value='0.003 MB of 0.003 MB uploaded\\r'), FloatProgress(value=1.0, max=1.0)))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + " View run swept-sound-11 at: https://wandb.ai/instructor/query-understanding/runs/zv4el5or
Synced 4 W&B file(s), 0 media file(s), 2 artifact file(s) and 0 other file(s)" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "Find logs at: ./wandb/run-20231221_145615-zv4el5or/logs" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "import json\n", + "import wandb\n", + "\n", + "run = wandb.init(\n", + " project=\"query-understanding\",\n", + " config=Query.model_json_schema()\n", + ")\n", + "\n", + "files = wandb.Artifact('data', type='dataset'\n", + ")\n", + "\n", + "with open(\"schema.json\", \"w+\") as f:\n", + " schema = Query.model_json_schema()\n", + " json.dump(schema, f, indent=2)\n", + "\n", + "with open(\"results.jsonl\", \"w+\") as f:\n", + " for questions in [\n", + " \"latest developments in artificial intelligence last 3 weeks\",\n", + " \"renewable energy trends past month\",\n", + " \"quantum computing advancements last 2 months\",\n", + " \"biotechnology updates last 10 days\"\n", + " ]:\n", + " query = expand_query(questions)\n", + " f.write(json.dumps(query.model_dump_json()) + \"\\n\")\n", + " \n", + "files.add_file('schema.json')\n", + "files.add_file('results.jsonl')\n", + "\n", + "run.log_artifact(files)\n", + "run.finish()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, { "cell_type": "markdown", "metadata": {}, From c2fc90265f6ed8446c21e9568ad02f25fa7ea619 Mon Sep 17 00:00:00 2001 From: Jason Liu Date: Thu, 21 Dec 2023 15:44:47 -0500 Subject: [PATCH 4/7] format --- tutorials/3.0.applications-rag.ipynb | 278 ++++++++++++++++++++------- 1 file changed, 206 insertions(+), 72 deletions(-) diff --git a/tutorials/3.0.applications-rag.ipynb b/tutorials/3.0.applications-rag.ipynb index 62ce23b..6d1c024 100644 --- a/tutorials/3.0.applications-rag.ipynb +++ b/tutorials/3.0.applications-rag.ipynb @@ -4,7 +4,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# Applying Structured Output to RAG applications " + "# Applying Structured Output to RAG applications\n" ] }, { @@ -23,7 +23,7 @@ "\n", "**Why is there a need for them?**\n", "\n", - "Pre-trained large language models do not learn over time. If you ask them a question they have not been trained on, they will often hallucinate. Therefore, we need to embed our own data to achieve a better output." + "Pre-trained large language models do not learn over time. If you ask them a question they have not been trained on, they will often hallucinate. Therefore, we need to embed our own data to achieve a better output.\n" ] }, { @@ -38,7 +38,7 @@ "\n", "- **Query-Document Mismatch:** It assumes that the query and document embeddings will align in the vector space, which is often not the case.\n", "- **Text Search Limitations:** The model is restricted to simple text queries without the nuances of advanced search features.\n", - "- **Limited Planning Ability:** It fails to consider additional contextual information that could refine the search results." + "- **Limited Planning Ability:** It fails to consider additional contextual information that could refine the search results.\n" ] }, { @@ -62,7 +62,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## Practical Examples" + "## Practical Examples\n" ] }, { @@ -78,7 +78,7 @@ "metadata": {}, "outputs": [], "source": [ - "import instructor \n", + "import instructor\n", "\n", "from openai import OpenAI\n", "from typing import List\n", @@ -93,8 +93,8 @@ "source": [ "### Example 1) Improving Extractions\n", "\n", - "One of the big limitations is that often times the query we embed and the text \n", - "A common method of using structured output is to extract information from a document and use it to answer a question. Directly, we can be creative in how we extract, summarize and generate potential questions in order for our embeddings to do better. \n", + "One of the big limitations is that often times the query we embed and the text\n", + "A common method of using structured output is to extract information from a document and use it to answer a question. Directly, we can be creative in how we extract, summarize and generate potential questions in order for our embeddings to do better.\n", "\n", "For example, instead of using just a text chunk we could try to:\n", "\n", @@ -102,7 +102,7 @@ "2. extract hypothetical questions\n", "3. generate a summary of the text\n", "\n", - "In the example below, we use the `instructor` library to extract the key words and themes from a text chunk and use them to answer a question." + "In the example below, we use the `instructor` library to extract the key words and themes from a text chunk and use them to answer a question.\n" ] }, { @@ -113,9 +113,14 @@ "source": [ "class Extraction(BaseModel):\n", " topic: str\n", - " summary: str \n", - " hypothetical_questions: List[str] = Field(default_factory=list, description=\"Hypothetical questions that this document could answer\")\n", - " keywords: List[str] = Field(default_factory=list, description=\"Keywords that this document is about\")" + " summary: str\n", + " hypothetical_questions: List[str] = Field(\n", + " default_factory=list,\n", + " description=\"Hypothetical questions that this document could answer\",\n", + " )\n", + " keywords: List[str] = Field(\n", + " default_factory=list, description=\"Keywords that this document is about\"\n", + " )" ] }, { @@ -224,13 +229,14 @@ " model=\"gpt-4-1106-preview\",\n", " stream=True,\n", " response_model=Iterable[Extraction],\n", - " messages=[{\n", - " \"role\": \"system\", \n", - " \"content\": \"Your role is to extract chunks from the following and create a set of topics.\"\n", - " }, {\n", - " \"role\": \"user\", \n", - " \"content\": text_chunk\n", - " }])\n", + " messages=[\n", + " {\n", + " \"role\": \"system\",\n", + " \"content\": \"Your role is to extract chunks from the following and create a set of topics.\",\n", + " },\n", + " {\"role\": \"user\", \"content\": text_chunk},\n", + " ],\n", + ")\n", "\n", "\n", "for extraction in extractions:\n", @@ -241,7 +247,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Now you can imagine if you were to embed the summaries, hypothetical questions, and keywords in a vector database, you can then use a vector search to find the best matching document for a given query. What you'll find is that the results are much better than if you were to just embed the text chunk! " + "Now you can imagine if you were to embed the summaries, hypothetical questions, and keywords in a vector database, you can then use a vector search to find the best matching document for a given query. What you'll find is that the results are much better than if you were to just embed the text chunk!\n" ] }, { @@ -261,10 +267,12 @@ "source": [ "from datetime import date\n", "\n", + "\n", "class DateRange(BaseModel):\n", " start: date\n", " end: date\n", "\n", + "\n", "class Query(BaseModel):\n", " rewritten_query: str\n", " published_daterange: DateRange" @@ -276,14 +284,14 @@ "source": [ "In this example, `DateRange` and `Query` are Pydantic models that structure the user's query with a date range and a list of domains to search within.\n", "\n", - "These models **restructure** the user's query by including a rewritten query, a range of published dates, and a list of domains to search in." + "These models **restructure** the user's query by including a rewritten query, a range of published dates, and a list of domains to search in.\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "Using the new restructured query, we can apply this pattern to our function calls to obtain results that are optimized for our backend." + "Using the new restructured query, we can apply this pattern to our function calls to obtain results that are optimized for our backend.\n" ] }, { @@ -309,16 +317,14 @@ " response_model=Query,\n", " messages=[\n", " {\n", - " \"role\": \"system\", \n", - " \"content\": f\"You're a query understanding system for the Metafor Systems search engine. Today is {date.today()}. Here are some tips: ...\"\n", + " \"role\": \"system\",\n", + " \"content\": f\"You're a query understanding system for the Metafor Systems search engine. Today is {date.today()}. Here are some tips: ...\",\n", " },\n", - " {\n", - " \"role\": \"user\", \n", - " \"content\": f\"query: {q}\"\n", - " }\n", + " {\"role\": \"user\", \"content\": f\"query: {q}\"},\n", " ],\n", " )\n", "\n", + "\n", "expand_query(\"What are some recent developments in AI?\")" ] }, @@ -326,7 +332,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "This isn't just about adding some date ranges. We can even use some chain of thought prompting to generate tailored searches that are deeply integrated with our backend. " + "This isn't just about adding some date ranges. We can even use some chain of thought prompting to generate tailored searches that are deeply integrated with our backend.\n" ] }, { @@ -353,6 +359,7 @@ " start: date\n", " end: date\n", "\n", + "\n", "class Query(BaseModel):\n", " rewritten_query: str = Field(\n", " description=\"Rewrite the query to make it more specific\"\n", @@ -368,22 +375,75 @@ " response_model=Query,\n", " messages=[\n", " {\n", - " \"role\": \"system\", \n", - " \"content\": f\"You're a query understanding system for the Metafor Systems search engine. Today is {date.today()}. Here are some tips: ...\"\n", + " \"role\": \"system\",\n", + " \"content\": f\"You're a query understanding system for the Metafor Systems search engine. Today is {date.today()}. Here are some tips: ...\",\n", " },\n", - " {\n", - " \"role\": \"user\", \n", - " \"content\": f\"query: {q}\"\n", - " }\n", + " {\"role\": \"user\", \"content\": f\"query: {q}\"},\n", " ],\n", " )\n", "\n", + "\n", "expand_query(\"What are some recent developments in AI?\")" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Using Weights and Biases to track experiments\n", + "\n", + "While running a function like this production is quite simple, a lot of time will be spend on iterating and improving the model. To do this, we can use Weights and Biases to track our experiments.\n", + "\n", + "In order to do so we wand manage a few things\n", + "\n", + "1. Save input and output pairs for later\n", + "2. Save the JSON schema for the response_model\n", + "3. Having snapshots of the model and data allow us to compare results over time, and as we make changes to the model we can see how the results change.\n" + ] + }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 43, + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "\n", + "\n", + "def flatten_dict(d, parent_key=\"\", sep=\"_\"):\n", + " \"\"\"\n", + " Flatten a nested dictionary.\n", + "\n", + " :param d: The nested dictionary to flatten.\n", + " :param parent_key: The base key to use for the flattened keys.\n", + " :param sep: Separator to use between keys.\n", + " :return: A flattened dictionary.\n", + " \"\"\"\n", + " items = []\n", + " for k, v in d.items():\n", + " new_key = f\"{parent_key}{sep}{k}\" if parent_key else k\n", + " if isinstance(v, dict):\n", + " items.extend(flatten_dict(v, new_key, sep=sep).items())\n", + " else:\n", + " items.append((new_key, v))\n", + " return dict(items)\n", + "\n", + "\n", + "def dicts_to_df(list_of_dicts):\n", + " \"\"\"\n", + " Convert a list of dictionaries to a pandas DataFrame.\n", + "\n", + " :param list_of_dicts: List of dictionaries, potentially nested.\n", + " :return: A pandas DataFrame representing the flattened data.\n", + " \"\"\"\n", + " # Flatten each dictionary and create a DataFrame\n", + " flattened_data = [flatten_dict(d) for d in list_of_dicts]\n", + " return pd.DataFrame(flattened_data)" + ] + }, + { + "cell_type": "code", + "execution_count": 40, "metadata": { "scrolled": true }, @@ -416,7 +476,7 @@ { "data": { "text/html": [ - "Run data is saved locally in /Users/jasonliu/dev/instructor/tutorials/wandb/run-20231221_145615-zv4el5or" + "Run data is saved locally in /Users/jasonliu/dev/instructor/tutorials/wandb/run-20231221_153734-idscpy5k" ], "text/plain": [ "" @@ -428,7 +488,7 @@ { "data": { "text/html": [ - "Syncing run swept-sound-11 to Weights & Biases (docs)
" + "Syncing run easy-feather-16 to Weights & Biases (docs)
" ], "text/plain": [ "" @@ -452,7 +512,7 @@ { "data": { "text/html": [ - " View run at https://wandb.ai/instructor/query-understanding/runs/zv4el5or" + " View run at https://wandb.ai/instructor/query-understanding/runs/idscpy5k" ], "text/plain": [ "" @@ -471,12 +531,12 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "eacf93c56db445ffafcfbf673ee44a91", + "model_id": "6440cf236ba24c3b839d1256cfada604", "version_major": 2, "version_minor": 0 }, "text/plain": [ - "VBox(children=(Label(value='0.003 MB of 0.003 MB uploaded\\r'), FloatProgress(value=1.0, max=1.0)))" + "VBox(children=(Label(value='0.007 MB of 0.007 MB uploaded (0.001 MB deduped)\\r'), FloatProgress(value=1.0, max…" ] }, "metadata": {}, @@ -485,7 +545,7 @@ { "data": { "text/html": [ - " View run swept-sound-11 at: https://wandb.ai/instructor/query-understanding/runs/zv4el5or
Synced 4 W&B file(s), 0 media file(s), 2 artifact file(s) and 0 other file(s)" + "W&B sync reduced upload amount by 9.0% " ], "text/plain": [ "" @@ -497,7 +557,19 @@ { "data": { "text/html": [ - "Find logs at: ./wandb/run-20231221_145615-zv4el5or/logs" + " View run easy-feather-16 at: https://wandb.ai/instructor/query-understanding/runs/idscpy5k
Synced 4 W&B file(s), 1 media file(s), 4 artifact file(s) and 0 other file(s)" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "Find logs at: ./wandb/run-20231221_153734-idscpy5k/logs" ], "text/plain": [ "" @@ -511,30 +583,70 @@ "import json\n", "import wandb\n", "\n", + "\n", + "class DateRange(BaseModel):\n", + " chain_of_thought: str = Field(\n", + " description=\"Think step by step to plan what is the best time range to search in\"\n", + " )\n", + " start: date\n", + " end: date\n", + "\n", + "\n", + "class Query(BaseModel):\n", + " rewritten_query: str = Field(\n", + " description=\"Rewrite the query to make it more specific\"\n", + " )\n", + " published_daterange: DateRange = Field(\n", + " description=\"Effective date range to search in\"\n", + " )\n", + "\n", + "\n", + "def expand_query(q) -> Query:\n", + " return client.chat.completions.create(\n", + " model=\"gpt-4-1106-preview\",\n", + " response_model=Query,\n", + " messages=[\n", + " {\n", + " \"role\": \"system\",\n", + " \"content\": f\"You're a query understanding system for the Metafor Systems search engine. Today is {date.today()}. Here are some tips: ...\",\n", + " },\n", + " {\"role\": \"user\", \"content\": f\"query: {q}\"},\n", + " ],\n", + " )\n", + "\n", + "\n", "run = wandb.init(\n", " project=\"query-understanding\",\n", - " config=Query.model_json_schema()\n", ")\n", "\n", - "files = wandb.Artifact('data', type='dataset'\n", - ")\n", + "test_queries = [\n", + " \"latest developments in artificial intelligence last 3 weeks\",\n", + " \"renewable energy trends past month\",\n", + " \"quantum computing advancements last 2 months\",\n", + " \"biotechnology updates last 10 days\",\n", + "]\n", + "\n", + "queries = [expand_query(q) for q in test_queries]\n", "\n", "with open(\"schema.json\", \"w+\") as f:\n", " schema = Query.model_json_schema()\n", " json.dump(schema, f, indent=2)\n", "\n", - "with open(\"results.jsonl\", \"w+\") as f:\n", - " for questions in [\n", - " \"latest developments in artificial intelligence last 3 weeks\",\n", - " \"renewable energy trends past month\",\n", - " \"quantum computing advancements last 2 months\",\n", - " \"biotechnology updates last 10 days\"\n", - " ]:\n", - " query = expand_query(questions)\n", - " f.write(json.dumps(query.model_dump_json()) + \"\\n\")\n", - " \n", - "files.add_file('schema.json')\n", - "files.add_file('results.jsonl')\n", + "with open(\"results.jsonlines\", \"w+\") as f:\n", + " for query in queries:\n", + " f.write(query.model_dump_json() + \"\\n\")\n", + "\n", + "df = dicts_to_df([q.model_dump() for q in queries])\n", + "df[\"input\"] = test_queries\n", + "df.to_csv(\"results.csv\")\n", + "\n", + "run.log({\"results\": wandb.Table(dataframe=df)})\n", + "\n", + "files = wandb.Artifact(\"data\", type=\"dataset\")\n", + "\n", + "files.add_file(\"schema.json\")\n", + "files.add_file(\"results.jsonlines\")\n", + "files.add_file(\"results.csv\")\n", "\n", "run.log_artifact(files)\n", "run.finish()" @@ -542,7 +654,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 42, "metadata": {}, "outputs": [], "source": [] @@ -568,13 +680,15 @@ "source": [ "from typing import Literal\n", "\n", + "\n", "class SearchClient(BaseModel):\n", " query: str\n", " keywords: List[str]\n", " email: str\n", - " source: Literal[\"gmail\", \"calendar\"] \n", + " source: Literal[\"gmail\", \"calendar\"]\n", " date_range: DateRange\n", "\n", + "\n", "class Retrival(BaseModel):\n", " queries: List[SearchClient]" ] @@ -626,8 +740,11 @@ " model=\"gpt-4-1106-preview\",\n", " response_model=Retrival,\n", " messages=[\n", - " {\"role\": \"system\", \"content\":f\"You are Jason's personal assistant. Today is {date.today()}\"},\n", - " {\"role\": \"user\", \"content\": \"What do I have today?\"}\n", + " {\n", + " \"role\": \"system\",\n", + " \"content\": f\"You are Jason's personal assistant. Today is {date.today()}\",\n", + " },\n", + " {\"role\": \"user\", \"content\": \"What do I have today?\"},\n", " ],\n", ")\n", "print(retrival.model_dump_json(indent=4))" @@ -637,7 +754,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "To make it more challenging, we will assign it multiple tasks, followed by a list of queries that are routed to various search backends, such as email and calendar. Not only do we dispatch to different backends, over which we have no control, but we are also likely to render them to the user in different ways." + "To make it more challenging, we will assign it multiple tasks, followed by a list of queries that are routed to various search backends, such as email and calendar. Not only do we dispatch to different backends, over which we have no control, but we are also likely to render them to the user in different ways.\n" ] }, { @@ -691,8 +808,14 @@ " model=\"gpt-4-1106-preview\",\n", " response_model=Retrival,\n", " messages=[\n", - " {\"role\": \"system\", \"content\": f\"You are Jason's personal assistant. Today is {date.today()}\"},\n", - " {\"role\": \"user\", \"content\": \"What meetings do I have today and are there any important emails I should be aware of?\"}\n", + " {\n", + " \"role\": \"system\",\n", + " \"content\": f\"You are Jason's personal assistant. Today is {date.today()}\",\n", + " },\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": \"What meetings do I have today and are there any important emails I should be aware of?\",\n", + " },\n", " ],\n", ")\n", "print(retrival.model_dump_json(indent=4))" @@ -702,9 +825,9 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "### Example 4) Decomposing questions \n", + "### Example 4) Decomposing questions\n", "\n", - "Lastly, a lightly more complex example of a problem that can be solved with structured output is decomposing questions. Where you ultimately want to decompose a question into a series of sub-questions that can be answered by a search backend. For example \n", + "Lastly, a lightly more complex example of a problem that can be solved with structured output is decomposing questions. Where you ultimately want to decompose a question into a series of sub-questions that can be answered by a search backend. For example\n", "\n", "\"Whats the difference in populations of jason's home country and canada?\"\n", "\n", @@ -715,7 +838,7 @@ "3. The population of Canada\n", "4. The difference between the two\n", "\n", - "This would not be done correctly as a single query, nor would it be done in parallel, however there are some opportunities try to be parallel since not all of the sub-questions are dependent on each other." + "This would not be done correctly as a single query, nor would it be done in parallel, however there are some opportunities try to be parallel since not all of the sub-questions are dependent on each other.\n" ] }, { @@ -764,20 +887,31 @@ "class Question(BaseModel):\n", " id: int = Field(..., description=\"A unique identifier for the question\")\n", " query: str = Field(..., description=\"The question decomposited as much as possible\")\n", - " subquestions: List[int] = Field(default_factory=list, description=\"The subquestions that this question is composed of\")\n", + " subquestions: List[int] = Field(\n", + " default_factory=list,\n", + " description=\"The subquestions that this question is composed of\",\n", + " )\n", "\n", "\n", "class QueryPlan(BaseModel):\n", " root_question: str = Field(..., description=\"The root question that the user asked\")\n", - " plan: List[Question] = Field(..., description=\"The plan to answer the root question and its subquestions\")\n", + " plan: List[Question] = Field(\n", + " ..., description=\"The plan to answer the root question and its subquestions\"\n", + " )\n", "\n", "\n", "retrival = client.chat.completions.create(\n", " model=\"gpt-4-1106-preview\",\n", " response_model=QueryPlan,\n", " messages=[\n", - " {\"role\": \"system\", \"content\":\"You are a query understanding system capable of decomposing a question into subquestions.\"},\n", - " {\"role\": \"user\", \"content\": \"What is the difference between the population of jason's home country and canada?\"}\n", + " {\n", + " \"role\": \"system\",\n", + " \"content\": \"You are a query understanding system capable of decomposing a question into subquestions.\",\n", + " },\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": \"What is the difference between the population of jason's home country and canada?\",\n", + " },\n", " ],\n", ")\n", "\n", @@ -788,7 +922,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "I hope in this section I've exposed you to some ways we can be creative in modeling structured outputs to leverage LLMS in building some lightweight components for our systems." + "I hope in this section I've exposed you to some ways we can be creative in modeling structured outputs to leverage LLMS in building some lightweight components for our systems.\n" ] } ], From feb2a532ca0e42aeaecaa20d4774f77a92a41e1b Mon Sep 17 00:00:00 2001 From: Jason Liu Date: Fri, 22 Dec 2023 15:23:52 -0500 Subject: [PATCH 5/7] clean up notebooks --- .gitignore | 5 + tutorials/3.0.applications-rag.ipynb | 331 +++++++++++---------------- tutorials/helpers.py | 32 +++ wandb/settings | 5 + 4 files changed, 180 insertions(+), 193 deletions(-) create mode 100644 tutorials/helpers.py create mode 100644 wandb/settings diff --git a/.gitignore b/.gitignore index ea3a62a..f58c18b 100644 --- a/.gitignore +++ b/.gitignore @@ -162,3 +162,8 @@ cython_debug/ examples/citation_with_extraction/fly.toml my_cache_directory/ tutorials/wandb/* +tutorials/results.csv +tutorials/results.jsonl +tutorials/results.jsonlines +tutorials/schema.json +wandb/settings diff --git a/tutorials/3.0.applications-rag.ipynb b/tutorials/3.0.applications-rag.ipynb index 6d1c024..2307db3 100644 --- a/tutorials/3.0.applications-rag.ipynb +++ b/tutorials/3.0.applications-rag.ipynb @@ -74,7 +74,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 23, "metadata": {}, "outputs": [], "source": [ @@ -107,7 +107,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 3, "metadata": {}, "outputs": [], "source": [ @@ -125,75 +125,43 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 4, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "{'hypothetical_questions': ['How does a simple RAG Model work?',\n", - " 'What are the key challenges faced by simple RAG '\n", - " 'systems?',\n", - " 'Can a simple RAG model handle complex queries '\n", - " 'effectively?'],\n", + "{'hypothetical_questions': ['What is the most basic implementation of '\n", + " 'Retrieval-Augmented Generation?',\n", + " 'Why might simple RAG not be adequate for complex '\n", + " 'user queries?'],\n", " 'keywords': ['Retrieval-Augmented Generation',\n", " 'RAG',\n", - " 'user query',\n", " 'vector database',\n", - " 'embeddings',\n", - " 'search limitations'],\n", - " 'summary': 'The simple Retrieval-Augmented Generation (RAG) model is an '\n", - " 'approach that uses a vector database to embed and search for user '\n", - " 'queries, such as Wikipedia articles. It provides answers by '\n", - " 'aligning query and document embeddings. However, it has several '\n", - " 'limitations, including query-document mismatch, reliance on a '\n", - " 'monolithic search backend, text search limitations, and a limited '\n", - " 'planning ability.',\n", - " 'topic': 'Simple RAG'}\n", - "{'hypothetical_questions': ['What is a query-document mismatch in the context '\n", - " 'of RAG models?',\n", - " 'Why might a simple RAG model struggle with '\n", - " 'specific user queries?'],\n", - " 'keywords': ['query-document mismatch',\n", - " 'RAG limitation',\n", - " 'embedding alignment'],\n", - " 'summary': \"A limitation where the simple RAG system's query and document \"\n", - " 'embeddings may not align properly, leading to ineffective '\n", - " \"retrieval of information specific to a user's query about, for \"\n", - " \"example, 'climate change effects on marine life'.\",\n", - " 'topic': 'Query-Document Mismatch'}\n", - "{'hypothetical_questions': ['Why is depending on a monolithic search backend a '\n", - " 'limitation for simple RAG?',\n", - " 'How can a monolithic search backend affect the '\n", - " 'quality of search results?'],\n", - " 'keywords': ['monolithic search backend', 'RAG system', 'data sources'],\n", - " 'summary': 'In simple RAG, the reliance on a single search method and backend '\n", - " \"can limit the system's ability to access diverse or specialized \"\n", - " \"data sources, such as when searching for 'latest research in \"\n", - " \"quantum computing'.\",\n", - " 'topic': 'Monolithic Search Backend'}\n", - "{'hypothetical_questions': ['How do text search limitations impact the '\n", - " 'effectiveness of simple RAG?',\n", - " 'Can simple RAG models understand the context of '\n", - " 'search terms?'],\n", - " 'keywords': ['text search limitations', 'RAG', 'advanced search'],\n", - " 'summary': 'The simple RAG model is limited to straightforward text queries '\n", - " 'without advanced search capabilities, failing to resolve nuanced '\n", - " \"queries like 'what problems did we fix last week' due to the \"\n", - " \"presence of generic terms such as 'problem' and 'last week' \"\n", - " 'throughout documents.',\n", - " 'topic': 'Text Search Limitations'}\n", - "{'hypothetical_questions': ['What does limited planning ability imply for a '\n", - " \"RAG model's search results?\",\n", - " 'Can simple RAG models provide context-specific '\n", - " 'information effectively?'],\n", - " 'keywords': ['limited planning ability', 'contextual information', 'RAG'],\n", - " 'summary': 'Simple RAG models struggle to incorporate additional context in '\n", - " 'their searches, which may result in less relevant or overly '\n", - " 'general responses to queries that require specific insights, like '\n", - " \"'Tips for first-time Europe travelers'.\",\n", - " 'topic': 'Limited Planning Ability'}\n" + " 'simple implementation'],\n", + " 'summary': 'The simplest form of RAG involves embedding a user query and '\n", + " 'performing a single search in a vector database, such as a '\n", + " 'Wikipedia article store. This method, however, often fails with '\n", + " 'complex queries and diverse data sources.',\n", + " 'topic': 'Simple Retrieval-Augmented Generation (RAG)'}\n", + "{'hypothetical_questions': ['What are the main drawbacks of the simple RAG '\n", + " 'model?',\n", + " 'How does Query-Document Mismatch affect search '\n", + " 'results in simple RAG?',\n", + " 'Why is relying on a monolithic search backend '\n", + " 'problematic for RAG?'],\n", + " 'keywords': ['limitations',\n", + " 'query-document mismatch',\n", + " 'monolithic search backend',\n", + " 'text search limitations',\n", + " 'limited planning ability'],\n", + " 'summary': 'The limitations of simple RAG include the Query-Document Mismatch '\n", + " 'which assumes a perfect alignment of embeddings, the reliance on '\n", + " 'a Monolithic Search Backend which limits flexibility, Text Search '\n", + " 'Limitations that impede nuanced search, and a Limited Planning '\n", + " 'Ability that overlooks additional context for refining results.',\n", + " 'topic': 'Limitations of Simple RAG'}\n" ] } ], @@ -261,7 +229,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 5, "metadata": {}, "outputs": [], "source": [ @@ -296,16 +264,16 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 6, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "Query(rewritten_query='recent developments in AI', published_daterange=DateRange(start=datetime.date(2023, 1, 1), end=datetime.date(2023, 12, 21)))" + "Query(rewritten_query='recent developments in AI', published_daterange=DateRange(start=datetime.date(2023, 1, 1), end=datetime.date(2023, 12, 22)))" ] }, - "execution_count": 5, + "execution_count": 6, "metadata": {}, "output_type": "execute_result" } @@ -325,7 +293,8 @@ " )\n", "\n", "\n", - "expand_query(\"What are some recent developments in AI?\")" + "query = expand_query(\"What are some recent developments in AI?\")\n", + "query" ] }, { @@ -398,55 +367,71 @@ "\n", "1. Save input and output pairs for later\n", "2. Save the JSON schema for the response_model\n", - "3. Having snapshots of the model and data allow us to compare results over time, and as we make changes to the model we can see how the results change.\n" + "3. Having snapshots of the model and data allow us to compare results over time, and as we make changes to the model we can see how the results change.\n", + "\n", + "This is particularly useful when we might want to blend a mix of synthetic and real data to evaluate our model. We can use the `wandb` library to track our experiments and save the results to a dashboard.\n" ] }, { "cell_type": "code", - "execution_count": 43, - "metadata": {}, - "outputs": [], - "source": [ - "import pandas as pd\n", - "\n", - "\n", - "def flatten_dict(d, parent_key=\"\", sep=\"_\"):\n", - " \"\"\"\n", - " Flatten a nested dictionary.\n", - "\n", - " :param d: The nested dictionary to flatten.\n", - " :param parent_key: The base key to use for the flattened keys.\n", - " :param sep: Separator to use between keys.\n", - " :return: A flattened dictionary.\n", - " \"\"\"\n", - " items = []\n", - " for k, v in d.items():\n", - " new_key = f\"{parent_key}{sep}{k}\" if parent_key else k\n", - " if isinstance(v, dict):\n", - " items.extend(flatten_dict(v, new_key, sep=sep).items())\n", - " else:\n", - " items.append((new_key, v))\n", - " return dict(items)\n", - "\n", - "\n", - "def dicts_to_df(list_of_dicts):\n", - " \"\"\"\n", - " Convert a list of dictionaries to a pandas DataFrame.\n", - "\n", - " :param list_of_dicts: List of dictionaries, potentially nested.\n", - " :return: A pandas DataFrame representing the flattened data.\n", - " \"\"\"\n", - " # Flatten each dictionary and create a DataFrame\n", - " flattened_data = [flatten_dict(d) for d in list_of_dicts]\n", - " return pd.DataFrame(flattened_data)" - ] - }, - { - "cell_type": "code", - "execution_count": 40, + "execution_count": 18, "metadata": { "scrolled": true }, + "outputs": [], + "source": [ + "import json\n", + "import wandb\n", + "from helpers import dicts_to_df\n", + "\n", + "\n", + "class DateRange(BaseModel):\n", + " chain_of_thought: str = Field(\n", + " description=\"Think step by step to plan what is the best time range to search in\"\n", + " )\n", + " start: date\n", + " end: date\n", + "\n", + "\n", + "class Query(BaseModel):\n", + " rewritten_query: str = Field(\n", + " description=\"Rewrite the query to make it more specific\"\n", + " )\n", + " published_daterange: DateRange = Field(\n", + " description=\"Effective date range to search in\"\n", + " )\n", + "\n", + " def report(self):\n", + " dct = self.model_dump()\n", + " dct[\"usage\"] = self._raw_response.usage.model_dump()\n", + " return dct\n", + "\n", + "\n", + "from openai import AsyncOpenAI\n", + "\n", + "# We'll use a different client for async calls\n", + "# To highlight the difference and how we can use both\n", + "aclient = instructor.patch(AsyncOpenAI())\n", + "\n", + "\n", + "async def expand_query(q) -> Query:\n", + " return await aclient.chat.completions.create(\n", + " model=\"gpt-4-1106-preview\",\n", + " response_model=Query,\n", + " messages=[\n", + " {\n", + " \"role\": \"system\",\n", + " \"content\": f\"You're a query understanding system for the Metafor Systems search engine. Today is {date.today()}. Here are some tips: ...\",\n", + " },\n", + " {\"role\": \"user\", \"content\": f\"query: {q}\"},\n", + " ],\n", + " )" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, "outputs": [ { "data": { @@ -476,7 +461,7 @@ { "data": { "text/html": [ - "Run data is saved locally in /Users/jasonliu/dev/instructor/tutorials/wandb/run-20231221_153734-idscpy5k" + "Run data is saved locally in /Users/jasonliu/dev/instructor/tutorials/wandb/run-20231222_152028-opuq58lr" ], "text/plain": [ "" @@ -488,7 +473,7 @@ { "data": { "text/html": [ - "Syncing run easy-feather-16 to Weights & Biases (docs)
" + "Syncing run major-firebrand-21 to Weights & Biases (docs)
" ], "text/plain": [ "" @@ -512,7 +497,7 @@ { "data": { "text/html": [ - " View run at https://wandb.ai/instructor/query-understanding/runs/idscpy5k" + " View run at https://wandb.ai/instructor/query-understanding/runs/opuq58lr" ], "text/plain": [ "" @@ -525,18 +510,39 @@ "name": "stderr", "output_type": "stream", "text": [ + "Retrying, exception: 1 validation error for Query\n", + "rewritten_query\n", + " Field required [type=missing, input_value={'rewitten_query': 'recen...', 'end': '2023-12-22'}}, input_type=dict]\n", + " For further information visit https://errors.pydantic.dev/2.5/v/missing\n", + "Traceback (most recent call last):\n", + " File \"/Users/jasonliu/dev/instructor/instructor/patch.py\", line 231, in retry_async\n", + " return await process_response_async(\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/Users/jasonliu/dev/instructor/instructor/patch.py\", line 201, in process_response_async\n", + " model = await response_model.from_response_async(\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/Users/jasonliu/dev/instructor/instructor/function_calls.py\", line 198, in from_response_async\n", + " return cls.model_validate_json(\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/Users/jasonliu/dev/instructor/.venv/lib/python3.11/site-packages/pydantic/main.py\", line 532, in model_validate_json\n", + " return cls.__pydantic_validator__.validate_json(json_data, strict=strict, context=context)\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + "pydantic_core._pydantic_core.ValidationError: 1 validation error for Query\n", + "rewritten_query\n", + " Field required [type=missing, input_value={'rewitten_query': 'recen...', 'end': '2023-12-22'}}, input_type=dict]\n", + " For further information visit https://errors.pydantic.dev/2.5/v/missing\n", "wandb: WARNING Source type is set to 'repo' but some required information is missing from the environment. A job will not be created from this run. See https://docs.wandb.ai/guides/launch/create-job\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "6440cf236ba24c3b839d1256cfada604", + "model_id": "96b112129c944465a35156a6ffbdfe54", "version_major": 2, "version_minor": 0 }, "text/plain": [ - "VBox(children=(Label(value='0.007 MB of 0.007 MB uploaded (0.001 MB deduped)\\r'), FloatProgress(value=1.0, max…" + "VBox(children=(Label(value='0.008 MB of 0.008 MB uploaded (0.001 MB deduped)\\r'), FloatProgress(value=1.0, max…" ] }, "metadata": {}, @@ -545,7 +551,7 @@ { "data": { "text/html": [ - "W&B sync reduced upload amount by 9.0% " + "W&B sync reduced upload amount by 7.9% " ], "text/plain": [ "" @@ -557,7 +563,7 @@ { "data": { "text/html": [ - " View run easy-feather-16 at: https://wandb.ai/instructor/query-understanding/runs/idscpy5k
Synced 4 W&B file(s), 1 media file(s), 4 artifact file(s) and 0 other file(s)" + " View run major-firebrand-21 at: https://wandb.ai/instructor/query-understanding/runs/opuq58lr
Synced 5 W&B file(s), 1 media file(s), 4 artifact file(s) and 0 other file(s)" ], "text/plain": [ "" @@ -569,7 +575,7 @@ { "data": { "text/html": [ - "Find logs at: ./wandb/run-20231221_153734-idscpy5k/logs" + "Find logs at: ./wandb/run-20231222_152028-opuq58lr/logs" ], "text/plain": [ "" @@ -580,39 +586,7 @@ } ], "source": [ - "import json\n", - "import wandb\n", - "\n", - "\n", - "class DateRange(BaseModel):\n", - " chain_of_thought: str = Field(\n", - " description=\"Think step by step to plan what is the best time range to search in\"\n", - " )\n", - " start: date\n", - " end: date\n", - "\n", - "\n", - "class Query(BaseModel):\n", - " rewritten_query: str = Field(\n", - " description=\"Rewrite the query to make it more specific\"\n", - " )\n", - " published_daterange: DateRange = Field(\n", - " description=\"Effective date range to search in\"\n", - " )\n", - "\n", - "\n", - "def expand_query(q) -> Query:\n", - " return client.chat.completions.create(\n", - " model=\"gpt-4-1106-preview\",\n", - " response_model=Query,\n", - " messages=[\n", - " {\n", - " \"role\": \"system\",\n", - " \"content\": f\"You're a query understanding system for the Metafor Systems search engine. Today is {date.today()}. Here are some tips: ...\",\n", - " },\n", - " {\"role\": \"user\", \"content\": f\"query: {q}\"},\n", - " ],\n", - " )\n", + "import asyncio\n", "\n", "\n", "run = wandb.init(\n", @@ -626,7 +600,7 @@ " \"biotechnology updates last 10 days\",\n", "]\n", "\n", - "queries = [expand_query(q) for q in test_queries]\n", + "queries = await asyncio.gather(*[expand_query(q) for q in test_queries])\n", "\n", "with open(\"schema.json\", \"w+\") as f:\n", " schema = Query.model_json_schema()\n", @@ -636,7 +610,7 @@ " for query in queries:\n", " f.write(query.model_dump_json() + \"\\n\")\n", "\n", - "df = dicts_to_df([q.model_dump() for q in queries])\n", + "df = dicts_to_df([q.report() for q in queries])\n", "df[\"input\"] = test_queries\n", "df.to_csv(\"results.csv\")\n", "\n", @@ -652,13 +626,6 @@ "run.finish()" ] }, - { - "cell_type": "code", - "execution_count": 42, - "metadata": {}, - "outputs": [], - "source": [] - }, { "cell_type": "markdown", "metadata": {}, @@ -674,7 +641,7 @@ }, { "cell_type": "code", - "execution_count": 27, + "execution_count": 26, "metadata": {}, "outputs": [], "source": [ @@ -682,7 +649,7 @@ "\n", "\n", "class SearchClient(BaseModel):\n", - " query: str\n", + " query: str = Field(description=\"The search query that will go into the search bar\")\n", " keywords: List[str]\n", " email: str\n", " source: Literal[\"gmail\", \"calendar\"]\n", @@ -706,43 +673,19 @@ }, { "cell_type": "code", - "execution_count": 28, + "execution_count": 27, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{\n", - " \"queries\": [\n", - " {\n", - " \"query\": \"schedule\",\n", - " \"keywords\": [\n", - " \"appointments\",\n", - " \"meetings\",\n", - " \"schedule\",\n", - " \"events\"\n", - " ],\n", - " \"email\": \"jason.assistant@busybot.com\",\n", - " \"source\": \"calendar\",\n", - " \"date_range\": {\n", - " \"start\": \"2023-11-18\",\n", - " \"end\": \"2023-11-18\"\n", - " }\n", - " }\n", - " ]\n", - "}\n" - ] - } - ], + "outputs": [], "source": [ "retrival = client.chat.completions.create(\n", - " model=\"gpt-4-1106-preview\",\n", + " model=\"gpt-3.5-turbo\",\n", " response_model=Retrival,\n", " messages=[\n", " {\n", " \"role\": \"system\",\n", - " \"content\": f\"You are Jason's personal assistant. Today is {date.today()}\",\n", + " \"content\": f\"\"\"You are Jason's personal assistant.\n", + " He has two emails jason@work.com jason@personal.com \n", + " Today is {date.today()}\"\"\",\n", " },\n", " {\"role\": \"user\", \"content\": \"What do I have today?\"},\n", " ],\n", @@ -810,11 +753,13 @@ " messages=[\n", " {\n", " \"role\": \"system\",\n", - " \"content\": f\"You are Jason's personal assistant. Today is {date.today()}\",\n", + " \"content\": f\"\"\"You are Jason's personal assistant.\n", + " He has two emails jason@work.com jason@personal.com \n", + " Today is {date.today()}\"\"\",\n", " },\n", " {\n", " \"role\": \"user\",\n", - " \"content\": \"What meetings do I have today and are there any important emails I should be aware of?\",\n", + " \"content\": \"What meetings do I have today and are there any important emails I should be aware of\",\n", " },\n", " ],\n", ")\n", diff --git a/tutorials/helpers.py b/tutorials/helpers.py new file mode 100644 index 0000000..3d7d5fe --- /dev/null +++ b/tutorials/helpers.py @@ -0,0 +1,32 @@ +import pandas as pd + + +def flatten_dict(d, parent_key="", sep="_"): + """ + Flatten a nested dictionary. + + :param d: The nested dictionary to flatten. + :param parent_key: The base key to use for the flattened keys. + :param sep: Separator to use between keys. + :return: A flattened dictionary. + """ + items = [] + for k, v in d.items(): + new_key = f"{parent_key}{sep}{k}" if parent_key else k + if isinstance(v, dict): + items.extend(flatten_dict(v, new_key, sep=sep).items()) + else: + items.append((new_key, v)) + return dict(items) + + +def dicts_to_df(list_of_dicts): + """ + Convert a list of dictionaries to a pandas DataFrame. + + :param list_of_dicts: List of dictionaries, potentially nested. + :return: A pandas DataFrame representing the flattened data. + """ + # Flatten each dictionary and create a DataFrame + flattened_data = [flatten_dict(d) for d in list_of_dicts] + return pd.DataFrame(flattened_data) diff --git a/wandb/settings b/wandb/settings new file mode 100644 index 0000000..fe5e2d8 --- /dev/null +++ b/wandb/settings @@ -0,0 +1,5 @@ +[default] +entity = instructor +project = query-understanding +base_url = https://api.wandb.ai + From 8dcc11136629d258e9500cdc4bbcad24312184c0 Mon Sep 17 00:00:00 2001 From: Jason Liu Date: Sat, 23 Dec 2023 12:24:45 -0500 Subject: [PATCH 6/7] bump --- tutorials/3.0.applications-rag.ipynb | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/tutorials/3.0.applications-rag.ipynb b/tutorials/3.0.applications-rag.ipynb index 2307db3..79a47a0 100644 --- a/tutorials/3.0.applications-rag.ipynb +++ b/tutorials/3.0.applications-rag.ipynb @@ -675,7 +675,33 @@ "cell_type": "code", "execution_count": 27, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{\n", + " \"queries\": [\n", + " {\n", + " \"query\": \"Today's events\",\n", + " \"keywords\": [\n", + " \"today\",\n", + " \"events\",\n", + " \"calendar\"\n", + " ],\n", + " \"email\": \"jason@work.com\",\n", + " \"source\": \"calendar\",\n", + " \"date_range\": {\n", + " \"chain_of_thought\": \"Since the user is asking about today's events, we should look for events scheduled for the current day, which is 2023-12-22. We don't need an end date as the user is only interested in today's events.\",\n", + " \"start\": \"2023-12-22\",\n", + " \"end\": \"2023-12-22\"\n", + " }\n", + " }\n", + " ]\n", + "}\n" + ] + } + ], "source": [ "retrival = client.chat.completions.create(\n", " model=\"gpt-3.5-turbo\",\n", From 5082faabf58508a6cb68114e5703ba6036662f94 Mon Sep 17 00:00:00 2001 From: Jason Liu Date: Sun, 24 Dec 2023 21:09:39 -0500 Subject: [PATCH 7/7] clean up --- tutorials/3.0.applications-rag.ipynb | 259 ++++++++--------- tutorials/3.1.validation-rag.ipynb | 403 ++++++++++++++++----------- 2 files changed, 356 insertions(+), 306 deletions(-) diff --git a/tutorials/3.0.applications-rag.ipynb b/tutorials/3.0.applications-rag.ipynb index 79a47a0..e5a0ab8 100644 --- a/tutorials/3.0.applications-rag.ipynb +++ b/tutorials/3.0.applications-rag.ipynb @@ -74,7 +74,7 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": 1, "metadata": {}, "outputs": [], "source": [ @@ -107,7 +107,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ @@ -132,41 +132,31 @@ "name": "stdout", "output_type": "stream", "text": [ - "{'hypothetical_questions': ['What is the most basic implementation of '\n", - " 'Retrieval-Augmented Generation?',\n", - " 'Why might simple RAG not be adequate for complex '\n", - " 'user queries?'],\n", - " 'keywords': ['Retrieval-Augmented Generation',\n", - " 'RAG',\n", - " 'vector database',\n", - " 'simple implementation'],\n", - " 'summary': 'The simplest form of RAG involves embedding a user query and '\n", - " 'performing a single search in a vector database, such as a '\n", - " 'Wikipedia article store. This method, however, often fails with '\n", - " 'complex queries and diverse data sources.',\n", - " 'topic': 'Simple Retrieval-Augmented Generation (RAG)'}\n", - "{'hypothetical_questions': ['What are the main drawbacks of the simple RAG '\n", - " 'model?',\n", - " 'How does Query-Document Mismatch affect search '\n", - " 'results in simple RAG?',\n", - " 'Why is relying on a monolithic search backend '\n", - " 'problematic for RAG?'],\n", - " 'keywords': ['limitations',\n", - " 'query-document mismatch',\n", - " 'monolithic search backend',\n", - " 'text search limitations',\n", - " 'limited planning ability'],\n", - " 'summary': 'The limitations of simple RAG include the Query-Document Mismatch '\n", - " 'which assumes a perfect alignment of embeddings, the reliance on '\n", - " 'a Monolithic Search Backend which limits flexibility, Text Search '\n", - " 'Limitations that impede nuanced search, and a Limited Planning '\n", - " 'Ability that overlooks additional context for refining results.',\n", - " 'topic': 'Limitations of Simple RAG'}\n" + "{\n", + " \"topic\": \"Simple RAG\",\n", + " \"summary\": \"The Simple RAG (Retrieval-Augmented Generation) is a basic implementation of an AI model that uses a single embedding search in a vector database to answer user queries. However, it has limitations in dealing with complex queries and diverse data sources due to factors like query-document mismatch, reliance on a monolithic search backend, restrictions to simple text searches, and lack of contextual understanding.\",\n", + " \"hypothetical_questions\": [\n", + " \"How does Simple RAG address complex user queries?\",\n", + " \"Why does Simple RAG struggle with multi-topic queries?\",\n", + " \"What are the main drawbacks of using a monolithic search backend in Simple RAG?\"\n", + " ],\n", + " \"keywords\": [\n", + " \"Simple RAG\",\n", + " \"retrieval-augmented generation\",\n", + " \"vector database\",\n", + " \"user query\",\n", + " \"embedding search\",\n", + " \"limitations\",\n", + " \"query-document mismatch\",\n", + " \"monolithic search backend\",\n", + " \"text search limitations\",\n", + " \"limited planning ability\"\n", + " ]\n", + "}\n" ] } ], "source": [ - "from pprint import pprint\n", "from typing import Iterable\n", "\n", "\n", @@ -208,7 +198,7 @@ "\n", "\n", "for extraction in extractions:\n", - " pprint(extraction.model_dump())" + " print(extraction.model_dump_json(indent=3))" ] }, { @@ -229,7 +219,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 6, "metadata": {}, "outputs": [], "source": [ @@ -264,18 +254,21 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 7, "metadata": {}, "outputs": [ { - "data": { - "text/plain": [ - "Query(rewritten_query='recent developments in AI', published_daterange=DateRange(start=datetime.date(2023, 1, 1), end=datetime.date(2023, 12, 22)))" - ] - }, - "execution_count": 6, - "metadata": {}, - "output_type": "execute_result" + "name": "stdout", + "output_type": "stream", + "text": [ + "{\n", + " \"rewritten_query\": \"recent developments in AI\",\n", + " \"published_daterange\": {\n", + " \"start\": \"2022-01-01\",\n", + " \"end\": \"2023-12-24\"\n", + " }\n", + "}\n" + ] } ], "source": [ @@ -294,7 +287,7 @@ "\n", "\n", "query = expand_query(\"What are some recent developments in AI?\")\n", - "query" + "print(query.model_dump_json(indent=2))" ] }, { @@ -306,24 +299,28 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 9, "metadata": {}, "outputs": [ { - "data": { - "text/plain": [ - "Query(rewritten_query='Latest advancements in Artificial Intelligence as of December 2023', published_daterange=DateRange(chain_of_thought=\"To get the most recent developments in AI, the date range should be quite recent. Considering today's date is 2023-12-21, a suitable range might be from the past three months to now.\", start=datetime.date(2023, 9, 21), end=datetime.date(2023, 12, 21)))" - ] - }, - "execution_count": 16, - "metadata": {}, - "output_type": "execute_result" + "name": "stdout", + "output_type": "stream", + "text": [ + "{\n", + " \"rewritten_query\": \"Recent developments in artificial intelligence after 2023\",\n", + " \"published_daterange\": {\n", + " \"chain_of_thought\": \"Given today's date, 2023-12-24, the recent developments would be those after 2023.\",\n", + " \"start\": \"2023-01-01\",\n", + " \"end\": \"2023-12-24\"\n", + " }\n", + "}\n" + ] } ], "source": [ "class DateRange(BaseModel):\n", " chain_of_thought: str = Field(\n", - " description=\"Think step by step to plan what is the best time range to search in\"\n", + " description=\"Think step by step to plan what is the best time range to search in, less than 40 words.\"\n", " )\n", " start: date\n", " end: date\n", @@ -352,7 +349,8 @@ " )\n", "\n", "\n", - "expand_query(\"What are some recent developments in AI?\")" + "query = expand_query(\"What are some recent developments in AI?\")\n", + "print(query.model_dump_json(indent=2))" ] }, { @@ -374,7 +372,7 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 10, "metadata": { "scrolled": true }, @@ -382,17 +380,10 @@ "source": [ "import json\n", "import wandb\n", + "from openai import AsyncOpenAI\n", "from helpers import dicts_to_df\n", "\n", "\n", - "class DateRange(BaseModel):\n", - " chain_of_thought: str = Field(\n", - " description=\"Think step by step to plan what is the best time range to search in\"\n", - " )\n", - " start: date\n", - " end: date\n", - "\n", - "\n", "class Query(BaseModel):\n", " rewritten_query: str = Field(\n", " description=\"Rewrite the query to make it more specific\"\n", @@ -407,8 +398,6 @@ " return dct\n", "\n", "\n", - "from openai import AsyncOpenAI\n", - "\n", "# We'll use a different client for async calls\n", "# To highlight the difference and how we can use both\n", "aclient = instructor.patch(AsyncOpenAI())\n", @@ -641,7 +630,7 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": 12, "metadata": {}, "outputs": [], "source": [ @@ -653,11 +642,7 @@ " keywords: List[str]\n", " email: str\n", " source: Literal[\"gmail\", \"calendar\"]\n", - " date_range: DateRange\n", - "\n", - "\n", - "class Retrival(BaseModel):\n", - " queries: List[SearchClient]" + " date_range: DateRange" ] }, { @@ -673,7 +658,7 @@ }, { "cell_type": "code", - "execution_count": 27, + "execution_count": 15, "metadata": {}, "outputs": [ { @@ -681,31 +666,29 @@ "output_type": "stream", "text": [ "{\n", - " \"queries\": [\n", - " {\n", - " \"query\": \"Today's events\",\n", - " \"keywords\": [\n", - " \"today\",\n", - " \"events\",\n", - " \"calendar\"\n", - " ],\n", - " \"email\": \"jason@work.com\",\n", - " \"source\": \"calendar\",\n", - " \"date_range\": {\n", - " \"chain_of_thought\": \"Since the user is asking about today's events, we should look for events scheduled for the current day, which is 2023-12-22. We don't need an end date as the user is only interested in today's events.\",\n", - " \"start\": \"2023-12-22\",\n", - " \"end\": \"2023-12-22\"\n", - " }\n", - " }\n", - " ]\n", + " \"query\": \"What do I have today?\",\n", + " \"keywords\": [\n", + " \"today\",\n", + " \"schedule\",\n", + " \"appointments\",\n", + " \"meetings\"\n", + " ],\n", + " \"email\": \"jason@work.com\",\n", + " \"source\": \"calendar\",\n", + " \"date_range\": {\n", + " \"chain_of_thought\": \"Today is 2023-12-24, so only search for entries on this date.\",\n", + " \"start\": \"2023-12-24\",\n", + " \"end\": \"2023-12-24\"\n", + " }\n", "}\n" ] } ], "source": [ "retrival = client.chat.completions.create(\n", - " model=\"gpt-3.5-turbo\",\n", - " response_model=Retrival,\n", + " model=\"gpt-4-1106-preview\",\n", + " stream=True,\n", + " response_model=Iterable[SearchClient],\n", " messages=[\n", " {\n", " \"role\": \"system\",\n", @@ -716,7 +699,9 @@ " {\"role\": \"user\", \"content\": \"What do I have today?\"},\n", " ],\n", ")\n", - "print(retrival.model_dump_json(indent=4))" + "\n", + "for r in retrival:\n", + " print(r.model_dump_json(indent=2))" ] }, { @@ -728,7 +713,7 @@ }, { "cell_type": "code", - "execution_count": 29, + "execution_count": 16, "metadata": {}, "outputs": [ { @@ -736,38 +721,33 @@ "output_type": "stream", "text": [ "{\n", - " \"queries\": [\n", - " {\n", - " \"query\": \"meetings\",\n", - " \"keywords\": [\n", - " \"meetings\",\n", - " \"appointments\",\n", - " \"schedule\",\n", - " \"calendar\"\n", - " ],\n", - " \"email\": \"user@email.com\",\n", - " \"source\": \"calendar\",\n", - " \"date_range\": {\n", - " \"start\": \"2023-11-18\",\n", - " \"end\": \"2023-11-18\"\n", - " }\n", - " },\n", - " {\n", - " \"query\": \"important emails\",\n", - " \"keywords\": [\n", - " \"important\",\n", - " \"priority\",\n", - " \"urgent\",\n", - " \"follow-up\"\n", - " ],\n", - " \"email\": \"user@email.com\",\n", - " \"source\": \"gmail\",\n", - " \"date_range\": {\n", - " \"start\": \"2023-11-18\",\n", - " \"end\": \"2023-11-18\"\n", - " }\n", - " }\n", - " ]\n", + " \"query\": \"meeting\",\n", + " \"keywords\": [\n", + " \"meeting\"\n", + " ],\n", + " \"email\": \"jason@work.com\",\n", + " \"source\": \"calendar\",\n", + " \"date_range\": {\n", + " \"chain_of_thought\": \"Searching for today's meetings means setting the date range to today's date, which is the current date.\",\n", + " \"start\": \"2023-12-24\",\n", + " \"end\": \"2023-12-24\"\n", + " }\n", + "}\n", + "{\n", + " \"query\": \"important\",\n", + " \"keywords\": [\n", + " \"important\",\n", + " \"urgent\",\n", + " \"asap\",\n", + " \"attention\"\n", + " ],\n", + " \"email\": \"jason@work.com\",\n", + " \"source\": \"gmail\",\n", + " \"date_range\": {\n", + " \"chain_of_thought\": \"For important emails, no specific date range, but it makes sense to check the past week leading up to today.\",\n", + " \"start\": \"2023-12-17\",\n", + " \"end\": \"2023-12-24\"\n", + " }\n", "}\n" ] } @@ -775,7 +755,8 @@ "source": [ "retrival = client.chat.completions.create(\n", " model=\"gpt-4-1106-preview\",\n", - " response_model=Retrival,\n", + " stream=True,\n", + " response_model=Iterable[SearchClient],\n", " messages=[\n", " {\n", " \"role\": \"system\",\n", @@ -789,7 +770,9 @@ " },\n", " ],\n", ")\n", - "print(retrival.model_dump_json(indent=4))" + "\n", + "for r in retrival:\n", + " print(r.model_dump_json(indent=2))" ] }, { @@ -814,7 +797,7 @@ }, { "cell_type": "code", - "execution_count": 31, + "execution_count": 20, "metadata": {}, "outputs": [ { @@ -831,19 +814,19 @@ " },\n", " {\n", " \"id\": 2,\n", - " \"query\": \"What is the population of Canada?\",\n", - " \"subquestions\": []\n", - " },\n", - " {\n", - " \"id\": 3,\n", - " \"query\": \"What is the population of {Jason's home country}?\",\n", + " \"query\": \"What is the population of Jason's home country?\",\n", " \"subquestions\": [\n", " 1\n", " ]\n", " },\n", " {\n", + " \"id\": 3,\n", + " \"query\": \"What is the population of Canada?\",\n", + " \"subquestions\": []\n", + " },\n", + " {\n", " \"id\": 4,\n", - " \"query\": \"What is the difference between the population of {Jason's home country} and the population of Canada?\",\n", + " \"query\": \"What is the difference between these two populations?\",\n", " \"subquestions\": [\n", " 2,\n", " 3\n", @@ -872,7 +855,7 @@ "\n", "\n", "retrival = client.chat.completions.create(\n", - " model=\"gpt-4-1106-preview\",\n", + " model=\"gpt-4\",\n", " response_model=QueryPlan,\n", " messages=[\n", " {\n", diff --git a/tutorials/3.1.validation-rag.ipynb b/tutorials/3.1.validation-rag.ipynb index 38ab7db..655d54c 100644 --- a/tutorials/3.1.validation-rag.ipynb +++ b/tutorials/3.1.validation-rag.ipynb @@ -5,7 +5,7 @@ "id": "5a01f3ac-5306-4a1b-9e47-a5d254bce93a", "metadata": {}, "source": [ - "# Validators\n" + "# Understanding Validators\n" ] }, { @@ -15,8 +15,6 @@ "source": [ "Pydantic offers an customizable and expressive validation framework for Python. Instructor leverages Pydantic's validation framework to provide a uniform developer experience for both code-based and LLM-based validation, as well as a reasking mechanism for correcting LLM outputs based on validation errors. To learn more check out the Pydantic [docs](https://docs.pydantic.dev/latest/) on validators.\n", "\n", - "Note: For the majority of this notebook we won't be calling openai, just using validators to see how we can control the validation of the objects.\n", - "\n", "Then we'll bring it all together into the context of RAG from the previous notebook.\n" ] }, @@ -47,14 +45,24 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "id": "d4bb6258-b03a-4621-8a73-29056a20ec0f", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1 validation error for UserDetail\n", + "name\n", + " Value error, Name must contain a space. [type=value_error, input_value='Jason', input_type=str]\n", + " For further information visit https://errors.pydantic.dev/2.5/v/value_error\n" + ] + } + ], "source": [ - "from pydantic import BaseModel\n", "from typing_extensions import Annotated\n", - "from pydantic import AfterValidator\n", + "from pydantic import BaseModel, AfterValidator\n", "\n", "\n", "def name_must_contain_space(v: str) -> str:\n", @@ -68,7 +76,10 @@ " name: Annotated[str, AfterValidator(name_must_contain_space)]\n", "\n", "\n", - "person = UserDetail(age=29, name=\"Jason\")" + "try:\n", + " person = UserDetail.model_validate({\"age\": 24, \"name\": \"Jason\"})\n", + "except Exception as e:\n", + " print(e)" ] }, { @@ -83,10 +94,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "id": "3242856f", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1 validation error for UserDetail\n", + "age\n", + " Input should be greater than 0 [type=greater_than, input_value=-10, input_type=int]\n", + " For further information visit https://errors.pydantic.dev/2.5/v/greater_than\n" + ] + } + ], "source": [ "from pydantic import Field\n", "\n", @@ -96,21 +118,38 @@ " name: str\n", "\n", "\n", - "person = UserDetail(age=-10, name=\"Jason\")" + "try:\n", + " person = UserDetail(age=-10, name=\"Jason\")\n", + "except Exception as e:\n", + " print(e)" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "id": "0035a329", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1 validation error for AssistantMessage\n", + "message\n", + " String should have at least 10 characters [type=string_too_short, input_value='Hey', input_type=str]\n", + " For further information visit https://errors.pydantic.dev/2.5/v/string_too_short\n" + ] + } + ], "source": [ "class AssistantMessage(BaseModel):\n", " message: str = Field(..., min_length=10)\n", "\n", "\n", - "message = AssistantMessage(message=\"Hey\")" + "try:\n", + " message = AssistantMessage(message=\"Hey\")\n", + "except Exception as e:\n", + " print(e)" ] }, { @@ -123,58 +162,52 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 7, "id": "ec043c23", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1 validation error for Response\n", + "message\n", + " Assertion failed, `hurt` was found in the message `I will hurt them.` [type=assertion_error, input_value='I will hurt them.', input_type=str]\n", + " For further information visit https://errors.pydantic.dev/2.5/v/assertion_error\n" + ] + } + ], "source": [ - "from pydantic import ValidationInfo, field_validator\n", + "from pydantic import ValidationInfo\n", "\n", "\n", + "def message_cannot_have_blacklisted_words(v: str, info: ValidationInfo) -> str:\n", + " blacklist = info.context.get(\"blacklist\", [])\n", + " for word in blacklist:\n", + " assert word not in v.lower(), f\"`{word}` was found in the message `{v}`\"\n", + " return v\n", + "\n", + "ModeratedStr = Annotated[str, AfterValidator(message_cannot_have_blacklisted_words)]\n", + "\n", "class Response(BaseModel):\n", - " message: str\n", - "\n", - " @field_validator(\"message\")\n", - " def message_cannot_have_blacklisted_words(cls, v: str, info: ValidationInfo) -> str:\n", - " blacklist = info.context.get(\"blacklist\", [])\n", - " for word in blacklist:\n", - " assert word not in v.lower(), f\"`{word}` was found in the message `{v}`\"\n", - " return v\n", + " message: ModeratedStr\n", "\n", "\n", - "Response.model_validate(\n", - " {\"message\": \"I will hurt them.\"},\n", - " context={\n", - " \"blacklist\": {\n", - " \"rob\",\n", - " \"steal\",\n", - " \"hurt\",\n", - " \"kill\",\n", - " \"attack\",\n", - " }\n", - " },\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "887dba80", - "metadata": {}, - "outputs": [], - "source": [ - "Response.model_validate(\n", - " {\"message\": \"My name is rob.\"},\n", - " context={\n", - " \"blacklist\": {\n", - " \"rob\",\n", - " \"steal\",\n", - " \"hurt\",\n", - " \"kill\",\n", - " \"attack\",\n", - " }\n", - " },\n", - ")" + "try:\n", + " Response.model_validate(\n", + " {\"message\": \"I will hurt them.\"},\n", + " context={\n", + " \"blacklist\": {\n", + " \"rob\",\n", + " \"steal\",\n", + " \"hurt\",\n", + " \"kill\",\n", + " \"attack\",\n", + " }\n", + " },\n", + " )\n", + "except Exception as e:\n", + " print(e)" ] }, { @@ -203,10 +236,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 13, "id": "82521112-5301-4442-acce-82b495bd838f", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1 validation error for Response\n", + "message\n", + " Value error, `I want to make them suffer the consequences` was flagged for harassment, harassment_threatening, violence, harassment/threatening [type=value_error, input_value='I want to make them suffer the consequences', input_type=str]\n", + " For further information visit https://errors.pydantic.dev/2.5/v/value_error\n" + ] + } + ], "source": [ "from typing import Annotated\n", "from pydantic import AfterValidator\n", @@ -226,7 +270,10 @@ " message: ModeratedStr\n", "\n", "\n", - "Response(message=\"I want to make them suffer the consequences\")" + "try:\n", + " Response(message=\"I want to make them suffer the consequences\")\n", + "except Exception as e:\n", + " print(e)" ] }, { @@ -284,27 +331,49 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 27, "id": "638fc368-5cf7-4ae7-9d3f-efea1b84eec0", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1 validation error for AnswerWithCitation\n", + "citation\n", + " Value error, Citation `Blueberries contain high levels of protein` not found in text, only use citations from the text. [type=value_error, input_value='Blueberries contain high levels of protein', input_type=str]\n", + " For further information visit https://errors.pydantic.dev/2.5/v/value_error\n" + ] + } + ], "source": [ "from pydantic import ValidationInfo\n", "\n", + "def citation_exists(v: str, info: ValidationInfo):\n", + " context = info.context\n", + " if context:\n", + " context = context.get(\"text_chunk\")\n", + " if v not in context:\n", + " raise ValueError(f\"Citation `{v}` not found in text, only use citations from the text.\")\n", + " return v\n", + "\n", + "Citation = Annotated[str, AfterValidator(citation_exists)]\n", + "\n", "\n", "class AnswerWithCitation(BaseModel):\n", " answer: str\n", - " citation: str\n", + " citation: Citation\n", "\n", - " @field_validator(\"citation\")\n", - " @classmethod\n", - " def citation_exists(cls, v: str, info: ValidationInfo):\n", - " context = info.context\n", - " if context:\n", - " context = context.get(\"text_chunk\")\n", - " if v not in context:\n", - " raise ValueError(f\"Citation `{v}` not found in text\")\n", - " return v" + "try:\n", + " AnswerWithCitation.model_validate(\n", + " {\n", + " \"answer\": \"Blueberries are packed with protein\",\n", + " \"citation\": \"Blueberries contain high levels of protein\",\n", + " },\n", + " context={\"text_chunk\": \"Blueberries are very rich in antioxidants\"},\n", + " )\n", + "except Exception as e:\n", + " print(e)" ] }, { @@ -312,23 +381,21 @@ "id": "3064b06b-7f85-40ec-8fe2-4fa2cce36585", "metadata": {}, "source": [ - "Here we assume that there is a \"text_chunk\" field that contains the text that the model is supposed to use as context. We then use the `field_validator` decorator to define a validator that checks if the citation is included in the text chunk. If it's not, we raise a `ValueError` with a message that will be returned to the user.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "0f3030b6-e6cf-45bf-a366-12de996fea40", - "metadata": {}, - "outputs": [], - "source": [ - "AnswerWithCitation.model_validate(\n", - " {\n", - " \"answer\": \"Blueberries are packed with protein\",\n", - " \"citation\": \"Blueberries contain high levels of protein\",\n", - " },\n", - " context={\"text_chunk\": \"Blueberries are very rich in antioxidants\"},\n", - ")" + "Here we assume that there is a \"text_chunk\" field that contains the text that the model is supposed to use as context. We then use the `field_validator` decorator to define a validator that checks if the citation is included in the text chunk. If it's not, we raise a `ValueError` with a message that will be returned to the user.\n", + "\n", + "\n", + "If we want to pass in the context through the `chat.completions.create`` endpoint, we can use the `validation_context` parameter\n", + "\n", + "```python\n", + "resp = client.chat.completions.create(\n", + " model=\"gpt-3.5-turbo\",\n", + " response_model=AnswerWithCitation,\n", + " messages=[\n", + " {\"role\": \"user\", \"content\": f\"Answer the question `{q}` using the text chunk\\n`{text_chunk}`\"},\n", + " ],\n", + " validation_context={\"text_chunk\": text_chunk},\n", + ")\n", + "```" ] }, { @@ -339,70 +406,6 @@ "In practice there are many ways to implement this: we could use a regex to check if the citation is included in the text chunk, or we could use a more sophisticated approach like a semantic similarity check. The important thing is that we have a way to validate that the model is using the provided context accurately.\n" ] }, - { - "cell_type": "code", - "execution_count": null, - "id": "04d2b691", - "metadata": {}, - "outputs": [], - "source": [ - "class AnswerWithCitation(BaseModel):\n", - " answer: str\n", - " citations: list[str]\n", - "\n", - " @field_validator(\"citations\")\n", - " @classmethod\n", - " def citation_exists(cls, v: str, info: ValidationInfo):\n", - " text_chunk = info.context.get(\"text_chunk\")\n", - " for citation in v:\n", - " if citation not in text_chunk:\n", - " raise ValueError(f\"Citation `{citation}` not found in text\")\n", - " return v" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "f8aae4f1", - "metadata": {}, - "outputs": [], - "source": [ - "class Citation(BaseModel):\n", - " span_start: str = Field(\n", - " ...,\n", - " description=\"The start of the citation, use a 3-4 word phrase that is unique to the citation\",\n", - " )\n", - " span_end: str = Field(\n", - " ...,\n", - " description=\"The end of the citation, use a 3-4 word phrase that is unique to the citation\",\n", - " )\n", - "\n", - " def check(self, text: str) -> bool:\n", - " index_start = text.find(self.span_start)\n", - " index_end = text.find(self.span_end)\n", - " if index_start == -1 or index_end == -1:\n", - " return False\n", - "\n", - " if index_start > index_end:\n", - " return False\n", - "\n", - " return True\n", - "\n", - "\n", - "class AnswerWithCitation(BaseModel):\n", - " answer: str\n", - " citations: list[Citation]\n", - "\n", - " @field_validator(\"citations\")\n", - " @classmethod\n", - " def citation_exists(cls, v: str, info: ValidationInfo):\n", - " text_chunk = info.context.get(\"text_chunk\")\n", - " for citation in v:\n", - " if not citation.check(text_chunk):\n", - " raise ValueError(f\"Citation `{citation}` not found in text\")\n", - " return v" - ] - }, { "cell_type": "markdown", "id": "5bbbaa11-32d2-4772-bc31-18d1d6d6c919", @@ -417,10 +420,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 15, "id": "97f544e7-2552-465c-89a9-a4820f00d658", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{\n", + " \"question\": \"What is the meaning of life?\",\n", + " \"answer\": \"According to the devil, the meaning of life is a life of sin and debauchery.\"\n", + "}\n" + ] + } + ], "source": [ "class QuestionAnswer(BaseModel):\n", " question: str\n", @@ -448,24 +462,58 @@ " ],\n", ")\n", "\n", - "resp.answer" + "print(resp.model_dump_json(indent=2))" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 20, "id": "0328bbc5", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Retrying, exception: 1 validation error for QuestionAnswer\n", + "answer\n", + " Assertion failed, The statement promotes sin and debauchery, which can be considered objectionable. [type=assertion_error, input_value='The meaning of life, acc... of sin and debauchery.', input_type=str]\n", + " For further information visit https://errors.pydantic.dev/2.5/v/assertion_error\n", + "Traceback (most recent call last):\n", + " File \"/Users/jasonliu/dev/instructor/instructor/patch.py\", line 277, in retry_sync\n", + " return process_response(\n", + " ^^^^^^^^^^^^^^^^^\n", + " File \"/Users/jasonliu/dev/instructor/instructor/patch.py\", line 164, in process_response\n", + " model = response_model.from_response(\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/Users/jasonliu/dev/instructor/instructor/function_calls.py\", line 137, in from_response\n", + " return cls.model_validate_json(\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/Users/jasonliu/dev/instructor/.venv/lib/python3.11/site-packages/pydantic/main.py\", line 532, in model_validate_json\n", + " return cls.__pydantic_validator__.validate_json(json_data, strict=strict, context=context)\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + "pydantic_core._pydantic_core.ValidationError: 1 validation error for QuestionAnswer\n", + "answer\n", + " Assertion failed, The statement promotes sin and debauchery, which can be considered objectionable. [type=assertion_error, input_value='The meaning of life, acc... of sin and debauchery.', input_type=str]\n", + " For further information visit https://errors.pydantic.dev/2.5/v/assertion_error\n" + ] + } + ], "source": [ + "from instructor import llm_validator\n", + "\n", + "\n", + "NotEvilAnswer = Annotated[\n", + " str,\n", + " AfterValidator(\n", + " llm_validator(\"don't say objectionable things\", openai_client=client)\n", + " ),\n", + "]\n", + "\n", + "\n", "class QuestionAnswer(BaseModel):\n", " question: str\n", - " answer: Annotated[\n", - " str,\n", - " AfterValidator(\n", - " llm_validator(\"don't say objectionable things\", openai_client=client)\n", - " ),\n", - " ]\n", + " answer: NotEvilAnswer\n", "\n", "\n", "resp = client.chat.completions.create(\n", @@ -482,9 +530,28 @@ " \"content\": f\"using the context: `{context}`\\n\\nAnswer the following question: `{question}`\",\n", " },\n", " ],\n", - ")\n", - "\n", - "resp.answer" + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "814d3554", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{\n", + " \"question\": \"What is the meaning of life?\",\n", + " \"answer\": \"The meaning of life is subjective and can vary depending on one's beliefs and perspectives. According to the devil, it is a life of sin and debauchery. However, this viewpoint may not be universally accepted and should be evaluated critically.\"\n", + "}\n" + ] + } + ], + "source": [ + "print(resp.model_dump_json(indent=2))" ] } ],