diff --git a/.gitignore b/.gitignore index 987816e..ea3a62a 100644 --- a/.gitignore +++ b/.gitignore @@ -161,4 +161,4 @@ cython_debug/ #.idea/ examples/citation_with_extraction/fly.toml my_cache_directory/ -tutorials/wandb/latest-run +tutorials/wandb/* diff --git a/tutorials/2.tips.ipynb b/tutorials/2.tips.ipynb index be5d94f..69bc771 100644 --- a/tutorials/2.tips.ipynb +++ b/tutorials/2.tips.ipynb @@ -32,21 +32,10 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": null, "id": "fdf5e1d9-31ad-4e8a-a55e-e2e70fff598d", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'age': 17, 'name': 'Harry Potter', 'house': }" - ] - }, - "execution_count": 10, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "import instructor\n", "from openai import OpenAI\n", @@ -88,39 +77,20 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": null, "id": "c609eb44", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Hello, I'm Harry Potter, I'm 17 years old and I'm from Gryffindor\n" - ] - } - ], + "outputs": [], "source": [ "resp.say_hello()" ] }, { "cell_type": "code", - "execution_count": 12, + "execution_count": null, "id": "03db160c-81e9-4373-bfec-7a107224b6dd", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'age': 17, 'name': 'Harry Potter', 'house': 'Gryffindor'}" - ] - }, - "execution_count": 12, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "class Character(BaseModel):\n", " age: int\n", @@ -148,7 +118,7 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": null, "id": "0e7938b8-4666-4df4-bd80-f53e8baf7550", "metadata": {}, "outputs": [], @@ -193,33 +163,7 @@ "execution_count": null, "id": "69a58d01-ab6f-41b6-bc0c-b0e55fdb6fe4", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'age': 38,\n", - " 'name': 'Severus Snape',\n", - " 'house': 'Slytherin',\n", - " 'properties': [{'index': '1', 'key': 'patronus', 'value': 'Doe'},\n", - " {'index': '2',\n", - " 'key': 'position',\n", - " 'value': 'Potions Master, Defense Against the Dark Arts teacher, Headmaster'},\n", - " {'index': '3',\n", - " 'key': 'loyalty',\n", - " 'value': 'Hogwarts, Albus Dumbledore, Order of the Phoenix, Lily Evans'},\n", - " {'index': '4',\n", - " 'key': 'skills',\n", - " 'value': 'Potions expertise, Occlumency, Legilimency'},\n", - " {'index': '5',\n", - " 'key': 'disguised_loyalty',\n", - " 'value': 'Death Eater (formerly, as a double agent)'}]}" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "class Property(BaseModel):\n", " index: str = Field(..., description=\"Monotonically increasing ID\")\n", @@ -260,16 +204,7 @@ "execution_count": null, "id": "1f2a2b14-a956-4f96-90c9-e11ca04ab7d1", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "age=38 name='Severus Snape' house='Slytherin'\n", - "age=115 name='Albus Dumbledore' house='Gryffindor'\n" - ] - } - ], + "outputs": [], "source": [ "from typing import Iterable\n", "\n", @@ -282,7 +217,7 @@ "\n", "resp = client.chat.completions.create(\n", " model=\"gpt-4-1106-preview\",\n", - " messages=[{\"role\": \"user\", \"content\": \"Snape and Dumbledore from Harry Potter\"}],\n", + " messages=[{\"role\": \"user\", \"content\": \"Five characters from Harry Potter\"}],\n", " response_model=Iterable[Character],\n", ")\n", "\n", @@ -295,16 +230,7 @@ "execution_count": null, "id": "a3091aba", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "age=38 name='Severus Snape' house='Slytherin'\n", - "age=115 name='Albus Dumbledore' house='Gryffindor'\n" - ] - } - ], + "outputs": [], "source": [ "from typing import Iterable\n", "\n", @@ -317,7 +243,7 @@ "\n", "resp = client.chat.completions.create(\n", " model=\"gpt-4-1106-preview\",\n", - " messages=[{\"role\": \"user\", \"content\": \"Snape and Dumbledore from Harry Potter\"}],\n", + " messages=[{\"role\": \"user\", \"content\": \"Five characters from Harry Potter\"}],\n", " stream=True,\n", " response_model=Iterable[Character],\n", ")\n", @@ -341,29 +267,17 @@ "execution_count": null, "id": "6de8768e-b36a-4a51-9cf9-940d178552f6", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "id=1 name='Harry Potter' friends=[2, 3, 4, 5]\n", - "id=2 name='Hermione Granger' friends=[1, 3, 4, 5]\n", - "id=3 name='Ron Weasley' friends=[1, 2, 4, 5]\n", - "id=4 name='Draco Malfoy' friends=[5]\n", - "id=5 name='Neville Longbottom' friends=[1, 2, 3, 4]\n" - ] - } - ], + "outputs": [], "source": [ "class Character(BaseModel):\n", " id: int\n", " name: str\n", - " friends: List[int]\n", + " friends_array: List[int] = Field(description=\"Relationships to their friends using the id\")\n", "\n", "\n", "resp = client.chat.completions.create(\n", " model=\"gpt-4-1106-preview\",\n", - " messages=[{\"role\": \"user\", \"content\": \"The 5 kids from Harry Potter\"}],\n", + " messages=[{\"role\": \"user\", \"content\": \"5 kids from Harry Potter\"}],\n", " stream=True,\n", " response_model=Iterable[Character],\n", ")\n", @@ -372,142 +286,6 @@ " print(character)" ] }, - { - "cell_type": "code", - "execution_count": null, - "id": "b31e10d7-ebd2-49b4-b2c4-20dd67ca135d", - "metadata": {}, - "outputs": [ - { - "data": { - "image/svg+xml": [ - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "1\n", - "\n", - "Harry Potter\n", - "\n", - "\n", - "\n", - "2\n", - "\n", - "Hermione Granger\n", - "\n", - "\n", - "\n", - "1->2\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "3\n", - "\n", - "Ron Weasley\n", - "\n", - "\n", - "\n", - "1->3\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "4\n", - "\n", - "Draco Malfoy\n", - "\n", - "\n", - "\n", - "1->4\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "5\n", - "\n", - "Neville Longbottom\n", - "\n", - "\n", - "\n", - "1->5\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "2->3\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "2->4\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "3->4\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "3->5\n", - "\n", - "\n", - "\n", - "\n", - "\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "from graphviz import Digraph\n", - "from IPython.display import display\n", - "\n", - "dot = Digraph()\n", - "\n", - "resp = client.chat.completions.create(\n", - " model=\"gpt-4-1106-preview\",\n", - " messages=[{\"role\": \"user\", \"content\": \"The 5 kids from Harry Potter\"}],\n", - " response_model=Iterable[Character],\n", - ")\n", - "\n", - "# Create nodes for each user\n", - "for user in resp:\n", - " dot.node(str(user.id), user.name)\n", - "\n", - "# Create edges for friends\n", - "for user in resp:\n", - " for friend_id in user.friends:\n", - " # To avoid duplicating edges, only add an edge if the friend ID is greater than the user ID\n", - " if friend_id > user.id:\n", - " dot.edge(str(user.id), str(friend_id))\n", - "\n", - "\n", - "# Render the graph to a file\n", - "display(dot)" - ] - }, { "cell_type": "markdown", "id": "523b5797-71a5-4a96-a4b7-21280fb73015", @@ -515,6 +293,108 @@ "source": [ "With the tools we've discussed, we can find numerous real-world applications in production settings. These include extracting action items from transcripts, generating fake data, filling out forms, and creating objects that correspond to generative UI. These simple tricks will be highly useful.\n" ] + }, + { + "cell_type": "markdown", + "id": "a9d20fd9-0cd0-4300-a8c1-d16388969e8e", + "metadata": {}, + "source": [ + "# Missing Data\n", + "\n", + "The Maybe pattern is a concept in functional programming used for error handling. Instead of raising exceptions or returning None, you can use a Maybe type to encapsulate both the result and potential errors.\n", + "\n", + "This pattern is particularly useful when making LLM calls, as providing language models with an escape hatch can effectively reduce hallucinations." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c04f44aa-dc4b-4499-a151-e812512e77e6", + "metadata": {}, + "outputs": [], + "source": [ + "from typing import Optional\n", + "\n", + "class Character(BaseModel):\n", + " age: int\n", + " name: str\n", + "\n", + "class MaybeCharacter(BaseModel):\n", + " result: Optional[Character] = Field(default=None)\n", + " error: bool = Field(default=False)\n", + " message: Optional[str]" + ] + }, + { + "cell_type": "code", + "execution_count": 63, + "id": "a2155190-e104-4ed6-a17f-e0732499dd51", + "metadata": {}, + "outputs": [], + "source": [ + "def extract(content: str) -> MaybeCharacter:\n", + " return client.chat.completions.create(\n", + " model=\"gpt-3.5-turbo\",\n", + " response_model=MaybeCharacter,\n", + " messages=[\n", + " {\"role\": \"user\", \"content\": f\"Extract `{content}`\"},\n", + " ],\n", + " )" + ] + }, + { + "cell_type": "code", + "execution_count": 64, + "id": "a7b59afa-9bf0-4dc0-a5ca-de584514f33b", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "MaybeCharacter(result=Character(age=17, name='Harry Potter'), error=False, message=None)" + ] + }, + "execution_count": 64, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "extract(\"Harry Potter\")" + ] + }, + { + "cell_type": "code", + "execution_count": 66, + "id": "b5ddd5c1-ca75-49a9-95ad-181170435291", + "metadata": {}, + "outputs": [ + { + "ename": "ValueError", + "evalue": "404 Error", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mValueError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[66], line 4\u001b[0m\n\u001b[1;32m 1\u001b[0m user \u001b[38;5;241m=\u001b[39m extract(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m404 Error\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m 3\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m user\u001b[38;5;241m.\u001b[39merror:\n\u001b[0;32m----> 4\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(user\u001b[38;5;241m.\u001b[39mmessage)\n", + "\u001b[0;31mValueError\u001b[0m: 404 Error" + ] + } + ], + "source": [ + "user = extract(\"404 Error\")\n", + "\n", + "if user.error:\n", + " raise ValueError(user.message)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1e14f7cb-d99c-4696-a1fa-e08319bf5d68", + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { diff --git a/tutorials/3.0.applications-rag.ipynb b/tutorials/3.0.applications-rag.ipynb index 0aadb5b..62ce23b 100644 --- a/tutorials/3.0.applications-rag.ipynb +++ b/tutorials/3.0.applications-rag.ipynb @@ -127,33 +127,68 @@ "name": "stdout", "output_type": "stream", "text": [ - "{'hypothetical_questions': [],\n", - " 'keywords': ['RAG',\n", - " 'Retrieval-Augmented Generation',\n", - " 'embedding',\n", - " 'query',\n", - " 'vector database'],\n", - " 'summary': 'The simplest implementation of RAG (Retrieval-Augmented '\n", - " 'Generation) involves embedding a user query and conducting a '\n", - " 'single embedding search in a vector database, often containing a '\n", - " 'vector store of Wikipedia articles. This approach is intended to '\n", - " 'retrieve relevant documents to aid in generating responses.',\n", - " 'topic': 'Simple RAG Description'}\n", - "{'hypothetical_questions': ['What are the common challenges encountered with '\n", - " 'the simple RAG approach?',\n", - " 'How does the monolithic search backend limit the '\n", - " 'simple RAG?'],\n", + "{'hypothetical_questions': ['How does a simple RAG Model work?',\n", + " 'What are the key challenges faced by simple RAG '\n", + " 'systems?',\n", + " 'Can a simple RAG model handle complex queries '\n", + " 'effectively?'],\n", + " 'keywords': ['Retrieval-Augmented Generation',\n", + " 'RAG',\n", + " 'user query',\n", + " 'vector database',\n", + " 'embeddings',\n", + " 'search limitations'],\n", + " 'summary': 'The simple Retrieval-Augmented Generation (RAG) model is an '\n", + " 'approach that uses a vector database to embed and search for user '\n", + " 'queries, such as Wikipedia articles. It provides answers by '\n", + " 'aligning query and document embeddings. However, it has several '\n", + " 'limitations, including query-document mismatch, reliance on a '\n", + " 'monolithic search backend, text search limitations, and a limited '\n", + " 'planning ability.',\n", + " 'topic': 'Simple RAG'}\n", + "{'hypothetical_questions': ['What is a query-document mismatch in the context '\n", + " 'of RAG models?',\n", + " 'Why might a simple RAG model struggle with '\n", + " 'specific user queries?'],\n", " 'keywords': ['query-document mismatch',\n", - " 'monolithic search backend',\n", - " 'text search limitations',\n", - " 'limited planning ability'],\n", - " 'summary': 'Simple RAG has several limitations including query-document '\n", - " 'mismatch where embeddings may not align, reliance on a monolithic '\n", - " 'search backend which reduces flexibility, text search limitations '\n", - " \"that can't grasp nuances of complex queries, and limited planning \"\n", - " 'ability that fails to consider additional contextual information '\n", - " 'for refining search results.',\n", - " 'topic': 'Limitations of Simple RAG'}\n" + " 'RAG limitation',\n", + " 'embedding alignment'],\n", + " 'summary': \"A limitation where the simple RAG system's query and document \"\n", + " 'embeddings may not align properly, leading to ineffective '\n", + " \"retrieval of information specific to a user's query about, for \"\n", + " \"example, 'climate change effects on marine life'.\",\n", + " 'topic': 'Query-Document Mismatch'}\n", + "{'hypothetical_questions': ['Why is depending on a monolithic search backend a '\n", + " 'limitation for simple RAG?',\n", + " 'How can a monolithic search backend affect the '\n", + " 'quality of search results?'],\n", + " 'keywords': ['monolithic search backend', 'RAG system', 'data sources'],\n", + " 'summary': 'In simple RAG, the reliance on a single search method and backend '\n", + " \"can limit the system's ability to access diverse or specialized \"\n", + " \"data sources, such as when searching for 'latest research in \"\n", + " \"quantum computing'.\",\n", + " 'topic': 'Monolithic Search Backend'}\n", + "{'hypothetical_questions': ['How do text search limitations impact the '\n", + " 'effectiveness of simple RAG?',\n", + " 'Can simple RAG models understand the context of '\n", + " 'search terms?'],\n", + " 'keywords': ['text search limitations', 'RAG', 'advanced search'],\n", + " 'summary': 'The simple RAG model is limited to straightforward text queries '\n", + " 'without advanced search capabilities, failing to resolve nuanced '\n", + " \"queries like 'what problems did we fix last week' due to the \"\n", + " \"presence of generic terms such as 'problem' and 'last week' \"\n", + " 'throughout documents.',\n", + " 'topic': 'Text Search Limitations'}\n", + "{'hypothetical_questions': ['What does limited planning ability imply for a '\n", + " \"RAG model's search results?\",\n", + " 'Can simple RAG models provide context-specific '\n", + " 'information effectively?'],\n", + " 'keywords': ['limited planning ability', 'contextual information', 'RAG'],\n", + " 'summary': 'Simple RAG models struggle to incorporate additional context in '\n", + " 'their searches, which may result in less relevant or overly '\n", + " 'general responses to queries that require specific insights, like '\n", + " \"'Tips for first-time Europe travelers'.\",\n", + " 'topic': 'Limited Planning Ability'}\n" ] } ], @@ -253,16 +288,16 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 5, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "Query(rewritten_query='recent developments in AI', published_daterange=DateRange(start=datetime.date(2023, 1, 1), end=datetime.date(2023, 12, 20)))" + "Query(rewritten_query='recent developments in AI', published_daterange=DateRange(start=datetime.date(2023, 1, 1), end=datetime.date(2023, 12, 21)))" ] }, - "execution_count": 6, + "execution_count": 5, "metadata": {}, "output_type": "execute_result" } @@ -296,16 +331,16 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 16, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "Query(rewritten_query='latest advancements in artificial intelligence', published_daterange=DateRange(chain_of_thought=\"Given that 'recent' suggests developments within the last few months to a year, the best time range to search would be from the beginning of the current year to the current date. This will ensure that the developments are timely and up-to-date.\", start=datetime.date(2023, 1, 1), end=datetime.date(2023, 12, 20)))" + "Query(rewritten_query='Latest advancements in Artificial Intelligence as of December 2023', published_daterange=DateRange(chain_of_thought=\"To get the most recent developments in AI, the date range should be quite recent. Considering today's date is 2023-12-21, a suitable range might be from the past three months to now.\", start=datetime.date(2023, 9, 21), end=datetime.date(2023, 12, 21)))" ] }, - "execution_count": 9, + "execution_count": 16, "metadata": {}, "output_type": "execute_result" } @@ -319,8 +354,12 @@ " end: date\n", "\n", "class Query(BaseModel):\n", - " rewritten_query: str\n", - " published_daterange: DateRange\n", + " rewritten_query: str = Field(\n", + " description=\"Rewrite the query to make it more specific\"\n", + " )\n", + " published_daterange: DateRange = Field(\n", + " description=\"Effective date range to search in\"\n", + " )\n", "\n", "\n", "def expand_query(q) -> Query:\n", @@ -342,6 +381,172 @@ "expand_query(\"What are some recent developments in AI?\")" ] }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "data": { + "text/html": [ + "wandb version 0.16.1 is available! To upgrade, please run:\n", + " $ pip install wandb --upgrade" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "Tracking run with wandb version 0.16.0" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "Run data is saved locally in /Users/jasonliu/dev/instructor/tutorials/wandb/run-20231221_145615-zv4el5or" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "Syncing run swept-sound-11 to Weights & Biases (docs)
" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + " View project at https://wandb.ai/instructor/query-understanding" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + " View run at https://wandb.ai/instructor/query-understanding/runs/zv4el5or" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "wandb: WARNING Source type is set to 'repo' but some required information is missing from the environment. A job will not be created from this run. See https://docs.wandb.ai/guides/launch/create-job\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "eacf93c56db445ffafcfbf673ee44a91", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "VBox(children=(Label(value='0.003 MB of 0.003 MB uploaded\\r'), FloatProgress(value=1.0, max=1.0)))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + " View run swept-sound-11 at: https://wandb.ai/instructor/query-understanding/runs/zv4el5or
Synced 4 W&B file(s), 0 media file(s), 2 artifact file(s) and 0 other file(s)" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "Find logs at: ./wandb/run-20231221_145615-zv4el5or/logs" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "import json\n", + "import wandb\n", + "\n", + "run = wandb.init(\n", + " project=\"query-understanding\",\n", + " config=Query.model_json_schema()\n", + ")\n", + "\n", + "files = wandb.Artifact('data', type='dataset'\n", + ")\n", + "\n", + "with open(\"schema.json\", \"w+\") as f:\n", + " schema = Query.model_json_schema()\n", + " json.dump(schema, f, indent=2)\n", + "\n", + "with open(\"results.jsonl\", \"w+\") as f:\n", + " for questions in [\n", + " \"latest developments in artificial intelligence last 3 weeks\",\n", + " \"renewable energy trends past month\",\n", + " \"quantum computing advancements last 2 months\",\n", + " \"biotechnology updates last 10 days\"\n", + " ]:\n", + " query = expand_query(questions)\n", + " f.write(json.dumps(query.model_dump_json()) + \"\\n\")\n", + " \n", + "files.add_file('schema.json')\n", + "files.add_file('results.jsonl')\n", + "\n", + "run.log_artifact(files)\n", + "run.finish()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, { "cell_type": "markdown", "metadata": {},