diff --git a/docs/docs/integrations/retrievers/self_query/elasticsearch_self_query.ipynb b/docs/docs/integrations/retrievers/self_query/elasticsearch_self_query.ipynb index ebe7fe347..00aaf80cb 100644 --- a/docs/docs/integrations/retrievers/self_query/elasticsearch_self_query.ipynb +++ b/docs/docs/integrations/retrievers/self_query/elasticsearch_self_query.ipynb @@ -28,19 +28,29 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 11, "id": "63a8af5b", "metadata": { "tags": [] }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[33mWARNING: You are using pip version 22.0.4; however, version 23.3 is available.\n", + "You should consider upgrading via the '/Users/joe/projects/elastic/langchain/libs/langchain/.venv/bin/python3 -m pip install --upgrade pip' command.\u001b[0m\u001b[33m\n", + "\u001b[0m" + ] + } + ], "source": [ - "#!pip install lark elasticsearch" + "#!pip install -qU lark elasticsearch" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "id": "cb4a5787", "metadata": { "tags": [] @@ -60,7 +70,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 2, "id": "bcbe04d9", "metadata": { "tags": [] @@ -115,7 +125,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 3, "id": "86e34dbf", "metadata": { "tags": [] @@ -164,17 +174,10 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 4, "id": "38a126e9", "metadata": {}, "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "query='dinosaur' filter=None limit=None\n" - ] - }, { "data": { "text/plain": [ @@ -184,7 +187,7 @@ " Document(page_content='A psychologist / detective gets lost in a series of dreams within dreams within dreams and Inception reused the idea', metadata={'year': 2006, 'director': 'Satoshi Kon', 'rating': 8.6})]" ] }, - "execution_count": 10, + "execution_count": 4, "metadata": {}, "output_type": "execute_result" } @@ -196,24 +199,17 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 5, "id": "b19d4da0", "metadata": {}, "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "query='women' filter=Comparison(comparator=, attribute='director', value='Greta Gerwig') limit=None\n" - ] - }, { "data": { "text/plain": [ "[Document(page_content='A bunch of normal-sized women are supremely wholesome and some men pine after them', metadata={'year': 2019, 'director': 'Greta Gerwig', 'rating': 8.3})]" ] }, - "execution_count": 11, + "execution_count": 5, "metadata": {}, "output_type": "execute_result" } @@ -237,7 +233,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 6, "id": "bff36b88-b506-4877-9c63-e5a1a8d78e64", "metadata": { "tags": [] @@ -256,19 +252,12 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 7, "id": "2758d229-4f97-499c-819f-888acaf8ee10", "metadata": { "tags": [] }, "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "query='dinosaur' filter=None limit=2\n" - ] - }, { "data": { "text/plain": [ @@ -276,7 +265,7 @@ " Document(page_content='Toys come alive and have a blast doing so', metadata={'year': 1995, 'genre': 'animated'})]" ] }, - "execution_count": 13, + "execution_count": 7, "metadata": {}, "output_type": "execute_result" } @@ -297,24 +286,17 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 8, "id": "e460da93", "metadata": {}, "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "query='animated toys' filter=Operation(operator=, arguments=[Operation(operator=, arguments=[Comparison(comparator=, attribute='genre', value='animated'), Comparison(comparator=, attribute='genre', value='comedy')]), Comparison(comparator=, attribute='year', value=1990)]) limit=None\n" - ] - }, { "data": { "text/plain": [ "[Document(page_content='Toys come alive and have a blast doing so', metadata={'year': 1995, 'genre': 'animated'})]" ] }, - "execution_count": 18, + "execution_count": 8, "metadata": {}, "output_type": "execute_result" } @@ -325,21 +307,10 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": null, "id": "0851fc42", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "ObjectApiResponse({'acknowledged': True})" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "vectorstore.client.indices.delete(index=\"elasticsearch-self-query-demo\")" ] @@ -361,7 +332,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.12" + "version": "3.10.3" } }, "nbformat": 4, diff --git a/docs/docs/integrations/vectorstores/elasticsearch.ipynb b/docs/docs/integrations/vectorstores/elasticsearch.ipynb index acc338c32..1717cb557 100644 --- a/docs/docs/integrations/vectorstores/elasticsearch.ipynb +++ b/docs/docs/integrations/vectorstores/elasticsearch.ipynb @@ -139,7 +139,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 1, "id": "67ab8afa-f7c6-4fbf-b596-cb512da949da", "metadata": { "id": "67ab8afa-f7c6-4fbf-b596-cb512da949da", @@ -172,7 +172,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "id": "aac9563e", "metadata": { "id": "aac9563e", @@ -186,7 +186,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 3, "id": "a3c3999a", "metadata": { "id": "a3c3999a", @@ -207,7 +207,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 4, "id": "12eb86d8", "metadata": { "id": "12eb86d8", @@ -218,7 +218,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "[Document(page_content='One of the most serious constitutional responsibilities a President has is nominating someone to serve on the United States Supreme Court. \\n\\nAnd I did that 4 days ago, when I nominated Circuit Court of Appeals Judge Ketanji Brown Jackson. One of our nation’s top legal minds, who will continue Justice Breyer’s legacy of excellence.', metadata={'source': '../../modules/state_of_the_union.txt'}), Document(page_content='One of the most serious constitutional responsibilities a President has is nominating someone to serve on the United States Supreme Court. \\n\\nAnd I did that 4 days ago, when I nominated Circuit Court of Appeals Judge Ketanji Brown Jackson. One of our nation’s top legal minds, who will continue Justice Breyer’s legacy of excellence.', metadata={'source': '../../modules/state_of_the_union.txt', 'date': '2016-01-01', 'rating': 2, 'author': 'John Doe'}), Document(page_content='One of the most serious constitutional responsibilities a President has is nominating someone to serve on the United States Supreme Court. \\n\\nAnd I did that 4 days ago, when I nominated Circuit Court of Appeals Judge Ketanji Brown Jackson. One of our nation’s top legal minds, who will continue Justice Breyer’s legacy of excellence.', metadata={'source': '../../modules/state_of_the_union.txt', 'date': '2010-01-01', 'rating': 1, 'author': 'John Doe'}), Document(page_content='As I said last year, especially to our younger transgender Americans, I will always have your back as your President, so you can be yourself and reach your God-given potential. \\n\\nWhile it often appears that we never agree, that isn’t true. I signed 80 bipartisan bills into law last year. From preventing government shutdowns to protecting Asian-Americans from still-too-common hate crimes to reforming military justice.', metadata={'source': '../../modules/state_of_the_union.txt'})]\n" + "[Document(page_content='One of the most serious constitutional responsibilities a President has is nominating someone to serve on the United States Supreme Court. \\n\\nAnd I did that 4 days ago, when I nominated Circuit Court of Appeals Judge Ketanji Brown Jackson. One of our nation’s top legal minds, who will continue Justice Breyer’s legacy of excellence.', metadata={'source': '../../modules/state_of_the_union.txt'}), Document(page_content='As I said last year, especially to our younger transgender Americans, I will always have your back as your President, so you can be yourself and reach your God-given potential. \\n\\nWhile it often appears that we never agree, that isn’t true. I signed 80 bipartisan bills into law last year. From preventing government shutdowns to protecting Asian-Americans from still-too-common hate crimes to reforming military justice.', metadata={'source': '../../modules/state_of_the_union.txt'}), Document(page_content='A former top litigator in private practice. A former federal public defender. And from a family of public school educators and police officers. A consensus builder. Since she’s been nominated, she’s received a broad range of support—from the Fraternal Order of Police to former judges appointed by Democrats and Republicans. \\n\\nAnd if we are to advance liberty and justice, we need to secure the Border and fix the immigration system.', metadata={'source': '../../modules/state_of_the_union.txt'}), Document(page_content='This is personal to me and Jill, to Kamala, and to so many of you. \\n\\nCancer is the #2 cause of death in America–second only to heart disease. \\n\\nLast month, I announced our plan to supercharge \\nthe Cancer Moonshot that President Obama asked me to lead six years ago. \\n\\nOur goal is to cut the cancer death rate by at least 50% over the next 25 years, turn more cancers from death sentences into treatable diseases. \\n\\nMore support for patients and families.', metadata={'source': '../../modules/state_of_the_union.txt'})]\n" ] } ], @@ -247,7 +247,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 5, "id": "5d076412", "metadata": {}, "outputs": [ @@ -284,12 +284,13 @@ "## Filtering Metadata\n", "With metadata added to the documents, you can add metadata filtering at query time. \n", "\n", - "### Example: Filter by keyword" + "### Example: Filter by Exact keyword\n", + "Notice: We are using the keyword subfield thats not analyzed" ] }, { "cell_type": "code", - "execution_count": 57, + "execution_count": 6, "id": "b2a4bd1b", "metadata": {}, "outputs": [ @@ -297,12 +298,42 @@ "name": "stdout", "output_type": "stream", "text": [ - "{'source': '../../modules/state_of_the_union.txt', 'date': '2010-01-01', 'rating': 1, 'author': 'John Doe', 'geo_location': {'lat': 40.12, 'lon': -71.34}}\n" + "{'source': '../../modules/state_of_the_union.txt', 'date': '2016-01-01', 'rating': 2, 'author': 'John Doe'}\n" ] } ], "source": [ - "docs = db.similarity_search(query, filter=[{ \"match\": { \"metadata.author\": \"John Doe\"}}])\n", + "docs = db.similarity_search(query, filter=[{ \"term\": { \"metadata.author.keyword\": \"John Doe\"}}])\n", + "print(docs[0].metadata)" + ] + }, + { + "cell_type": "markdown", + "id": "1898ab77", + "metadata": {}, + "source": [ + "### Example: Filter by Partial Match\n", + "This example shows how to filter by partial match. This is useful when you don't know the exact value of the metadata field. For example, if you want to filter by the metadata field `author` and you don't know the exact value of the author, you can use a partial match to filter by the author's last name. Fuzzy matching is also supported.\n", + "\n", + "\"Jon\" matches on \"John Doe\" as \"Jon\" is a close match to \"John\" token." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "f3d294ff", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'source': '../../modules/state_of_the_union.txt', 'date': '2016-01-01', 'rating': 2, 'author': 'John Doe'}\n" + ] + } + ], + "source": [ + "docs = db.similarity_search(query, filter=[{ \"match\": { \"metadata.author\": { \"query\": \"Jon\", \"fuzziness\": \"AUTO\" } }}])\n", "print(docs[0].metadata)" ] }, diff --git a/libs/langchain/langchain/retrievers/self_query/elasticsearch.py b/libs/langchain/langchain/retrievers/self_query/elasticsearch.py index 8f93ea034..c908faa64 100644 --- a/libs/langchain/langchain/retrievers/self_query/elasticsearch.py +++ b/libs/langchain/langchain/retrievers/self_query/elasticsearch.py @@ -39,7 +39,7 @@ class ElasticsearchTranslator(Visitor): Comparator.LT: "lt", Comparator.LTE: "lte", Comparator.CONTAIN: "match", - Comparator.LIKE: "fuzzy", + Comparator.LIKE: "match", } return map_dict[func] @@ -67,15 +67,19 @@ class ElasticsearchTranslator(Visitor): } } - if comparison.comparator == Comparator.LIKE: + if comparison.comparator == Comparator.CONTAIN: return { self._format_func(comparison.comparator): { - field: {"value": comparison.value, "fuzziness": "AUTO"} + field: {"query": comparison.value} } } - if comparison.comparator == Comparator.CONTAIN: - return {self._format_func(comparison.comparator): {field: comparison.value}} + if comparison.comparator == Comparator.LIKE: + return { + self._format_func(comparison.comparator): { + field: {"query": comparison.value, "fuzziness": "AUTO"} + } + } # we assume that if the value is a string, # we want to use the keyword field diff --git a/libs/langchain/langchain/vectorstores/elastic_vector_search.py b/libs/langchain/langchain/vectorstores/elastic_vector_search.py index be7695d89..48c492c22 100644 --- a/libs/langchain/langchain/vectorstores/elastic_vector_search.py +++ b/libs/langchain/langchain/vectorstores/elastic_vector_search.py @@ -156,12 +156,22 @@ class ElasticVectorSearch(VectorStore): self.index_name = index_name _ssl_verify = ssl_verify or {} try: - self.client = elasticsearch.Elasticsearch(elasticsearch_url, **_ssl_verify) + self.client = elasticsearch.Elasticsearch( + elasticsearch_url, + **_ssl_verify, + headers={"user-agent": self.get_user_agent()}, + ) except ValueError as e: raise ValueError( f"Your elasticsearch client string is mis-formatted. Got error: {e} " ) + @staticmethod + def get_user_agent() -> str: + from langchain import __version__ + + return f"langchain-py-dvs/{__version__}" + @property def embeddings(self) -> Embeddings: return self.embedding diff --git a/libs/langchain/tests/integration_tests/vectorstores/test_elasticsearch.py b/libs/langchain/tests/integration_tests/vectorstores/test_elasticsearch.py index 8824d94e3..fd549cf7f 100644 --- a/libs/langchain/tests/integration_tests/vectorstores/test_elasticsearch.py +++ b/libs/langchain/tests/integration_tests/vectorstores/test_elasticsearch.py @@ -531,7 +531,7 @@ class TestElasticsearch: }, } }, - settings={"index": {"default_pipeline": "pipeline"}}, + settings={"index": {"default_pipeline": "test_pipeline"}}, ) # adding documents to the index diff --git a/libs/langchain/tests/unit_tests/retrievers/self_query/test_elasticsearch.py b/libs/langchain/tests/unit_tests/retrievers/self_query/test_elasticsearch.py index 924bb1a8c..25c4526c8 100644 --- a/libs/langchain/tests/unit_tests/retrievers/self_query/test_elasticsearch.py +++ b/libs/langchain/tests/unit_tests/retrievers/self_query/test_elasticsearch.py @@ -49,14 +49,14 @@ def test_visit_comparison_range_lte() -> None: def test_visit_comparison_range_match() -> None: comp = Comparison(comparator=Comparator.CONTAIN, attribute="foo", value="1") - expected = {"match": {"metadata.foo": "1"}} + expected = {"match": {"metadata.foo": {"query": "1"}}} actual = DEFAULT_TRANSLATOR.visit_comparison(comp) assert expected == actual def test_visit_comparison_range_like() -> None: comp = Comparison(comparator=Comparator.LIKE, attribute="foo", value="bar") - expected = {"fuzzy": {"metadata.foo": {"value": "bar", "fuzziness": "AUTO"}}} + expected = {"match": {"metadata.foo": {"query": "bar", "fuzziness": "AUTO"}}} actual = DEFAULT_TRANSLATOR.visit_comparison(comp) assert expected == actual @@ -200,9 +200,9 @@ def test_visit_structured_query_complex() -> None: "should": [ {"range": {"metadata.bar": {"lt": 1}}}, { - "fuzzy": { + "match": { "metadata.bar": { - "value": "10", + "query": "10", "fuzziness": "AUTO", } }