diff --git a/simplemind/providers/openai.py b/simplemind/providers/openai.py index 25cc3d3..5718f6e 100644 --- a/simplemind/providers/openai.py +++ b/simplemind/providers/openai.py @@ -182,13 +182,32 @@ class OpenAI(BaseProvider): response_model: Type[T], *, llm_model: str | None = None, + image_url: str | None = None, **kwargs, ) -> T: """Get a structured response from the OpenAI API.""" # Ensure messages are provided in kwargs messages = [ - {"role": "user", "content": prompt}, + { + "role": "user", + "content": [ + { + "type": "text", + "text": prompt + } + ] + }, ] + + """Add an image (url or base64-encoded) to the message if provided.""" + if image_url: + messages[0]['content'].append({ + "type": "image_url", + "image_url": { + "url": image_url + } + }) + response = self.structured_client.chat.completions.create( messages=messages, model=llm_model or self.DEFAULT_MODEL, @@ -199,12 +218,30 @@ class OpenAI(BaseProvider): @logger def generate_text( - self, prompt: str, *, llm_model: str | None = None, **kwargs + self, prompt: str, *, llm_model: str | None = None, image_url: str | None = None, **kwargs ): """Generate text using the OpenAI API.""" messages = [ - {"role": "user", "content": prompt}, + { + "role": "user", + "content": [ + { + "type": "text", + "text": prompt + } + ] + }, ] + + """Add an image (url or base64-encoded) to the message if provided.""" + if image_url: + messages[0]['content'].append({ + "type": "image_url", + "image_url": { + "url": image_url + } + }) + response = self.client.chat.completions.create( messages=messages, model=llm_model or self.DEFAULT_MODEL, @@ -214,15 +251,33 @@ class OpenAI(BaseProvider): @logger def generate_stream_text( - self, prompt: str, *, llm_model: str | None = None, **kwargs + self, prompt: str, *, llm_model: str | None = None, image_url: str | None = None, **kwargs ) -> Iterator[str]: """Generate streaming text using the OpenAI API. Yields chunks of text as they are generated by the model. """ messages = [ - {"role": "user", "content": prompt}, + { + "role": "user", + "content": [ + { + "type": "text", + "text": prompt + } + ] + }, ] + + """Add an image (url or base64-encoded) to the message if provided.""" + if image_url: + messages[0]['content'].append({ + "type": "image_url", + "image_url": { + "url": image_url + } + }) + response = self.client.chat.completions.create( messages=messages, model=llm_model or self.DEFAULT_MODEL,