This is an automated email from the ASF dual-hosted git repository.
wenjin272 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/flink-agents.git
The following commit(s) were added to refs/heads/main by this push:
new 99103da67 [integration][java][python] Send user images to Ollama
(#1178)
99103da67 is described below
commit 99103da672f897e9c241b51922f2c31a8a728e43
Author: Edson <[email protected]>
AuthorDate: Thu Oct 1 03:13:35 2026 -0400
[integration][java][python] Send user images to Ollama (#1178)
Generated-by: Claude Code 2.1.259 (Claude Opus 5.5)
---
docs/content/docs/development/chat_models.md | 8 +-
.../integration/test/MultimodalChatAgent.java | 92 +++++++++++
.../test/MultimodalChatIntegrationTest.java | 90 +++++++++++
.../ollama/OllamaChatModelConnection.java | 53 +++++++
.../ollama/OllamaMultimodalLiveTest.java | 108 +++++++++++++
.../chatmodels/ollama/OllamaMultimodalTest.java | 175 +++++++++++++++++++++
.../e2e_tests_integration/multimodal_chat_agent.py | 93 +++++++++++
.../e2e_tests_integration/multimodal_chat_test.py | 122 ++++++++++++++
.../integrations/chat_models/ollama_chat_model.py | 55 ++++++-
.../chat_models/tests/test_ollama_multimodal.py | 136 ++++++++++++++++
.../tests/test_ollama_multimodal_live.py | 85 ++++++++++
11 files changed, 1011 insertions(+), 6 deletions(-)
diff --git a/docs/content/docs/development/chat_models.md
b/docs/content/docs/development/chat_models.md
index 11a1b8a88..75251d615 100644
--- a/docs/content/docs/development/chat_models.md
+++ b/docs/content/docs/development/chat_models.md
@@ -62,9 +62,9 @@ a question followed by an image:
**Provider support:** The OpenAI Chat Completions integration, and the Azure
OpenAI
and vLLM integrations built on it, send media blocks to the model; see
-[Multimodal Input](#multimodal-input) for what each block becomes. The other
-built-in integrations, including the OpenAI Responses integration, currently
send
-only the text portion of a message.
+[Multimodal Input](#multimodal-input) for what each block becomes. Ollama sends
+Base64 images; see its section. The other built-in integrations, including the
+OpenAI Responses integration, currently send only the text portion of a
message.
{{< tabs "Message content blocks" >}}
@@ -844,6 +844,8 @@ public class MyAgent extends Agent {
{{< /tabs >}}
+**Multimodal input.** The `ImageBlock`s of a user message are sent as the
message's `images`, in block order, next to its text; Ollama's API does not
interleave images with text. The images must be Base64 data, and the model must
support vision. Images given by URL, other media, and media in system,
assistant or tool messages raise `UnsupportedContentBlockException` (Java) or
`UnsupportedContentBlockError` (Python) before the request is sent.
+
#### Available Models
Visit the [Ollama Models Library](https://ollama.com/library) for the complete
and up-to-date list of available chat models.
diff --git
a/e2e-test/flink-agents-end-to-end-tests-integration/src/test/java/org/apache/flink/agents/integration/test/MultimodalChatAgent.java
b/e2e-test/flink-agents-end-to-end-tests-integration/src/test/java/org/apache/flink/agents/integration/test/MultimodalChatAgent.java
new file mode 100644
index 000000000..f0205a6bd
--- /dev/null
+++
b/e2e-test/flink-agents-end-to-end-tests-integration/src/test/java/org/apache/flink/agents/integration/test/MultimodalChatAgent.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.agents.integration.test;
+
+import org.apache.flink.agents.api.Event;
+import org.apache.flink.agents.api.EventType;
+import org.apache.flink.agents.api.InputEvent;
+import org.apache.flink.agents.api.OutputEvent;
+import org.apache.flink.agents.api.agents.Agent;
+import org.apache.flink.agents.api.annotation.Action;
+import org.apache.flink.agents.api.annotation.ChatModelConnection;
+import org.apache.flink.agents.api.annotation.ChatModelSetup;
+import org.apache.flink.agents.api.chat.messages.ChatMessage;
+import org.apache.flink.agents.api.chat.messages.ImageBlock;
+import org.apache.flink.agents.api.chat.messages.TextBlock;
+import org.apache.flink.agents.api.context.RunnerContext;
+import org.apache.flink.agents.api.event.ChatRequestEvent;
+import org.apache.flink.agents.api.event.ChatResponseEvent;
+import org.apache.flink.agents.api.resource.ResourceDescriptor;
+import org.apache.flink.agents.api.resource.ResourceName;
+
+import java.util.List;
+
+/**
+ * Agent that receives a Base64 PNG as its input, asks an Ollama vision model
about it, and outputs
+ * the answer, so the image travels the whole agent path: input event, chat
request event, the
+ * built-in chat action and the Ollama connection.
+ */
+public class MultimodalChatAgent extends Agent {
+
+ /** A small Ollama vision model; override with {@code
OLLAMA_VISION_MODEL}. */
+ public static final String VISION_MODEL = resolveVisionModel();
+
+ private static String resolveVisionModel() {
+ String model = System.getenv("OLLAMA_VISION_MODEL");
+ return model == null || model.isBlank() ? "qwen3.5:2b" : model;
+ }
+
+ @ChatModelConnection
+ public static ResourceDescriptor visionConnection() {
+ return
ResourceDescriptor.Builder.newBuilder(ResourceName.ChatModel.OLLAMA_CONNECTION)
+ .addInitialArgument("endpoint", "http://localhost:11434";)
+ .addInitialArgument("requestTimeout", 240)
+ .build();
+ }
+
+ @ChatModelSetup
+ public static ResourceDescriptor visionModel() {
+ return
ResourceDescriptor.Builder.newBuilder(ResourceName.ChatModel.OLLAMA_SETUP)
+ .addInitialArgument("connection", "visionConnection")
+ .addInitialArgument("model", VISION_MODEL)
+ // Not every vision model supports thinking.
+ .addInitialArgument("think", false)
+ .build();
+ }
+
+ @Action(EventType.InputEvent)
+ public static void describeImage(Event event, RunnerContext ctx) {
+ String image = (String) InputEvent.fromEvent(event).getInput();
+ ctx.sendEvent(
+ new ChatRequestEvent(
+ "visionModel",
+ List.of(
+ ChatMessage.user(
+ List.of(
+ TextBlock.of(
+ "What color is this
image? Answer in one"
+ + " word."),
+
ImageBlock.fromBase64("image/png", image))))));
+ }
+
+ @Action(EventType.ChatResponseEvent)
+ public static void outputAnswer(Event event, RunnerContext ctx) {
+ ctx.sendEvent(new
OutputEvent(ChatResponseEvent.fromEvent(event).getResponse().getText()));
+ }
+}
diff --git
a/e2e-test/flink-agents-end-to-end-tests-integration/src/test/java/org/apache/flink/agents/integration/test/MultimodalChatIntegrationTest.java
b/e2e-test/flink-agents-end-to-end-tests-integration/src/test/java/org/apache/flink/agents/integration/test/MultimodalChatIntegrationTest.java
new file mode 100644
index 000000000..7bb4af54f
--- /dev/null
+++
b/e2e-test/flink-agents-end-to-end-tests-integration/src/test/java/org/apache/flink/agents/integration/test/MultimodalChatIntegrationTest.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.agents.integration.test;
+
+import org.apache.flink.agents.api.AgentsExecutionEnvironment;
+import org.apache.flink.api.java.functions.KeySelector;
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.util.CloseableIterator;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Assumptions;
+import org.junit.jupiter.api.Test;
+
+import javax.imageio.ImageIO;
+
+import java.awt.Color;
+import java.awt.image.BufferedImage;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Base64;
+import java.util.List;
+
+import static
org.apache.flink.agents.integration.test.MultimodalChatAgent.VISION_MODEL;
+
+/** Runs {@link MultimodalChatAgent} on Flink with a real image and an Ollama
vision model. */
+public class MultimodalChatIntegrationTest extends OllamaPreparationUtils {
+
+ private final boolean ollamaReady;
+
+ public MultimodalChatIntegrationTest() throws IOException {
+ ollamaReady = pullModel(VISION_MODEL);
+ }
+
+ @Test
+ public void testImageReachesTheModelThroughTheAgent() throws Exception {
+ Assumptions.assumeTrue(ollamaReady, String.format("%s is not ready",
VISION_MODEL));
+
+ StreamExecutionEnvironment env =
StreamExecutionEnvironment.getExecutionEnvironment();
+ env.setParallelism(1);
+ DataStream<String> images = env.fromData(redSquarePng());
+
+ AgentsExecutionEnvironment agentsEnv =
+ AgentsExecutionEnvironment.getExecutionEnvironment(env);
+ DataStream<Object> answers =
+ agentsEnv
+ .fromDataStream(images, (KeySelector<String, String>)
image -> "image")
+ .apply(new MultimodalChatAgent())
+ .toDataStream();
+ CloseableIterator<Object> results = answers.collectAsync();
+ agentsEnv.execute();
+
+ List<String> responses = new ArrayList<>();
+ while (results.hasNext()) {
+ responses.add(String.valueOf(results.next()));
+ }
+ // The model's wording varies; the point is that the image reached it
and it answered.
+ Assertions.assertEquals(1, responses.size(), "Unexpected responses: "
+ responses);
+ Assertions.assertFalse(responses.get(0).isBlank(), "The model returned
an empty answer");
+ }
+
+ /** A 64x64 red PNG; Qwen vision processors reject sides under 32 pixels.
*/
+ private static String redSquarePng() throws IOException {
+ BufferedImage image = new BufferedImage(64, 64,
BufferedImage.TYPE_INT_RGB);
+ for (int x = 0; x < 64; x++) {
+ for (int y = 0; y < 64; y++) {
+ image.setRGB(x, y, Color.RED.getRGB());
+ }
+ }
+ ByteArrayOutputStream png = new ByteArrayOutputStream();
+ ImageIO.write(image, "png", png);
+ return Base64.getEncoder().encodeToString(png.toByteArray());
+ }
+}
diff --git
a/integrations/chat-models/ollama/src/main/java/org/apache/flink/agents/integrations/chatmodels/ollama/OllamaChatModelConnection.java
b/integrations/chat-models/ollama/src/main/java/org/apache/flink/agents/integrations/chatmodels/ollama/OllamaChatModelConnection.java
index 22bc58fb7..ec631c83b 100644
---
a/integrations/chat-models/ollama/src/main/java/org/apache/flink/agents/integrations/chatmodels/ollama/OllamaChatModelConnection.java
+++
b/integrations/chat-models/ollama/src/main/java/org/apache/flink/agents/integrations/chatmodels/ollama/OllamaChatModelConnection.java
@@ -28,8 +28,13 @@ import
io.github.ollama4j.models.request.OllamaChatEndpointCaller;
import io.github.ollama4j.models.request.ThinkMode;
import io.github.ollama4j.tools.OllamaToolCallsFunction;
import io.github.ollama4j.tools.Tools;
+import org.apache.flink.agents.api.chat.messages.Base64Source;
import org.apache.flink.agents.api.chat.messages.ChatMessage;
+import org.apache.flink.agents.api.chat.messages.ContentBlock;
+import org.apache.flink.agents.api.chat.messages.ImageBlock;
+import org.apache.flink.agents.api.chat.messages.MediaBlock;
import org.apache.flink.agents.api.chat.messages.MessageRole;
+import
org.apache.flink.agents.api.chat.messages.UnsupportedContentBlockException;
import org.apache.flink.agents.api.chat.model.BaseChatModelConnection;
import org.apache.flink.agents.api.resource.ResourceContext;
import org.apache.flink.agents.api.resource.ResourceDescriptor;
@@ -168,6 +173,7 @@ public class OllamaChatModelConnection extends
BaseChatModelConnection {
* @param message the framework message
* @return the corresponding Ollama message
* @throws RuntimeException if the role cannot be mapped to an Ollama role
+ * @throws UnsupportedContentBlockException if the message has media
Ollama cannot take
*/
private OllamaChatMessage convertToOllamaChatMessages(ChatMessage message)
{
final MessageRole role = message.getRole();
@@ -181,6 +187,10 @@ public class OllamaChatModelConnection extends
BaseChatModelConnection {
// Without the calls, the history shows tool results the model
never requested.
ollamaMessage.setToolCalls(toOllamaToolCalls(toolCalls));
}
+ final List<byte[]> images = toOllamaImages(message);
+ if (!images.isEmpty()) {
+ ollamaMessage.setImages(images);
+ }
return ollamaMessage;
} catch (RoleNotFoundException e) {
throw new RuntimeException(e);
@@ -228,6 +238,45 @@ public class OllamaChatModelConnection extends
BaseChatModelConnection {
}
}
+ /**
+ * The images of a user message, in block order. Ollama takes inline image
data only, attached
+ * to the message rather than interleaved with its text; any other media
fails explicitly.
+ */
+ private static List<byte[]> toOllamaImages(ChatMessage message) {
+ final List<byte[]> images = new ArrayList<>();
+ for (ContentBlock block : message.getBlocks()) {
+ if (!(block instanceof MediaBlock)) {
+ continue;
+ }
+ if (message.getRole() != MessageRole.USER) {
+ throw UnsupportedContentBlockException.forBlock(
+ "Ollama",
+ block,
+ "only user messages can carry media, not "
+ + message.getRole().getValue()
+ + " messages");
+ }
+ if (!(block instanceof ImageBlock)) {
+ throw UnsupportedContentBlockException.forBlock(
+ "Ollama", block, "Ollama accepts images only");
+ }
+ final ImageBlock image = (ImageBlock) block;
+ if (!(image.getSource() instanceof Base64Source)) {
+ throw UnsupportedContentBlockException.forBlock(
+ "Ollama", block, "Ollama takes base64 image data, not
a URL");
+ }
+ try {
+ // ollama4j base64-encodes the bytes again when it serializes
the request.
+ images.add(
+ Base64.getDecoder().decode(((Base64Source)
image.getSource()).getData()));
+ } catch (IllegalArgumentException e) {
+ throw new IllegalArgumentException(
+ "An image block's base64 data could not be decoded.",
e);
+ }
+ }
+ return images;
+ }
+
/**
* Whether Ollama can constrain generation to a schema for {@code
effectiveModel}.
*
@@ -315,6 +364,10 @@ public class OllamaChatModelConnection extends
BaseChatModelConnection {
}
return chatMessage;
+ } catch (RuntimeException e) {
+ // Unchanged, so callers can catch documented errors such as
+ // UnsupportedContentBlockException.
+ throw e;
} catch (Exception e) {
throw new RuntimeException(e);
}
diff --git
a/integrations/chat-models/ollama/src/test/java/org/apache/flink/agents/integrations/chatmodels/ollama/OllamaMultimodalLiveTest.java
b/integrations/chat-models/ollama/src/test/java/org/apache/flink/agents/integrations/chatmodels/ollama/OllamaMultimodalLiveTest.java
new file mode 100644
index 000000000..5f6e5d879
--- /dev/null
+++
b/integrations/chat-models/ollama/src/test/java/org/apache/flink/agents/integrations/chatmodels/ollama/OllamaMultimodalLiveTest.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.agents.integrations.chatmodels.ollama;
+
+import org.apache.flink.agents.api.chat.messages.ChatMessage;
+import org.apache.flink.agents.api.chat.messages.ImageBlock;
+import org.apache.flink.agents.api.chat.messages.TextBlock;
+import org.apache.flink.agents.api.resource.ResourceContext;
+import org.apache.flink.agents.api.resource.ResourceDescriptor;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.Test;
+
+import javax.imageio.ImageIO;
+
+import java.awt.Color;
+import java.awt.image.BufferedImage;
+import java.io.ByteArrayOutputStream;
+import java.util.Base64;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.jupiter.api.Assumptions.assumeTrue;
+
+/**
+ * Sends a real image through the Ollama connection, to check that the server
accepts what {@link
+ * OllamaMultimodalTest} pins.
+ *
+ * <p>Skipped unless {@code OLLAMA_VISION_MODEL} names a vision model the
server already has (for
+ * example {@code qwen3.5:2b} or {@code qwen2.5vl:3b}); {@code
OLLAMA_ENDPOINT} defaults to {@code
+ * http://localhost:11434}. Thinking is disabled, and the image is 64x64
because Qwen vision
+ * processors reject sides under 32 pixels.
+ */
+class OllamaMultimodalLiveTest {
+
+ private static final String MODEL = System.getenv("OLLAMA_VISION_MODEL");
+
+ @BeforeAll
+ static void requireModel() {
+ assumeTrue(MODEL != null && !MODEL.isBlank(), "OLLAMA_VISION_MODEL is
not set");
+ }
+
+ @Test
+ @DisplayName("A base64 image is accepted")
+ void testImage() throws Exception {
+ String endpoint = System.getenv("OLLAMA_ENDPOINT");
+ ResourceDescriptor descriptor =
+
ResourceDescriptor.Builder.newBuilder(OllamaChatModelConnection.class.getName())
+ .addInitialArgument(
+ "endpoint",
+ endpoint == null || endpoint.isBlank()
+ ? "http://localhost:11434";
+ : endpoint)
+ .build();
+ OllamaChatModelConnection connection =
+ new OllamaChatModelConnection(
+ descriptor, ResourceContext.fromGetResource((name,
type) -> null));
+ Map<String, Object> params = new HashMap<>();
+ params.put("model", MODEL);
+ // Not every vision model supports thinking (qwen2.5vl does not).
+ params.put("think", false);
+
+ ChatMessage response =
+ connection.chat(
+ List.of(
+ ChatMessage.user(
+ List.of(
+ TextBlock.of(
+ "What color is this
image? Answer in one"
+ + " word."),
+ ImageBlock.fromBase64(
+ "image/png",
redSquarePng())))),
+ List.of(),
+ params);
+
+ assertThat(response.getText()).isNotBlank();
+ }
+
+ private static String redSquarePng() throws Exception {
+ BufferedImage image = new BufferedImage(64, 64,
BufferedImage.TYPE_INT_RGB);
+ for (int x = 0; x < 64; x++) {
+ for (int y = 0; y < 64; y++) {
+ image.setRGB(x, y, Color.RED.getRGB());
+ }
+ }
+ ByteArrayOutputStream png = new ByteArrayOutputStream();
+ ImageIO.write(image, "png", png);
+ return Base64.getEncoder().encodeToString(png.toByteArray());
+ }
+}
diff --git
a/integrations/chat-models/ollama/src/test/java/org/apache/flink/agents/integrations/chatmodels/ollama/OllamaMultimodalTest.java
b/integrations/chat-models/ollama/src/test/java/org/apache/flink/agents/integrations/chatmodels/ollama/OllamaMultimodalTest.java
new file mode 100644
index 000000000..010a3a047
--- /dev/null
+++
b/integrations/chat-models/ollama/src/test/java/org/apache/flink/agents/integrations/chatmodels/ollama/OllamaMultimodalTest.java
@@ -0,0 +1,175 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.agents.integrations.chatmodels.ollama;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import io.github.ollama4j.models.chat.OllamaChatMessage;
+import io.github.ollama4j.models.chat.OllamaChatRequest;
+import io.github.ollama4j.utils.Utils;
+import org.apache.flink.agents.api.chat.messages.AudioBlock;
+import org.apache.flink.agents.api.chat.messages.ChatMessage;
+import org.apache.flink.agents.api.chat.messages.ContentBlock;
+import org.apache.flink.agents.api.chat.messages.DocumentBlock;
+import org.apache.flink.agents.api.chat.messages.ImageBlock;
+import org.apache.flink.agents.api.chat.messages.MessageRole;
+import org.apache.flink.agents.api.chat.messages.TextBlock;
+import
org.apache.flink.agents.api.chat.messages.UnsupportedContentBlockException;
+import org.apache.flink.agents.api.chat.messages.VideoBlock;
+import org.apache.flink.agents.api.resource.ResourceContext;
+import org.apache.flink.agents.api.resource.ResourceDescriptor;
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.EnumSource;
+
+import java.nio.charset.StandardCharsets;
+import java.util.List;
+import java.util.Map;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+/** Tests how content blocks reach an Ollama chat request. */
+class OllamaMultimodalTest {
+
+ private static final String URL = "https://example.com/cat.png?sig=secret";;
+ private static final String FIRST = "Zmlyc3Q="; // "first"
+ private static final String SECOND = "c2Vjb25k"; // "second"
+
+ private static OllamaChatRequest request(ChatMessage message) {
+ ResourceDescriptor descriptor =
+
ResourceDescriptor.Builder.newBuilder(OllamaChatModelConnection.class.getName())
+ .addInitialArgument("endpoint",
"http://localhost:11434";)
+ .build();
+ return new OllamaChatModelConnection(
+ descriptor, ResourceContext.fromGetResource((a, b) ->
null))
+ .buildRequest(List.of(message), List.of(), Map.of("model",
"llava"), null);
+ }
+
+ @Test
+ @DisplayName("A text-only message carries no images")
+ void testTextOnlyMessageHasNoImages() {
+ OllamaChatMessage message =
+
request(ChatMessage.user(List.of(TextBlock.of("hi")))).getMessages().get(0);
+
+ assertThat(message.getResponse()).isEqualTo("hi");
+ assertThat(message.getImages()).isNull();
+ }
+
+ @Test
+ @DisplayName("Base64 images are attached in block order, next to the text
projection")
+ void testBase64ImagesAttachedInBlockOrder() throws Exception {
+ OllamaChatRequest request =
+ request(
+ ChatMessage.user(
+ List.of(
+ TextBlock.of("Compare "),
+ ImageBlock.fromBase64("image/png",
FIRST),
+ TextBlock.of("and"),
+ ImageBlock.fromBase64("image/jpeg",
SECOND))));
+ OllamaChatMessage message = request.getMessages().get(0);
+
+ assertThat(message.getResponse()).isEqualTo("Compare and");
+ assertThat(message.getImages())
+ .containsExactly(
+ "first".getBytes(StandardCharsets.UTF_8),
+ "second".getBytes(StandardCharsets.UTF_8));
+ // On the wire the images are the original base64 strings again.
+ JsonNode wire = Utils.getObjectMapper().valueToTree(request);
+ assertThat(wire.at("/messages/0/images/0").asText()).isEqualTo(FIRST);
+ assertThat(wire.at("/messages/0/images/1").asText()).isEqualTo(SECOND);
+ }
+
+ @Test
+ @DisplayName("Media Ollama cannot take fails without leaking the source")
+ void testUnsupportedMediaFailsExplicitly() {
+ List<ContentBlock> unsupported =
+ List.of(
+ ImageBlock.fromUrl("image/png", URL),
+ AudioBlock.fromBase64("audio/wav", FIRST),
+ VideoBlock.fromUrl("video/mp4", URL),
+ DocumentBlock.fromBase64("application/pdf", FIRST));
+ for (ContentBlock block : unsupported) {
+ assertThatThrownBy(() ->
request(ChatMessage.user(List.of(TextBlock.of("hi"), block))))
+ .isInstanceOf(UnsupportedContentBlockException.class)
+ .hasMessageStartingWith("Ollama cannot send a")
+ .hasMessageContaining(" " + block.getType() + " block")
+ .hasMessageNotContaining("secret")
+ .hasMessageNotContaining(FIRST);
+ }
+ }
+
+ @ParameterizedTest
+ @EnumSource(
+ value = MessageRole.class,
+ names = {"SYSTEM", "ASSISTANT", "TOOL"})
+ @DisplayName("Images outside user messages fail explicitly")
+ void testImagesOutsideUserMessagesFail(MessageRole role) {
+ ChatMessage message =
+ new ChatMessage(
+ role,
+ List.of(TextBlock.of("see"),
ImageBlock.fromBase64("image/png", FIRST)));
+
+ assertThatThrownBy(() -> request(message))
+ .isInstanceOf(UnsupportedContentBlockException.class)
+ .hasMessageContaining("only user messages can carry media");
+ }
+
+ @Test
+ @DisplayName("Image data that is not base64 fails with a clear error")
+ void testInvalidBase64Fails() {
+ ChatMessage message =
ChatMessage.user(List.of(ImageBlock.fromBase64("image/png", "!!")));
+
+ assertThatThrownBy(() -> request(message))
+ .isInstanceOf(IllegalArgumentException.class)
+ .isNotInstanceOf(UnsupportedContentBlockException.class)
+ .hasMessage("An image block's base64 data could not be
decoded.");
+ }
+
+ @Test
+ @DisplayName("chat() surfaces media errors with their own types")
+ void testChatKeepsMediaErrorTypes() {
+ assertThatThrownBy(
+ () ->
+ chat(
+ ChatMessage.user(
+ List.of(
+ AudioBlock.fromBase64(
+ "audio/wav",
FIRST)))))
+ .isExactlyInstanceOf(UnsupportedContentBlockException.class);
+ assertThatThrownBy(
+ () ->
+ chat(
+ ChatMessage.user(
+
List.of(ImageBlock.fromBase64("image/png", "!!")))))
+ .isExactlyInstanceOf(IllegalArgumentException.class)
+ .hasMessage("An image block's base64 data could not be
decoded.");
+ }
+
+ /** Goes through the public chat(); the errors are raised before any
request is sent. */
+ private static ChatMessage chat(ChatMessage message) {
+ ResourceDescriptor descriptor =
+
ResourceDescriptor.Builder.newBuilder(OllamaChatModelConnection.class.getName())
+ .addInitialArgument("endpoint",
"http://localhost:11434";)
+ .build();
+ return new OllamaChatModelConnection(
+ descriptor, ResourceContext.fromGetResource((a, b) ->
null))
+ .chat(List.of(message), List.of(), Map.of("model", "llava"));
+ }
+}
diff --git
a/python/flink_agents/e2e_tests/e2e_tests_integration/multimodal_chat_agent.py
b/python/flink_agents/e2e_tests/e2e_tests_integration/multimodal_chat_agent.py
new file mode 100644
index 000000000..bb867cc6c
--- /dev/null
+++
b/python/flink_agents/e2e_tests/e2e_tests_integration/multimodal_chat_agent.py
@@ -0,0 +1,93 @@
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#################################################################################
+"""Agent that sends an image arriving with its input to an Ollama vision
model."""
+
+import os
+
+from pyflink.common import Row
+
+from flink_agents.api.agents.agent import Agent
+from flink_agents.api.chat_message import ChatMessage, ImageBlock, TextBlock
+from flink_agents.api.decorators import (
+ action,
+ chat_model_connection,
+ chat_model_setup,
+)
+from flink_agents.api.events.chat_event import ChatRequestEvent,
ChatResponseEvent
+from flink_agents.api.events.event import Event, InputEvent, OutputEvent
+from flink_agents.api.events.event_type import EventType
+from flink_agents.api.resource import ResourceDescriptor, ResourceName
+from flink_agents.api.runner_context import RunnerContext
+
+# A small Ollama vision model; override with OLLAMA_VISION_MODEL.
+VISION_MODEL = os.environ.get("OLLAMA_VISION_MODEL") or "qwen3.5:2b"
+
+
+class MultimodalChatAgent(Agent):
+ """Asks a vision model about the Base64 PNG in each input row.
+
+ The image travels the whole agent path: input event, chat request event,
+ the built-in chat action and the Ollama connection.
+ """
+
+ @chat_model_connection
+ @staticmethod
+ def vision_connection() -> ResourceDescriptor:
+ """Connection to the local Ollama server."""
+ return ResourceDescriptor(
+ clazz=ResourceName.ChatModel.OLLAMA_CONNECTION,
request_timeout=240.0
+ )
+
+ @chat_model_setup
+ @staticmethod
+ def vision_model() -> ResourceDescriptor:
+ """The vision model, with thinking off since not every one supports
it."""
+ return ResourceDescriptor(
+ clazz=ResourceName.ChatModel.OLLAMA_SETUP,
+ connection="vision_connection",
+ model=os.environ.get("OLLAMA_VISION_MODEL") or VISION_MODEL,
+ think=False,
+ )
+
+ @action(EventType.InputEvent)
+ @staticmethod
+ def describe_image(event: Event, ctx: RunnerContext) -> None:
+ """Send the input image with a question to the vision model."""
+ image = InputEvent.from_event(event).input[0]
+ ctx.send_event(
+ ChatRequestEvent(
+ model="vision_model",
+ messages=[
+ ChatMessage.user(
+ [
+ TextBlock(
+ text="What color is this image? Answer in one
word."
+ ),
+ ImageBlock.from_base64("image/png", image),
+ ]
+ )
+ ],
+ )
+ )
+
+ @action(EventType.ChatResponseEvent)
+ @staticmethod
+ def output_answer(event: Event, ctx: RunnerContext) -> None:
+ """Output the model's answer."""
+ answer = ChatResponseEvent.from_event(event).response.text
+ ctx.send_event(OutputEvent(output=Row(answer=answer)))
diff --git
a/python/flink_agents/e2e_tests/e2e_tests_integration/multimodal_chat_test.py
b/python/flink_agents/e2e_tests/e2e_tests_integration/multimodal_chat_test.py
new file mode 100644
index 000000000..8d379364b
--- /dev/null
+++
b/python/flink_agents/e2e_tests/e2e_tests_integration/multimodal_chat_test.py
@@ -0,0 +1,122 @@
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#################################################################################
+"""Runs MultimodalChatAgent on Flink with a real image and an Ollama vision
model.
+
+Mirrors the Java MultimodalChatIntegrationTest. Skipped when the Ollama client
+or the vision model is not available.
+"""
+
+import base64
+import json
+import os
+import struct
+import sysconfig
+import zlib
+from pathlib import Path
+
+import pytest
+from pyflink.common import Row
+from pyflink.common.typeinfo import BasicTypeInfo, ExternalTypeInfo,
RowTypeInfo
+from pyflink.datastream import KeySelector, StreamExecutionEnvironment
+from pyflink.table import DataTypes, Schema, StreamTableEnvironment,
TableDescriptor
+
+from flink_agents.api.execution_environment import AgentsExecutionEnvironment
+from flink_agents.e2e_tests.e2e_tests_integration.multimodal_chat_agent import
(
+ VISION_MODEL,
+ MultimodalChatAgent,
+)
+from flink_agents.e2e_tests.test_utils import pull_model
+
+os.environ["PYTHONPATH"] = sysconfig.get_paths()["purelib"]
+
+client = pull_model(VISION_MODEL)
+
+
+class ImageKeySelector(KeySelector):
+ """Keys every row the same way; there is one image."""
+
+ def get_key(self, value: Row) -> str:
+ """Return a constant key."""
+ return "image"
+
+
+def _red_square_png() -> str:
+ """A 64x64 red PNG; Qwen vision processors reject sides under 32 pixels."""
+ rows = b"".join(b"\x00" + b"\xff\x00\x00" * 64 for _ in range(64))
+
+ def chunk(kind: bytes, data: bytes) -> bytes:
+ body = kind + data
+ return struct.pack(">I", len(data)) + body + struct.pack(">I",
zlib.crc32(body))
+
+ png = (
+ b"\x89PNG\r\n\x1a\n"
+ + chunk(b"IHDR", struct.pack(">IIBBBBB", 64, 64, 8, 2, 0, 0, 0))
+ + chunk(b"IDAT", zlib.compress(rows))
+ + chunk(b"IEND", b"")
+ )
+ return base64.b64encode(png).decode()
+
+
[email protected](
+ client is None, reason="Ollama client is not available or vision model is
missing"
+)
+def test_image_reaches_the_model_through_the_agent(
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ """The image in the input row reaches the vision model and it answers."""
+ monkeypatch.setenv("OLLAMA_VISION_MODEL", VISION_MODEL)
+ stream_env = StreamExecutionEnvironment.get_execution_environment()
+ stream_env.set_parallelism(1)
+ t_env =
StreamTableEnvironment.create(stream_execution_environment=stream_env)
+ table = t_env.from_elements(
+ elements=[(_red_square_png(),)],
+ schema=DataTypes.ROW([DataTypes.FIELD("image", DataTypes.STRING())]),
+ )
+ env = AgentsExecutionEnvironment.get_execution_environment(
+ env=stream_env, t_env=t_env
+ )
+
+ output_type_info = RowTypeInfo([BasicTypeInfo.STRING_TYPE_INFO()],
["answer"])
+ schema = Schema.new_builder().column("answer", DataTypes.STRING()).build()
+ output_table = (
+ env.from_table(input=table, key_selector=ImageKeySelector())
+ .apply(MultimodalChatAgent())
+ .to_table(schema=schema,
output_type=ExternalTypeInfo(output_type_info))
+ )
+
+ result_dir = tmp_path / "results"
+ result_dir.mkdir(parents=True, exist_ok=True)
+ t_env.create_temporary_table(
+ "sink",
+ TableDescriptor.for_connector("filesystem")
+ .option("path", str(result_dir.absolute()))
+ .format("json")
+ .schema(schema)
+ .build(),
+ )
+ output_table.execute_insert("sink").wait()
+
+ answers = []
+ for file in result_dir.iterdir():
+ if file.is_file():
+ with file.open() as f:
+ answers.extend(json.loads(line)["answer"] for line in f if
line.strip())
+ # The model's wording varies; the point is that the image reached it and it
+ # answered.
+ assert len(answers) == 1, answers
+ assert answers[0].strip()
diff --git a/python/flink_agents/integrations/chat_models/ollama_chat_model.py
b/python/flink_agents/integrations/chat_models/ollama_chat_model.py
index 4ce21b366..d4d16e06c 100644
--- a/python/flink_agents/integrations/chat_models/ollama_chat_model.py
+++ b/python/flink_agents/integrations/chat_models/ollama_chat_model.py
@@ -15,15 +15,24 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#################################################################################
+import base64
+import binascii
import uuid
from typing import Any, Dict, List, Literal, Sequence
-from ollama import Client, Message
+from ollama import Client, Image, Message
from pydantic import BaseModel, Field
from typing_extensions import override
from flink_agents.api.agents.types import OutputSchema
-from flink_agents.api.chat_message import ChatMessage, MessageRole
+from flink_agents.api.chat_message import (
+ Base64Source,
+ ChatMessage,
+ ImageBlock,
+ MediaBlock,
+ MessageRole,
+ UnsupportedContentBlockError,
+)
from flink_agents.api.chat_models.chat_model import (
BaseChatModelConnection,
BaseChatModelSetup,
@@ -237,7 +246,11 @@ class OllamaChatModelConnection(BaseChatModelConnection):
def __convert_to_ollama_messages(messages: Sequence[ChatMessage]) ->
List[Message]:
ollama_messages = []
for message in messages:
- ollama_message = Message(role=message.role.value,
content=message.text)
+ ollama_message = Message(
+ role=message.role.value,
+ content=message.text,
+ images=_ollama_images(message),
+ )
if len(message.tool_calls) > 0:
ollama_tool_calls = []
for tool_call in message.tool_calls:
@@ -254,6 +267,42 @@ class OllamaChatModelConnection(BaseChatModelConnection):
return ollama_messages
+def _unsupported(block: MediaBlock, reason: str) ->
UnsupportedContentBlockError:
+ return UnsupportedContentBlockError.for_block("Ollama", block, reason)
+
+
+def _ollama_images(message: ChatMessage) -> List[Image] | None:
+ """Return the images of a user message, in block order.
+
+ Ollama takes inline image data only, attached to the message rather than
+ interleaved with its text; any other media raises
+ UnsupportedContentBlockError. Returns None when the message has no media,
+ so a text-only request is unchanged.
+ """
+ images = []
+ for block in message.blocks:
+ if not isinstance(block, MediaBlock):
+ continue
+ if message.role != MessageRole.USER:
+ reason = (
+ f"only user messages can carry media, not {message.role.value}
messages"
+ )
+ raise _unsupported(block, reason)
+ if not isinstance(block, ImageBlock):
+ raise _unsupported(block, "Ollama accepts images only")
+ if not isinstance(block.source, Base64Source):
+ raise _unsupported(block, "Ollama takes base64 image data, not a
URL")
+ try:
+ # Bytes rather than the string: the client re-encodes bytes,
whereas a
+ # string is first tried as a file path.
+ data = base64.b64decode(block.source.data, validate=True)
+ except binascii.Error as e:
+ msg = "An image block's base64 data could not be decoded."
+ raise ValueError(msg) from e
+ images.append(Image(value=data))
+ return images or None
+
+
class OllamaChatModelSetup(BaseChatModelSetup):
"""Ollama chat model setup which manages chat configuration and will
internally
call ollama chat model connection to do chat.
diff --git
a/python/flink_agents/integrations/chat_models/tests/test_ollama_multimodal.py
b/python/flink_agents/integrations/chat_models/tests/test_ollama_multimodal.py
new file mode 100644
index 000000000..da8b8c84a
--- /dev/null
+++
b/python/flink_agents/integrations/chat_models/tests/test_ollama_multimodal.py
@@ -0,0 +1,136 @@
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#################################################################################
+"""How content blocks reach an Ollama chat request.
+
+Mirrors the Java OllamaMultimodalTest.
+"""
+
+from typing import Any, Dict, List
+from unittest.mock import MagicMock
+
+import pytest
+
+from flink_agents.api.chat_message import (
+ AudioBlock,
+ ChatMessage,
+ ContentBlock,
+ DocumentBlock,
+ ImageBlock,
+ MessageRole,
+ TextBlock,
+ UnsupportedContentBlockError,
+ VideoBlock,
+)
+from flink_agents.integrations.chat_models.ollama_chat_model import (
+ OllamaChatModelConnection,
+)
+
+URL = "https://example.com/cat.png?sig=secret";
+FIRST = "Zmlyc3Q=" # "first"
+SECOND = "c2Vjb25k" # "second"
+
+
+def _sent_messages(message: ChatMessage) -> List[Dict[str, Any]]:
+ """Chat through a mocked client and return the messages as serialized."""
+ conn = OllamaChatModelConnection()
+ response = MagicMock()
+ response.message.role = "assistant"
+ response.message.content = "ok"
+ response.message.tool_calls = None
+ response.prompt_eval_count = 1
+ response.eval_count = 2
+ mock_client = MagicMock()
+ mock_client.chat.return_value = response
+ conn._OllamaChatModelConnection__client = mock_client
+ conn.chat([message], model="llava")
+ return [
+ sent.model_dump(exclude_none=True)
+ for sent in mock_client.chat.call_args.kwargs["messages"]
+ ]
+
+
+def test_text_only_message_has_no_images() -> None:
+ """A message without media is sent without an images field."""
+ sent = _sent_messages(ChatMessage.user([TextBlock(text="hi")]))[0]
+
+ assert sent["content"] == "hi"
+ assert "images" not in sent
+
+
+def test_base64_images_attached_in_block_order() -> None:
+ """Images follow block order, next to the text projection."""
+ sent = _sent_messages(
+ ChatMessage.user(
+ [
+ TextBlock(text="Compare "),
+ ImageBlock.from_base64("image/png", FIRST),
+ TextBlock(text="and"),
+ ImageBlock.from_base64("image/jpeg", SECOND),
+ ]
+ )
+ )[0]
+
+ assert sent["content"] == "Compare and"
+ assert sent["images"] == [FIRST, SECOND]
+
+
[email protected](
+ "block",
+ [
+ ImageBlock.from_url("image/png", URL),
+ AudioBlock.from_base64("audio/wav", FIRST),
+ VideoBlock.from_url("video/mp4", URL),
+ DocumentBlock.from_base64("application/pdf", FIRST),
+ ],
+ ids=["image-url", "audio", "video", "document"],
+)
+def test_unsupported_media_fails_explicitly(block: ContentBlock) -> None:
+ """Media Ollama cannot take fails, naming the block but not its source."""
+ with pytest.raises(UnsupportedContentBlockError) as error:
+ _sent_messages(ChatMessage.user([TextBlock(text="hi"), block]))
+
+ article = "an" if block.type[0] in "aeiou" else "a"
+ assert str(error.value).startswith(
+ f"Ollama cannot send {article} {block.type} block"
+ )
+ assert "secret" not in str(error.value)
+ assert FIRST not in str(error.value)
+
+
[email protected](
+ "role", [MessageRole.SYSTEM, MessageRole.ASSISTANT, MessageRole.TOOL]
+)
+def test_images_outside_user_messages_fail(role: MessageRole) -> None:
+ """Only user messages can carry images."""
+ message = ChatMessage(
+ role=role,
+ blocks=[TextBlock(text="see"), ImageBlock.from_base64("image/png",
FIRST)],
+ )
+
+ with pytest.raises(UnsupportedContentBlockError, match="only user
messages"):
+ _sent_messages(message)
+
+
+def test_invalid_base64_fails() -> None:
+ """Image data that is not base64 fails with a clear error."""
+ message = ChatMessage.user([ImageBlock.from_base64("image/png", "!!")])
+
+ with pytest.raises(ValueError, match="could not be decoded") as error:
+ _sent_messages(message)
+
+ assert not isinstance(error.value, UnsupportedContentBlockError)
diff --git
a/python/flink_agents/integrations/chat_models/tests/test_ollama_multimodal_live.py
b/python/flink_agents/integrations/chat_models/tests/test_ollama_multimodal_live.py
new file mode 100644
index 000000000..8b042f87b
--- /dev/null
+++
b/python/flink_agents/integrations/chat_models/tests/test_ollama_multimodal_live.py
@@ -0,0 +1,85 @@
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#################################################################################
+"""A real image through the Ollama connection.
+
+Checks that the server accepts what test_ollama_multimodal.py pins. Skipped
+unless OLLAMA_VISION_MODEL names a vision model (for example qwen3.5:2b);
+the model is pulled if the server lacks it. Mirrors the Java
+OllamaMultimodalLiveTest.
+"""
+
+import base64
+import os
+import struct
+import zlib
+
+import pytest
+
+from flink_agents.api.chat_message import ChatMessage, ImageBlock, TextBlock
+from flink_agents.integrations.chat_models.ollama_chat_model import (
+ OllamaChatModelConnection,
+)
+
+pytestmark = pytest.mark.integration
+
+VISION_MODEL = os.environ.get("OLLAMA_VISION_MODEL")
+
+
+def _client_ready() -> bool:
+ if not VISION_MODEL:
+ return False
+ from flink_agents.e2e_tests.test_utils import pull_model
+
+ return pull_model(VISION_MODEL) is not None
+
+
+def _red_square_png() -> str:
+ """A 64x64 red PNG; Qwen vision processors reject sides under 32 pixels."""
+ rows = b"".join(b"\x00" + b"\xff\x00\x00" * 64 for _ in range(64))
+
+ def chunk(kind: bytes, data: bytes) -> bytes:
+ body = kind + data
+ return struct.pack(">I", len(data)) + body + struct.pack(">I",
zlib.crc32(body))
+
+ png = (
+ b"\x89PNG\r\n\x1a\n"
+ + chunk(b"IHDR", struct.pack(">IIBBBBB", 64, 64, 8, 2, 0, 0, 0))
+ + chunk(b"IDAT", zlib.compress(rows))
+ + chunk(b"IEND", b"")
+ )
+ return base64.b64encode(png).decode()
+
+
[email protected](not _client_ready(), reason="OLLAMA_VISION_MODEL is not
set")
+def test_image() -> None:
+ """A base64 image is accepted."""
+ connection = OllamaChatModelConnection(request_timeout=120.0)
+ response = connection.chat(
+ [
+ ChatMessage.user(
+ [
+ TextBlock(text="What color is this image? Answer in one
word."),
+ ImageBlock.from_base64("image/png", _red_square_png()),
+ ]
+ )
+ ],
+ model=VISION_MODEL,
+ # Not every vision model supports thinking (qwen2.5vl does not).
+ think=False,
+ )
+ assert response.text.strip()
![https://example.com/cat.png?sig=secret"](https://example.com/cat.png?sig=secret"){kind=link}