This is an automated email from the ASF dual-hosted git repository.
davsclaus pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/camel.git
The following commit(s) were added to refs/heads/main by this push:
new 81ef9b3dfc36 CAMEL-26216: camel-pinecone - QUERY defaults to 10
matches, UPSERT generates a missing id, the vector takes any numbers, the
pinecone:rag transformer reads the query response, and test-infra binds the
index ports (#29063)
81ef9b3dfc36 is described below
commit 81ef9b3dfc362b56b51db171f16f0bf70a811cdb
Author: Andrea Cosentino <[email protected]>
AuthorDate: Wed Nov 4 10:36:45 2026 +0100
CAMEL-26216: camel-pinecone - QUERY defaults to 10 matches, UPSERT
generates a missing id, the vector takes any numbers, the pinecone:rag
transformer reads the query response, and test-infra binds the index ports
(#29063)
Co-Authored-By: Claude Opus 5.5 <[email protected]>
---
.../apache/camel/catalog/components/pinecone.json | 10 +-
.../camel/catalog/docs/pinecone-component.adoc | 4 +-
.../camel/catalog/transformers/pinecone-rag.json | 3 +-
.../apache/camel/component/pinecone/pinecone.json | 10 +-
.../org/apache/camel/transformer/pinecone-rag.json | 3 +-
.../src/main/docs/pinecone-component.adoc | 4 +-
.../pinecone/PineconeVectorDbEndpoint.java | 13 +-
.../pinecone/PineconeVectorDbHeaders.java | 10 +-
.../pinecone/PineconeVectorDbProducer.java | 47 +++-
...neconeReverseEmbeddingsDataTypeTransformer.java | 20 +-
.../pinecone/PineconeVectorDbProducerTest.java | 291 +++++++++++++++++++++
...neReverseEmbeddingsDataTypeTransformerTest.java | 17 +-
.../ROOT/pages/camel-4x-upgrade-guide-4_24.adoc | 8 +
.../PineconeVectorDbEndpointBuilderFactory.java | 10 +-
.../PineconeLocalContainerInfraService.java | 19 +-
15 files changed, 422 insertions(+), 47 deletions(-)
diff --git
a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/components/pinecone.json
b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/components/pinecone.json
index 5645fa59db76..fe55acd56013 100644
---
a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/components/pinecone.json
+++
b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/components/pinecone.json
@@ -51,15 +51,15 @@
"CamelPineconeCollectionDimension": { "index": 8, "kind": "header",
"displayName": "", "group": "producer", "label": "", "required": false,
"javaType": "int", "deprecated": false, "deprecationNote": "", "autowired":
false, "secret": false, "description": "Collection Dimension", "constantName":
"org.apache.camel.component.pinecone.PineconeVectorDbHeaders#COLLECTION_DIMENSION"
},
"CamelPineconeCollectionCloud": { "index": 9, "kind": "header",
"displayName": "", "group": "producer", "label": "", "required": false,
"javaType": "String", "enum": [ "aws", "gcp", "azure" ], "deprecated": false,
"deprecationNote": "", "autowired": false, "secret": false, "description":
"Collection Cloud Vendor", "constantName":
"org.apache.camel.component.pinecone.PineconeVectorDbHeaders#COLLECTION_CLOUD"
},
"CamelPineconeCollectionCloudRegion": { "index": 10, "kind": "header",
"displayName": "", "group": "producer", "label": "", "required": false,
"javaType": "String", "enum": [ "aws", "gcp", "azure" ], "deprecated": false,
"deprecationNote": "", "autowired": false, "secret": false, "description":
"Collection Cloud Vendor Region", "constantName":
"org.apache.camel.component.pinecone.PineconeVectorDbHeaders#COLLECTION_CLOUD_REGION"
},
- "CamelPineconeIndexId": { "index": 11, "kind": "header", "displayName":
"", "group": "producer", "label": "", "required": false, "javaType": "String",
"deprecated": false, "deprecationNote": "", "autowired": false, "secret":
false, "description": "Index Upsert Id", "constantName":
"org.apache.camel.component.pinecone.PineconeVectorDbHeaders#INDEX_ID" },
- "CamelPineconeQueryTopK": { "index": 12, "kind": "header", "displayName":
"", "group": "producer", "label": "", "required": false, "javaType": "Integer",
"deprecated": false, "deprecationNote": "", "autowired": false, "secret":
false, "description": "Query Top K", "constantName":
"org.apache.camel.component.pinecone.PineconeVectorDbHeaders#QUERY_TOP_K" },
+ "CamelPineconeIndexId": { "index": 11, "kind": "header", "displayName":
"", "group": "producer", "label": "", "required": false, "javaType": "String",
"deprecated": false, "deprecationNote": "", "autowired": false, "secret":
false, "description": "The id of the vector for UPSERT, UPDATE and QUERY_BY_ID.
Without it, UPSERT generates a random UUID and sets this header to it",
"constantName":
"org.apache.camel.component.pinecone.PineconeVectorDbHeaders#INDEX_ID" },
+ "CamelPineconeQueryTopK": { "index": 12, "kind": "header", "displayName":
"", "group": "producer", "label": "", "required": false, "javaType": "Integer",
"deprecated": false, "deprecationNote": "", "autowired": false, "secret":
false, "defaultValue": "10", "description": "The number of matches QUERY and
QUERY_BY_ID return", "constantName":
"org.apache.camel.component.pinecone.PineconeVectorDbHeaders#QUERY_TOP_K" },
"CamelPineconeNamespace": { "index": 13, "kind": "header", "displayName":
"", "group": "producer", "label": "", "required": false, "javaType": "String",
"deprecated": false, "deprecationNote": "", "autowired": false, "secret":
false, "description": "Namespace for actions (query\/upsert\/etc)",
"constantName":
"org.apache.camel.component.pinecone.PineconeVectorDbHeaders#NAMESPACE" },
- "CamelPineconeQueryFilter": { "index": 14, "kind": "header",
"displayName": "", "group": "producer", "label": "", "required": false,
"javaType": "String", "deprecated": false, "deprecationNote": "", "autowired":
false, "secret": false, "description": "Query Filter", "constantName":
"org.apache.camel.component.pinecone.PineconeVectorDbHeaders#QUERY_FILTER" },
+ "CamelPineconeQueryFilter": { "index": 14, "kind": "header",
"displayName": "", "group": "producer", "label": "", "required": false,
"javaType": "com.google.protobuf.Struct", "deprecated": false,
"deprecationNote": "", "autowired": false, "secret": false, "description":
"Query Filter", "constantName":
"org.apache.camel.component.pinecone.PineconeVectorDbHeaders#QUERY_FILTER" },
"CamelPineconeQueryIncludeValues": { "index": 15, "kind": "header",
"displayName": "", "group": "producer", "label": "", "required": false,
"javaType": "boolean", "deprecated": false, "deprecationNote": "", "autowired":
false, "secret": false, "description": "Query Include Values", "constantName":
"org.apache.camel.component.pinecone.PineconeVectorDbHeaders#QUERY_INCLUDE_VALUES"
},
- "CamelPineconeQueryIncludeMetadata": { "index": 16, "kind": "header",
"displayName": "", "group": "producer", "label": "", "required": false,
"javaType": "com.google.protobuf.Struct", "deprecated": false,
"deprecationNote": "", "autowired": false, "secret": false, "description":
"Query Include Metadata", "constantName":
"org.apache.camel.component.pinecone.PineconeVectorDbHeaders#QUERY_INCLUDE_METADATA"
}
+ "CamelPineconeQueryIncludeMetadata": { "index": 16, "kind": "header",
"displayName": "", "group": "producer", "label": "", "required": false,
"javaType": "boolean", "deprecated": false, "deprecationNote": "", "autowired":
false, "secret": false, "description": "Query Include Metadata",
"constantName":
"org.apache.camel.component.pinecone.PineconeVectorDbHeaders#QUERY_INCLUDE_METADATA"
}
},
"bodyTypes": {
- "producerIn": { "carriers": [ "list", "null" ], "byOption": [ { "option":
"action", "value": "UPSERT", "carriers": [ "list" ] }, { "option": "action",
"value": "UPDATE", "carriers": [ "list" ] }, { "option": "action", "value":
"QUERY", "carriers": [ "list" ] }, { "option": "action", "value": "FETCH",
"carriers": [ "list" ] }, { "option": "action", "value": "DELETE_BY_ID",
"carriers": [ "list" ] }, { "option": "action", "value": "QUERY_BY_ID",
"carriers": [ "any" ] }, { "option": "act [...]
+ "producerIn": { "carriers": [ "list", "null" ], "byOption": [ { "option":
"action", "value": "UPSERT", "carriers": [ "list" ] }, { "option": "action",
"value": "UPDATE", "carriers": [ "list" ] }, { "option": "action", "value":
"QUERY", "carriers": [ "list" ] }, { "option": "action", "value": "FETCH",
"carriers": [ "list" ] }, { "option": "action", "value": "DELETE_BY_ID",
"carriers": [ "list" ] }, { "option": "action", "value": "QUERY_BY_ID",
"carriers": [ "any" ] }, { "option": "act [...]
"producerOut": { "carriers": [ "pojo", "unchanged" ], "byOption": [ {
"option": "action", "value": "UPSERT", "carriers": [ "pojo" ], "javaType":
"io.pinecone.proto.UpsertResponse" }, { "option": "action", "value": "UPDATE",
"carriers": [ "pojo" ], "javaType": "io.pinecone.proto.UpdateResponse" }, {
"option": "action", "value": "QUERY", "carriers": [ "pojo" ], "javaType":
"io.pinecone.unsigned_indices_model.QueryResponseWithUnsignedIndices" }, {
"option": "action", "value": "QUERY_BY_ [...]
},
"properties": {
diff --git
a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/docs/pinecone-component.adoc
b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/docs/pinecone-component.adoc
index 2c46492fd874..35a185854b8a 100644
---
a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/docs/pinecone-component.adoc
+++
b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/docs/pinecone-component.adoc
@@ -33,7 +33,7 @@ include::partial$component-endpoint-headers.adoc[]
== Usage
-The `action` option, or the `CamelPineconeAction` header, selects the
operation. The first route below creates a serverless index. The second route
queries the index with a vector, and the `CamelPineconeQueryTopK` header sets
how many matches it returns:
+The `action` option, or the `CamelPineconeAction` header, selects the
operation. The first route below creates a serverless index. The second route
queries the index with a vector, and the `CamelPineconeQueryTopK` header sets
how many matches it returns (10 without it):
[source,yaml]
----
@@ -74,4 +74,6 @@ The `action` option, or the `CamelPineconeAction` header,
selects the operation.
message: "${body}"
----
+The vector of UPSERT, UPDATE and QUERY is the message body: a List of numbers,
such as the List of Double a JSON array unmarshals to, or a `float[]` or a
`double[]`; each number is converted to a float. Another body fails, and so
does a List with an element that is not a number. UPSERT stores the vector
under the id in the `CamelPineconeIndexId` header; without the header, it
generates a random UUID and sets the header to it.
+The `pinecone:rag` data type transformer turns the result of a QUERY or
QUERY_BY_ID into the List of the texts of its matches, read from their
`text_segment` metadata, which the query returns when the
`CamelPineconeQueryIncludeMetadata` header is `true`.
diff --git
a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/transformers/pinecone-rag.json
b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/transformers/pinecone-rag.json
index a5c574cdd5e9..44dd4755b35d 100644
---
a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/transformers/pinecone-rag.json
+++
b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/transformers/pinecone-rag.json
@@ -13,10 +13,11 @@
"bodyTypes": {
"transformerIn": {
"carriers": [
+ "pojo",
"list",
"null"
],
- "description": "The List of ScoredVectorWithUnsignedIndices of a QUERY,
the matchesList of its QueryResponseWithUnsignedIndices, with the text in the
text_segment metadata. A null body, or the QueryResponseWithUnsignedIndices
itself, gives an empty List"
+ "description": "The QueryResponseWithUnsignedIndices of a QUERY or
QUERY_BY_ID, or the List of ScoredVectorWithUnsignedIndices of its matches,
with the text in the text_segment metadata, which the query returns with the
CamelPineconeQueryIncludeMetadata header set to true. A match without the text
gives an empty String, and a null body an empty List"
},
"transformerOut": {
"carriers": [
diff --git
a/components/camel-ai/camel-pinecone/src/generated/resources/META-INF/org/apache/camel/component/pinecone/pinecone.json
b/components/camel-ai/camel-pinecone/src/generated/resources/META-INF/org/apache/camel/component/pinecone/pinecone.json
index 5645fa59db76..fe55acd56013 100644
---
a/components/camel-ai/camel-pinecone/src/generated/resources/META-INF/org/apache/camel/component/pinecone/pinecone.json
+++
b/components/camel-ai/camel-pinecone/src/generated/resources/META-INF/org/apache/camel/component/pinecone/pinecone.json
@@ -51,15 +51,15 @@
"CamelPineconeCollectionDimension": { "index": 8, "kind": "header",
"displayName": "", "group": "producer", "label": "", "required": false,
"javaType": "int", "deprecated": false, "deprecationNote": "", "autowired":
false, "secret": false, "description": "Collection Dimension", "constantName":
"org.apache.camel.component.pinecone.PineconeVectorDbHeaders#COLLECTION_DIMENSION"
},
"CamelPineconeCollectionCloud": { "index": 9, "kind": "header",
"displayName": "", "group": "producer", "label": "", "required": false,
"javaType": "String", "enum": [ "aws", "gcp", "azure" ], "deprecated": false,
"deprecationNote": "", "autowired": false, "secret": false, "description":
"Collection Cloud Vendor", "constantName":
"org.apache.camel.component.pinecone.PineconeVectorDbHeaders#COLLECTION_CLOUD"
},
"CamelPineconeCollectionCloudRegion": { "index": 10, "kind": "header",
"displayName": "", "group": "producer", "label": "", "required": false,
"javaType": "String", "enum": [ "aws", "gcp", "azure" ], "deprecated": false,
"deprecationNote": "", "autowired": false, "secret": false, "description":
"Collection Cloud Vendor Region", "constantName":
"org.apache.camel.component.pinecone.PineconeVectorDbHeaders#COLLECTION_CLOUD_REGION"
},
- "CamelPineconeIndexId": { "index": 11, "kind": "header", "displayName":
"", "group": "producer", "label": "", "required": false, "javaType": "String",
"deprecated": false, "deprecationNote": "", "autowired": false, "secret":
false, "description": "Index Upsert Id", "constantName":
"org.apache.camel.component.pinecone.PineconeVectorDbHeaders#INDEX_ID" },
- "CamelPineconeQueryTopK": { "index": 12, "kind": "header", "displayName":
"", "group": "producer", "label": "", "required": false, "javaType": "Integer",
"deprecated": false, "deprecationNote": "", "autowired": false, "secret":
false, "description": "Query Top K", "constantName":
"org.apache.camel.component.pinecone.PineconeVectorDbHeaders#QUERY_TOP_K" },
+ "CamelPineconeIndexId": { "index": 11, "kind": "header", "displayName":
"", "group": "producer", "label": "", "required": false, "javaType": "String",
"deprecated": false, "deprecationNote": "", "autowired": false, "secret":
false, "description": "The id of the vector for UPSERT, UPDATE and QUERY_BY_ID.
Without it, UPSERT generates a random UUID and sets this header to it",
"constantName":
"org.apache.camel.component.pinecone.PineconeVectorDbHeaders#INDEX_ID" },
+ "CamelPineconeQueryTopK": { "index": 12, "kind": "header", "displayName":
"", "group": "producer", "label": "", "required": false, "javaType": "Integer",
"deprecated": false, "deprecationNote": "", "autowired": false, "secret":
false, "defaultValue": "10", "description": "The number of matches QUERY and
QUERY_BY_ID return", "constantName":
"org.apache.camel.component.pinecone.PineconeVectorDbHeaders#QUERY_TOP_K" },
"CamelPineconeNamespace": { "index": 13, "kind": "header", "displayName":
"", "group": "producer", "label": "", "required": false, "javaType": "String",
"deprecated": false, "deprecationNote": "", "autowired": false, "secret":
false, "description": "Namespace for actions (query\/upsert\/etc)",
"constantName":
"org.apache.camel.component.pinecone.PineconeVectorDbHeaders#NAMESPACE" },
- "CamelPineconeQueryFilter": { "index": 14, "kind": "header",
"displayName": "", "group": "producer", "label": "", "required": false,
"javaType": "String", "deprecated": false, "deprecationNote": "", "autowired":
false, "secret": false, "description": "Query Filter", "constantName":
"org.apache.camel.component.pinecone.PineconeVectorDbHeaders#QUERY_FILTER" },
+ "CamelPineconeQueryFilter": { "index": 14, "kind": "header",
"displayName": "", "group": "producer", "label": "", "required": false,
"javaType": "com.google.protobuf.Struct", "deprecated": false,
"deprecationNote": "", "autowired": false, "secret": false, "description":
"Query Filter", "constantName":
"org.apache.camel.component.pinecone.PineconeVectorDbHeaders#QUERY_FILTER" },
"CamelPineconeQueryIncludeValues": { "index": 15, "kind": "header",
"displayName": "", "group": "producer", "label": "", "required": false,
"javaType": "boolean", "deprecated": false, "deprecationNote": "", "autowired":
false, "secret": false, "description": "Query Include Values", "constantName":
"org.apache.camel.component.pinecone.PineconeVectorDbHeaders#QUERY_INCLUDE_VALUES"
},
- "CamelPineconeQueryIncludeMetadata": { "index": 16, "kind": "header",
"displayName": "", "group": "producer", "label": "", "required": false,
"javaType": "com.google.protobuf.Struct", "deprecated": false,
"deprecationNote": "", "autowired": false, "secret": false, "description":
"Query Include Metadata", "constantName":
"org.apache.camel.component.pinecone.PineconeVectorDbHeaders#QUERY_INCLUDE_METADATA"
}
+ "CamelPineconeQueryIncludeMetadata": { "index": 16, "kind": "header",
"displayName": "", "group": "producer", "label": "", "required": false,
"javaType": "boolean", "deprecated": false, "deprecationNote": "", "autowired":
false, "secret": false, "description": "Query Include Metadata",
"constantName":
"org.apache.camel.component.pinecone.PineconeVectorDbHeaders#QUERY_INCLUDE_METADATA"
}
},
"bodyTypes": {
- "producerIn": { "carriers": [ "list", "null" ], "byOption": [ { "option":
"action", "value": "UPSERT", "carriers": [ "list" ] }, { "option": "action",
"value": "UPDATE", "carriers": [ "list" ] }, { "option": "action", "value":
"QUERY", "carriers": [ "list" ] }, { "option": "action", "value": "FETCH",
"carriers": [ "list" ] }, { "option": "action", "value": "DELETE_BY_ID",
"carriers": [ "list" ] }, { "option": "action", "value": "QUERY_BY_ID",
"carriers": [ "any" ] }, { "option": "act [...]
+ "producerIn": { "carriers": [ "list", "null" ], "byOption": [ { "option":
"action", "value": "UPSERT", "carriers": [ "list" ] }, { "option": "action",
"value": "UPDATE", "carriers": [ "list" ] }, { "option": "action", "value":
"QUERY", "carriers": [ "list" ] }, { "option": "action", "value": "FETCH",
"carriers": [ "list" ] }, { "option": "action", "value": "DELETE_BY_ID",
"carriers": [ "list" ] }, { "option": "action", "value": "QUERY_BY_ID",
"carriers": [ "any" ] }, { "option": "act [...]
"producerOut": { "carriers": [ "pojo", "unchanged" ], "byOption": [ {
"option": "action", "value": "UPSERT", "carriers": [ "pojo" ], "javaType":
"io.pinecone.proto.UpsertResponse" }, { "option": "action", "value": "UPDATE",
"carriers": [ "pojo" ], "javaType": "io.pinecone.proto.UpdateResponse" }, {
"option": "action", "value": "QUERY", "carriers": [ "pojo" ], "javaType":
"io.pinecone.unsigned_indices_model.QueryResponseWithUnsignedIndices" }, {
"option": "action", "value": "QUERY_BY_ [...]
},
"properties": {
diff --git
a/components/camel-ai/camel-pinecone/src/generated/resources/META-INF/services/org/apache/camel/transformer/pinecone-rag.json
b/components/camel-ai/camel-pinecone/src/generated/resources/META-INF/services/org/apache/camel/transformer/pinecone-rag.json
index a5c574cdd5e9..44dd4755b35d 100644
---
a/components/camel-ai/camel-pinecone/src/generated/resources/META-INF/services/org/apache/camel/transformer/pinecone-rag.json
+++
b/components/camel-ai/camel-pinecone/src/generated/resources/META-INF/services/org/apache/camel/transformer/pinecone-rag.json
@@ -13,10 +13,11 @@
"bodyTypes": {
"transformerIn": {
"carriers": [
+ "pojo",
"list",
"null"
],
- "description": "The List of ScoredVectorWithUnsignedIndices of a QUERY,
the matchesList of its QueryResponseWithUnsignedIndices, with the text in the
text_segment metadata. A null body, or the QueryResponseWithUnsignedIndices
itself, gives an empty List"
+ "description": "The QueryResponseWithUnsignedIndices of a QUERY or
QUERY_BY_ID, or the List of ScoredVectorWithUnsignedIndices of its matches,
with the text in the text_segment metadata, which the query returns with the
CamelPineconeQueryIncludeMetadata header set to true. A match without the text
gives an empty String, and a null body an empty List"
},
"transformerOut": {
"carriers": [
diff --git
a/components/camel-ai/camel-pinecone/src/main/docs/pinecone-component.adoc
b/components/camel-ai/camel-pinecone/src/main/docs/pinecone-component.adoc
index 2c46492fd874..35a185854b8a 100644
--- a/components/camel-ai/camel-pinecone/src/main/docs/pinecone-component.adoc
+++ b/components/camel-ai/camel-pinecone/src/main/docs/pinecone-component.adoc
@@ -33,7 +33,7 @@ include::partial$component-endpoint-headers.adoc[]
== Usage
-The `action` option, or the `CamelPineconeAction` header, selects the
operation. The first route below creates a serverless index. The second route
queries the index with a vector, and the `CamelPineconeQueryTopK` header sets
how many matches it returns:
+The `action` option, or the `CamelPineconeAction` header, selects the
operation. The first route below creates a serverless index. The second route
queries the index with a vector, and the `CamelPineconeQueryTopK` header sets
how many matches it returns (10 without it):
[source,yaml]
----
@@ -74,4 +74,6 @@ The `action` option, or the `CamelPineconeAction` header,
selects the operation.
message: "${body}"
----
+The vector of UPSERT, UPDATE and QUERY is the message body: a List of numbers,
such as the List of Double a JSON array unmarshals to, or a `float[]` or a
`double[]`; each number is converted to a float. Another body fails, and so
does a List with an element that is not a number. UPSERT stores the vector
under the id in the `CamelPineconeIndexId` header; without the header, it
generates a random UUID and sets the header to it.
+The `pinecone:rag` data type transformer turns the result of a QUERY or
QUERY_BY_ID into the List of the texts of its matches, read from their
`text_segment` metadata, which the query returns when the
`CamelPineconeQueryIncludeMetadata` header is `true`.
diff --git
a/components/camel-ai/camel-pinecone/src/main/java/org/apache/camel/component/pinecone/PineconeVectorDbEndpoint.java
b/components/camel-ai/camel-pinecone/src/main/java/org/apache/camel/component/pinecone/PineconeVectorDbEndpoint.java
index 654f6bbccbb0..fa93992e9abf 100644
---
a/components/camel-ai/camel-pinecone/src/main/java/org/apache/camel/component/pinecone/PineconeVectorDbEndpoint.java
+++
b/components/camel-ai/camel-pinecone/src/main/java/org/apache/camel/component/pinecone/PineconeVectorDbEndpoint.java
@@ -62,12 +62,13 @@ import org.apache.camel.support.DefaultEndpoint;
@BodyType.When(option = "action",
value = "DELETE_COLLECTION",
carriers =
Carrier.ANY) },
description = "By the action in the action
option or the CamelPineconeAction header: the"
- + " vector as a List of Float
or a float array for UPSERT, UPDATE and QUERY,"
- + " with the id in the
CamelPineconeIndexId header, and the ids as a List"
- + " of String for FETCH and
DELETE_BY_ID. The other actions do not read the"
- + " body. Another body fails,
a List of Double too: the pinecone:embeddings"
- + " data type transformer
builds the vector from the output of"
- + " langchain4j-embeddings"),
+ + " vector as a List of
numbers, such as the List of Double a JSON array"
+ + " unmarshals to, or a float
or double array for UPSERT, UPDATE and QUERY,"
+ + " with the id in the
CamelPineconeIndexId header (UPSERT generates one"
+ + " without it), and the ids
as a List of String for FETCH and DELETE_BY_ID."
+ + " The other actions do not
read the body. Another body fails: the"
+ + " pinecone:embeddings data
type transformer builds the vector from the"
+ + " output of
langchain4j-embeddings"),
producerOut = @BodyType(value = { Carrier.POJO, Carrier.UNCHANGED },
byOption = {
@BodyType.When(option = "action",
value = "UPSERT", carriers = Carrier.POJO,
diff --git
a/components/camel-ai/camel-pinecone/src/main/java/org/apache/camel/component/pinecone/PineconeVectorDbHeaders.java
b/components/camel-ai/camel-pinecone/src/main/java/org/apache/camel/component/pinecone/PineconeVectorDbHeaders.java
index ebd4c162fb67..222d31fe84dc 100644
---
a/components/camel-ai/camel-pinecone/src/main/java/org/apache/camel/component/pinecone/PineconeVectorDbHeaders.java
+++
b/components/camel-ai/camel-pinecone/src/main/java/org/apache/camel/component/pinecone/PineconeVectorDbHeaders.java
@@ -53,22 +53,24 @@ public class PineconeVectorDbHeaders {
@Metadata(description = "Collection Cloud Vendor Region", javaType =
"String", enums = "aws,gcp,azure")
public static final String COLLECTION_CLOUD_REGION =
"CamelPineconeCollectionCloudRegion";
- @Metadata(description = "Index Upsert Id", javaType = "String")
+ @Metadata(description = "The id of the vector for UPSERT, UPDATE and
QUERY_BY_ID. Without it, UPSERT generates a random UUID"
+ + " and sets this header to it",
+ javaType = "String")
public static final String INDEX_ID = "CamelPineconeIndexId";
- @Metadata(description = "Query Top K", javaType = "Integer")
+ @Metadata(description = "The number of matches QUERY and QUERY_BY_ID
return", javaType = "Integer", defaultValue = "10")
public static final String QUERY_TOP_K = "CamelPineconeQueryTopK";
@Metadata(description = "Namespace for actions (query/upsert/etc)",
javaType = "String")
public static final String NAMESPACE = "CamelPineconeNamespace";
- @Metadata(description = "Query Filter", javaType = "String")
+ @Metadata(description = "Query Filter", javaType =
"com.google.protobuf.Struct")
public static final String QUERY_FILTER = "CamelPineconeQueryFilter";
@Metadata(description = "Query Include Values", javaType = "boolean")
public static final String QUERY_INCLUDE_VALUES =
"CamelPineconeQueryIncludeValues";
- @Metadata(description = "Query Include Metadata", javaType =
"com.google.protobuf.Struct")
+ @Metadata(description = "Query Include Metadata", javaType = "boolean")
public static final String QUERY_INCLUDE_METADATA =
"CamelPineconeQueryIncludeMetadata";
}
diff --git
a/components/camel-ai/camel-pinecone/src/main/java/org/apache/camel/component/pinecone/PineconeVectorDbProducer.java
b/components/camel-ai/camel-pinecone/src/main/java/org/apache/camel/component/pinecone/PineconeVectorDbProducer.java
index 24b54ad7133f..4ea721b30af8 100644
---
a/components/camel-ai/camel-pinecone/src/main/java/org/apache/camel/component/pinecone/PineconeVectorDbProducer.java
+++
b/components/camel-ai/camel-pinecone/src/main/java/org/apache/camel/component/pinecone/PineconeVectorDbProducer.java
@@ -16,7 +16,9 @@
*/
package org.apache.camel.component.pinecone;
+import java.util.ArrayList;
import java.util.List;
+import java.util.UUID;
import java.util.concurrent.ExecutorService;
import com.google.protobuf.Struct;
@@ -29,6 +31,7 @@ import io.pinecone.proto.UpsertResponse;
import io.pinecone.unsigned_indices_model.QueryResponseWithUnsignedIndices;
import org.apache.camel.CamelContext;
import org.apache.camel.Exchange;
+import org.apache.camel.InvalidPayloadException;
import org.apache.camel.Message;
import org.apache.camel.NoSuchHeaderException;
import org.apache.camel.support.DefaultProducer;
@@ -37,6 +40,10 @@ import
org.openapitools.db_control.client.model.DeletionProtection;
import org.openapitools.db_control.client.model.IndexModel;
public class PineconeVectorDbProducer extends DefaultProducer {
+ // the number of matches of QUERY and QUERY_BY_ID without the
CamelPineconeQueryTopK header, as camel-weaviate
+ private static final int DEFAULT_QUERY_TOP_K = 10;
+ private static final String VECTOR_TYPES = "a List of numbers, such as
Float or Double, a float[] or a double[]";
+
private Pinecone client;
private ExecutorService executor;
@@ -205,12 +212,17 @@ public class PineconeVectorDbProducer extends
DefaultProducer {
private void upsert(Exchange exchange) throws Exception {
final Message in = exchange.getMessage();
- List elements = in.getMandatoryBody(List.class);
+ List<Float> elements = toVector(exchange);
String indexName = getEndpoint().getConfiguration().getIndexName();
if (in.getHeader(PineconeVectorDbHeaders.INDEX_NAME, String.class) !=
null) {
indexName = in.getHeader(PineconeVectorDbHeaders.INDEX_NAME,
String.class);
}
String indexId = in.getHeader(PineconeVectorDbHeaders.INDEX_ID,
String.class);
+ if (indexId == null) {
+ // a random id, as the pinecone:embeddings transformer gives, in
the header so the caller knows it
+ indexId = UUID.randomUUID().toString();
+ in.setHeader(PineconeVectorDbHeaders.INDEX_ID, indexId);
+ }
String namespace = in.getHeader(PineconeVectorDbHeaders.NAMESPACE,
String.class);
Index index = this.client.getIndexConnection(indexName);
@@ -223,7 +235,7 @@ public class PineconeVectorDbProducer extends
DefaultProducer {
private void update(Exchange exchange) throws Exception {
final Message in = exchange.getMessage();
- List elements = in.getMandatoryBody(List.class);
+ List<Float> elements = toVector(exchange);
String indexName = getEndpoint().getConfiguration().getIndexName();
if (in.getHeader(PineconeVectorDbHeaders.INDEX_NAME, String.class) !=
null) {
@@ -292,14 +304,14 @@ public class PineconeVectorDbProducer extends
DefaultProducer {
private void query(Exchange exchange) throws Exception {
final Message in = exchange.getMessage();
- List elements = in.getMandatoryBody(List.class);
+ List<Float> elements = toVector(exchange);
String indexName = getEndpoint().getConfiguration().getIndexName();
if (in.getHeader(PineconeVectorDbHeaders.INDEX_NAME, String.class) !=
null) {
indexName = in.getHeader(PineconeVectorDbHeaders.INDEX_NAME,
String.class);
}
- int topK = in.getHeader(PineconeVectorDbHeaders.QUERY_TOP_K,
Integer.class);
+ int topK = in.getHeader(PineconeVectorDbHeaders.QUERY_TOP_K,
DEFAULT_QUERY_TOP_K, Integer.class);
Index index = this.client.getIndexConnection(indexName);
// Optional arguments, can be null
@@ -326,7 +338,7 @@ public class PineconeVectorDbProducer extends
DefaultProducer {
indexName = in.getHeader(PineconeVectorDbHeaders.INDEX_NAME,
String.class);
}
- int topK = in.getHeader(PineconeVectorDbHeaders.QUERY_TOP_K,
Integer.class);
+ int topK = in.getHeader(PineconeVectorDbHeaders.QUERY_TOP_K,
DEFAULT_QUERY_TOP_K, Integer.class);
Index index = this.client.getIndexConnection(indexName);
String indexId = in.getHeader(PineconeVectorDbHeaders.INDEX_ID,
String.class);
@@ -355,6 +367,31 @@ public class PineconeVectorDbProducer extends
DefaultProducer {
return getEndpoint().getCamelContext();
}
+ /**
+ * The vector in the body of UPSERT, UPDATE and QUERY: a List of numbers,
such as the List of Double a JSON array
+ * unmarshals to, or another body Camel converts to a List, such as a
float[] or a double[]. The Pinecone SDK takes
+ * a List of Float.
+ */
+ private static List<Float> toVector(Exchange exchange) throws
InvalidPayloadException {
+ final Message in = exchange.getMessage();
+ List<?> elements = in.getBody(List.class);
+ if (elements == null) {
+ throw new InvalidPayloadException(exchange, List.class, in, "the
vector must be " + VECTOR_TYPES);
+ }
+ List<Float> vector = new ArrayList<>(elements.size());
+ for (Object element : elements) {
+ if (!(element instanceof Number number)) {
+ // the type only, not the value, which comes from the message
+ throw new IllegalArgumentException(
+ "Vector element at index " + vector.size() + " is "
+ + (element == null ? "null"
: "a " + element.getClass().getName())
+ + ": the vector must be " +
VECTOR_TYPES);
+ }
+ vector.add(number.floatValue());
+ }
+ return vector;
+ }
+
private void populateCollectionResponse(CollectionModel r, Exchange
exchange) {
Message out = exchange.getMessage();
out.setBody(r);
diff --git
a/components/camel-ai/camel-pinecone/src/main/java/org/apache/camel/component/pinecone/transform/PineconeReverseEmbeddingsDataTypeTransformer.java
b/components/camel-ai/camel-pinecone/src/main/java/org/apache/camel/component/pinecone/transform/PineconeReverseEmbeddingsDataTypeTransformer.java
index cbd6486a5804..78cd3308408c 100644
---
a/components/camel-ai/camel-pinecone/src/main/java/org/apache/camel/component/pinecone/transform/PineconeReverseEmbeddingsDataTypeTransformer.java
+++
b/components/camel-ai/camel-pinecone/src/main/java/org/apache/camel/component/pinecone/transform/PineconeReverseEmbeddingsDataTypeTransformer.java
@@ -21,6 +21,7 @@ import java.util.List;
import java.util.stream.Collectors;
import com.google.protobuf.Value;
+import io.pinecone.unsigned_indices_model.QueryResponseWithUnsignedIndices;
import io.pinecone.unsigned_indices_model.ScoredVectorWithUnsignedIndices;
import org.apache.camel.Message;
import org.apache.camel.spi.BodyType;
@@ -35,16 +36,23 @@ import org.apache.camel.spi.Transformer;
*/
@DataTypeTransformer(name = "pinecone:rag",
description = "Maps a List of retrieved LangChain4j
Embeddings with similarity search to a List of String for LangChain4j RAG")
-@BodyTypes(transformerIn = @BodyType(value = { Carrier.LIST, Carrier.NULL },
- description = "The List of
ScoredVectorWithUnsignedIndices of a QUERY, the matchesList of"
- + " its
QueryResponseWithUnsignedIndices, with the text in the text_segment"
- + " metadata. A null body,
or the QueryResponseWithUnsignedIndices"
- + " itself, gives an empty
List"),
+@BodyTypes(transformerIn = @BodyType(value = { Carrier.POJO, Carrier.LIST,
Carrier.NULL },
+ description = "The
QueryResponseWithUnsignedIndices of a QUERY or QUERY_BY_ID, or the List"
+ + " of
ScoredVectorWithUnsignedIndices of its matches, with the text in the"
+ + " text_segment metadata,
which the query returns with the"
+ + "
CamelPineconeQueryIncludeMetadata header set to true. A match without"
+ + " the text gives an empty
String, and a null body an empty List"),
transformerOut = @BodyType(value = Carrier.LIST, javaType =
"java.util.List<java.lang.String>"))
public class PineconeReverseEmbeddingsDataTypeTransformer extends Transformer {
@Override
public void transform(Message message, DataType fromType, DataType toType)
throws Exception {
- List<ScoredVectorWithUnsignedIndices> embeddings =
message.getBody(List.class);
+ List<ScoredVectorWithUnsignedIndices> embeddings;
+ if (message.getBody() instanceof QueryResponseWithUnsignedIndices
response) {
+ // the body a QUERY or QUERY_BY_ID leaves
+ embeddings = response.getMatchesList();
+ } else {
+ embeddings = message.getBody(List.class);
+ }
if (embeddings == null || embeddings.isEmpty()) {
message.setBody(Collections.emptyList());
diff --git
a/components/camel-ai/camel-pinecone/src/test/java/org/apache/camel/component/pinecone/PineconeVectorDbProducerTest.java
b/components/camel-ai/camel-pinecone/src/test/java/org/apache/camel/component/pinecone/PineconeVectorDbProducerTest.java
new file mode 100644
index 000000000000..9fa7f5f6ab5d
--- /dev/null
+++
b/components/camel-ai/camel-pinecone/src/test/java/org/apache/camel/component/pinecone/PineconeVectorDbProducerTest.java
@@ -0,0 +1,291 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.camel.component.pinecone;
+
+import java.io.IOException;
+import java.math.BigDecimal;
+import java.net.InetSocketAddress;
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+import java.util.List;
+import java.util.UUID;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.stream.Stream;
+
+import com.google.protobuf.Struct;
+import com.google.protobuf.Value;
+import com.sun.net.httpserver.HttpExchange;
+import com.sun.net.httpserver.HttpServer;
+import io.grpc.Server;
+import io.grpc.ServerBuilder;
+import io.grpc.stub.StreamObserver;
+import io.pinecone.proto.QueryRequest;
+import io.pinecone.proto.QueryResponse;
+import io.pinecone.proto.ScoredVector;
+import io.pinecone.proto.UpdateRequest;
+import io.pinecone.proto.UpdateResponse;
+import io.pinecone.proto.UpsertRequest;
+import io.pinecone.proto.UpsertResponse;
+import io.pinecone.proto.VectorServiceGrpc;
+import io.pinecone.unsigned_indices_model.QueryResponseWithUnsignedIndices;
+import org.apache.camel.CamelContext;
+import org.apache.camel.Exchange;
+import org.apache.camel.InvalidPayloadException;
+import org.apache.camel.builder.RouteBuilder;
+import org.apache.camel.spi.DataType;
+import org.apache.camel.test.junit6.CamelTestSupport;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.Named;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.MethodSource;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+/**
+ * CAMEL-26216: UPSERT, UPDATE and QUERY take the vector as a List of numbers,
such as the List of Double a JSON array
+ * unmarshals to, or a float or double array, QUERY and QUERY_BY_ID return 10
matches without the CamelPineconeQueryTopK
+ * header, UPSERT without the CamelPineconeIndexId header stores the vector
under a random id, and the pinecone:rag data
+ * type transformer reads the matches of a QUERY. A fake Pinecone, its control
plane on a JDK HttpServer and the data
+ * plane of the index on a gRPC server, records the requests of the client, so
no Pinecone is needed.
+ */
+class PineconeVectorDbProducerTest extends CamelTestSupport {
+
+ private static final List<Float> VECTOR = List.of(0.1f, 0.2f, 0.3f);
+
+ // the client keeps a connection per index name for the JVM, so each test
has its own index
+ private final String indexName = "camel-" + UUID.randomUUID();
+ private final List<Object> requests = new CopyOnWriteArrayList<>();
+ private HttpServer controlPlane;
+ private Server dataPlane;
+
+ @Override
+ protected CamelContext createCamelContext() throws Exception {
+ dataPlane = ServerBuilder.forPort(0).addService(new
VectorServiceGrpc.VectorServiceImplBase() {
+ @Override
+ public void upsert(UpsertRequest request,
StreamObserver<UpsertResponse> reply) {
+ requests.add(request);
+
reply.onNext(UpsertResponse.newBuilder().setUpsertedCount(request.getVectorsCount()).build());
+ reply.onCompleted();
+ }
+
+ @Override
+ public void update(UpdateRequest request,
StreamObserver<UpdateResponse> reply) {
+ requests.add(request);
+ reply.onNext(UpdateResponse.getDefaultInstance());
+ reply.onCompleted();
+ }
+
+ @Override
+ public void query(QueryRequest request,
StreamObserver<QueryResponse> reply) {
+ requests.add(request);
+ Struct metadata = Struct.newBuilder()
+ .putFields("text_segment",
Value.newBuilder().setStringValue("hello").build())
+ .build();
+ reply.onNext(QueryResponse.newBuilder()
+
.addMatches(ScoredVector.newBuilder().setId("vector-1").setScore(0.9f).setMetadata(metadata))
+ .build());
+ reply.onCompleted();
+ }
+ }).build().start();
+ controlPlane = HttpServer.create(new InetSocketAddress("localhost",
0), 0);
+ controlPlane.createContext("/indexes/", this::describeIndex);
+ controlPlane.start();
+
+ CamelContext context = super.createCamelContext();
+ PineconeVectorDbConfiguration configuration
+ = context.getComponent(PineconeVectorDb.SCHEME,
PineconeVectorDbComponent.class).getConfiguration();
+ configuration.setToken("test");
+ configuration.setHost("http://localhost:"; +
controlPlane.getAddress().getPort());
+ configuration.setTls(false);
+ configuration.setIndexName(indexName);
+ return context;
+ }
+
+ @Override
+ protected RouteBuilder createRouteBuilder() {
+ return new RouteBuilder() {
+ @Override
+ public void configure() {
+ from("direct:rag")
+ .to("pinecone:rag?action=QUERY")
+ .transformDataType(new DataType("pinecone:rag"));
+ }
+ };
+ }
+
+ @AfterEach
+ void stopFakePinecone() {
+ controlPlane.stop(0);
+ dataPlane.shutdownNow();
+ }
+
+ static Stream<Named<Object>> vectorBodies() {
+ return Stream.of(
+ Named.of("List of Float", List.of(0.1f, 0.2f, 0.3f)),
+ Named.of("List of Double", List.of(0.1d, 0.2d, 0.3d)),
+ Named.of("List of BigDecimal", List.of(new BigDecimal("0.1"),
new BigDecimal("0.2"), new BigDecimal("0.3"))),
+ Named.of("float[]", new float[] { 0.1f, 0.2f, 0.3f }),
+ Named.of("double[]", new double[] { 0.1d, 0.2d, 0.3d }),
+ Named.of("Double[]", new Double[] { 0.1d, 0.2d, 0.3d }));
+ }
+
+ @ParameterizedTest
+ @MethodSource("vectorBodies")
+ void upsertTakesTheVector(Object body) {
+ Exchange result = fluentTemplate.to("pinecone:test?action=UPSERT")
+ .withHeader(PineconeVectorDbHeaders.INDEX_ID, "vector-1")
+ .withBody(body)
+ .request(Exchange.class);
+
+ assertNull(result.getException());
+ UpsertRequest upsert = assertInstanceOf(UpsertRequest.class,
singleRequest());
+ assertEquals("vector-1", upsert.getVectors(0).getId());
+ assertEquals(VECTOR, upsert.getVectors(0).getValuesList());
+ }
+
+ @ParameterizedTest
+ @MethodSource("vectorBodies")
+ void updateTakesTheVector(Object body) {
+ Exchange result = fluentTemplate.to("pinecone:test?action=UPDATE")
+ .withHeader(PineconeVectorDbHeaders.INDEX_ID, "vector-1")
+ .withBody(body)
+ .request(Exchange.class);
+
+ assertNull(result.getException());
+ UpdateRequest update = assertInstanceOf(UpdateRequest.class,
singleRequest());
+ assertEquals("vector-1", update.getId());
+ assertEquals(VECTOR, update.getValuesList());
+ }
+
+ @ParameterizedTest
+ @MethodSource("vectorBodies")
+ void queryTakesTheVector(Object body) {
+ Exchange result = fluentTemplate.to("pinecone:test?action=QUERY")
+ .withHeader(PineconeVectorDbHeaders.QUERY_TOP_K, 3)
+ .withBody(body)
+ .request(Exchange.class);
+
+ assertNull(result.getException());
+ QueryRequest query = assertInstanceOf(QueryRequest.class,
singleRequest());
+ assertEquals(VECTOR, query.getVectorList());
+ assertEquals(3, query.getTopK());
+ }
+
+ @Test
+ void queryWithoutTheTopKHeaderReturnsTenMatches() {
+ Exchange result = fluentTemplate.to("pinecone:test?action=QUERY")
+ .withBody(VECTOR)
+ .request(Exchange.class);
+
+ assertNull(result.getException());
+ assertInstanceOf(QueryResponseWithUnsignedIndices.class,
result.getMessage().getBody());
+ assertEquals(10, assertInstanceOf(QueryRequest.class,
singleRequest()).getTopK());
+ }
+
+ @Test
+ void queryByIdWithoutTheTopKHeaderReturnsTenMatches() {
+ Exchange result = fluentTemplate.to("pinecone:test?action=QUERY_BY_ID")
+ .withHeader(PineconeVectorDbHeaders.INDEX_ID, "vector-1")
+ .request(Exchange.class);
+
+ assertNull(result.getException());
+ QueryRequest query = assertInstanceOf(QueryRequest.class,
singleRequest());
+ assertEquals("vector-1", query.getId());
+ assertEquals(10, query.getTopK());
+ }
+
+ @Test
+ void upsertWithoutTheIdHeaderGeneratesOne() {
+ Exchange result = fluentTemplate.to("pinecone:test?action=UPSERT")
+ .withBody(VECTOR)
+ .request(Exchange.class);
+
+ assertNull(result.getException());
+ String id =
result.getMessage().getHeader(PineconeVectorDbHeaders.INDEX_ID, String.class);
+ assertNotNull(id);
+ assertEquals(id, UUID.fromString(id).toString());
+ UpsertRequest upsert = assertInstanceOf(UpsertRequest.class,
singleRequest());
+ assertEquals(id, upsert.getVectors(0).getId());
+ assertEquals(VECTOR, upsert.getVectors(0).getValuesList());
+ }
+
+ @Test
+ void theRagTransformerReadsTheMatchesOfAQuery() {
+ Object texts = fluentTemplate.to("direct:rag")
+ .withHeader(PineconeVectorDbHeaders.QUERY_TOP_K, 3)
+ .withHeader(PineconeVectorDbHeaders.QUERY_INCLUDE_METADATA,
true)
+ .withBody(VECTOR)
+ .request();
+
+ assertEquals(List.of("hello"), texts);
+ }
+
+ @Test
+ void aBodyThatIsNotAVectorFails() {
+ Exchange text = fluentTemplate.to("pinecone:test?action=UPSERT")
+ .withHeader(PineconeVectorDbHeaders.INDEX_ID, "vector-1")
+ .withBody("0.1, 0.2, 0.3")
+ .request(Exchange.class);
+ InvalidPayloadException invalidPayload =
assertInstanceOf(InvalidPayloadException.class, text.getException());
+ assertEquals(List.class, invalidPayload.getType());
+ assertTrue(invalidPayload.getMessage()
+ .contains("the vector must be a List of numbers, such as Float
or Double, a float[] or a double[]"),
+ invalidPayload.getMessage());
+
+ // the message names the type of the element, not its value
+ Exchange listOfText = fluentTemplate.to("pinecone:test?action=QUERY")
+ .withBody(List.of("secret-0.1", "0.2"))
+ .request(Exchange.class);
+ IllegalArgumentException notANumber =
assertInstanceOf(IllegalArgumentException.class, listOfText.getException());
+ assertTrue(notANumber.getMessage().startsWith("Vector element at index
0 is a java.lang.String"),
+ notANumber.getMessage());
+ assertFalse(notANumber.getMessage().contains("secret"),
notANumber.getMessage());
+
+ Exchange nullElement = fluentTemplate.to("pinecone:test?action=UPDATE")
+ .withHeader(PineconeVectorDbHeaders.INDEX_ID, "vector-1")
+ .withBody(Arrays.asList(0.1d, null, 0.3d))
+ .request(Exchange.class);
+ IllegalArgumentException isNull =
assertInstanceOf(IllegalArgumentException.class, nullElement.getException());
+ assertTrue(isNull.getMessage().startsWith("Vector element at index 1
is null"), isNull.getMessage());
+
+ assertEquals(List.of(), requests);
+ }
+
+ private Object singleRequest() {
+ assertEquals(1, requests.size(), requests::toString);
+ return requests.get(0);
+ }
+
+ private void describeIndex(HttpExchange http) throws IOException {
+ // the index, which the data plane of the fake serves
+ String json = "{\"name\":\"" + indexName +
"\",\"dimension\":3,\"metric\":\"cosine\","
+ + "\"host\":\"localhost:" + dataPlane.getPort() + "\","
+ +
"\"spec\":{\"serverless\":{\"cloud\":\"aws\",\"region\":\"us-east-1\"}},"
+ + "\"status\":{\"ready\":true,\"state\":\"Ready\"}}";
+ byte[] bytes = json.getBytes(StandardCharsets.UTF_8);
+ http.getResponseHeaders().add("Content-Type", "application/json");
+ http.sendResponseHeaders(200, bytes.length);
+ http.getResponseBody().write(bytes);
+ http.close();
+ }
+}
diff --git
a/components/camel-ai/camel-pinecone/src/test/java/org/apache/camel/component/pinecone/transform/PineconeReverseEmbeddingsDataTypeTransformerTest.java
b/components/camel-ai/camel-pinecone/src/test/java/org/apache/camel/component/pinecone/transform/PineconeReverseEmbeddingsDataTypeTransformerTest.java
index d97516e489cb..e4b5dd850174 100644
---
a/components/camel-ai/camel-pinecone/src/test/java/org/apache/camel/component/pinecone/transform/PineconeReverseEmbeddingsDataTypeTransformerTest.java
+++
b/components/camel-ai/camel-pinecone/src/test/java/org/apache/camel/component/pinecone/transform/PineconeReverseEmbeddingsDataTypeTransformerTest.java
@@ -43,9 +43,10 @@ class PineconeReverseEmbeddingsDataTypeTransformerTest {
@Test
void theBodyTypesSayTheBody() throws Exception {
- // CAMEL-25928: the matches of a query, or null, as transformerIn
says, and the List of String transformerOut says
+ // CAMEL-25928: the response of a query or its matches, or null, as
transformerIn says, and the List of String
+ // transformerOut says
BodyTypes bodyTypes =
PineconeReverseEmbeddingsDataTypeTransformer.class.getAnnotation(BodyTypes.class);
- assertArrayEquals(new Carrier[] { Carrier.LIST, Carrier.NULL },
bodyTypes.transformerIn().value());
+ assertArrayEquals(new Carrier[] { Carrier.POJO, Carrier.LIST,
Carrier.NULL }, bodyTypes.transformerIn().value());
assertArrayEquals(new Carrier[] { Carrier.LIST },
bodyTypes.transformerOut().value());
assertEquals("java.util.List<java.lang.String>",
bodyTypes.transformerOut().javaType());
@@ -55,12 +56,14 @@ class PineconeReverseEmbeddingsDataTypeTransformerTest {
// a null body gives an empty List
assertEquals(List.of(), transform(null));
- // the QueryResponseWithUnsignedIndices a QUERY leaves is not a List:
the transformer gives an empty List
+ // CAMEL-26216: the QueryResponseWithUnsignedIndices a QUERY or
QUERY_BY_ID leaves gives the texts of its matches,
+ // as its matches do
QueryResponseWithUnsignedIndices response = new
QueryResponseWithUnsignedIndices(
- QueryResponse.newBuilder().addMatches(scoredVector("1",
"hello")).build());
- assertEquals(List.of(), transform(response));
- // its matches are the body the transformer needs
- assertEquals(List.of("hello"), transform(response.getMatchesList()));
+ QueryResponse.newBuilder().addMatches(scoredVector("1",
"hello")).addMatches(scoredVector("2", null)).build());
+ assertEquals(List.of("hello", ""), transform(response));
+ assertEquals(List.of("hello", ""),
transform(response.getMatchesList()));
+ // a response without matches gives an empty List
+ assertEquals(List.of(), transform(new
QueryResponseWithUnsignedIndices(QueryResponse.getDefaultInstance())));
}
private Object transform(Object body) throws Exception {
diff --git
a/docs/user-manual/modules/ROOT/pages/camel-4x-upgrade-guide-4_24.adoc
b/docs/user-manual/modules/ROOT/pages/camel-4x-upgrade-guide-4_24.adoc
index f23049e8871e..f7bfdc3326ae 100644
--- a/docs/user-manual/modules/ROOT/pages/camel-4x-upgrade-guide-4_24.adoc
+++ b/docs/user-manual/modules/ROOT/pages/camel-4x-upgrade-guide-4_24.adoc
@@ -5582,6 +5582,14 @@ other operations do. Previously they failed without the
headers.
* A `java.util.Date` in the `CamelAwsCwMetricTimestamp` header is now used.
Previously it was ignored. The catalog
lists the header as an `Instant`.
+=== camel-pinecone - default topK, generated UPSERT id and the pinecone:rag
transformer
+
+QUERY and QUERY_BY_ID without the `CamelPineconeQueryTopK` header now return
10 matches, as camel-weaviate does. Before, they failed with a
`NullPointerException`.
+
+UPSERT without the `CamelPineconeIndexId` header now stores the vector under a
random UUID, as the `pinecone:embeddings` data type transformer does, and sets
the `CamelPineconeIndexId` header to that id. Before, it failed with a
`PineconeValidationException`.
+
+The `pinecone:rag` data type transformer now reads the
`QueryResponseWithUnsignedIndices` body that a QUERY or QUERY_BY_ID leaves, and
returns the texts of its matches. Before, it returned an empty List for that
body. A route that passes the `matchesList` of the response to the transformer
gets the same result as before.
+
=== camel-aws2-lambda - the code of createFunction and updateFunction, and the
payload of invokeFunction
* createFunction and updateFunction: the `CamelAwsLambdaZipFile` header, or
the S3 location of the
diff --git
a/dsl/camel-endpointdsl/src/generated/java/org/apache/camel/builder/endpoint/dsl/PineconeVectorDbEndpointBuilderFactory.java
b/dsl/camel-endpointdsl/src/generated/java/org/apache/camel/builder/endpoint/dsl/PineconeVectorDbEndpointBuilderFactory.java
index 09af9ee571a1..4c64d5d39b6f 100644
---
a/dsl/camel-endpointdsl/src/generated/java/org/apache/camel/builder/endpoint/dsl/PineconeVectorDbEndpointBuilderFactory.java
+++
b/dsl/camel-endpointdsl/src/generated/java/org/apache/camel/builder/endpoint/dsl/PineconeVectorDbEndpointBuilderFactory.java
@@ -525,7 +525,8 @@ public interface PineconeVectorDbEndpointBuilderFactory {
return "CamelPineconeCollectionCloudRegion";
}
/**
- * Index Upsert Id.
+ * The id of the vector for UPSERT, UPDATE and QUERY_BY_ID. Without it,
+ * UPSERT generates a random UUID and sets this header to it.
*
* The option is a: {@code String} type.
*
@@ -537,10 +538,11 @@ public interface PineconeVectorDbEndpointBuilderFactory {
return "CamelPineconeIndexId";
}
/**
- * Query Top K.
+ * The number of matches QUERY and QUERY_BY_ID return.
*
* The option is a: {@code Integer} type.
*
+ * Default: 10
* Group: producer
*
* @return the name of the header {@code PineconeQueryTopK}.
@@ -563,7 +565,7 @@ public interface PineconeVectorDbEndpointBuilderFactory {
/**
* Query Filter.
*
- * The option is a: {@code String} type.
+ * The option is a: {@code com.google.protobuf.Struct} type.
*
* Group: producer
*
@@ -587,7 +589,7 @@ public interface PineconeVectorDbEndpointBuilderFactory {
/**
* Query Include Metadata.
*
- * The option is a: {@code com.google.protobuf.Struct} type.
+ * The option is a: {@code boolean} type.
*
* Group: producer
*
diff --git
a/test-infra/camel-test-infra-pinecone/src/main/java/org/apache/camel/test/infra/pinecone/services/PineconeLocalContainerInfraService.java
b/test-infra/camel-test-infra-pinecone/src/main/java/org/apache/camel/test/infra/pinecone/services/PineconeLocalContainerInfraService.java
index de2e9b20c4ef..fb449f35c606 100644
---
a/test-infra/camel-test-infra-pinecone/src/main/java/org/apache/camel/test/infra/pinecone/services/PineconeLocalContainerInfraService.java
+++
b/test-infra/camel-test-infra-pinecone/src/main/java/org/apache/camel/test/infra/pinecone/services/PineconeLocalContainerInfraService.java
@@ -28,6 +28,7 @@ import
org.apache.camel.test.infra.common.services.ContainerService;
import org.apache.camel.test.infra.pinecone.common.PineconeProperties;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.testcontainers.containers.wait.strategy.Wait;
import org.testcontainers.pinecone.PineconeLocalContainer;
import org.testcontainers.utility.DockerImageName;
@@ -38,6 +39,13 @@ public class PineconeLocalContainerInfraService implements
PineconeInfraService,
private static final Logger LOG =
LoggerFactory.getLogger(PineconeLocalContainerInfraService.class);
+ // the control plane, and the ports pinecone-local serves the indexes it
creates on, one port each. They are bound
+ // to the same host ports, as pinecone-local gives each index host as
localhost:<port>, so a container serves at
+ // most five indexes: a test that creates more needs a wider range
+ private static final int CONTROL_PLANE_PORT = 5080;
+ private static final int FIRST_INDEX_PORT = 5081;
+ private static final int LAST_INDEX_PORT = 5085;
+
private final PineconeLocalContainer container;
public PineconeLocalContainerInfraService() {
@@ -59,9 +67,18 @@ public class PineconeLocalContainerInfraService implements
PineconeInfraService,
super(DockerImageName.parse(imageName)
.asCompatibleSubstituteFor("pinecone-io/pinecone-local"));
+ // wait for the control plane only, not for the index ports
bound below
+ waitingFor(Wait.forListeningPorts(CONTROL_PLANE_PORT));
withStartupTimeout(Duration.ofMinutes(3L));
- ContainerEnvironmentUtil.configurePort(this, fixedPort, 5080);
+ ContainerEnvironmentUtil.configurePort(this, fixedPort,
CONTROL_PLANE_PORT);
+
+ // the control plane gives each index as localhost:<port>,
which the Pinecone client connects to for
+ // the data plane (upsert, query, fetch), so the index ports
are bound to the same ports of the host
+ withEnv("PINECONE_HOST", "localhost");
+ for (int port = FIRST_INDEX_PORT; port <= LAST_INDEX_PORT;
port++) {
+ addFixedExposedPort(port, port);
+ }
}
}
