This is an automated email from the ASF dual-hosted git repository.
davsclaus pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/camel.git
The following commit(s) were added to refs/heads/main by this push:
new de5cdb96825c CAMEL-25777: camel-core - the tokenizer EIP works in XML
and YAML routes, as each tokenizer names itself (#28085)
de5cdb96825c is described below
commit de5cdb96825c4754aed3beaebe6205fa15f1aebf
Author: Claus Ibsen <[email protected]>
AuthorDate: Mon Oct 19 21:24:58 2026 +0200
CAMEL-25777: camel-core - the tokenizer EIP works in XML and YAML routes,
as each tokenizer names itself (#28085)
Co-authored-by: Claude Opus 5.5 (1M context) <[email protected]>
---
.../LangChain4JTokenizerDefinitionTest.java | 79 ++++++++++++++++++++++
.../apache/camel/model/TokenizerDefinition.java | 6 +-
.../LangChain4jCharacterTokenizerDefinition.java | 2 +
.../LangChain4jLineTokenizerDefinition.java | 2 +
.../LangChain4jParagraphTokenizerDefinition.java | 2 +
.../LangChain4jSentenceTokenizerDefinition.java | 2 +
.../LangChain4jWordTokenizerDefinition.java | 2 +
7 files changed, 94 insertions(+), 1 deletion(-)
diff --git
a/components/camel-ai/camel-langchain4j-tokenizer/src/test/java/org/apache/camel/component/langchain4j/tokenizer/LangChain4JTokenizerDefinitionTest.java
b/components/camel-ai/camel-langchain4j-tokenizer/src/test/java/org/apache/camel/component/langchain4j/tokenizer/LangChain4JTokenizerDefinitionTest.java
new file mode 100644
index 000000000000..6025cb4e578a
--- /dev/null
+++
b/components/camel-ai/camel-langchain4j-tokenizer/src/test/java/org/apache/camel/component/langchain4j/tokenizer/LangChain4JTokenizerDefinitionTest.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.camel.component.langchain4j.tokenizer;
+
+import org.apache.camel.RoutesBuilder;
+import org.apache.camel.builder.RouteBuilder;
+import org.apache.camel.component.mock.MockEndpoint;
+import org.apache.camel.model.RouteDefinition;
+import org.apache.camel.model.TokenizerDefinition;
+import
org.apache.camel.model.tokenizer.LangChain4jCharacterTokenizerDefinition;
+import org.apache.camel.model.tokenizer.LangChain4jLineTokenizerDefinition;
+import
org.apache.camel.model.tokenizer.LangChain4jParagraphTokenizerDefinition;
+import org.apache.camel.model.tokenizer.LangChain4jSentenceTokenizerDefinition;
+import org.apache.camel.model.tokenizer.LangChain4jWordTokenizerDefinition;
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+/**
+ * CAMEL-25777: the XML and YAML DSL create the tokenizer with the no-arg
constructors and set its implementation, but
+ * not the tokenizer name that the Java DSL builder sets, so their routes
failed to start. The route here is built the
+ * same way, from the model (a component cannot depend on the YAML DSL).
+ */
+public class LangChain4JTokenizerDefinitionTest extends
LangChain4JTokenizerTestSupport {
+
+ @Test
+ public void testTokenizerAsTheXmlAndYamlDslBuildIt() throws
InterruptedException {
+ MockEndpoint mock = getMockEndpoint("mock:result");
+ mock.expectedMessageCount(4);
+
+ template.sendBody("direct:start", TEXT);
+
+ mock.assertIsSatisfied();
+ }
+
+ @Test
+ public void testEveryTokenizerHasItsName() {
+ // the names of the tokenizer services, which the Java DSL builder
gives them too
+ assertEquals("langChain4jCharacterTokenizer", new
LangChain4jCharacterTokenizerDefinition().tokenizerName());
+ assertEquals("langChain4jLineTokenizer", new
LangChain4jLineTokenizerDefinition().tokenizerName());
+ assertEquals("langChain4jParagraphTokenizer", new
LangChain4jParagraphTokenizerDefinition().tokenizerName());
+ assertEquals("langChain4jSentenceTokenizer", new
LangChain4jSentenceTokenizerDefinition().tokenizerName());
+ assertEquals("langChain4jWordTokenizer", new
LangChain4jWordTokenizerDefinition().tokenizerName());
+ }
+
+ @Override
+ protected RoutesBuilder createRouteBuilder() {
+ return new RouteBuilder() {
+ @Override
+ public void configure() {
+ // what the XML and YAML DSL parse
<tokenizer><langChain4jParagraphTokenizer .../></tokenizer> into
+ LangChain4jParagraphTokenizerDefinition paragraph = new
LangChain4jParagraphTokenizerDefinition();
+ paragraph.setMaxTokens("1024");
+ paragraph.setMaxOverlap("10");
+ paragraph.setTokenizerType("OPEN_AI");
+ TokenizerDefinition tokenizer = new TokenizerDefinition();
+ tokenizer.setTokenizerImplementation(paragraph);
+
+ RouteDefinition route = from("direct:start");
+ route.addOutput(tokenizer);
+ route.split().body().to("mock:result");
+ }
+ };
+ }
+}
diff --git
a/core/camel-core-model/src/main/java/org/apache/camel/model/TokenizerDefinition.java
b/core/camel-core-model/src/main/java/org/apache/camel/model/TokenizerDefinition.java
index 1994e9e521c1..c94a2cd2df23 100644
---
a/core/camel-core-model/src/main/java/org/apache/camel/model/TokenizerDefinition.java
+++
b/core/camel-core-model/src/main/java/org/apache/camel/model/TokenizerDefinition.java
@@ -66,9 +66,13 @@ public class TokenizerDefinition extends
NoOutputDefinition<TokenizerDefinition>
}
/**
- * Gets the tokenizer name
+ * Gets the tokenizer name, which is the name of the tokenizer
implementation when not set: the XML and YAML DSL set
+ * only the implementation (CAMEL-25777)
*/
public String tokenizerName() {
+ if (tokenizerName == null && tokenizerImplementation != null) {
+ return tokenizerImplementation.tokenizerName();
+ }
return tokenizerName;
}
diff --git
a/core/camel-core-model/src/main/java/org/apache/camel/model/tokenizer/LangChain4jCharacterTokenizerDefinition.java
b/core/camel-core-model/src/main/java/org/apache/camel/model/tokenizer/LangChain4jCharacterTokenizerDefinition.java
index 012fd9672920..ea30e473a35d 100644
---
a/core/camel-core-model/src/main/java/org/apache/camel/model/tokenizer/LangChain4jCharacterTokenizerDefinition.java
+++
b/core/camel-core-model/src/main/java/org/apache/camel/model/tokenizer/LangChain4jCharacterTokenizerDefinition.java
@@ -33,6 +33,8 @@ import org.apache.camel.spi.Metadata;
public class LangChain4jCharacterTokenizerDefinition extends
LangChain4jTokenizerDefinition {
public LangChain4jCharacterTokenizerDefinition() {
+ // the name the XML and YAML DSL do not set, as the builder does
(CAMEL-25777)
+ setTokenizerName(toName("CharacterTokenizer"));
}
public
LangChain4jCharacterTokenizerDefinition(LangChain4jTokenizerDefinition source) {
diff --git
a/core/camel-core-model/src/main/java/org/apache/camel/model/tokenizer/LangChain4jLineTokenizerDefinition.java
b/core/camel-core-model/src/main/java/org/apache/camel/model/tokenizer/LangChain4jLineTokenizerDefinition.java
index 11833a62e8a8..2d8698560b3a 100644
---
a/core/camel-core-model/src/main/java/org/apache/camel/model/tokenizer/LangChain4jLineTokenizerDefinition.java
+++
b/core/camel-core-model/src/main/java/org/apache/camel/model/tokenizer/LangChain4jLineTokenizerDefinition.java
@@ -33,6 +33,8 @@ import org.apache.camel.spi.Metadata;
public class LangChain4jLineTokenizerDefinition extends
LangChain4jTokenizerDefinition {
public LangChain4jLineTokenizerDefinition() {
+ // the name the XML and YAML DSL do not set, as the builder does
(CAMEL-25777)
+ setTokenizerName(toName("LineTokenizer"));
}
public LangChain4jLineTokenizerDefinition(LangChain4jTokenizerDefinition
source) {
diff --git
a/core/camel-core-model/src/main/java/org/apache/camel/model/tokenizer/LangChain4jParagraphTokenizerDefinition.java
b/core/camel-core-model/src/main/java/org/apache/camel/model/tokenizer/LangChain4jParagraphTokenizerDefinition.java
index 2a0a864a600f..d91e9547d8c0 100644
---
a/core/camel-core-model/src/main/java/org/apache/camel/model/tokenizer/LangChain4jParagraphTokenizerDefinition.java
+++
b/core/camel-core-model/src/main/java/org/apache/camel/model/tokenizer/LangChain4jParagraphTokenizerDefinition.java
@@ -33,6 +33,8 @@ import org.apache.camel.spi.Metadata;
public class LangChain4jParagraphTokenizerDefinition extends
LangChain4jTokenizerDefinition {
public LangChain4jParagraphTokenizerDefinition() {
+ // the name the XML and YAML DSL do not set, as the builder does
(CAMEL-25777)
+ setTokenizerName(toName("ParagraphTokenizer"));
}
public
LangChain4jParagraphTokenizerDefinition(LangChain4jTokenizerDefinition source) {
diff --git
a/core/camel-core-model/src/main/java/org/apache/camel/model/tokenizer/LangChain4jSentenceTokenizerDefinition.java
b/core/camel-core-model/src/main/java/org/apache/camel/model/tokenizer/LangChain4jSentenceTokenizerDefinition.java
index 8e94a09f4adf..8c4acc34bdc3 100644
---
a/core/camel-core-model/src/main/java/org/apache/camel/model/tokenizer/LangChain4jSentenceTokenizerDefinition.java
+++
b/core/camel-core-model/src/main/java/org/apache/camel/model/tokenizer/LangChain4jSentenceTokenizerDefinition.java
@@ -33,6 +33,8 @@ import org.apache.camel.spi.Metadata;
public class LangChain4jSentenceTokenizerDefinition extends
LangChain4jTokenizerDefinition {
public LangChain4jSentenceTokenizerDefinition() {
+ // the name the XML and YAML DSL do not set, as the builder does
(CAMEL-25777)
+ setTokenizerName(toName("SentenceTokenizer"));
}
public
LangChain4jSentenceTokenizerDefinition(LangChain4jTokenizerDefinition source) {
diff --git
a/core/camel-core-model/src/main/java/org/apache/camel/model/tokenizer/LangChain4jWordTokenizerDefinition.java
b/core/camel-core-model/src/main/java/org/apache/camel/model/tokenizer/LangChain4jWordTokenizerDefinition.java
index 7029eb5f59c5..d30ddf15fe3c 100644
---
a/core/camel-core-model/src/main/java/org/apache/camel/model/tokenizer/LangChain4jWordTokenizerDefinition.java
+++
b/core/camel-core-model/src/main/java/org/apache/camel/model/tokenizer/LangChain4jWordTokenizerDefinition.java
@@ -33,6 +33,8 @@ import org.apache.camel.spi.Metadata;
public class LangChain4jWordTokenizerDefinition extends
LangChain4jTokenizerDefinition {
public LangChain4jWordTokenizerDefinition() {
+ // the name the XML and YAML DSL do not set, as the builder does
(CAMEL-25777)
+ setTokenizerName(toName("WordTokenizer"));
}
public LangChain4jWordTokenizerDefinition(LangChain4jTokenizerDefinition
source) {
