Author: davsclaus
Date: Tue May 1 13:38:27 2012
New Revision: 1332649
URL: http://svn.apache.org/viewvc?rev=1332649&view=rev
Log:
CAMEL-5326: Added group option to tokenizer to group N parts together as a
combined message. For example to split large files by N line numbers.
Added:
camel/trunk/camel-core/src/main/java/org/apache/camel/util/GroupIterator.java
camel/trunk/camel-core/src/test/java/org/apache/camel/processor/SplitGroupMultiLinesTest.java
camel/trunk/camel-core/src/test/java/org/apache/camel/util/GroupIteratorTest.java
camel/trunk/components/camel-spring/src/test/java/org/apache/camel/spring/processor/SpringSplitGroupMultiLinesTest.java
- copied, changed from r1332534,
camel/trunk/components/camel-spring/src/test/java/org/apache/camel/spring/processor/SpringAOPAfterTest.java
camel/trunk/components/camel-spring/src/test/resources/org/apache/camel/spring/processor/SplitGroupMultiLinesTest.xml
- copied, changed from r1332534,
camel/trunk/components/camel-spring/src/test/resources/org/apache/camel/spring/processor/aopafter.xml
Modified:
camel/trunk/camel-core/src/main/java/org/apache/camel/builder/ExpressionBuilder.java
camel/trunk/camel-core/src/main/java/org/apache/camel/builder/ExpressionClause.java
camel/trunk/camel-core/src/main/java/org/apache/camel/builder/ExpressionClauseSupport.java
camel/trunk/camel-core/src/main/java/org/apache/camel/language/tokenizer/TokenizeLanguage.java
camel/trunk/camel-core/src/main/java/org/apache/camel/model/language/TokenizerExpression.java
Modified:
camel/trunk/camel-core/src/main/java/org/apache/camel/builder/ExpressionBuilder.java
URL:
http://svn.apache.org/viewvc/camel/trunk/camel-core/src/main/java/org/apache/camel/builder/ExpressionBuilder.java?rev=1332649&r1=1332648&r2=1332649&view=diff
==============================================================================
---
camel/trunk/camel-core/src/main/java/org/apache/camel/builder/ExpressionBuilder.java
(original)
+++
camel/trunk/camel-core/src/main/java/org/apache/camel/builder/ExpressionBuilder.java
Tue May 1 13:38:27 2012
@@ -23,6 +23,7 @@ import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.Date;
+import java.util.Iterator;
import java.util.List;
import java.util.Scanner;
import java.util.regex.Pattern;
@@ -45,6 +46,7 @@ import org.apache.camel.support.TokenPai
import org.apache.camel.support.TokenXMLPairExpressionIterator;
import org.apache.camel.util.ExchangeHelper;
import org.apache.camel.util.FileUtil;
+import org.apache.camel.util.GroupIterator;
import org.apache.camel.util.IOHelper;
import org.apache.camel.util.ObjectHelper;
import org.apache.camel.util.OgnlHelper;
@@ -1113,6 +1115,22 @@ public final class ExpressionBuilder {
};
}
+ public static Expression groupIteratorExpression(final Expression
expression, final String token, final int group) {
+ return new ExpressionAdapter() {
+ public Object evaluate(Exchange exchange) {
+ // evaluate expression as iterator
+ Iterator it = expression.evaluate(exchange, Iterator.class);
+ ObjectHelper.notNull(it, "expression: " + expression + "
evaluated on " + exchange + " must return an java.util.Iterator");
+ return new GroupIterator(exchange.getContext(), it, token,
group);
+ }
+
+ @Override
+ public String toString() {
+ return "group " + expression + " " + group + " times";
+ }
+ };
+ }
+
/**
* Returns a sort expression which will sort the expression with the given
comparator.
* <p/>
Modified:
camel/trunk/camel-core/src/main/java/org/apache/camel/builder/ExpressionClause.java
URL:
http://svn.apache.org/viewvc/camel/trunk/camel-core/src/main/java/org/apache/camel/builder/ExpressionClause.java?rev=1332649&r1=1332648&r2=1332649&view=diff
==============================================================================
---
camel/trunk/camel-core/src/main/java/org/apache/camel/builder/ExpressionClause.java
(original)
+++
camel/trunk/camel-core/src/main/java/org/apache/camel/builder/ExpressionClause.java
Tue May 1 13:38:27 2012
@@ -419,6 +419,29 @@ public class ExpressionClause<T> extends
}
/**
+ * Evaluates a token expression on the message body
+ *
+ * @param token the token
+ * @param regex whether the token is a regular expression or not
+ * @param group to group by the given number
+ * @return the builder to continue processing the DSL
+ */
+ public T tokenize(String token, boolean regex, int group) {
+ return delegate.tokenize(token, regex);
+ }
+
+ /**
+ * Evaluates a token expression on the message body
+ *
+ * @param token the token
+ * @param group to group by the given number
+ * @return the builder to continue processing the DSL
+ */
+ public T tokenize(String token, int group) {
+ return delegate.tokenize(token, group);
+ }
+
+ /**
* Evaluates a token expression on the given header
*
* @param token the token
Modified:
camel/trunk/camel-core/src/main/java/org/apache/camel/builder/ExpressionClauseSupport.java
URL:
http://svn.apache.org/viewvc/camel/trunk/camel-core/src/main/java/org/apache/camel/builder/ExpressionClauseSupport.java?rev=1332649&r1=1332648&r2=1332649&view=diff
==============================================================================
---
camel/trunk/camel-core/src/main/java/org/apache/camel/builder/ExpressionClauseSupport.java
(original)
+++
camel/trunk/camel-core/src/main/java/org/apache/camel/builder/ExpressionClauseSupport.java
Tue May 1 13:38:27 2012
@@ -448,6 +448,17 @@ public class ExpressionClauseSupport<T>
* Evaluates a token expression on the message body
*
* @param token the token
+ * @param group to group by the given number
+ * @return the builder to continue processing the DSL
+ */
+ public T tokenize(String token, int group) {
+ return tokenize(token, null, false, group);
+ }
+
+ /**
+ * Evaluates a token expression on the message body
+ *
+ * @param token the token
* @param regex whether the token is a regular expression or not
* @return the builder to continue processing the DSL
*/
@@ -456,6 +467,18 @@ public class ExpressionClauseSupport<T>
}
/**
+ * Evaluates a token expression on the message body
+ *
+ * @param token the token
+ * @param regex whether the token is a regular expression or not
+ * @param group to group by the given number
+ * @return the builder to continue processing the DSL
+ */
+ public T tokenize(String token, boolean regex, int group) {
+ return tokenize(token, null, regex, group);
+ }
+
+ /**
* Evaluates a token expression on the given header
*
* @param token the token
@@ -484,6 +507,25 @@ public class ExpressionClauseSupport<T>
}
/**
+ * Evaluates a token expression on the given header
+ *
+ * @param token the token
+ * @param headerName name of header to tokenize
+ * @param regex whether the token is a regular expression or not
+ * @param group to group by number of parts
+ * @return the builder to continue processing the DSL
+ */
+ public T tokenize(String token, String headerName, boolean regex, int
group) {
+ TokenizerExpression expression = new TokenizerExpression();
+ expression.setToken(token);
+ expression.setHeaderName(headerName);
+ expression.setRegex(regex);
+ expression.setGroup(group);
+ setExpressionType(expression);
+ return result;
+ }
+
+ /**
* Evaluates a token pair expression on the message body
*
* @param startToken the start token
Modified:
camel/trunk/camel-core/src/main/java/org/apache/camel/language/tokenizer/TokenizeLanguage.java
URL:
http://svn.apache.org/viewvc/camel/trunk/camel-core/src/main/java/org/apache/camel/language/tokenizer/TokenizeLanguage.java?rev=1332649&r1=1332648&r2=1332649&view=diff
==============================================================================
---
camel/trunk/camel-core/src/main/java/org/apache/camel/language/tokenizer/TokenizeLanguage.java
(original)
+++
camel/trunk/camel-core/src/main/java/org/apache/camel/language/tokenizer/TokenizeLanguage.java
Tue May 1 13:38:27 2012
@@ -46,6 +46,7 @@ public class TokenizeLanguage implements
private boolean regex;
private boolean xml;
private boolean includeTokens;
+ private int group;
public static Expression tokenize(String token) {
return tokenize(token, false);
@@ -104,12 +105,20 @@ public class TokenizeLanguage implements
}
// use the regular tokenizer
+ Expression answer;
Expression exp = headerName == null ?
ExpressionBuilder.bodyExpression() :
ExpressionBuilder.headerExpression(headerName);
if (regex) {
- return ExpressionBuilder.regexTokenizeExpression(exp, token);
+ answer = ExpressionBuilder.regexTokenizeExpression(exp, token);
} else {
- return ExpressionBuilder.tokenizeExpression(exp, token);
+ answer = ExpressionBuilder.tokenizeExpression(exp, token);
}
+
+ // if group then wrap answer in group expression
+ if (group > 0) {
+ answer = ExpressionBuilder.groupIteratorExpression(answer, token,
group);
+ }
+
+ return answer;
}
public Expression createExpression(String expression) {
@@ -175,6 +184,14 @@ public class TokenizeLanguage implements
this.includeTokens = includeTokens;
}
+ public int getGroup() {
+ return group;
+ }
+
+ public void setGroup(int group) {
+ this.group = group;
+ }
+
public boolean isSingleton() {
return false;
}
Modified:
camel/trunk/camel-core/src/main/java/org/apache/camel/model/language/TokenizerExpression.java
URL:
http://svn.apache.org/viewvc/camel/trunk/camel-core/src/main/java/org/apache/camel/model/language/TokenizerExpression.java?rev=1332649&r1=1332648&r2=1332649&view=diff
==============================================================================
---
camel/trunk/camel-core/src/main/java/org/apache/camel/model/language/TokenizerExpression.java
(original)
+++
camel/trunk/camel-core/src/main/java/org/apache/camel/model/language/TokenizerExpression.java
Tue May 1 13:38:27 2012
@@ -49,6 +49,8 @@ public class TokenizerExpression extends
private Boolean xml;
@XmlAttribute
private Boolean includeTokens;
+ @XmlAttribute
+ private Integer group;
public TokenizerExpression() {
}
@@ -114,8 +116,21 @@ public class TokenizerExpression extends
this.includeTokens = includeTokens;
}
+ public Integer getGroup() {
+ return group;
+ }
+
+ public void setGroup(Integer group) {
+ this.group = group;
+ }
+
@Override
public Expression createExpression(CamelContext camelContext) {
+ // special for new line tokens, if defined from XML then its 2
characters, so we replace that back to a single char
+ if (token.startsWith("\\n")) {
+ token = '\n' + token.substring(2);
+ }
+
TokenizeLanguage language = new TokenizeLanguage();
language.setToken(token);
language.setEndToken(endToken);
@@ -130,6 +145,12 @@ public class TokenizerExpression extends
if (includeTokens != null) {
language.setIncludeTokens(includeTokens);
}
+ if (group != null) {
+ if (group <= 0) {
+ throw new IllegalArgumentException("Group must be a positive
number, was: " + group);
+ }
+ language.setGroup(group);
+ }
return language.createExpression();
}
Added:
camel/trunk/camel-core/src/main/java/org/apache/camel/util/GroupIterator.java
URL:
http://svn.apache.org/viewvc/camel/trunk/camel-core/src/main/java/org/apache/camel/util/GroupIterator.java?rev=1332649&view=auto
==============================================================================
---
camel/trunk/camel-core/src/main/java/org/apache/camel/util/GroupIterator.java
(added)
+++
camel/trunk/camel-core/src/main/java/org/apache/camel/util/GroupIterator.java
Tue May 1 13:38:27 2012
@@ -0,0 +1,142 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.camel.util;
+
+import java.io.ByteArrayOutputStream;
+import java.io.Closeable;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Iterator;
+import java.util.Scanner;
+
+import org.apache.camel.CamelContext;
+import org.apache.camel.NoTypeConversionAvailableException;
+
+/**
+ * Group based {@link Iterator} which groups the given {@link Iterator} a
number of times
+ * and then return a combined response as a String.
+ * <p/>
+ * This implementation uses as internal byte array buffer, to combine the
response.
+ * The token is inserted between the individual parts.
+ * <p/>
+ * For example if you group by new line, then a new line token is inserted
between the lines.
+ */
+public final class GroupIterator implements Iterator, Closeable {
+
+ private final CamelContext camelContext;
+ private final Iterator it;
+ private final String token;
+ private final int group;
+ private boolean closed;
+ private final ByteArrayOutputStream bos = new ByteArrayOutputStream();
+
+ /**
+ * Creates a new group iterator
+ *
+ * @param camelContext the camel context
+ * @param it the iterator to group
+ * @param token then token used to separate between the parts, use
<tt>null</tt> to not add the token
+ * @param group number of parts to group together
+ * @throws IllegalArgumentException is thrown if group is not a positive
number
+ */
+ public GroupIterator(CamelContext camelContext, Iterator it, String token,
int group) {
+ this.camelContext = camelContext;
+ this.it = it;
+ this.token = token;
+ this.group = group;
+ if (group <= 0) {
+ throw new IllegalArgumentException("Group must be a positive
number, was: " + group);
+ }
+ }
+
+ @Override
+ public void close() throws IOException {
+ if (it instanceof Closeable) {
+ IOHelper.close((Closeable) it);
+ } else if (it instanceof Scanner) {
+ // special for Scanner as it does not implement Closeable
+ ((Scanner) it).close();
+ }
+ // close the buffer as well
+ bos.close();
+ // we are now closed
+ closed = true;
+ }
+
+ @Override
+ public boolean hasNext() {
+ if (closed) {
+ return false;
+ }
+
+ boolean answer = it.hasNext();
+ if (!answer) {
+ // auto close
+ try {
+ close();
+ } catch (IOException e) {
+ // ignore
+ }
+ }
+ return answer;
+ }
+
+ @Override
+ public Object next() {
+ try {
+ return doNext();
+ } catch (Exception e) {
+ throw ObjectHelper.wrapRuntimeCamelException(e);
+ }
+ }
+
+ private Object doNext() throws IOException,
NoTypeConversionAvailableException {
+ int count = 0;
+ Object data = "";
+ while (count < group && it.hasNext()) {
+ data = it.next();
+
+ // include token in between
+ if (data != null && count > 0 && token != null) {
+ bos.write(token.getBytes());
+ }
+ if (data instanceof InputStream) {
+ InputStream is = (InputStream) data;
+ IOHelper.copy(is, bos);
+ } else if (data instanceof byte[]) {
+ byte[] bytes = (byte[]) data;
+ bos.write(bytes);
+ } else if (data != null) {
+ // convert to input stream
+ InputStream is =
camelContext.getTypeConverter().mandatoryConvertTo(InputStream.class, data);
+ IOHelper.copy(is, bos);
+ }
+
+ count++;
+ }
+
+ // prepare and return answer as String
+ String answer = bos.toString();
+ bos.reset();
+ return answer;
+ }
+
+ @Override
+ public void remove() {
+ it.remove();
+ }
+}
Added:
camel/trunk/camel-core/src/test/java/org/apache/camel/processor/SplitGroupMultiLinesTest.java
URL:
http://svn.apache.org/viewvc/camel/trunk/camel-core/src/test/java/org/apache/camel/processor/SplitGroupMultiLinesTest.java?rev=1332649&view=auto
==============================================================================
---
camel/trunk/camel-core/src/test/java/org/apache/camel/processor/SplitGroupMultiLinesTest.java
(added)
+++
camel/trunk/camel-core/src/test/java/org/apache/camel/processor/SplitGroupMultiLinesTest.java
Tue May 1 13:38:27 2012
@@ -0,0 +1,50 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.camel.processor;
+
+import org.apache.camel.ContextTestSupport;
+import org.apache.camel.builder.RouteBuilder;
+
+/**
+ *
+ */
+public class SplitGroupMultiLinesTest extends ContextTestSupport {
+
+ public void testSplitMultiLines() throws Exception {
+ getMockEndpoint("mock:group").expectedBodiesReceived("ABC\nDEF\nGHI",
"JKL\nMN");
+
+ template.sendBody("direct:start", "ABC\nDEF\nGHI\nJKL\nMN");
+
+ assertMockEndpointsSatisfied();
+ }
+
+ @Override
+ protected RouteBuilder createRouteBuilder() throws Exception {
+ return new RouteBuilder() {
+ @Override
+ public void configure() throws Exception {
+ // START SNIPPET: e1
+ from("direct:start")
+ // split by new line and group by 3
+ .split().tokenize("\n", 3).streaming()
+ .to("mock:group");
+ // END SNIPPET: e1
+ }
+ };
+ }
+
+}
Added:
camel/trunk/camel-core/src/test/java/org/apache/camel/util/GroupIteratorTest.java
URL:
http://svn.apache.org/viewvc/camel/trunk/camel-core/src/test/java/org/apache/camel/util/GroupIteratorTest.java?rev=1332649&view=auto
==============================================================================
---
camel/trunk/camel-core/src/test/java/org/apache/camel/util/GroupIteratorTest.java
(added)
+++
camel/trunk/camel-core/src/test/java/org/apache/camel/util/GroupIteratorTest.java
Tue May 1 13:38:27 2012
@@ -0,0 +1,59 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.camel.util;
+
+import java.util.Scanner;
+
+import org.apache.camel.CamelContext;
+import org.apache.camel.TestSupport;
+import org.apache.camel.impl.DefaultCamelContext;
+
+/**
+ *
+ */
+public class GroupIteratorTest extends TestSupport {
+
+ private CamelContext context;
+
+ @Override
+ public void setUp() throws Exception {
+ super.setUp();
+ context = new DefaultCamelContext();
+ context.start();
+ }
+
+ @Override
+ public void tearDown() throws Exception {
+ context.stop();
+ super.tearDown();
+ }
+
+ public void testGroupIterator() throws Exception {
+ String s = "ABC\nDEF\nGHI\nJKL\nMNO\nPQR\nSTU\nVW";
+ Scanner scanner = new Scanner(s);
+ scanner.useDelimiter("\n");
+
+ GroupIterator gi = new GroupIterator(context, scanner, "\n", 3);
+
+ assertTrue(gi.hasNext());
+ assertEquals("ABC\nDEF\nGHI", gi.next());
+ assertEquals("JKL\nMNO\nPQR", gi.next());
+ assertEquals("STU\nVW", gi.next());
+ assertFalse(gi.hasNext());
+ }
+
+}
Copied:
camel/trunk/components/camel-spring/src/test/java/org/apache/camel/spring/processor/SpringSplitGroupMultiLinesTest.java
(from r1332534,
camel/trunk/components/camel-spring/src/test/java/org/apache/camel/spring/processor/SpringAOPAfterTest.java)
URL:
http://svn.apache.org/viewvc/camel/trunk/components/camel-spring/src/test/java/org/apache/camel/spring/processor/SpringSplitGroupMultiLinesTest.java?p2=camel/trunk/components/camel-spring/src/test/java/org/apache/camel/spring/processor/SpringSplitGroupMultiLinesTest.java&p1=camel/trunk/components/camel-spring/src/test/java/org/apache/camel/spring/processor/SpringAOPAfterTest.java&r1=1332534&r2=1332649&rev=1332649&view=diff
==============================================================================
---
camel/trunk/components/camel-spring/src/test/java/org/apache/camel/spring/processor/SpringAOPAfterTest.java
(original)
+++
camel/trunk/components/camel-spring/src/test/java/org/apache/camel/spring/processor/SpringSplitGroupMultiLinesTest.java
Tue May 1 13:38:27 2012
@@ -17,15 +17,16 @@
package org.apache.camel.spring.processor;
import org.apache.camel.CamelContext;
-import org.apache.camel.processor.AOPAfterTest;
+import org.apache.camel.processor.SplitGroupMultiLinesTest;
+
import static
org.apache.camel.spring.processor.SpringTestHelper.createSpringCamelContext;
/**
* @version
*/
-public class SpringAOPAfterTest extends AOPAfterTest {
+public class SpringSplitGroupMultiLinesTest extends SplitGroupMultiLinesTest {
protected CamelContext createCamelContext() throws Exception {
- return createSpringCamelContext(this,
"org/apache/camel/spring/processor/aopafter.xml");
+ return createSpringCamelContext(this,
"org/apache/camel/spring/processor/SplitGroupMultiLinesTest.xml");
}
}
\ No newline at end of file
Copied:
camel/trunk/components/camel-spring/src/test/resources/org/apache/camel/spring/processor/SplitGroupMultiLinesTest.xml
(from r1332534,
camel/trunk/components/camel-spring/src/test/resources/org/apache/camel/spring/processor/aopafter.xml)
URL:
http://svn.apache.org/viewvc/camel/trunk/components/camel-spring/src/test/resources/org/apache/camel/spring/processor/SplitGroupMultiLinesTest.xml?p2=camel/trunk/components/camel-spring/src/test/resources/org/apache/camel/spring/processor/SplitGroupMultiLinesTest.xml&p1=camel/trunk/components/camel-spring/src/test/resources/org/apache/camel/spring/processor/aopafter.xml&r1=1332534&r2=1332649&rev=1332649&view=diff
==============================================================================
---
camel/trunk/components/camel-spring/src/test/resources/org/apache/camel/spring/processor/aopafter.xml
(original)
+++
camel/trunk/components/camel-spring/src/test/resources/org/apache/camel/spring/processor/SplitGroupMultiLinesTest.xml
Tue May 1 13:38:27 2012
@@ -22,16 +22,17 @@
http://camel.apache.org/schema/spring
http://camel.apache.org/schema/spring/camel-spring.xsd
">
- <!-- START SNIPPET: e1 -->
- <camelContext xmlns="http://camel.apache.org/schema/spring";>
- <route>
- <from uri="direct:start"/>
- <aop afterUri="mock:after">
- <transform><constant>Bye World</constant></transform>
- <to uri="mock:result"/>
- </aop>
- </route>
- </camelContext>
- <!-- END SNIPPET: e1 -->
+ <!-- START SNIPPET: e1 -->
+ <camelContext xmlns="http://camel.apache.org/schema/spring";>
+ <route>
+ <from uri="direct:start"/>
+ <split streaming="true">
+ <!-- split by new line and group by 3 -->
+ <tokenize token="\n" group="3"/>
+ <to uri="mock:group"/>
+ </split>
+ </route>
+ </camelContext>
+ <!-- END SNIPPET: e1 -->
</beans>
