This is an automated email from the ASF dual-hosted git repository.
kulagaivan pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/camel.git
The following commit(s) were added to refs/heads/main by this push:
new 257cd5a15df make PythonLanguage and PythonExpression cache compiled
PyCode objects (#15019)
257cd5a15df is described below
commit 257cd5a15dfdc31781c46da40d2cfa5b034a846f
Author: Ivan Kulaga <kulagaivanandreev...@gmail.com>
AuthorDate: Mon Aug 5 19:44:21 2024 +0500
make PythonLanguage and PythonExpression cache compiled PyCode objects
(#15019)
Instead of calling PythonInterpreter's eval(String s) method each time we
evaluate python expression, we can call compile(String script) method, cache
the PyCode result and call eval(PyObject code). This way we avoid making
unnecessary Py.compile_flags calls.
This is also needed for camel-quarkus-python extension
(https://github.com/apache/camel-quarkus/issues/4408) because if we're
compiling expressions each time on evaluation, there is nothing to pre-process
at the build stage.
---
.../camel/language/python/PythonExpression.java | 15 +++++-
.../camel/language/python/PythonLanguage.java | 54 ++++++++++++++++++++-
.../python/PythonScriptingLanguageTest.java | 56 ++++++++++++++++++++++
.../apache/camel/language/python/PythonTest.java | 17 +++++++
.../camel-python/src/test/resources/mypython.py | 18 +++++++
5 files changed, 156 insertions(+), 4 deletions(-)
diff --git
a/components/camel-python/src/main/java/org/apache/camel/language/python/PythonExpression.java
b/components/camel-python/src/main/java/org/apache/camel/language/python/PythonExpression.java
index 2caa1bc9c92..7202f12b159 100644
---
a/components/camel-python/src/main/java/org/apache/camel/language/python/PythonExpression.java
+++
b/components/camel-python/src/main/java/org/apache/camel/language/python/PythonExpression.java
@@ -19,6 +19,7 @@ package org.apache.camel.language.python;
import org.apache.camel.Exchange;
import org.apache.camel.ExpressionIllegalSyntaxException;
import org.apache.camel.support.ExpressionSupport;
+import org.python.core.PyCode;
import org.python.core.PyObject;
import org.python.util.PythonInterpreter;
@@ -26,10 +27,18 @@ public class PythonExpression extends ExpressionSupport {
private final String expressionString;
private final Class<?> type;
+ private final PythonInterpreter compiler;
+ private final PyCode compiledExpression;
public PythonExpression(String expressionString, Class<?> type) {
this.expressionString = expressionString;
this.type = type;
+ this.compiler = new PythonInterpreter();
+ try {
+ this.compiledExpression = compiler.compile(expressionString);
+ } catch (Exception e) {
+ throw new ExpressionIllegalSyntaxException(expressionString, e);
+ }
}
public static PythonExpression python(String expression) {
@@ -38,7 +47,7 @@ public class PythonExpression extends ExpressionSupport {
@Override
public <T> T evaluate(Exchange exchange, Class<T> type) {
- try (PythonInterpreter compiler = new PythonInterpreter()) {
+ try {
compiler.set("exchange", exchange);
compiler.set("context", exchange.getContext());
compiler.set("exchangeId", exchange.getExchangeId());
@@ -47,13 +56,15 @@ public class PythonExpression extends ExpressionSupport {
compiler.set("properties", exchange.getAllProperties());
compiler.set("body", exchange.getMessage().getBody());
- PyObject out = compiler.eval(expressionString);
+ PyObject out = compiler.eval(compiledExpression);
if (out != null) {
String value = out.toString();
return
exchange.getContext().getTypeConverter().convertTo(type, value);
}
} catch (Exception e) {
throw new ExpressionIllegalSyntaxException(expressionString, e);
+ } finally {
+ compiler.cleanup();
}
return null;
}
diff --git
a/components/camel-python/src/main/java/org/apache/camel/language/python/PythonLanguage.java
b/components/camel-python/src/main/java/org/apache/camel/language/python/PythonLanguage.java
index 5ee5decc299..fbad8206d2b 100644
---
a/components/camel-python/src/main/java/org/apache/camel/language/python/PythonLanguage.java
+++
b/components/camel-python/src/main/java/org/apache/camel/language/python/PythonLanguage.java
@@ -16,6 +16,8 @@
*/
package org.apache.camel.language.python;
+import java.util.Collections;
+import java.util.HashMap;
import java.util.Map;
import org.apache.camel.Expression;
@@ -23,13 +25,27 @@ import org.apache.camel.ExpressionIllegalSyntaxException;
import org.apache.camel.Predicate;
import org.apache.camel.spi.ScriptingLanguage;
import org.apache.camel.spi.annotations.Language;
+import org.apache.camel.support.LRUCacheFactory;
import org.apache.camel.support.TypedLanguageSupport;
+import org.python.core.PyCode;
import org.python.core.PyObject;
import org.python.util.PythonInterpreter;
@Language("python")
public class PythonLanguage extends TypedLanguageSupport implements
ScriptingLanguage {
+ private final Map<String, PyCode> compiledScriptsCache;
+
+ private final PythonInterpreter compiler = new PythonInterpreter();
+
+ private PythonLanguage(Map<String, PyCode> compiledScriptsCache) {
+ this.compiledScriptsCache = compiledScriptsCache;
+ }
+
+ public PythonLanguage() {
+ this(LRUCacheFactory.newLRUSoftCache(16, 1000, true));
+ }
+
@Override
public Predicate createPredicate(String expression) {
return createPythonExpression(expression, Boolean.class);
@@ -47,19 +63,53 @@ public class PythonLanguage extends TypedLanguageSupport
implements ScriptingLan
@Override
public <T> T evaluate(String script, Map<String, Object> bindings,
Class<T> resultType) {
script = loadResource(script);
- try (PythonInterpreter compiler = new PythonInterpreter()) {
+
+ PyCode code = getCompiledScriptFromCache(script);
+
+ if (code == null) {
+ try {
+ code = compiler.compile(script);
+ addCompiledScriptToCache(script, code);
+ } catch (Exception e) {
+ throw new ExpressionIllegalSyntaxException(script, e);
+ }
+ }
+
+ try {
if (bindings != null) {
bindings.forEach(compiler::set);
}
- PyObject out = compiler.eval(script);
+ PyObject out = compiler.eval(code);
if (out != null) {
String value = out.toString();
return
getCamelContext().getTypeConverter().convertTo(resultType, value);
}
} catch (Exception e) {
throw new ExpressionIllegalSyntaxException(script, e);
+ } finally {
+ compiler.cleanup();
}
return null;
}
+ private void addCompiledScriptToCache(String script, PyCode
compiledScript) {
+ compiledScriptsCache.put(script, compiledScript);
+ }
+
+ private PyCode getCompiledScriptFromCache(String script) {
+ return compiledScriptsCache.get(script);
+ }
+
+ public static class Builder {
+ private final Map<String, PyCode> cache = new HashMap<>();
+
+ public void addScript(String script, PyCode compiledScript) {
+ cache.put(script, compiledScript);
+ }
+
+ public PythonLanguage build() {
+ return new PythonLanguage(Collections.unmodifiableMap(cache));
+ }
+ }
+
}
diff --git
a/components/camel-python/src/test/java/org/apache/camel/language/python/PythonScriptingLanguageTest.java
b/components/camel-python/src/test/java/org/apache/camel/language/python/PythonScriptingLanguageTest.java
new file mode 100644
index 00000000000..140e58a1bf9
--- /dev/null
+++
b/components/camel-python/src/test/java/org/apache/camel/language/python/PythonScriptingLanguageTest.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.camel.language.python;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.camel.spi.Language;
+import org.apache.camel.spi.ScriptingLanguage;
+import org.apache.camel.test.junit5.CamelTestSupport;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+public class PythonScriptingLanguageTest extends CamelTestSupport {
+
+ @Test
+ public void testScripting() {
+ Language lan = context.resolveLanguage("python");
+ Assertions.assertTrue(lan instanceof ScriptingLanguage);
+ ScriptingLanguage slan = (ScriptingLanguage) lan;
+
+ int num = slan.evaluate("2 * 3", null, int.class);
+ Assertions.assertEquals(6, num);
+
+ Map<String, Object> bindings = new HashMap<>();
+ bindings.put("context", context());
+ String id = slan.evaluate("context.name", bindings, String.class);
+ Assertions.assertEquals(context.getName(), id);
+ }
+
+ @Test
+ public void testExternalScripting() {
+ Language lan = context.resolveLanguage("python");
+ Assertions.assertTrue(lan instanceof ScriptingLanguage);
+ ScriptingLanguage slan = (ScriptingLanguage) lan;
+
+ Map<String, Object> bindings = new HashMap<>();
+ bindings.put("body", 3);
+ String text = slan.evaluate("resource:classpath:mypython.py",
bindings, String.class);
+ Assertions.assertEquals("The result is 6", text);
+ }
+}
diff --git
a/components/camel-python/src/test/java/org/apache/camel/language/python/PythonTest.java
b/components/camel-python/src/test/java/org/apache/camel/language/python/PythonTest.java
index d91fc598c8a..3444caf7198 100644
---
a/components/camel-python/src/test/java/org/apache/camel/language/python/PythonTest.java
+++
b/components/camel-python/src/test/java/org/apache/camel/language/python/PythonTest.java
@@ -16,9 +16,13 @@
*/
package org.apache.camel.language.python;
+import org.apache.camel.Exchange;
import org.apache.camel.test.junit5.LanguageTestSupport;
import org.junit.jupiter.api.Test;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
public class PythonTest extends LanguageTestSupport {
@Test
@@ -32,6 +36,19 @@ public class PythonTest extends LanguageTestSupport {
assertExpression("headers['foo']", "bar");
}
+ @Test
+ void testPythonExpressionRepeatableEvaluation() {
+ PythonLanguage python = new PythonLanguage();
+ PythonExpression expression = (PythonExpression)
python.createExpression("body == 5");
+ Exchange exchange = createExchange();
+
+ exchange.getIn().setBody(5);
+ assertTrue(expression.evaluate(exchange, Boolean.class));
+
+ exchange.getIn().setBody(6);
+ assertFalse(expression.evaluate(exchange, Boolean.class));
+ }
+
@Override
protected String getLanguageName() {
return "python";
diff --git a/components/camel-python/src/test/resources/mypython.py
b/components/camel-python/src/test/resources/mypython.py
new file mode 100644
index 00000000000..193f253d912
--- /dev/null
+++ b/components/camel-python/src/test/resources/mypython.py
@@ -0,0 +1,18 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"The result is " + str(body * 2)
\ No newline at end of file
