This is an automated email from the ASF dual-hosted git repository. davsclaus pushed a commit to branch camel-3.20.x in repository https://gitbox.apache.org/repos/asf/camel.git
commit 00616410e4e2ac6d9f56e9369abdfa4a55f3fb48 Author: Claus Ibsen <claus.ib...@gmail.com> AuthorDate: Thu May 4 13:57:05 2023 +0200 CAMEL-19309: camel-jbang - camel run --source-dir --- .../java/org/apache/camel/spi/RoutesLoader.java | 14 ++++- .../camel/impl/engine/DefaultRoutesLoader.java | 45 ++++++++++----- .../apache/camel/main/DefaultRoutesCollector.java | 65 +++++++++++++++++----- .../org/apache/camel/main/RoutesConfigurer.java | 56 ++++++++++++++++--- .../org/apache/camel/support/ResourceHelper.java | 19 +++++-- .../modules/ROOT/pages/camel-jbang.adoc | 38 +++++++++++++ .../apache/camel/dsl/jbang/core/commands/Run.java | 61 +++++++++++++++----- .../java/org/apache/camel/main/KameletMain.java | 5 ++ 8 files changed, 246 insertions(+), 57 deletions(-) diff --git a/core/camel-api/src/main/java/org/apache/camel/spi/RoutesLoader.java b/core/camel-api/src/main/java/org/apache/camel/spi/RoutesLoader.java index 984d004aa40..0bc6deb2fd5 100644 --- a/core/camel-api/src/main/java/org/apache/camel/spi/RoutesLoader.java +++ b/core/camel-api/src/main/java/org/apache/camel/spi/RoutesLoader.java @@ -131,6 +131,16 @@ public interface RoutesLoader extends CamelContextAware { */ Collection<RoutesBuilder> findRoutesBuilders(Collection<Resource> resources) throws Exception; + /** + * Find {@link RoutesBuilder} from the give list of {@link Resource}. + * + * @param resources the resource to be loaded. + * @param optional whether parsing the resource is optional, such as there is no supported parser for the given + * resource extension + * @return a collection {@link RoutesBuilder} + */ + Collection<RoutesBuilder> findRoutesBuilders(Collection<Resource> resources, boolean optional) throws Exception; + /** * Pre-parses the {@link RoutesBuilder} from {@link Resource}. * @@ -138,8 +148,10 @@ public interface RoutesLoader extends CamelContextAware { * specify configurations that affect the bootstrap, such as by camel-jbang and camel-yaml-dsl. * * @param resource the resource to be pre parsed. + * @param optional whether parsing the resource is optional, such as there is no supported parser for the given + * resource extension */ - default void preParseRoute(Resource resource) throws Exception { + default void preParseRoute(Resource resource, boolean optional) throws Exception { // noop } diff --git a/core/camel-base-engine/src/main/java/org/apache/camel/impl/engine/DefaultRoutesLoader.java b/core/camel-base-engine/src/main/java/org/apache/camel/impl/engine/DefaultRoutesLoader.java index 32e0c46c419..5407c04f4ac 100644 --- a/core/camel-base-engine/src/main/java/org/apache/camel/impl/engine/DefaultRoutesLoader.java +++ b/core/camel-base-engine/src/main/java/org/apache/camel/impl/engine/DefaultRoutesLoader.java @@ -69,9 +69,7 @@ public class DefaultRoutesLoader extends ServiceSupport implements RoutesLoader, @Override public void doStop() throws Exception { super.doStop(); - ServiceHelper.stopService(loaders.values()); - loaders.clear(); } @@ -87,27 +85,41 @@ public class DefaultRoutesLoader extends ServiceSupport implements RoutesLoader, @Override public Collection<RoutesBuilder> findRoutesBuilders(Collection<Resource> resources) throws Exception { + return findRoutesBuilders(resources, false); + } + + @Override + public Collection<RoutesBuilder> findRoutesBuilders(Collection<Resource> resources, boolean optional) throws Exception { List<RoutesBuilder> answer = new ArrayList<>(resources.size()); // first we need to parse for modeline to gather all the configurations if (camelContext.isModeline()) { ModelineFactory factory = camelContext.adapt(ExtendedCamelContext.class).getModelineFactory(); for (Resource resource : resources) { - RoutesBuilderLoader loader = resolveRoutesBuilderLoader(resource); - // gather resources for modeline - factory.parseModeline(resource); - // pre-parse before loading - loader.preParseRoute(resource); + if (resource.exists()) { + try (RoutesBuilderLoader loader = resolveRoutesBuilderLoader(resource, optional)) { + if (loader != null) { + // gather resources for modeline + factory.parseModeline(resource); + // pre-parse before loading + loader.preParseRoute(resource); + } + } + } } } // now group resources by loader Map<RoutesBuilderLoader, List<Resource>> groups = new LinkedHashMap<>(); for (Resource resource : resources) { - RoutesBuilderLoader loader = resolveRoutesBuilderLoader(resource); - List<Resource> list = groups.getOrDefault(loader, new ArrayList<>()); - list.add(resource); - groups.put(loader, list); + if (resource.exists()) { + RoutesBuilderLoader loader = resolveRoutesBuilderLoader(resource, optional); + if (loader != null) { + List<Resource> list = groups.getOrDefault(loader, new ArrayList<>()); + list.add(resource); + groups.put(loader, list); + } + } } // now load all the same resources for each loader @@ -134,8 +146,11 @@ public class DefaultRoutesLoader extends ServiceSupport implements RoutesLoader, } @Override - public void preParseRoute(Resource resource) throws Exception { - resolveRoutesBuilderLoader(resource).preParseRoute(resource); + public void preParseRoute(Resource resource, boolean optional) throws Exception { + RoutesBuilderLoader loader = resolveRoutesBuilderLoader(resource, optional); + if (loader != null) { + loader.preParseRoute(resource); + } } @Override @@ -196,7 +211,7 @@ public class DefaultRoutesLoader extends ServiceSupport implements RoutesLoader, return answer; } - protected RoutesBuilderLoader resolveRoutesBuilderLoader(Resource resource) throws Exception { + protected RoutesBuilderLoader resolveRoutesBuilderLoader(Resource resource, boolean optional) throws Exception { // the loader to use is derived from the file extension final String extension = FileUtil.onlyExt(resource.getLocation(), false); @@ -206,7 +221,7 @@ public class DefaultRoutesLoader extends ServiceSupport implements RoutesLoader, } RoutesBuilderLoader loader = getRoutesLoader(extension); - if (loader == null) { + if (!optional && loader == null) { throw new IllegalArgumentException( "Cannot find RoutesBuilderLoader in classpath supporting file extension: " + extension); } diff --git a/core/camel-main/src/main/java/org/apache/camel/main/DefaultRoutesCollector.java b/core/camel-main/src/main/java/org/apache/camel/main/DefaultRoutesCollector.java index 5ca9236339c..46e62a013f4 100644 --- a/core/camel-main/src/main/java/org/apache/camel/main/DefaultRoutesCollector.java +++ b/core/camel-main/src/main/java/org/apache/camel/main/DefaultRoutesCollector.java @@ -20,6 +20,7 @@ import java.lang.reflect.Modifier; import java.util.ArrayList; import java.util.Collection; import java.util.List; +import java.util.StringJoiner; import org.apache.camel.CamelContext; import org.apache.camel.ExtendedCamelContext; @@ -29,6 +30,7 @@ import org.apache.camel.builder.LambdaRouteBuilder; import org.apache.camel.builder.RouteBuilder; import org.apache.camel.spi.PackageScanResourceResolver; import org.apache.camel.spi.Resource; +import org.apache.camel.spi.RoutesLoader; import org.apache.camel.util.AntPathMatcher; import org.apache.camel.util.ObjectHelper; import org.apache.camel.util.StopWatch; @@ -138,24 +140,40 @@ public class DefaultRoutesCollector implements RoutesCollector { String excludePattern, String includePattern) { - final ExtendedCamelContext ecc = camelContext.adapt(ExtendedCamelContext.class); final List<RoutesBuilder> answer = new ArrayList<>(); - final String[] includes = includePattern != null ? includePattern.split(",") : null; - StopWatch watch = new StopWatch(); - Collection<Resource> accepted = findRouteResourcesFromDirectory(camelContext, excludePattern, includePattern); - try { - Collection<RoutesBuilder> builders = ecc.getRoutesLoader().findRoutesBuilders(accepted); - if (!builders.isEmpty()) { - log.debug("Found {} route builder from locations: {}", builders.size(), includes); - answer.addAll(builders); + + // include pattern may indicate a resource is optional, so we need to scan twice + String pattern = includePattern; + String optionalPattern = null; + if (pattern != null && pattern.contains("?optional=true")) { + StringJoiner sj1 = new StringJoiner(","); + StringJoiner sj2 = new StringJoiner(","); + for (String p : pattern.split(",")) { + if (p.endsWith("?optional=true")) { + sj2.add(p.substring(0, p.length() - 14)); + } else { + sj1.add(p); + } + } + pattern = sj1.length() > 0 ? sj1.toString() : null; + optionalPattern = sj2.length() > 0 ? sj2.toString() : null; + } + + if (optionalPattern == null) { + // only mandatory pattern + doCollectRoutesFromDirectory(camelContext, answer, excludePattern, pattern, false); + } else { + // find optional first + doCollectRoutesFromDirectory(camelContext, answer, excludePattern, optionalPattern, true); + if (pattern != null) { + // and then any mandatory + doCollectRoutesFromDirectory(camelContext, answer, excludePattern, pattern, false); } - } catch (Exception e) { - throw RuntimeCamelException.wrapRuntimeException(e); } + if (!answer.isEmpty()) { - log.debug("Loaded {} ({} millis) additional RoutesBuilder from: {}, pattern: {}", answer.size(), watch.taken(), - includes, + log.debug("Loaded {} ({} millis) additional RoutesBuilder from: {}", answer.size(), watch.taken(), includePattern); } else { log.debug("No additional RoutesBuilder discovered from: {}", includePattern); @@ -164,6 +182,24 @@ public class DefaultRoutesCollector implements RoutesCollector { return answer; } + protected void doCollectRoutesFromDirectory( + CamelContext camelContext, List<RoutesBuilder> builders, + String excludePattern, String includePattern, boolean optional) { + + ExtendedCamelContext ecc = camelContext.adapt(ExtendedCamelContext.class); + RoutesLoader loader = ecc.getRoutesLoader(); + Collection<Resource> accepted = findRouteResourcesFromDirectory(camelContext, excludePattern, includePattern); + try { + Collection<RoutesBuilder> found = loader.findRoutesBuilders(accepted, optional); + if (!found.isEmpty()) { + log.debug("Found {} route builder from locations: {}", builders.size(), found); + builders.addAll(found); + } + } catch (Exception e) { + throw RuntimeCamelException.wrapRuntimeException(e); + } + } + @Override public Collection<Resource> findRouteResourcesFromDirectory( CamelContext camelContext, @@ -181,6 +217,9 @@ public class DefaultRoutesCollector implements RoutesCollector { Collection<Resource> accepted = new ArrayList<>(); for (String include : includes) { + if (include.endsWith("?optional=true")) { + include = include.substring(0, include.length() - 14); + } log.debug("Finding additional routes from: {}", include); try { for (Resource resource : resolver.findResources(include)) { diff --git a/core/camel-main/src/main/java/org/apache/camel/main/RoutesConfigurer.java b/core/camel-main/src/main/java/org/apache/camel/main/RoutesConfigurer.java index de7ea91ddcc..8fd745a4d80 100644 --- a/core/camel-main/src/main/java/org/apache/camel/main/RoutesConfigurer.java +++ b/core/camel-main/src/main/java/org/apache/camel/main/RoutesConfigurer.java @@ -20,6 +20,7 @@ import java.util.ArrayList; import java.util.Collection; import java.util.List; import java.util.Set; +import java.util.StringJoiner; import org.apache.camel.CamelContext; import org.apache.camel.ExtendedCamelContext; @@ -51,6 +52,7 @@ public class RoutesConfigurer { private String javaRoutesIncludePattern; private String routesExcludePattern; private String routesIncludePattern; + private String routesSourceDir; public List<RoutesBuilder> getRoutesBuilders() { return routesBuilders; @@ -108,6 +110,14 @@ public class RoutesConfigurer { this.routesIncludePattern = routesIncludePattern; } + public String getRoutesSourceDir() { + return routesSourceDir; + } + + public void setRoutesSourceDir(String routesSourceDir) { + this.routesSourceDir = routesSourceDir; + } + public RoutesCollector getRoutesCollector() { return routesCollector; } @@ -258,19 +268,51 @@ public class RoutesConfigurer { try { LOG.debug("RoutesCollectorEnabled: {}", getRoutesCollector()); + // include pattern may indicate a resource is optional, so we need to scan twice + String pattern = getRoutesIncludePattern(); + String optionalPattern = null; + if (pattern != null && pattern.contains("?optional=true")) { + StringJoiner sj1 = new StringJoiner(","); + StringJoiner sj2 = new StringJoiner(","); + for (String p : pattern.split(",")) { + if (p.endsWith("?optional=true")) { + sj2.add(p.substring(0, p.length() - 14)); + } else { + sj1.add(p); + } + } + pattern = sj1.length() > 0 ? sj1.toString() : null; + optionalPattern = sj2.length() > 0 ? sj2.toString() : null; + } + // we can only scan for modeline for routes that we can load from directory as modelines // are comments in the source files - resources = getRoutesCollector().findRouteResourcesFromDirectory( - camelContext, - getRoutesExcludePattern(), - getRoutesIncludePattern()); - + if (optionalPattern == null) { + resources = getRoutesCollector().findRouteResourcesFromDirectory(camelContext, getRoutesExcludePattern(), + pattern); + doConfigureModeline(camelContext, resources, false); + } else { + // we have optional resources + resources = getRoutesCollector().findRouteResourcesFromDirectory(camelContext, getRoutesExcludePattern(), + optionalPattern); + doConfigureModeline(camelContext, resources, true); + // and then mandatory after + if (pattern != null) { + resources = getRoutesCollector().findRouteResourcesFromDirectory(camelContext, getRoutesExcludePattern(), + pattern); + doConfigureModeline(camelContext, resources, false); + } + } } catch (Exception e) { throw RuntimeCamelException.wrapRuntimeException(e); } + } + protected void doConfigureModeline(CamelContext camelContext, Collection<Resource> resources, boolean optional) + throws Exception { ExtendedCamelContext ecc = camelContext.adapt(ExtendedCamelContext.class); ModelineFactory factory = ecc.getModelineFactory(); + RoutesLoader loader = camelContext.adapt(ExtendedCamelContext.class).getRoutesLoader(); for (Resource resource : resources) { LOG.debug("Parsing modeline: {}", resource); @@ -279,10 +321,8 @@ public class RoutesConfigurer { // the resource may also have additional configurations which we need to detect via pre-parsing for (Resource resource : resources) { LOG.debug("Pre-parsing: {}", resource); - RoutesLoader loader = camelContext.adapt(ExtendedCamelContext.class).getRoutesLoader(); - loader.preParseRoute(resource); + loader.preParseRoute(resource, optional); } - } } diff --git a/core/camel-support/src/main/java/org/apache/camel/support/ResourceHelper.java b/core/camel-support/src/main/java/org/apache/camel/support/ResourceHelper.java index 0cf84430eff..07ddf729fc0 100644 --- a/core/camel-support/src/main/java/org/apache/camel/support/ResourceHelper.java +++ b/core/camel-support/src/main/java/org/apache/camel/support/ResourceHelper.java @@ -258,12 +258,13 @@ public final class ResourceHelper { } /** - * Find resources from the file system using Ant-style path patterns. + * Find resources from the file system using Ant-style path patterns (skips hidden files, or files from hidden + * folders). * * @param root the starting file * @param pattern the Ant pattern - * @return a list of files matching the given pattern - * @throws Exception + * @return set of files matching the given pattern + * @throws Exception is thrown if IO error */ public static Set<Path> findInFileSystem(Path root, String pattern) throws Exception { try (Stream<Path> path = Files.walk(root)) { @@ -273,9 +274,15 @@ public final class ResourceHelper { Path relative = root.relativize(entry); String str = relative.toString().replaceAll(Pattern.quote(File.separator), AntPathMatcher.DEFAULT_PATH_SEPARATOR); - boolean match = AntPathMatcher.INSTANCE.match(pattern, str); - LOG.debug("Found resource: {} matching pattern: {} -> {}", entry, pattern, match); - return match; + // skip files in hidden folders + boolean hidden = str.startsWith(".") || str.contains(AntPathMatcher.DEFAULT_PATH_SEPARATOR + "."); + if (!hidden) { + boolean match = AntPathMatcher.INSTANCE.match(pattern, str); + LOG.debug("Found resource: {} matching pattern: {} -> {}", entry, pattern, match); + return match; + } else { + return false; + } }) .collect(Collectors.toCollection(LinkedHashSet::new)); } diff --git a/docs/user-manual/modules/ROOT/pages/camel-jbang.adoc b/docs/user-manual/modules/ROOT/pages/camel-jbang.adoc index 56979938d1f..2e6a0bda793 100644 --- a/docs/user-manual/modules/ROOT/pages/camel-jbang.adoc +++ b/docs/user-manual/modules/ROOT/pages/camel-jbang.adoc @@ -152,6 +152,18 @@ This is very limited as the CLI argument is a bit cumbersome to use than files. NOTE: Using `--code` is only usable for very quick and small prototypes. +=== Running Routes from source directory + +You can also run dev mode when running Camel with `--source-dir`, such as: + +[source,bash] +---- +camel run --source-dir=mycode +---- + +This starts Camel where it will load the files from the _source dir_ (also sub folders). + + === Dev mode with live reload You can enable dev mode that comes with live reload of the route(s) when the source file is updated (saved), @@ -174,6 +186,32 @@ camel run hello.java --dev NOTE: The live reload is meant for development purposes, and if you encounter problems with reloading such as JVM class loading issues, then you may need to restart the integration. +You can also run dev mode when running Camel with `--source-dir`, such as: + +[source,bash] +---- +camel run --source-dir=mycode --dev +---- + +This starts Camel where it will load the files from the _source dir_ (also sub folders). +And in _dev mode_ then you can add new files, update existing files, and delete files, and Camel +will automatically hot-reload on the fly. + +Using _source dir_ is more flexible than having to specify the files in the CLI as shown below: + +[source,bash] +---- +camel run mycode/foo.java mycode/bar.java --dev +---- + +In this situation then Camel will only watch and reload these two files (foo.java and bar.java). +So for example if you add a new file cheese.xml, then this file is not reloaded. On the other hand +if you use `--source-dir` then any files in this directory (and sub folders) are automatic detected +and reloaded. You can also delete files to remove routes. + +NOTE: You cannot use both files and source dir together. +The following is not allowed: `camel run abc.java --source-dir=mycode`. + === Developer Console You can enable the developer console, which presents a variety of information to the developer. diff --git a/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/Run.java b/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/Run.java index e51afd3a77c..2008b256a61 100644 --- a/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/Run.java +++ b/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/Run.java @@ -104,6 +104,11 @@ public class Run extends CamelCommand { List<String> files = new ArrayList<>(); + @Option(names = { "--source-dir" }, + description = "Source directory for loading Camel file(s) to run. When using this, then files cannot be specified at the same time." + + " Multiple directories can be specified separated by comma.") + String sourceDir; + @Option(names = { "--background" }, defaultValue = "false", description = "Run in the background") boolean background; @@ -279,6 +284,12 @@ public class Run extends CamelCommand { } private int run() throws Exception { + if (!files.isEmpty() && sourceDir != null) { + // cannot have both files and source dir at the same time + System.err.println("Cannot specify both file(s) and source-dir at the same time."); + return 1; + } + File work = new File(WORK_DIR); removeDir(work); work.mkdirs(); @@ -298,11 +309,11 @@ public class Run extends CamelCommand { } // if no specific file to run then try to auto-detect - if (files.isEmpty()) { + if (files.isEmpty() && sourceDir == null) { String routes = profileProperties != null ? profileProperties.getProperty("camel.main.routesIncludePattern") : null; if (routes == null) { if (!silentRun) { - System.out + System.err .println("Cannot run because " + getProfile() + ".properties file does not exist or camel.main.routesIncludePattern is not configured"); return 1; @@ -335,6 +346,9 @@ public class Run extends CamelCommand { // allow quick shutdown during development writeSetting(main, profileProperties, "camel.main.shutdownTimeout", "5"); } + if (sourceDir != null) { + writeSetting(main, profileProperties, "camel.jbang.sourceDir", sourceDir); + } if (trace) { writeSetting(main, profileProperties, "camel.main.tracing", "true"); } @@ -481,6 +495,18 @@ public class Run extends CamelCommand { } writeSetting(main, profileProperties, "camel.main.name", name); + if (sourceDir != null) { + // must be an existing directory + File dir = new File(sourceDir); + if (!dir.exists() && !dir.isDirectory()) { + System.err.println("Directory does not exist: " + sourceDir); + return 1; + } + // make it a pattern as we load all files from this directory + // (optional=true as there may be non Camel routes files as well) + js.add("file:" + sourceDir + "/**?optional=true"); + } + if (js.length() > 0) { main.addInitialProperty("camel.main.routesIncludePattern", js.toString()); writeSettings("camel.main.routesIncludePattern", js.toString()); @@ -508,21 +534,28 @@ public class Run extends CamelCommand { } // we can only reload if file based - if (dev && sjReload.length() > 0) { - String reload = sjReload.toString(); + if (dev && (sourceDir != null || sjReload.length() > 0)) { main.addInitialProperty("camel.main.routesReloadEnabled", "true"); - // use current dir, however if we run a file that are in another folder, then we should track that folder instead - String reloadDir = "."; - for (String r : reload.split(",")) { - String path = FileUtil.onlyPath(r); - if (path != null) { - reloadDir = path; - break; + if (sourceDir != null) { + main.addInitialProperty("camel.main.routesReloadDirectory", sourceDir); + main.addInitialProperty("camel.main.routesReloadPattern", "*"); + main.addInitialProperty("camel.main.routesReloadDirectoryRecursive", "true"); + } else { + String pattern = sjReload.toString(); + String reloadDir = "."; + // use current dir, however if we run a file that are in another folder, then we should track that folder instead + for (String r : sjReload.toString().split(",")) { + String path = FileUtil.onlyPath(r); + if (path != null) { + reloadDir = path; + break; + } } + main.addInitialProperty("camel.main.routesReloadDirectory", reloadDir); + main.addInitialProperty("camel.main.routesReloadPattern", pattern); + main.addInitialProperty("camel.main.routesReloadDirectoryRecursive", + isReloadRecursive(pattern) ? "true" : "false"); } - main.addInitialProperty("camel.main.routesReloadDirectory", reloadDir); - main.addInitialProperty("camel.main.routesReloadPattern", reload); - main.addInitialProperty("camel.main.routesReloadDirectoryRecursive", isReloadRecursive(reload) ? "true" : "false"); // do not shutdown the JVM but stop routes when max duration is triggered main.addInitialProperty("camel.main.durationMaxAction", "stop"); } diff --git a/dsl/camel-kamelet-main/src/main/java/org/apache/camel/main/KameletMain.java b/dsl/camel-kamelet-main/src/main/java/org/apache/camel/main/KameletMain.java index ed732da5306..d315309fab6 100644 --- a/dsl/camel-kamelet-main/src/main/java/org/apache/camel/main/KameletMain.java +++ b/dsl/camel-kamelet-main/src/main/java/org/apache/camel/main/KameletMain.java @@ -388,6 +388,11 @@ public class KameletMain extends MainCommandLineSupport { if (console) { VertxHttpServer.registerConsole(answer); } + String sourceDir = getInitialProperties().getProperty("camel.jbang.sourceDir"); + if (sourceDir != null) { + // TODO: + } + // always enable developer console as it is needed by camel-cli-connector configure().withDevConsoleEnabled(true); // and enable a bunch of other stuff that gives more details for developers