This is an automated email from the ASF dual-hosted git repository. jackie pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push: new bb2eb49475 Replace custom fmpp plugin with fmpp-maven-plugin (#12737) bb2eb49475 is described below commit bb2eb49475dcdefecc3c3a9e459868687e3b41ef Author: Xiaotian (Jackie) Jiang <17555551+jackie-ji...@users.noreply.github.com> AuthorDate: Thu Mar 28 10:24:44 2024 -0700 Replace custom fmpp plugin with fmpp-maven-plugin (#12737) --- contrib/pinot-fmpp-maven-plugin/pom.xml | 127 ---------- .../main/java/org/apache/pinot/fmpp/FMPPMojo.java | 270 --------------------- .../org/apache/pinot/fmpp/MavenDataLoader.java | 55 ----- pinot-common/pom.xml | 57 +++-- pinot-common/src/main/codegen/config.fmpp | 2 + pom.xml | 48 ++-- 6 files changed, 59 insertions(+), 500 deletions(-) diff --git a/contrib/pinot-fmpp-maven-plugin/pom.xml b/contrib/pinot-fmpp-maven-plugin/pom.xml deleted file mode 100644 index 0fbdc44f48..0000000000 --- a/contrib/pinot-fmpp-maven-plugin/pom.xml +++ /dev/null @@ -1,127 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!-- - - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. - ---> -<project xmlns="http://maven.apache.org/POM/4.0.0" - xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" - xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - <modelVersion>4.0.0</modelVersion> - <parent> - <artifactId>pinot</artifactId> - <groupId>org.apache.pinot</groupId> - <version>1.2.0-SNAPSHOT</version> - <relativePath>../..</relativePath> - </parent> - - <artifactId>pinot-fmpp-maven-plugin</artifactId> - <name>Pinot FMPP plugin</name> - <url>https://pinot.apache.org/</url> - <packaging>maven-plugin</packaging> - <properties> - <pinot.root>${basedir}/../..</pinot.root> - <maven.version>3.8.2</maven.version> - <fmpp.version>0.9.16</fmpp.version> - <freemarker.version>2.3.32</freemarker.version> - </properties> - - <dependencies> - <dependency> - <groupId>commons-io</groupId> - <artifactId>commons-io</artifactId> - </dependency> - <dependency> - <groupId>org.apache.maven</groupId> - <artifactId>maven-core</artifactId> - <version>${maven.version}</version> - <exclusions> - <exclusion> - <groupId>org.codehaus.plexus</groupId> - <artifactId>plexus-utils</artifactId> - </exclusion> - <exclusion> - <groupId>org.eclipse.sisu</groupId> - <artifactId>org.eclipse.sisu.plexus</artifactId> - </exclusion> - </exclusions> - </dependency> - <dependency> - <groupId>org.apache.maven</groupId> - <artifactId>maven-plugin-api</artifactId> - <version>${maven.version}</version> - <exclusions> - <exclusion> - <groupId>org.eclipse.sisu</groupId> - <artifactId>org.eclipse.sisu.plexus</artifactId> - </exclusion> - </exclusions> - </dependency> - <dependency> - <groupId>net.sourceforge.fmpp</groupId> - <artifactId>fmpp</artifactId> - <version>${fmpp.version}</version> - <exclusions> - <exclusion> - <groupId>org.freemarker</groupId> - <artifactId>freemarker</artifactId> - </exclusion> - </exclusions> - </dependency> - <dependency> - <groupId>org.freemarker</groupId> - <artifactId>freemarker</artifactId> - <version>${freemarker.version}</version> - </dependency> - </dependencies> - <build> - <plugins> - <plugin> - <!-- Checkstyle shouldn't apply to Mojo classes --> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-checkstyle-plugin</artifactId> - <configuration> - <skip>true</skip> - </configuration> - </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-plugin-plugin</artifactId> - <configuration> - <goalPrefix>pinot-fmpp</goalPrefix> - </configuration> - <executions> - <execution> - <id>default-descriptor</id> - <goals> - <goal>descriptor</goal> - </goals> - <phase>process-classes</phase> - </execution> - <execution> - <id>help-descriptor</id> - <goals> - <goal>helpmojo</goal> - </goals> - <phase>process-classes</phase> - </execution> - </executions> - </plugin> - </plugins> - </build> -</project> diff --git a/contrib/pinot-fmpp-maven-plugin/src/main/java/org/apache/pinot/fmpp/FMPPMojo.java b/contrib/pinot-fmpp-maven-plugin/src/main/java/org/apache/pinot/fmpp/FMPPMojo.java deleted file mode 100644 index 787ac7606c..0000000000 --- a/contrib/pinot-fmpp-maven-plugin/src/main/java/org/apache/pinot/fmpp/FMPPMojo.java +++ /dev/null @@ -1,270 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pinot.fmpp; - -import com.google.common.base.Joiner; -import com.google.common.base.Stopwatch; -import fmpp.Engine; -import fmpp.ProgressListener; -import fmpp.progresslisteners.TerseConsoleProgressListener; -import fmpp.setting.Settings; -import fmpp.util.MiscUtil; -import java.io.File; -import java.io.IOException; -import java.nio.file.Files; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.TimeUnit; -import org.apache.commons.io.FileUtils; -import org.apache.maven.plugin.AbstractMojo; -import org.apache.maven.plugin.MojoExecutionException; -import org.apache.maven.plugin.MojoFailureException; -import org.apache.maven.project.MavenProject; - -import static java.lang.String.format; - - -/** - * a maven plugin to run the freemarker generation incrementally - * (if output has not changed, the files are not touched) - * - * @goal generate - * @phase generate-sources - */ -public class FMPPMojo extends AbstractMojo { - - /** - * Used to add new source directories to the build. - * - * @parameter default-value="${project}" - * @required - * @readonly - **/ - private MavenProject project; - - /** - * Where to find the FreeMarker template files. - * - * @parameter default-value="src/main/resources/fmpp/templates/" - * @required - */ - private File templates; - - /** - * Where to write the generated files of the output files. - * - * @parameter default-value="${project.build.directory}/generated-sources/fmpp/" - * @required - */ - private File output; - - /** - * Location of the FreeMarker config file. - * - * @parameter default-value="src/main/resources/fmpp/config.fmpp" - * @required - */ - private File config; - - /** - * compilation scope to be added to ("compile" or "test") - * - * @parameter default-value="compile" - * @required - */ - private String scope; - - /** - * FMPP data model build parameter. - * - * @see <a href="http://fmpp.sourceforge.net/settings.html#key_data">FMPP Data Model Building</a> - * @parameter default-value="" - */ - private String data; - - /** - * if maven properties are added as data - * - * @parameter default-value="true" - * @required - */ - private boolean addMavenDataLoader; - - @Override - public void execute() - throws MojoExecutionException, MojoFailureException { - if (project == null) { - throw new MojoExecutionException("This plugin can only be used inside a project."); - } - String outputPath = output.getAbsolutePath(); - if ((!output.exists() && !output.mkdirs()) || !output.isDirectory()) { - throw new MojoFailureException("can not write to output dir: " + outputPath); - } - String templatesPath = templates.getAbsolutePath(); - if (!templates.exists() || !templates.isDirectory()) { - throw new MojoFailureException("templates not found in dir: " + outputPath); - } - - // add the output directory path to the project source directories - switch (scope) { - case "compile": - project.addCompileSourceRoot(outputPath); - break; - case "test": - project.addTestCompileSourceRoot(outputPath); - break; - default: - throw new MojoFailureException("scope must be compile or test"); - } - - final Stopwatch sw = Stopwatch.createStarted(); - try { - getLog().info( - format("Freemarker generation:\n scope: %s,\n config: %s,\n templates: %s", scope, config.getAbsolutePath(), - templatesPath)); - final File tmp = Files.createTempDirectory("freemarker-tmp").toFile(); - String tmpPath = tmp.getAbsolutePath(); - final String tmpPathNormalized = tmpPath.endsWith(File.separator) ? tmpPath : tmpPath + File.separator; - Settings settings = new Settings(new File(".")); - settings.set(Settings.NAME_SOURCE_ROOT, templatesPath); - settings.set(Settings.NAME_OUTPUT_ROOT, tmp.getAbsolutePath()); - settings.load(config); - settings.addProgressListener(new TerseConsoleProgressListener()); - settings.addProgressListener(new ProgressListener() { - @Override - public void notifyProgressEvent(Engine engine, int event, File src, int pMode, Throwable error, Object param) - throws Exception { - if (event == EVENT_END_PROCESSING_SESSION) { - getLog().info(format("Freemarker generation took %dms", sw.elapsed(TimeUnit.MILLISECONDS))); - sw.reset(); - Report report = moveIfChanged(tmp, tmpPathNormalized); - if (!tmp.delete()) { - throw new MojoFailureException(format("can not delete %s", tmp)); - } - getLog().info(format("Incremental output update took %dms", sw.elapsed(TimeUnit.MILLISECONDS))); - getLog().info(format("new: %d", report.newFiles)); - getLog().info(format("changed: %d", report.changedFiles)); - getLog().info(format("unchanged: %d", report.unchangedFiles)); - } - } - }); - List<String> dataValues = new ArrayList<>(); - if (addMavenDataLoader) { - getLog().info("Adding maven data loader"); - settings.setEngineAttribute(MavenDataLoader.MAVEN_DATA_ATTRIBUTE, new MavenDataLoader.MavenData(project)); - dataValues.add(format("maven: %s()", MavenDataLoader.class.getName())); - } - if (data != null) { - dataValues.add(data); - } - if (!dataValues.isEmpty()) { - String dataString = Joiner.on(",").join(dataValues); - getLog().info("Setting data loader " + dataString); - - settings.add(Settings.NAME_DATA, dataString); - } - settings.execute(); - } catch (Exception e) { - throw new MojoFailureException(MiscUtil.causeMessages(e), e); - } - } - - private static final class Report { - int changedFiles; - int unchangedFiles; - int newFiles; - - Report(int changedFiles, int unchangedFiles, int newFiles) { - super(); - this.changedFiles = changedFiles; - this.unchangedFiles = unchangedFiles; - this.newFiles = newFiles; - } - - public Report() { - this(0, 0, 0); - } - - void add(Report other) { - changedFiles += other.changedFiles; - unchangedFiles += other.unchangedFiles; - newFiles += other.newFiles; - } - - public void addChanged() { - ++changedFiles; - } - - public void addNew() { - ++newFiles; - } - - public void addUnchanged() { - ++unchangedFiles; - } - } - - private Report moveIfChanged(File root, String tmpPath) - throws MojoFailureException, IOException { - Report report = new Report(); - for (File file : root.listFiles()) { - if (file.isDirectory()) { - report.add(moveIfChanged(file, tmpPath)); - if (!file.delete()) { - throw new MojoFailureException(format("can not delete %s", file)); - } - } else { - String absPath = file.getAbsolutePath(); - if (!absPath.startsWith(tmpPath)) { - throw new MojoFailureException(format("%s should start with %s", absPath, tmpPath)); - } - String relPath = absPath.substring(tmpPath.length()); - File outputFile = new File(output, relPath); - if (!outputFile.exists()) { - report.addNew(); - } else if (!FileUtils.contentEquals(file, outputFile)) { - getLog().info(format("%s has changed", relPath)); - if (!outputFile.delete()) { - throw new MojoFailureException(format("can not delete %s", outputFile)); - } - report.addChanged(); - } else { - report.addUnchanged(); - } - if (!outputFile.exists()) { - File parentDir = outputFile.getParentFile(); - if (parentDir.exists() && !parentDir.isDirectory()) { - throw new MojoFailureException( - format("can not move %s to %s as %s is not a dir", file, outputFile, parentDir)); - } - if (!parentDir.exists() && !parentDir.mkdirs()) { - throw new MojoFailureException( - format("can not move %s to %s as dir %s can not be created", file, outputFile, parentDir)); - } - FileUtils.moveFile(file, outputFile); - } else { - if (!file.delete()) { - throw new MojoFailureException(format("can not delete %s", file)); - } - } - } - } - return report; - } -} diff --git a/contrib/pinot-fmpp-maven-plugin/src/main/java/org/apache/pinot/fmpp/MavenDataLoader.java b/contrib/pinot-fmpp-maven-plugin/src/main/java/org/apache/pinot/fmpp/MavenDataLoader.java deleted file mode 100644 index df85ad891b..0000000000 --- a/contrib/pinot-fmpp-maven-plugin/src/main/java/org/apache/pinot/fmpp/MavenDataLoader.java +++ /dev/null @@ -1,55 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pinot.fmpp; - -import fmpp.Engine; -import fmpp.tdd.DataLoader; -import java.util.List; -import org.apache.maven.project.MavenProject; - - -/** - * A data loader for Maven - */ -public class MavenDataLoader implements DataLoader { - public static final class MavenData { - private final MavenProject project; - - public MavenData(MavenProject project) { - this.project = project; - } - - public MavenProject getProject() { - return project; - } - } - - public static final String MAVEN_DATA_ATTRIBUTE = "maven.data"; - - @Override - public Object load(Engine e, List args) - throws Exception { - if (!args.isEmpty()) { - throw new IllegalArgumentException("maven model data loader has no parameters"); - } - - MavenData data = (MavenData) e.getAttribute(MAVEN_DATA_ATTRIBUTE); - return data; - } -} diff --git a/pinot-common/pom.xml b/pinot-common/pom.xml index b6e47114e0..f9a3f64c41 100644 --- a/pinot-common/pom.xml +++ b/pinot-common/pom.xml @@ -42,6 +42,7 @@ <filtering>true</filtering> </resource> </resources> + <plugins> <plugin> <groupId>org.apache.maven.plugins</groupId> @@ -55,19 +56,21 @@ </properties> </configuration> </plugin> + <plugin> <groupId>org.xolstice.maven.plugins</groupId> <artifactId>protobuf-maven-plugin</artifactId> </plugin> + + <!-- Following plugins and their configurations are used to generate the custom Calcite's SQL parser --> + <!-- Extracts the Parser.jj from Calcite to ${project.build.directory}, where all the Freemarker templates are --> <plugin> - <!-- Extract parser grammar template from calcite-core.jar and put - it under ${project.build.directory} where all freemarker templates are. --> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-dependency-plugin</artifactId> <executions> <execution> <id>unpack-parser-template</id> - <phase>initialize</phase> + <phase>generate-sources</phase> <goals> <goal>unpack</goal> </goals> @@ -76,6 +79,7 @@ <artifactItem> <groupId>org.apache.calcite</groupId> <artifactId>calcite-core</artifactId> + <version>${calcite.version}</version> <type>jar</type> <overWrite>true</overWrite> <outputDirectory>${project.build.directory}/</outputDirectory> @@ -86,44 +90,50 @@ </execution> </executions> </plugin> + <!-- Copy the templates present in the codegen directory containing custom SQL rules to + ${project.build.directory}/codegen --> <plugin> - <groupId>org.apache.pinot</groupId> - <artifactId>pinot-fmpp-maven-plugin</artifactId> - <version>${project.version}</version> + <artifactId>maven-resources-plugin</artifactId> <executions> <execution> - <id>generate-fmpp-sources</id> + <id>copy-fmpp-resources</id> <phase>generate-sources</phase> <goals> - <goal>generate</goal> + <goal>copy-resources</goal> </goals> <configuration> - <config>${project.basedir}/src/main/codegen/config.fmpp</config> - <output>${project.build.directory}/generated-sources/fmpp</output> - <templates>${project.build.directory}/codegen/templates</templates> - <data>tdd(${project.basedir}/src/main/codegen/config.fmpp), default:tdd(${project.build.directory}/codegen/default_config.fmpp)</data> + <outputDirectory>${project.build.directory}/codegen</outputDirectory> + <resources> + <resource> + <directory>src/main/codegen</directory> + <filtering>false</filtering> + </resource> + </resources> </configuration> </execution> </executions> </plugin> + <!-- "Plugs in" the Calcite's Parser.jj with the variables present in config.fmpp. These contain the custom rules + as well as the class to which the custom implementation will get generated --> <plugin> - <groupId>org.codehaus.mojo</groupId> - <artifactId>build-helper-maven-plugin</artifactId> + <groupId>com.googlecode.fmpp-maven-plugin</groupId> + <artifactId>fmpp-maven-plugin</artifactId> <executions> <execution> - <id>add-generated-sources</id> - <phase>process-sources</phase> + <id>generate-fmpp-sources</id> + <phase>generate-sources</phase> <goals> - <goal>add-source</goal> + <goal>generate</goal> </goals> <configuration> - <sources> - <source>${project.build.directory}/generated-sources/javacc</source> - </sources> + <cfgFile>${project.build.directory}/codegen/config.fmpp</cfgFile> + <outputDirectory>${project.build.directory}/generated-sources</outputDirectory> + <templateDirectory>${project.build.directory}/codegen/templates</templateDirectory> </configuration> </execution> </executions> </plugin> + <!-- Creates a Java class for the custom parser from Parser.jj --> <plugin> <groupId>org.codehaus.mojo</groupId> <artifactId>javacc-maven-plugin</artifactId> @@ -135,18 +145,19 @@ <goal>javacc</goal> </goals> <configuration> - <sourceDirectory>${project.build.directory}/generated-sources/fmpp</sourceDirectory> + <sourceDirectory>${project.build.directory}/generated-sources/javacc</sourceDirectory> <includes> - <include>**/Parser.jj</include> + <include>Parser.jj</include> </includes> <lookAhead>2</lookAhead> <isStatic>false</isStatic> - <outputDirectory>${project.build.directory}/generated-sources/javacc</outputDirectory> + <outputDirectory>${project.build.directory}/generated-sources</outputDirectory> </configuration> </execution> </executions> </plugin> </plugins> + <pluginManagement> <plugins> <plugin> diff --git a/pinot-common/src/main/codegen/config.fmpp b/pinot-common/src/main/codegen/config.fmpp index 178029a3b8..c2fb71ea01 100644 --- a/pinot-common/src/main/codegen/config.fmpp +++ b/pinot-common/src/main/codegen/config.fmpp @@ -18,6 +18,8 @@ # data: { + default: tdd("../default_config.fmpp") + # Data declarations for this parser. # # Default declarations are in default_config.fmpp; if you do not include a diff --git a/pom.xml b/pom.xml index 3550227eea..fc4f5c7648 100644 --- a/pom.xml +++ b/pom.xml @@ -56,8 +56,6 @@ <module>pinot-connectors</module> <module>pinot-segment-local</module> <module>pinot-compatibility-verifier</module> - <module>contrib/pinot-fmpp-maven-plugin</module> - <module>pinot-query-planner</module> <module>pinot-query-runtime</module> </modules> @@ -1777,6 +1775,29 @@ <encoding>${project.build.sourceEncoding}</encoding> </configuration> </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-remote-resources-plugin</artifactId> + <version>3.2.0</version> + </plugin> + <plugin> + <groupId>com.googlecode.fmpp-maven-plugin</groupId> + <artifactId>fmpp-maven-plugin</artifactId> + <version>1.0</version> + </plugin> + <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>javacc-maven-plugin</artifactId> + <version>3.1.0</version> + <dependencies> + <dependency> + <groupId>net.java.dev.javacc</groupId> + <artifactId>javacc</artifactId> + <!-- Higher version JavaCC throws exception when generating custom Calcite parser --> + <version>5.0</version> + </dependency> + </dependencies> + </plugin> <plugin> <groupId>net.alchim31.maven</groupId> <artifactId>scala-maven-plugin</artifactId> @@ -2218,7 +2239,6 @@ <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-remote-resources-plugin</artifactId> - <version>1.6.0</version> <executions> <execution> <goals> @@ -2235,28 +2255,6 @@ </execution> </executions> </plugin> - <plugin> - <groupId>org.codehaus.mojo</groupId> - <artifactId>build-helper-maven-plugin</artifactId> - <version>3.5.0</version> - </plugin> - <plugin> - <groupId>org.codehaus.mojo</groupId> - <artifactId>javacc-maven-plugin</artifactId> - <version>2.6</version> - <dependencies> - <dependency> - <groupId>net.java.dev.javacc</groupId> - <artifactId>javacc</artifactId> - <version>7.0.13</version> - </dependency> - </dependencies> - </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-plugin-plugin</artifactId> - <version>3.6.0</version> - </plugin> </plugins> </build> <reporting> --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org