This is an automated email from the ASF dual-hosted git repository.
pfzhan pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
The following commit(s) were added to refs/heads/kylin5 by this push:
new 7fe50b2d58 [KYLIN-5475] Use javaCC to parse self define DML (#2102)
7fe50b2d58 is described below
commit 7fe50b2d5833ad4b8a09d6a27027563dedce8287
Author: Yang Jiang <[email protected]>
AuthorDate: Wed Apr 2 17:09:17 2025 +0800
[KYLIN-5475] Use javaCC to parse self define DML (#2102)
* [KYLIN-5475] Use javaCC to parse self define DML
---
.../kylin/query/util/ManipulationSqlParser.jj | 152 +++++++++++++++++++++
.../apache/kylin/query/util/DMLParserResult.java | 46 +++++++
.../query/util/ManipulationSqlParserTest.java | 102 ++++++++++++++
3 files changed, 300 insertions(+)
diff --git
a/src/query-common/src/main/codegen/javacc/org/apache/kylin/query/util/ManipulationSqlParser.jj
b/src/query-common/src/main/codegen/javacc/org/apache/kylin/query/util/ManipulationSqlParser.jj
new file mode 100644
index 0000000000..7921dfee2b
--- /dev/null
+++
b/src/query-common/src/main/codegen/javacc/org/apache/kylin/query/util/ManipulationSqlParser.jj
@@ -0,0 +1,152 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+options {
+ IGNORE_CASE = true;
+ STATIC = false;
+ UNICODE_INPUT=true;
+}
+
+PARSER_BEGIN(ManipulationSqlParser)
+package org.apache.kylin.query.util;
+
+import java.io.StringReader;
+import java.util.ArrayList;
+import java.util.List;
+import java.io.Reader;
+import org.apache.kylin.query.util.DMLParserResult;
+
+
+public class ManipulationSqlParser {
+ public static DMLParserResult evaluate(String src) throws ParseException {
+ Reader reader = new StringReader(src);
+ return new ManipulationSqlParser(reader).parse();
+ }
+}
+
+PARSER_END(ManipulationSqlParser)
+
+// Operator type
+< DEFAULT >
+TOKEN :
+{
+ < SHOW : "SHOW" > |
+ < LOAD : "LOAD"> |
+ < DELETE: "DELETE" > |
+ < CANCEL: "CANCEL" >
+}
+
+// Operator unit
+< DEFAULT >
+TOKEN :
+{
+ < JOB : "JOB" > |
+ < PARTITION : "PARTITION"> |
+ < MODEL: "MODEL" >
+}
+
+< DEFAULT >
+TOKEN: {
+ // identifier
+ < DATE:
+ (["0"-"9"]){4}
+ "-"
+ (["0"-"9"]){2}
+ "-"
+ (["0"-"9"]){2}
+ > |
+ < DEFINENAME: (["a"-"z","A"-"Z","0"-"9"])
+ (["a"-"z","A"-"Z","0"-"9","_"])*
+ > |
+ < JOBID: (["a"-"z","A"-"Z","0"-"9"])
+ (["a"-"z","A"-"Z","0"-"9","-"])*
+ >
+}
+
+<DEFAULT> SKIP :
+{
+ " "
+| "\t"
+| "\n"
+| "\r"
+| "\f"
+}
+
+DMLParserResult parse():
+{
+ DMLParserResult res = new DMLParserResult();
+ List<String> argList = new ArrayList<String>();
+ Token x, y, sta, end;
+}
+{
+ // SHOW JOB job_id
+ <SHOW> <JOB> x = <JOBID> <EOF>
+ {
+ argList.add(x.image);
+ res.setOperator(DMLParserResult.OPERATOR.SHOW);
+ res.setUnit(DMLParserResult.UNIT.JOB);
+ res.setArg(argList);
+ return res;
+} |
+ // LOAD PARTITION poject_name.model_test (2023-01-01, 2023-02-10)
+ LOOKAHEAD(2) <LOAD> <PARTITION> x = <DEFINENAME> "." y = <DEFINENAME> "("
sta = <DATE> "," end = <DATE> ")" <EOF>
+ {
+ argList.add(x.image);
+ argList.add(y.image);
+ argList.add(sta.image);
+ argList.add(end.image);
+ res.setOperator(DMLParserResult.OPERATOR.LOAD);
+ res.setUnit(DMLParserResult.UNIT.PARTITION);
+ res.setArg(argList);
+ return res;
+ } |
+ // LOAD MODEL poject_name.model_test
+ <LOAD> <MODEL> x = <DEFINENAME> "." y = <DEFINENAME> <EOF>
+ {
+ argList.add(x.image);
+ argList.add(y.image);
+ res.setOperator(DMLParserResult.OPERATOR.LOAD);
+ res.setUnit(DMLParserResult.UNIT.MODEL);
+ res.setArg(argList);
+ return res;
+ } |
+ // DELETE MODEL poject_name.model_name
+ <DELETE> <MODEL> x = <DEFINENAME> "." y = <DEFINENAME> <EOF>
+ {
+ argList.add(x.image);
+ argList.add(y.image);
+ res.setOperator(DMLParserResult.OPERATOR.DELETE);
+ res.setUnit(DMLParserResult.UNIT.MODEL);
+ res.setArg(argList);
+ return res;
+ } |
+ // CANCEL JOB job_id
+ <CANCEL> <JOB> x = <JOBID> <EOF>
+ {
+ argList.add(x.image);
+ res.setOperator(DMLParserResult.OPERATOR.CANCEL);
+ res.setUnit(DMLParserResult.UNIT.JOB);
+ res.setArg(argList);
+ return res;
+ }
+
+ {
+ return res;
+ }
+
+}
\ No newline at end of file
diff --git
a/src/query-common/src/main/java/org/apache/kylin/query/util/DMLParserResult.java
b/src/query-common/src/main/java/org/apache/kylin/query/util/DMLParserResult.java
new file mode 100644
index 0000000000..148b8e6670
--- /dev/null
+++
b/src/query-common/src/main/java/org/apache/kylin/query/util/DMLParserResult.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.query.util;
+
+import java.util.List;
+
+import lombok.Getter;
+import lombok.Setter;
+
+@Getter
+@Setter
+public class DMLParserResult {
+ // All DML consists with `OPERATOR` + `UNIT` + multi args
+ public enum OPERATOR {
+ SHOW, LOAD, DELETE, CANCEL;
+ }
+
+ public enum UNIT {
+ JOB, PARTITION, MODEL;
+ }
+
+ OPERATOR operator;
+ UNIT unit;
+ List<String> arg;
+
+ // Notice operator not be Null
+ public boolean isValid() {
+ return operator != null;
+ }
+}
diff --git
a/src/query/src/test/java/org/apache/kylin/query/util/ManipulationSqlParserTest.java
b/src/query/src/test/java/org/apache/kylin/query/util/ManipulationSqlParserTest.java
new file mode 100644
index 0000000000..0b7943c5de
--- /dev/null
+++
b/src/query/src/test/java/org/apache/kylin/query/util/ManipulationSqlParserTest.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.query.util;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+public class ManipulationSqlParserTest {
+
+ @Test
+ public void showJob() throws ParseException {
+ String sql = "SHOW JOB
70ba0c50-15b1-a40c-39a9-5f67f0f813c7-713ab396-1038-eecd-824f-3da66d531db6";
+ DMLParserResult evaluate = ManipulationSqlParser.evaluate(sql);
+ Assert.assertEquals(DMLParserResult.OPERATOR.SHOW, evaluate.operator);
+ Assert.assertEquals(DMLParserResult.UNIT.JOB, evaluate.unit);
+ Assert.assertEquals(1, evaluate.arg.size());
+
Assert.assertEquals("70ba0c50-15b1-a40c-39a9-5f67f0f813c7-713ab396-1038-eecd-824f-3da66d531db6",
+ evaluate.arg.get(0));
+ }
+
+ @Test
+ public void triggerJob() throws ParseException {
+ // 1. trigger partial build job with partition col
+ String sql = "LOAD PARTITION project_name.model_test (2023-01-01,
2023-02-10)";
+ DMLParserResult evaluate = ManipulationSqlParser.evaluate(sql);
+
+ Assert.assertEquals(DMLParserResult.OPERATOR.LOAD, evaluate.operator);
+ Assert.assertEquals(DMLParserResult.UNIT.PARTITION, evaluate.unit);
+ Assert.assertEquals(4, evaluate.arg.size());
+ Assert.assertEquals("project_name", evaluate.arg.get(0));
+ Assert.assertEquals("model_test", evaluate.arg.get(1));
+ Assert.assertEquals("2023-01-01", evaluate.arg.get(2));
+ Assert.assertEquals("2023-02-10", evaluate.arg.get(3));
+
+ // 2. trigger full build job
+ String sql2 = "LOAD MODEL project_name.model_test2";
+ DMLParserResult evaluate2 = ManipulationSqlParser.evaluate(sql2);
+
+ Assert.assertEquals(evaluate2.operator, DMLParserResult.OPERATOR.LOAD);
+ Assert.assertEquals(evaluate2.unit, DMLParserResult.UNIT.MODEL);
+ Assert.assertEquals(2, evaluate2.arg.size());
+ Assert.assertEquals("project_name", evaluate2.arg.get(0));
+ Assert.assertEquals("model_test2", evaluate2.arg.get(1));
+
+ }
+
+ @Test
+ public void cancelJob() throws ParseException {
+ String sql = "CANCEL JOB
70ba0c50-15b1-a40c-39a9-5f67f0f813c7-713ab396-1038-eecd-824f-3da66d531db6";
+ DMLParserResult evaluate = ManipulationSqlParser.evaluate(sql);
+
+ Assert.assertEquals(DMLParserResult.OPERATOR.CANCEL,
evaluate.operator);
+ Assert.assertEquals(DMLParserResult.UNIT.JOB, evaluate.unit);
+ Assert.assertEquals(1, evaluate.arg.size());
+
Assert.assertEquals("70ba0c50-15b1-a40c-39a9-5f67f0f813c7-713ab396-1038-eecd-824f-3da66d531db6",
+ evaluate.arg.get(0));
+ }
+
+ @Test
+ public void deleteModel() throws ParseException {
+ String sql = "DELETE MODEL project_name.model_test";
+ DMLParserResult evaluate = ManipulationSqlParser.evaluate(sql);
+
+ Assert.assertEquals(DMLParserResult.OPERATOR.DELETE,
evaluate.operator);
+ Assert.assertEquals(DMLParserResult.UNIT.MODEL, evaluate.unit);
+ Assert.assertEquals(2, evaluate.arg.size());
+ Assert.assertEquals("project_name", evaluate.arg.get(0));
+ Assert.assertEquals("model_test", evaluate.arg.get(1));
+ }
+
+ @Test
+ public void testError() {
+ String sql = "DELETE MODELs model_test";
+ Assert.assertThrows(ParseException.class, () ->
ManipulationSqlParser.evaluate(sql));
+ }
+
+ @Test
+ public void validDMLParserResult() {
+ DMLParserResult test = new DMLParserResult();
+ test.unit = DMLParserResult.UNIT.JOB;
+ Assert.assertFalse(test.isValid());
+
+ test.operator = DMLParserResult.OPERATOR.SHOW;
+ Assert.assertTrue(test.isValid());
+ }
+}
