(kylin) 28/30: KYLIN-5877 Upgrade table schema of recommendation

[liyang]

This is an automated email from the ASF dual-hosted git repository.

liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git

commit ad294fde01a6e512ad4d3e2943fff21ec85a118c
Author: Zhiting Guo <zhiting....@kyligence.io>
AuthorDate: Wed Nov 22 17:58:24 2023 +0800

    KYLIN-5877 Upgrade table schema of recommendation
---
 .../src/main/resources/metadata-jdbc-mysql.properties        |  3 ++-
 .../src/main/resources/metadata-jdbc-postgresql.properties   |  3 ++-
 .../metadata/recommendation/candidate/JdbcRawRecStore.java   | 12 ++++++++++--
 .../java/org/apache/kylin/tool/RecCandidateToolTest.java     |  4 ++--
 4 files changed, 16 insertions(+), 6 deletions(-)

diff --git a/src/core-common/src/main/resources/metadata-jdbc-mysql.properties 
b/src/core-common/src/main/resources/metadata-jdbc-mysql.properties
index 6bff22a85e..ab37221216 100644
--- a/src/core-common/src/main/resources/metadata-jdbc-mysql.properties
+++ b/src/core-common/src/main/resources/metadata-jdbc-mysql.properties
@@ -216,7 +216,8 @@ create.rawrecommendation.store.table=CREATE TABLE IF NOT 
EXISTS `%s` ( \
   `reserved_field_1` VARCHAR(50), \
   `reserved_field_2` longblob, \
   `reserved_field_3` longblob, \
-   primary key(id) \
+   primary key(id), \
+   UNIQUE KEY (`unique_flag`) \
 ) ENGINE=INNODB DEFAULT CHARSET=utf8;
 
 create.rawrecommendation.store.index=ALTER TABLE %s ADD UNIQUE %s_idx 
(project, model_id, unique_flag, semantic_version);
diff --git 
a/src/core-common/src/main/resources/metadata-jdbc-postgresql.properties 
b/src/core-common/src/main/resources/metadata-jdbc-postgresql.properties
index f5577a3e12..d58b644af8 100644
--- a/src/core-common/src/main/resources/metadata-jdbc-postgresql.properties
+++ b/src/core-common/src/main/resources/metadata-jdbc-postgresql.properties
@@ -209,7 +209,8 @@ create.rawrecommendation.store.table=CREATE TABLE IF NOT 
EXISTS %s ( \
   reserved_field_1 VARCHAR(50), \
   reserved_field_2 bytea, \
   reserved_field_3 bytea, \
-  primary key(id) \
+  primary key(id), \
+  unique (unique_flag) \
 );
 
 create.rawrecommendation.store.index=CREATE UNIQUE INDEX %s_idx ON %s using 
btree (project, model_id, unique_flag, semantic_version);
diff --git 
a/src/core-metadata/src/main/java/org/apache/kylin/metadata/recommendation/candidate/JdbcRawRecStore.java
 
b/src/core-metadata/src/main/java/org/apache/kylin/metadata/recommendation/candidate/JdbcRawRecStore.java
index 294d8eb6f3..5ea76322c0 100644
--- 
a/src/core-metadata/src/main/java/org/apache/kylin/metadata/recommendation/candidate/JdbcRawRecStore.java
+++ 
b/src/core-metadata/src/main/java/org/apache/kylin/metadata/recommendation/candidate/JdbcRawRecStore.java
@@ -84,7 +84,7 @@ import lombok.extern.slf4j.Slf4j;
 @Slf4j
 public class JdbcRawRecStore {
 
-    public static final String RECOMMENDATION_CANDIDATE = "_rec_candidate";
+    public static final String RECOMMENDATION_CANDIDATE = "_rec_candidate_v2";
     private static final int NON_EXIST_MODEL_SEMANTIC_VERSION = 
Integer.MIN_VALUE;
     private static final int LAG_SEMANTIC_VERSION = 1;
     private static final List<RawRecItem.RawRecState> FINAL_STATE = 
Arrays.asList(RawRecItem.RawRecState.APPLIED,
@@ -447,12 +447,20 @@ public class JdbcRawRecStore {
             val baseId = manager.getMaxId() + 100;
 
             Map<Integer, Integer> idChangedMap = new HashMap<>();
+            Map<String, RawRecItem> layoutRecommendations = new HashMap<>();
+            Map<String, List<String>> md5ToUuid = new HashMap<>();
 
             val index = new AtomicInteger(0);
 
             List<RawRecItem> tmpRawRecItems = 
rawRecItems.stream().map(rawRecItem -> {
                 rawRecItem.setProject(project);
-                rawRecItem.setModelID(modelId);
+                if (!modelId.equals(rawRecItem.getModelID())) {
+                    rawRecItem.setModelID(modelId);
+                    String content = RawRecUtil.getContent(rawRecItem);
+                    String md5 = RawRecUtil.computeMD5(content);
+                    Pair<String, RawRecItem> recItemPair = 
RawRecUtil.getRawRecItemFromMap(md5, content, md5ToUuid, layoutRecommendations);
+                    rawRecItem.setUniqueFlag(recItemPair.getFirst());
+                }
 
                 String uniqueFlag = rawRecItem.getUniqueFlag();
                 RawRecItem originalRawRecItem = 
manager.getRawRecItemByUniqueFlag(rawRecItem.getProject(),
diff --git 
a/src/tool/src/test/java/org/apache/kylin/tool/RecCandidateToolTest.java 
b/src/tool/src/test/java/org/apache/kylin/tool/RecCandidateToolTest.java
index 830332920a..ba0fc75935 100644
--- a/src/tool/src/test/java/org/apache/kylin/tool/RecCandidateToolTest.java
+++ b/src/tool/src/test/java/org/apache/kylin/tool/RecCandidateToolTest.java
@@ -175,7 +175,7 @@ public class RecCandidateToolTest extends 
NLocalFileMetadataTestCase {
         jdbcRawRecStore.deleteAll();
         assertEquals(0, jdbcRawRecStore.queryAll().size());
         tool = new RecCandidateTool();
-        tool.execute(new String[] { "-restore", "-dir", 
file.getAbsolutePath(), "-table", "test_opt_rec_candidate" });
+        tool.execute(new String[] { "-restore", "-dir", 
file.getAbsolutePath(), "-table", "test_opt_rec_candidate_v2" });
         assertEquals(3, jdbcRawRecStore.queryAll().size());
     }
 
@@ -197,7 +197,7 @@ public class RecCandidateToolTest extends 
NLocalFileMetadataTestCase {
         jdbcRawRecStore.deleteAll();
         assertEquals(0, jdbcRawRecStore.queryAll().size());
         tool = new RecCandidateTool();
-        tool.execute(new String[] { "-restore", "-dir", 
file.getAbsolutePath(), "-table", "test_opt_rec_candidate" });
+        tool.execute(new String[] { "-restore", "-dir", 
file.getAbsolutePath(), "-table", "test_opt_rec_candidate_v2" });
         assertEquals(3, jdbcRawRecStore.queryAll().size());
     }