This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit fd87fe92fe48d09b5b362b7566dd0686d3777605
Author: huangsheng <huangshen...@163.com>
AuthorDate: Fri Apr 28 17:34:34 2023 +0800
KYLIN-5651 supports obtaining table comment from Hive
---
.../rest/response/PreReloadTableResponse.java | 4 +++
.../org/apache/kylin/metadata/model/TableDesc.java | 6 ++++
.../metadata/model/schema/ReloadTableContext.java | 4 ++-
.../apache/kylin/rest/service/TableService.java | 4 +++
.../kylin/rest/service/TableReloadServiceTest.java | 17 +++++++++
.../spark/source/NSparkMetadataExplorer.java | 1 +
.../kylin/engine/spark/source/NSparkTableMeta.java | 6 ++--
.../spark/source/NSparkTableMetaBuilder.java | 10 ++++--
.../spark/source/NSparkTableMetaExplorer.java | 4 +++
.../spark/source/NSparkTableMetaExplorerTest.scala | 40 ++++++++++++++++++++++
10 files changed, 91 insertions(+), 5 deletions(-)
diff --git
a/src/common-service/src/main/java/org/apache/kylin/rest/response/PreReloadTableResponse.java
b/src/common-service/src/main/java/org/apache/kylin/rest/response/PreReloadTableResponse.java
index 1c91b9458f..973508cf8c 100644
---
a/src/common-service/src/main/java/org/apache/kylin/rest/response/PreReloadTableResponse.java
+++
b/src/common-service/src/main/java/org/apache/kylin/rest/response/PreReloadTableResponse.java
@@ -61,6 +61,9 @@ public class PreReloadTableResponse {
@JsonProperty("snapshot_deleted")
private boolean snapshotDeleted = false;
+ @JsonProperty("table_comment_changed")
+ private boolean tableCommentChanged = false;
+
@JsonProperty("update_base_index_count")
private int updateBaseIndexCount;
@@ -81,6 +84,7 @@ public class PreReloadTableResponse {
this.addLayoutsCount = otherResponse.addLayoutsCount;
this.refreshLayoutsCount = otherResponse.refreshLayoutsCount;
this.snapshotDeleted = otherResponse.snapshotDeleted;
+ this.tableCommentChanged = otherResponse.tableCommentChanged;
this.details = otherResponse.details;
}
diff --git
a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java
b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java
index eb4c5f2e8c..76dba6c87b 100644
---
a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java
+++
b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java
@@ -199,6 +199,11 @@ public class TableDesc extends RootPersistentEntity
implements Serializable, ISo
@JsonProperty("partition_desc")
private PartitionDesc partitionDesc;
+ @Getter
+ @Setter
+ @JsonProperty("table_comment")
+ private String tableComment;
+
public TableDesc() {
}
@@ -234,6 +239,7 @@ public class TableDesc extends RootPersistentEntity
implements Serializable, ISo
this.isTransactional = other.isTransactional;
this.isRangePartition = other.isRangePartition;
this.partitionDesc = other.partitionDesc;
+ this.tableComment = other.tableComment;
setMvcc(other.getMvcc());
}
diff --git
a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/schema/ReloadTableContext.java
b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/schema/ReloadTableContext.java
index 7502b8cb0f..3774177bd4 100644
---
a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/schema/ReloadTableContext.java
+++
b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/schema/ReloadTableContext.java
@@ -54,6 +54,8 @@ public class ReloadTableContext {
private Set<String> effectedJobs = Sets.newHashSet();
+ private boolean isTableCommentChanged = false;
+
private TableDesc tableDesc;
private TableExtDesc tableExtDesc;
@@ -100,6 +102,6 @@ public class ReloadTableContext {
public boolean isNeedProcess() {
return CollectionUtils.isNotEmpty(addColumns) ||
CollectionUtils.isNotEmpty(removeColumns)
- || CollectionUtils.isNotEmpty(changeTypeColumns);
+ || CollectionUtils.isNotEmpty(changeTypeColumns) ||
isTableCommentChanged;
}
}
diff --git
a/src/datasource-service/src/main/java/org/apache/kylin/rest/service/TableService.java
b/src/datasource-service/src/main/java/org/apache/kylin/rest/service/TableService.java
index 5647d6c412..d8f49d96c3 100644
---
a/src/datasource-service/src/main/java/org/apache/kylin/rest/service/TableService.java
+++
b/src/datasource-service/src/main/java/org/apache/kylin/rest/service/TableService.java
@@ -306,6 +306,7 @@ public class TableService extends BasicService {
nTableDesc.setUuid(origTable.getUuid());
nTableDesc.setLastModified(origTable.getLastModified());
nTableDesc.setIncrementLoading(origTable.isIncrementLoading());
+ nTableDesc.setTableComment(tableDesc.getTableComment());
}
tableMetaMgr.saveSourceTable(nTableDesc);
@@ -1125,6 +1126,7 @@ public class TableService extends BasicService {
result.setRemoveDimCount(context.getRemoveAffectedModels().values().stream()
.map(AffectedModelContext::getDimensions).mapToLong(Set::size).sum());
result.setDataTypeChangeColumnCount(context.getChangeTypeColumns().size());
+ result.setTableCommentChanged(context.isTableCommentChanged());
val schemaChanged = result.getAddColumnCount() > 0 ||
result.getRemoveColumnCount() > 0
|| result.getDataTypeChangeColumnCount() > 0;
@@ -1605,6 +1607,8 @@ public class TableService extends BasicService {
context.setAddColumns(diff.entriesOnlyOnLeft().keySet());
context.setRemoveColumns(diff.entriesOnlyOnRight().keySet());
context.setChangeTypeColumns(diff.entriesDiffering().keySet());
+
context.setTableCommentChanged(!Objects.equals(originTableDesc.getTableComment(),
newTableDesc.getTableComment()));
+
if (!context.isNeedProcess()) {
return context;
}
diff --git
a/src/modeling-service/src/test/java/org/apache/kylin/rest/service/TableReloadServiceTest.java
b/src/modeling-service/src/test/java/org/apache/kylin/rest/service/TableReloadServiceTest.java
index 7da97168db..4c3168110b 100644
---
a/src/modeling-service/src/test/java/org/apache/kylin/rest/service/TableReloadServiceTest.java
+++
b/src/modeling-service/src/test/java/org/apache/kylin/rest/service/TableReloadServiceTest.java
@@ -1206,6 +1206,23 @@ public class TableReloadServiceTest extends
CSVSourceTestCase {
() -> tableService.preProcessBeforeReloadWithFailFast(PROJECT,
tableIdentity));
}
+ @Test
+ public void testReloadAddTableComment() throws Exception {
+ val tableManager = NTableMetadataManager.getInstance(getTestConfig(),
PROJECT);
+ val tableDesc = tableManager.getTableDesc("EDW.TEST_CAL_DT");
+ Assert.assertNull(tableDesc.getTableComment());
+
+ String resPath =
KylinConfig.getInstanceFromEnv().getMetadataUrl().getIdentifier();
+ String tablePath = resPath + "/../data/tableDesc/" + "EDW.TEST_CAL_DT"
+ ".json";
+ val tableMeta = JsonUtil.readValue(new File(tablePath),
TableDesc.class);
+ tableMeta.setTableComment("Table Comment");
+ JsonUtil.writeValueIndent(new FileOutputStream(tablePath), tableMeta);
+
+ tableService.innerReloadTable(PROJECT, "EDW.TEST_CAL_DT", true, null);
+ val newTableDesc = tableManager.getTableDesc("EDW.TEST_CAL_DT");
+ Assert.assertEquals("Table Comment", newTableDesc.getTableComment());
+ }
+
@Test
public void testReloadChangeColumn() throws Exception {
removeColumn("EDW.TEST_CAL_DT", "CAL_DT_UPD_USER");
diff --git
a/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/source/NSparkMetadataExplorer.java
b/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/source/NSparkMetadataExplorer.java
index 31dcc1e5c5..3ade78d63a 100644
---
a/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/source/NSparkMetadataExplorer.java
+++
b/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/source/NSparkMetadataExplorer.java
@@ -251,6 +251,7 @@ public class NSparkMetadataExplorer implements
ISourceMetadataExplorer, ISampleD
tableDesc.setSourceType(ISourceAware.ID_SPARK);
tableDesc.setTransactional(tableMeta.isTransactional);
tableDesc.setRangePartition(tableMeta.isRangePartition);
+ tableDesc.setTableComment(tableMeta.tableComment);
Set<String> partColumnSet =
Optional.ofNullable(tableMeta.partitionColumns) //
.orElseGet(Collections::emptyList).stream().map(field ->
field.name) //
diff --git
a/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/source/NSparkTableMeta.java
b/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/source/NSparkTableMeta.java
index 29f48bfbfc..44da3b6177 100644
---
a/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/source/NSparkTableMeta.java
+++
b/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/source/NSparkTableMeta.java
@@ -68,6 +68,7 @@ public class NSparkTableMeta {
boolean isRangePartition;
String s3Role;
String s3Endpoint;
+ String tableComment;
public List<SparkTableColumnMeta> getAllColumns() {
return allColumns;
@@ -77,7 +78,7 @@ public class NSparkTableMeta {
String owner, String provider, String tableType, String
createTime, String lastAccessTime, long fileSize,
long fileNum, boolean isNative, List<SparkTableColumnMeta>
allColumns,
List<SparkTableColumnMeta> partitionColumns, boolean
isTransactional, boolean isRangePartition,
- String s3Role, String s3Endpoint) {
+ String s3Role, String s3Endpoint, String tableComment) {
this.tableName = tableName;
this.sdLocation = sdLocation;
this.sdInputFormat = sdInputFormat;
@@ -96,6 +97,7 @@ public class NSparkTableMeta {
this.isRangePartition = isRangePartition;
this.s3Role = s3Role;
this.s3Endpoint = s3Endpoint;
+ this.tableComment = tableComment;
}
@Override
@@ -106,6 +108,6 @@ public class NSparkTableMeta {
+ ", createTime='" + createTime + '\'' + ", lastAccessTime=" +
lastAccessTime + ", fileSize=" + fileSize
+ ", fileNum=" + fileNum + ", isNative=" + isNative + ",
allColumns=" + allColumns
+ ", partitionColumns=" + partitionColumns + ",
isTransactional=" + isTransactional
- + ", isRangePartition=" + isRangePartition + ", s3Role=" +
s3Role + ", s3Endpoint=" + s3Endpoint + '}';
+ + ", isRangePartition=" + isRangePartition + ", s3Role=" +
s3Role + ", s3Endpoint=" + s3Endpoint + ", tableComment=" + tableComment + '}';
}
}
diff --git
a/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/source/NSparkTableMetaBuilder.java
b/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/source/NSparkTableMetaBuilder.java
index 52385e9c35..8a04039156 100644
---
a/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/source/NSparkTableMetaBuilder.java
+++
b/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/source/NSparkTableMetaBuilder.java
@@ -41,6 +41,12 @@ public class NSparkTableMetaBuilder {
private boolean isRangePartition = false;
private String s3Role;
private String s3Endpoint;
+ private String tableComment;
+
+ public NSparkTableMetaBuilder setTableComment(String tableComment) {
+ this.tableComment = tableComment;
+ return this;
+ }
public NSparkTableMetaBuilder setTableName(String tableName) {
this.tableName = tableName;
@@ -135,6 +141,6 @@ public class NSparkTableMetaBuilder {
public NSparkTableMeta createSparkTableMeta() {
return new NSparkTableMeta(tableName, sdLocation, sdInputFormat,
sdOutputFormat, owner, provider, tableType,
createTime, lastAccessTime, fileSize, fileNum, isNative,
allColumns, partitionColumns, isTransactional,
- isRangePartition, s3Role, s3Endpoint);
+ isRangePartition, s3Role, s3Endpoint, tableComment);
}
-}
\ No newline at end of file
+}
diff --git
a/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/source/NSparkTableMetaExplorer.java
b/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/source/NSparkTableMetaExplorer.java
index 332b2c20b3..c52222a225 100644
---
a/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/source/NSparkTableMetaExplorer.java
+++
b/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/source/NSparkTableMetaExplorer.java
@@ -122,6 +122,10 @@ public class NSparkTableMetaExplorer implements
Serializable {
builder.setTableType(tableMetadata.tableType().name());
builder.setPartitionColumns(getColumns(tableMetadata,
tableMetadata.partitionSchema()));
builder.setIsRangePartition(isRangePartition(tableMetadata));
+
+ if (tableMetadata.comment().isDefined()) {
+ builder.setTableComment(tableMetadata.comment().get());
+ }
if (tableMetadata.storage().inputFormat().isDefined()) {
builder.setSdInputFormat(tableMetadata.storage().inputFormat().get());
}
diff --git
a/src/spark-project/engine-spark/src/test/scala/org/apache/kylin/engine/spark/source/NSparkTableMetaExplorerTest.scala
b/src/spark-project/engine-spark/src/test/scala/org/apache/kylin/engine/spark/source/NSparkTableMetaExplorerTest.scala
index dd6244be8f..fa672c5f63 100644
---
a/src/spark-project/engine-spark/src/test/scala/org/apache/kylin/engine/spark/source/NSparkTableMetaExplorerTest.scala
+++
b/src/spark-project/engine-spark/src/test/scala/org/apache/kylin/engine/spark/source/NSparkTableMetaExplorerTest.scala
@@ -160,6 +160,46 @@ class NSparkTableMetaExplorerTest extends
SparderBaseFunSuite with SharedSparkSe
importUnsupportedCol(BinaryType)
}
+
+ test("Test load hive table and get table comment") {
+ SparderEnv.setSparkSession(spark)
+
+ val view = CatalogTable(
+ identifier = TableIdentifier("hive_table_comment"),
+ comment = Option("Table Comment"),
+ tableType = CatalogTableType.MANAGED,
+ storage = CatalogStorageFormat.empty,
+ schema = new StructType()
+ .add("a", "string", nullable = true, new
MetadataBuilder().putString("__CHAR_VARCHAR_TYPE_STRING", "char(10)").build())
+ .add("b", "string", nullable = true, new
MetadataBuilder().putString("__CHAR_VARCHAR_TYPE_STRING",
"varchar(33)").build())
+ .add("c", "int"),
+ properties = Map()
+ )
+ spark.sessionState.catalog.createTable(view, ignoreIfExists = false, false)
+
+ withTable("hive_table_comment") {
+ val meta = new NSparkTableMetaExplorer().getSparkTableMeta("",
"hive_table_comment")
+ assert(meta.tableComment.equals("Table Comment"))
+ }
+
+ val view2 = CatalogTable(
+ identifier = TableIdentifier("hive_no_table_comment"),
+ tableType = CatalogTableType.MANAGED,
+ storage = CatalogStorageFormat.empty,
+ schema = new StructType()
+ .add("a", "string", nullable = true, new
MetadataBuilder().putString("__CHAR_VARCHAR_TYPE_STRING", "char(10)").build())
+ .add("b", "string", nullable = true, new
MetadataBuilder().putString("__CHAR_VARCHAR_TYPE_STRING",
"varchar(33)").build())
+ .add("c", "int"),
+ properties = Map()
+ )
+ spark.sessionState.catalog.createTable(view2, ignoreIfExists = false,
false)
+
+ withTable("hive_no_table_comment") {
+ val meta = new NSparkTableMetaExplorer().getSparkTableMeta("",
"hive_no_table_comment")
+ assert(meta.tableComment == null)
+ }
+ }
+
def createTmpCatalog(st: StructType): CatalogTable = {
CatalogTable(
identifier = TableIdentifier("hive_table_types"),
