This is an automated email from the ASF dual-hosted git repository.
caolu pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
The following commit(s) were added to refs/heads/kylin5 by this push:
new 2ced249833 KYLIN-6053 Add OpenAPI to show internaltable details
2ced249833 is described below
commit 2ced2498338c7894cc6fdd115ce37121576472c6
Author: Yan Xin <[email protected]>
AuthorDate: Fri Nov 29 11:22:08 2024 +0800
KYLIN-6053 Add OpenAPI to show internaltable details
---
.../kylin/metadata/table/InternalTableManager.java | 25 ++++++++++
.../rest/service/InternalTableServiceTest.java | 56 ++++++++++++++++------
.../kylin/rest/service/InternalTableService.java | 20 +++-----
.../rest/controller/InternalTableController.java | 11 +----
.../open/OpenInternalTableController.java | 2 +-
5 files changed, 75 insertions(+), 39 deletions(-)
diff --git
a/src/core-metadata/src/main/java/org/apache/kylin/metadata/table/InternalTableManager.java
b/src/core-metadata/src/main/java/org/apache/kylin/metadata/table/InternalTableManager.java
index ab6da16e8c..4848a219ef 100644
---
a/src/core-metadata/src/main/java/org/apache/kylin/metadata/table/InternalTableManager.java
+++
b/src/core-metadata/src/main/java/org/apache/kylin/metadata/table/InternalTableManager.java
@@ -19,13 +19,20 @@
package org.apache.kylin.metadata.table;
import static java.util.stream.Collectors.groupingBy;
+import static
org.apache.kylin.common.persistence.RawResourceFilter.Operator.EQUAL_CASE_INSENSITIVE;
+import static
org.apache.kylin.common.persistence.RawResourceFilter.Operator.LIKE_CASE_INSENSITIVE;
+import java.util.ArrayList;
+import java.util.Collections;
import java.util.List;
import java.util.Map;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.MetadataType;
+import org.apache.kylin.common.persistence.RawResourceFilter;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.metadata.cachesync.CachedCrudAssist;
@@ -125,4 +132,22 @@ public class InternalTableManager {
void modify(InternalTableDesc copyForWrite);
}
+ public List<InternalTableDesc> getInternalTableDescInfos(String
databaseName, String tableName, boolean isFuzzy) {
+ RawResourceFilter combinedFilter = new RawResourceFilter();
+ if (!StringUtils.isBlank(databaseName)) {
+ if (isFuzzy) {
+ combinedFilter.addConditions("dbName",
Collections.singletonList(databaseName), LIKE_CASE_INSENSITIVE);
+ } else {
+ combinedFilter.addConditions("dbName",
Collections.singletonList(databaseName), EQUAL_CASE_INSENSITIVE);
+ }
+ }
+ if (!StringUtils.isBlank(tableName)) {
+ if (isFuzzy) {
+ combinedFilter.addConditions("name",
Collections.singletonList(tableName), LIKE_CASE_INSENSITIVE);
+ } else {
+ combinedFilter.addConditions("name",
Collections.singletonList(tableName), EQUAL_CASE_INSENSITIVE);
+ }
+ }
+ return
internalTableCrud.listByFilter(combinedFilter).stream().distinct().collect(Collectors.toList());
+ }
}
diff --git
a/src/data-loading-service/src/test/java/org/apache/kylin/rest/service/InternalTableServiceTest.java
b/src/data-loading-service/src/test/java/org/apache/kylin/rest/service/InternalTableServiceTest.java
index c81b251c93..14980cff88 100644
---
a/src/data-loading-service/src/test/java/org/apache/kylin/rest/service/InternalTableServiceTest.java
+++
b/src/data-loading-service/src/test/java/org/apache/kylin/rest/service/InternalTableServiceTest.java
@@ -282,7 +282,8 @@ public class InternalTableServiceTest extends
AbstractTestCase {
partitionCols, dateFormat, tblProperties,
InternalTableDesc.StorageType.PARQUET.name());
// check internal table metadata
- List<InternalTableDescResponse> internalTables =
internalTableService.getTableList(PROJECT, false, false, "");
+ List<InternalTableDescResponse> internalTables =
internalTableService.getTableList(PROJECT, false, false, "",
+ "");
Assertions.assertEquals(1, internalTables.size());
InternalTableDescResponse response = internalTables.get(0);
Assertions.assertEquals(DATE_COL, response.getTimePartitionCol());
@@ -580,7 +581,7 @@ public class InternalTableServiceTest extends
AbstractTestCase {
internalTableService.createInternalTable(PROJECT, table.getName(),
table.getDatabase(), null, null,
new HashMap<>(), InternalTableDesc.StorageType.PARQUET.name());
- List<InternalTableDescResponse> tables =
internalTableService.getTableList(PROJECT, false, false, "");
+ List<InternalTableDescResponse> tables =
internalTableService.getTableList(PROJECT, false, false, "", "");
Assertions.assertEquals(1, tables.size());
Assertions.assertNull(tables.get(0).getTimePartitionCol());
@@ -590,7 +591,7 @@ public class InternalTableServiceTest extends
AbstractTestCase {
internalTableService.updateInternalTable(PROJECT,
tables.get(0).getTableName(), tables.get(0).getDatabaseName(),
new String[] { DATE_COL }, "yyyy-MM-dd", new HashMap<>(),
InternalTableDesc.StorageType.PARQUET.name());
- tables = internalTableService.getTableList(PROJECT, false, false, "");
+ tables = internalTableService.getTableList(PROJECT, false, false, "",
"");
Assertions.assertEquals(1, tables.size());
Assertions.assertEquals(DATE_COL, tables.get(0).getTimePartitionCol());
@@ -782,7 +783,6 @@ public class InternalTableServiceTest extends
AbstractTestCase {
TableDesc table = tManager.getTableDesc(TABLE_INDENTITY);
String[] partitionCols = new String[] { DATE_COL };
Map<String, String> tblProperties = new HashMap<>();
- String tableIdentity = table.getDatabase() + "." + table.getName();
boolean isFuzzy = true;
boolean needDetails = true;
when(tableService.getPartitionColumnFormat(any(), any(), any(),
any())).thenReturn("yyyy-MM-dd");
@@ -796,7 +796,7 @@ public class InternalTableServiceTest extends
AbstractTestCase {
Assertions.assertTrue(internalTableFolder.exists() &&
internalTableFolder.isDirectory());
List<InternalTableDescResponse> tables =
internalTableService.getTableList(PROJECT, isFuzzy, needDetails,
- tableIdentity);
+ table.getDatabase(), table.getName());
Assertions.assertEquals(tables.get(0).getTableName(), table.getName());
Assertions.assertEquals(DATE_COL, tables.get(0).getTimePartitionCol());
List<ColumnDesc> tableColumn = tables.get(0).getColumns();
@@ -809,19 +809,17 @@ public class InternalTableServiceTest extends
AbstractTestCase {
Assertions.assertEquals(columns[i].getDatatype(),
columninfo.getDatatype());
}
- tableIdentity = "E.S";
tables.clear();
- tables = internalTableService.getTableList(PROJECT, isFuzzy,
needDetails, tableIdentity);
+ tables = internalTableService.getTableList(PROJECT, isFuzzy,
needDetails, "E", "S");
Assertions.assertEquals(tables.get(0).getTableName(), table.getName());
Assertions.assertEquals(DATE_COL, tables.get(0).getTimePartitionCol());
tableColumn = tables.get(0).getColumns();
Assertions.assertEquals(tableColumn.size(), columns.length);
- tableIdentity = "DEFAULT2.DUMMY";
KylinException notExistException = null;
tables.clear();
try {
- tables = internalTableService.getTableList(PROJECT, isFuzzy,
needDetails, tableIdentity);
+ tables = internalTableService.getTableList(PROJECT, isFuzzy,
needDetails, "DEFAULT2", "DUMMY");
} catch (KylinException e) {
notExistException = e;
}
@@ -830,10 +828,9 @@ public class InternalTableServiceTest extends
AbstractTestCase {
null != notExistException &&
notExistException.getErrorCode().getCodeString().equals("KE-010007014"));
//Illegal string
- tableIdentity = "DEFAULT.1";
notExistException = null;
try {
- tables = internalTableService.getTableList(PROJECT, isFuzzy,
needDetails, tableIdentity);
+ tables = internalTableService.getTableList(PROJECT, isFuzzy,
needDetails, "DEFAULT", "1");
} catch (KylinException e) {
notExistException = e;
}
@@ -850,7 +847,6 @@ public class InternalTableServiceTest extends
AbstractTestCase {
TableDesc table = tManager.getTableDesc(TABLE_INDENTITY);
String[] partitionCols = new String[] { DATE_COL };
Map<String, String> tblProperties = new HashMap<>();
- String tableIdentity = table.getDatabase() + "." + table.getName();
boolean isFuzzy = false;
boolean needDetails = true;
when(tableService.getPartitionColumnFormat(any(), any(), any(),
any())).thenReturn("yyyy-MM-dd");
@@ -864,7 +860,7 @@ public class InternalTableServiceTest extends
AbstractTestCase {
Assertions.assertTrue(internalTableFolder.exists() &&
internalTableFolder.isDirectory());
List<InternalTableDescResponse> tables =
internalTableService.getTableList(PROJECT, isFuzzy, needDetails,
- tableIdentity);
+ table.getDatabase(), table.getName());
Assertions.assertEquals(tables.get(0).getTableName(), table.getName());
Assertions.assertEquals(DATE_COL, tables.get(0).getTimePartitionCol());
List<ColumnDesc> tableColumn = tables.get(0).getColumns();
@@ -877,11 +873,10 @@ public class InternalTableServiceTest extends
AbstractTestCase {
Assertions.assertEquals(columns[i].getDatatype(),
columninfo.getDatatype());
}
- tableIdentity = "DEFAULT.T";
KylinException notExistException = null;
tables.clear();
try {
- tables = internalTableService.getTableList(PROJECT, isFuzzy,
needDetails, tableIdentity);
+ tables = internalTableService.getTableList(PROJECT, isFuzzy,
needDetails, "DEFAULT", "T");
} catch (KylinException e) {
notExistException = e;
}
@@ -889,4 +884,35 @@ public class InternalTableServiceTest extends
AbstractTestCase {
Assertions.assertTrue(
null != notExistException &&
notExistException.getErrorCode().getCodeString().equals("KE-010007014"));
}
+
+ @Test
+ public void testGetInternalTableInfoByFuzzyKey() throws Exception {
+ KylinConfig config = KylinConfig.getInstanceFromEnv();
+ NTableMetadataManager tManager =
NTableMetadataManager.getInstance(config, PROJECT);
+ InternalTableManager internalTableManager =
InternalTableManager.getInstance(config, PROJECT);
+ TableDesc table = tManager.getTableDesc(TABLE_INDENTITY);
+ String[] partitionCols = new String[] { DATE_COL };
+ Map<String, String> tblProperties = new HashMap<>();
+ when(tableService.getPartitionColumnFormat(any(), any(), any(),
any())).thenReturn("yyyy-MM-dd");
+
+ internalTableService.createInternalTable(PROJECT, table.getName(),
table.getDatabase(), partitionCols,
+ "yyyy-MM-dd", tblProperties,
InternalTableDesc.StorageType.PARQUET.name());
+
+ List<InternalTableDesc> tables =
internalTableManager.getInternalTableDescInfos("", "", true);
+ Assert.assertEquals(1, tables.size());
+ tables = internalTableManager.getInternalTableDescInfos("", "", false);
+ Assert.assertEquals(1, tables.size());
+ tables = internalTableManager.getInternalTableDescInfos("D", "",
false);
+ Assert.assertEquals(0, tables.size());
+ tables = internalTableManager.getInternalTableDescInfos("D", "T",
true);
+ Assert.assertEquals(1, tables.size());//DEFAULT.TEST_KYLIN_FACT
+ tables = internalTableManager.getInternalTableDescInfos("D", "T",
false);
+ Assert.assertEquals(0, tables.size());
+ tables = internalTableManager.getInternalTableDescInfos("",
"TEST_KYLIN_FACT", false);
+ Assert.assertEquals(1, tables.size());
+ tables = internalTableManager.getInternalTableDescInfos("", "TEST",
true);
+ Assert.assertEquals(1, tables.size());
+ tables = internalTableManager.getInternalTableDescInfos("D", "", true);
+ Assert.assertEquals(1, tables.size());
+ }
}
diff --git
a/src/datasource-service/src/main/java/org/apache/kylin/rest/service/InternalTableService.java
b/src/datasource-service/src/main/java/org/apache/kylin/rest/service/InternalTableService.java
index cc6913b8c3..706efe96a7 100644
---
a/src/datasource-service/src/main/java/org/apache/kylin/rest/service/InternalTableService.java
+++
b/src/datasource-service/src/main/java/org/apache/kylin/rest/service/InternalTableService.java
@@ -402,23 +402,17 @@ public class InternalTableService extends BasicService {
}
public List<InternalTableDescResponse> getTableList(String project,
boolean isFuzzy, boolean needDetails,
- String fuzzyKey) {
+ String databaseName, String tableName) {
InternalTableManager internalTableManager =
getManager(InternalTableManager.class, project);
- List<InternalTableDesc> tableList =
internalTableManager.listAllTables();
- if (StringUtils.isNotBlank(fuzzyKey)) {
- if (isFuzzy) {
- String regex = fuzzyKey.replace(".", ".*");
- tableList = tableList.stream().filter(table ->
table.getIdentity().matches("(?i).*" + regex + ".*"))
- .collect(Collectors.toList());
- } else {
- tableList = tableList.stream()
- .filter(table ->
StringUtils.equalsIgnoreCase(table.getIdentity(), fuzzyKey))
- .collect(Collectors.toList());
- }
+ List<InternalTableDesc> tableList;
+ if (StringUtils.isNotBlank(databaseName) ||
StringUtils.isNotBlank(tableName)) {
+ tableList =
internalTableManager.getInternalTableDescInfos(databaseName, tableName,
isFuzzy);
if (tableList.isEmpty()) {
throw new KylinException(INTERNAL_TABLE_NOT_EXIST,
- String.format(Locale.ROOT,
MsgPicker.getMsg().getInternalTableNotFound(), fuzzyKey));
+ String.format(Locale.ROOT,
MsgPicker.getMsg().getInternalTableNotFound(), tableName));
}
+ } else {
+ tableList = internalTableManager.listAllTables();
}
return tableList.stream().map(table ->
InternalTableDescResponse.convertToResponse(table, needDetails))
.collect(Collectors.toList());
diff --git
a/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/InternalTableController.java
b/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/InternalTableController.java
index d514bfc6e9..85e1d3e6b9 100644
---
a/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/InternalTableController.java
+++
b/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/InternalTableController.java
@@ -168,16 +168,7 @@ public class InternalTableController extends
NBasicController {
@RequestParam(value = "page_size", required = false, defaultValue
= "10") Integer limit) {
project = checkProjectName(project);
- String fuzzyKey = "";
- if (StringUtils.isNotBlank(database) && StringUtils.isNotBlank(table))
{
- fuzzyKey = StringUtils.trim(database) + "." +
StringUtils.trim(table);
- } else if (StringUtils.isBlank(database)) {
- fuzzyKey = StringUtils.trim(table);
- } else if (StringUtils.isBlank(table)) {
- fuzzyKey = StringUtils.trim(database);
- }
-
- val rep = internalTableService.getTableList(project, isFuzzy,
needDetails, fuzzyKey);
+ val rep = internalTableService.getTableList(project, isFuzzy,
needDetails, database, table);
return new EnvelopeResponse<>(KylinException.CODE_SUCCESS,
DataResult.get(rep, offset, limit), "");
}
diff --git
a/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/open/OpenInternalTableController.java
b/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/open/OpenInternalTableController.java
index c54d7ed853..31755e0188 100644
---
a/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/open/OpenInternalTableController.java
+++
b/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/open/OpenInternalTableController.java
@@ -94,7 +94,7 @@ public class OpenInternalTableController extends
NBasicController {
if (StringUtils.isBlank(table) && StringUtils.isBlank(database) &&
isFuzzy) {
throw new KylinException(EMPTY_PARAMETER,
MsgPicker.getMsg().getTableOrDatabaseNameCannotEmpty());
}
- return internalTableController.getTableList(projectName, isFuzzy,
needDetails, database, table, offset, limit);
+ return internalTableController.getTableList(projectName, needDetails,
isFuzzy, database, table, offset, limit);
}
@ApiOperation(value = "get_table_detail", tags = { "AI" })
