This is an automated email from the ASF dual-hosted git repository.
pfzhan pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
The following commit(s) were added to refs/heads/kylin5 by this push:
new 9db7d8de0a KYLIN-6031 Add OpenAPI to show internalTable info (#2280)
9db7d8de0a is described below
commit 9db7d8de0a6743077fcb1bb11d75e081b89e8c34
Author: Guoliang Sun <[email protected]>
AuthorDate: Tue Feb 25 14:36:38 2025 +0800
KYLIN-6031 Add OpenAPI to show internalTable info (#2280)
Co-authored-by: Yan Xin <[email protected]>
---
.../org/apache/kylin/common/msg/CnMessage.java | 30 ++++
.../java/org/apache/kylin/common/msg/Message.java | 24 +++
.../rest/service/InternalTableServiceTest.java | 162 ++++++++++++++++++---
.../job/service/InternalTableLoadingService.java | 4 +-
.../rest/response/InternalTableDescResponse.java | 32 ++++
.../kylin/rest/service/InternalTableService.java | 55 +++----
.../apache/kylin/query/QueryGroupingSetsTest.java | 3 +
.../rest/controller/InternalTableController.java | 43 ++++--
.../open/OpenInternalTableController.java | 26 ++--
9 files changed, 303 insertions(+), 76 deletions(-)
diff --git
a/src/core-common/src/main/java/org/apache/kylin/common/msg/CnMessage.java
b/src/core-common/src/main/java/org/apache/kylin/common/msg/CnMessage.java
index 0da315c13d..4616c3e352 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/msg/CnMessage.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/msg/CnMessage.java
@@ -940,6 +940,36 @@ public class CnMessage extends Message {
return "无法找到内表 \"%s\"。 请检查后重试。";
}
+ @Override
+ public String getInternalTableEmpty() {
+ return "无法更新非空内表 \"%s\"。 请检查后重试。";
+ }
+
+ @Override
+ public String getSameInternalTableNameExist() {
+ return "表：\"%s\" 已经存在，请检查后重试。";
+ }
+
+ @Override
+ public String getInternalTablePath() {
+ return "创建内表路径失败，请检查后重试。";
+ }
+
+ @Override
+ public String getInternalTableUnpartitioned() {
+ return "不支持增量构建为分区内表，请检查后重试。";
+ }
+
+ @Override
+ public String getInternalTableNullPartitionFormat() {
+ return "日期分区格式不能为空， 请检查后重试。";
+ }
+
+ @Override
+ public String getInternalTableNoDataCol() {
+ return "在分区列中无法找到日期列，请检查后重试。";
+ }
+
@Override
public String getTimeExceedPartitionRange() {
return "刷新时间超出已加载范围 :\"%s\" ~ \"%s\"，请检查后重试。 ";
diff --git
a/src/core-common/src/main/java/org/apache/kylin/common/msg/Message.java
b/src/core-common/src/main/java/org/apache/kylin/common/msg/Message.java
index 230cffe1d3..7091589691 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/msg/Message.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/msg/Message.java
@@ -446,6 +446,30 @@ public class Message {
return "Can’t find internal table \"%s\". Please check and try again.";
}
+ public String getInternalTableEmpty() {
+ return "Can't update non empty internal table \"%s\", Please check and
try again.";
+ }
+
+ public String getSameInternalTableNameExist() {
+ return "Internal table %s already exists, please choose a different
name.";
+ }
+
+ public String getInternalTablePath() {
+ return "Failed to create internal table location.";
+ }
+
+ public String getInternalTableUnpartitioned() {
+ return "Incremental build is not supported for unPartitioned table.";
+ }
+
+ public String getInternalTableNullPartitionFormat() {
+ return "date_partition_format can not be null, please check again.";
+ }
+
+ public String getInternalTableNoDataCol() {
+ return "couldn't find date_col present in partition_cols, please check
again. ";
+ }
+
public String getTimeExceedPartitionRange() {
return "Refresh time exceed loaded range :\"%s\" ~ \"%s\", Please
check and try again. ";
}
diff --git
a/src/data-loading-service/src/test/java/org/apache/kylin/rest/service/InternalTableServiceTest.java
b/src/data-loading-service/src/test/java/org/apache/kylin/rest/service/InternalTableServiceTest.java
index 4d3f5db4da..c81b251c93 100644
---
a/src/data-loading-service/src/test/java/org/apache/kylin/rest/service/InternalTableServiceTest.java
+++
b/src/data-loading-service/src/test/java/org/apache/kylin/rest/service/InternalTableServiceTest.java
@@ -51,7 +51,9 @@ import
org.apache.kylin.engine.spark.utils.SparkJobFactoryUtils;
import org.apache.kylin.job.execution.ExecutableManager;
import org.apache.kylin.job.execution.ExecutableState;
import org.apache.kylin.job.service.InternalTableLoadingService;
+import org.apache.kylin.job.util.JobContextUtil;
import org.apache.kylin.junit.annotation.MetadataInfo;
+import org.apache.kylin.metadata.model.ColumnDesc;
import org.apache.kylin.metadata.model.NTableMetadataManager;
import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.metadata.table.InternalTableDesc;
@@ -111,6 +113,7 @@ public class InternalTableServiceTest extends
AbstractTestCase {
@BeforeEach
void setUp() throws Exception {
+ JobContextUtil.cleanUp();
MockitoAnnotations.openMocks(this);
SecurityContextHolder.getContext()
.setAuthentication(new TestingAuthenticationToken("ADMIN",
"ADMIN", Constant.ROLE_ADMIN));
@@ -279,7 +282,7 @@ public class InternalTableServiceTest extends
AbstractTestCase {
partitionCols, dateFormat, tblProperties,
InternalTableDesc.StorageType.PARQUET.name());
// check internal table metadata
- List<InternalTableDescResponse> internalTables =
internalTableService.getTableList(PROJECT);
+ List<InternalTableDescResponse> internalTables =
internalTableService.getTableList(PROJECT, false, false, "");
Assertions.assertEquals(1, internalTables.size());
InternalTableDescResponse response = internalTables.get(0);
Assertions.assertEquals(DATE_COL, response.getTimePartitionCol());
@@ -577,7 +580,7 @@ public class InternalTableServiceTest extends
AbstractTestCase {
internalTableService.createInternalTable(PROJECT, table.getName(),
table.getDatabase(), null, null,
new HashMap<>(), InternalTableDesc.StorageType.PARQUET.name());
- List<InternalTableDescResponse> tables =
internalTableService.getTableList(PROJECT);
+ List<InternalTableDescResponse> tables =
internalTableService.getTableList(PROJECT, false, false, "");
Assertions.assertEquals(1, tables.size());
Assertions.assertNull(tables.get(0).getTimePartitionCol());
@@ -587,7 +590,7 @@ public class InternalTableServiceTest extends
AbstractTestCase {
internalTableService.updateInternalTable(PROJECT,
tables.get(0).getTableName(), tables.get(0).getDatabaseName(),
new String[] { DATE_COL }, "yyyy-MM-dd", new HashMap<>(),
InternalTableDesc.StorageType.PARQUET.name());
- tables = internalTableService.getTableList(PROJECT);
+ tables = internalTableService.getTableList(PROJECT, false, false, "");
Assertions.assertEquals(1, tables.size());
Assertions.assertEquals(DATE_COL, tables.get(0).getTimePartitionCol());
@@ -669,10 +672,9 @@ public class InternalTableServiceTest extends
AbstractTestCase {
TransactionException exception =
Assertions.assertThrows(TransactionException.class,
() -> internalTableService.createInternalTable(PROJECT,
table.getName(), table.getDatabase(),
partitionCols, "yyyy-MM-dd", tblProperties,
InternalTableDesc.StorageType.PARQUET.name()));
-
- Assertions.assertEquals("KE-010007011(Internal Table Operation Failed)
\n"
- + "org.apache.kylin.common.exception.KylinException:
KE-010007011(Internal Table Operation Failed):Table is already an internal
table",
- exception.getCause().toString());
+ Assertions.assertEquals(String.format(Locale.ROOT,
MsgPicker.getMsg().getSameInternalTableNameExist(),
+ table.getName()),
+ exception.getCause().getMessage());
if (!internalTableFolder.delete()) {
Assertions.fail();
}
@@ -712,25 +714,23 @@ public class InternalTableServiceTest extends
AbstractTestCase {
() -> internalTableService.updateInternalTable(PROJECT,
tableName, db, partitionCols, dateFormat,
tblProperties,
InternalTableDesc.StorageType.PARQUET.name()));
- Assertions.assertEquals("KE-010007011(Internal Table Operation Failed)
\n"
- + "org.apache.kylin.common.exception.KylinException:
KE-010007011(Internal Table Operation Failed):Non-empty internal table can not
be updated",
- exception.getCause().toString());
+ Assertions.assertEquals(String.format(Locale.ROOT,
MsgPicker.getMsg().getInternalTableEmpty(),
+ table.getDatabase() + "." + table.getName()),
exception.getCause().getMessage());
}
@Test
void testCreateInternalPathFailedErrorCode() throws Exception {
when(tableService.getPartitionColumnFormat(any(), any(), any(),
any())).thenReturn("yyyy-MM-dd");
- FileSystem mockFileSystem = Mockito.mock(FileSystem.class);
+ FileSystem mockFileSystem = mock(FileSystem.class);
try (MockedStatic<HadoopUtil> mockedHadoopUtil =
Mockito.mockStatic(HadoopUtil.class)) {
mockedHadoopUtil.when(HadoopUtil::getWorkingFileSystem).thenReturn(mockFileSystem);
- Mockito.doThrow(new IOException("Simulated IO
error")).when(mockFileSystem).mkdirs(Mockito.any(Path.class));
+ doThrow(new IOException("Simulated IO
error")).when(mockFileSystem).mkdirs(any(Path.class));
KylinException exception =
Assertions.assertThrows(KylinException.class, () -> {
String path = "mocked/path/to/internal_table";
internalTableService.createInternalTablePath(path);
});
- Assertions.assertEquals("KE-010007011(Internal Table Operation
Failed) \n"
- + "org.apache.kylin.common.exception.KylinException:
KE-010007011(Internal Table Operation Failed):Failed to create internal table
location",
- exception.toString());
+ Assertions.assertEquals(String.format(Locale.ROOT,
MsgPicker.getMsg().getInternalTablePath()),
+ exception.getMessage());
}
}
@@ -745,9 +745,8 @@ public class InternalTableServiceTest extends
AbstractTestCase {
TransactionException exception =
Assertions.assertThrows(TransactionException.class,
() -> internalTableService.loadIntoInternalTable(PROJECT,
table.getName(), table.getDatabase(), true,
false, startDate, endDate, null));
- Assertions.assertEquals("KE-010007011(Internal Table Operation Failed)
\n"
- + "org.apache.kylin.common.exception.KylinException:
KE-010007011(Internal Table Operation Failed):Incremental build is not
supported for unPartitioned table",
- exception.getCause().toString());
+ Assertions.assertEquals(String.format(Locale.ROOT,
MsgPicker.getMsg().getInternalTableUnpartitioned()),
+ exception.getCause().getMessage());
}
@Test
@@ -758,9 +757,9 @@ public class InternalTableServiceTest extends
AbstractTestCase {
when(tableService.getPartitionColumnFormat(any(), any(), any(),
any())).thenReturn("yyyy-MM-dd");
KylinException exception =
Assertions.assertThrows(KylinException.class,
() -> internalTableService.checkParameters(null, table,
"yyyy-MM-dd"));
- Assertions.assertEquals("KE-010007013(Internal Table Parameter
Invalid) \n"
- + "org.apache.kylin.common.exception.KylinException:
KE-010007013(Internal Table Parameter Invalid):Can’t find the partition column.
Please check and try again.",
- exception.toString());
+ Assertions.assertEquals(
+ String.format(Locale.ROOT,
MsgPicker.getMsg().getPartitionColumnNotExist(), table.getIdentity()),
+ exception.getMessage());
}
@Test
@@ -771,8 +770,123 @@ public class InternalTableServiceTest extends
AbstractTestCase {
when(tableService.getPartitionColumnFormat(any(), any(), any(),
any())).thenReturn("yyyy-MM-dd");
KylinException exception =
Assertions.assertThrows(KylinException.class,
() -> internalTableService.checkParameters(new String[] {
"TRANS_ID", "CAL_DT" }, table, "yyyy-MM"));
- Assertions.assertEquals("KE-010007013(Internal Table Parameter
Invalid) \n"
- + "org.apache.kylin.common.exception.KylinException:
KE-010007013(Internal Table Parameter Invalid):Date partition format
\"yyyy-MM\" is not correct.",
- exception.toString());
+ Assertions.assertEquals(String.format(Locale.ROOT,
MsgPicker.getMsg().getIncorrectDateformat(), "yyyy-MM"),
+ exception.getMessage());
+ }
+
+ @Test
+ void testInternalTableShowDetailsFuzzy() throws Exception {
+ KylinConfig config = KylinConfig.getInstanceFromEnv();
+ NTableMetadataManager tManager =
NTableMetadataManager.getInstance(config, PROJECT);
+ InternalTableManager internalTableManager =
InternalTableManager.getInstance(config, PROJECT);
+ TableDesc table = tManager.getTableDesc(TABLE_INDENTITY);
+ String[] partitionCols = new String[] { DATE_COL };
+ Map<String, String> tblProperties = new HashMap<>();
+ String tableIdentity = table.getDatabase() + "." + table.getName();
+ boolean isFuzzy = true;
+ boolean needDetails = true;
+ when(tableService.getPartitionColumnFormat(any(), any(), any(),
any())).thenReturn("yyyy-MM-dd");
+
+ internalTableService.createInternalTable(PROJECT, table.getName(),
table.getDatabase(), partitionCols,
+ "yyyy-MM-dd", tblProperties,
InternalTableDesc.StorageType.PARQUET.name());
+ InternalTableDesc internalTable =
internalTableManager.getInternalTableDesc(TABLE_INDENTITY);
+ Assertions.assertNotNull(internalTable);
+ String workingDir =
config.getHdfsWorkingDirectory().replace("file://", "");
+ File internalTableFolder = new File(workingDir, INTERNAL_DIR);
+ Assertions.assertTrue(internalTableFolder.exists() &&
internalTableFolder.isDirectory());
+
+ List<InternalTableDescResponse> tables =
internalTableService.getTableList(PROJECT, isFuzzy, needDetails,
+ tableIdentity);
+ Assertions.assertEquals(tables.get(0).getTableName(), table.getName());
+ Assertions.assertEquals(DATE_COL, tables.get(0).getTimePartitionCol());
+ List<ColumnDesc> tableColumn = tables.get(0).getColumns();
+ ColumnDesc[] columns = internalTable.getColumns();
+ Assertions.assertEquals(tableColumn.size(), columns.length);
+ for (int i = 0; i < columns.length; i++) {
+ ColumnDesc columninfo = tableColumn.get(i);
+ Assertions.assertEquals(columns[i].getName(),
columninfo.getName());
+ Assertions.assertEquals(columns[i].getId(), columninfo.getId());
+ Assertions.assertEquals(columns[i].getDatatype(),
columninfo.getDatatype());
+ }
+
+ tableIdentity = "E.S";
+ tables.clear();
+ tables = internalTableService.getTableList(PROJECT, isFuzzy,
needDetails, tableIdentity);
+ Assertions.assertEquals(tables.get(0).getTableName(), table.getName());
+ Assertions.assertEquals(DATE_COL, tables.get(0).getTimePartitionCol());
+ tableColumn = tables.get(0).getColumns();
+ Assertions.assertEquals(tableColumn.size(), columns.length);
+
+ tableIdentity = "DEFAULT2.DUMMY";
+ KylinException notExistException = null;
+ tables.clear();
+ try {
+ tables = internalTableService.getTableList(PROJECT, isFuzzy,
needDetails, tableIdentity);
+ } catch (KylinException e) {
+ notExistException = e;
+ }
+ Assertions.assertTrue(tables.isEmpty());
+ Assertions.assertTrue(
+ null != notExistException &&
notExistException.getErrorCode().getCodeString().equals("KE-010007014"));
+
+ //Illegal string
+ tableIdentity = "DEFAULT.1";
+ notExistException = null;
+ try {
+ tables = internalTableService.getTableList(PROJECT, isFuzzy,
needDetails, tableIdentity);
+ } catch (KylinException e) {
+ notExistException = e;
+ }
+ Assertions.assertTrue(tables.isEmpty());
+ Assertions.assertTrue(
+ null != notExistException &&
notExistException.getErrorCode().getCodeString().equals("KE-010007014"));
+ }
+
+ @Test
+ void testInternalTableShowDetails() throws Exception {
+ KylinConfig config = KylinConfig.getInstanceFromEnv();
+ NTableMetadataManager tManager =
NTableMetadataManager.getInstance(config, PROJECT);
+ InternalTableManager internalTableManager =
InternalTableManager.getInstance(config, PROJECT);
+ TableDesc table = tManager.getTableDesc(TABLE_INDENTITY);
+ String[] partitionCols = new String[] { DATE_COL };
+ Map<String, String> tblProperties = new HashMap<>();
+ String tableIdentity = table.getDatabase() + "." + table.getName();
+ boolean isFuzzy = false;
+ boolean needDetails = true;
+ when(tableService.getPartitionColumnFormat(any(), any(), any(),
any())).thenReturn("yyyy-MM-dd");
+
+ internalTableService.createInternalTable(PROJECT, table.getName(),
table.getDatabase(), partitionCols,
+ "yyyy-MM-dd", tblProperties,
InternalTableDesc.StorageType.PARQUET.name());
+ InternalTableDesc internalTable =
internalTableManager.getInternalTableDesc(TABLE_INDENTITY);
+ Assertions.assertNotNull(internalTable);
+ String workingDir =
config.getHdfsWorkingDirectory().replace("file://", "");
+ File internalTableFolder = new File(workingDir, INTERNAL_DIR);
+ Assertions.assertTrue(internalTableFolder.exists() &&
internalTableFolder.isDirectory());
+
+ List<InternalTableDescResponse> tables =
internalTableService.getTableList(PROJECT, isFuzzy, needDetails,
+ tableIdentity);
+ Assertions.assertEquals(tables.get(0).getTableName(), table.getName());
+ Assertions.assertEquals(DATE_COL, tables.get(0).getTimePartitionCol());
+ List<ColumnDesc> tableColumn = tables.get(0).getColumns();
+ ColumnDesc[] columns = internalTable.getColumns();
+ Assertions.assertEquals(tableColumn.size(), columns.length);
+ for (int i = 0; i < columns.length; i++) {
+ ColumnDesc columninfo = tableColumn.get(i);
+ Assertions.assertEquals(columns[i].getName(),
columninfo.getName());
+ Assertions.assertEquals(columns[i].getId(), columninfo.getId());
+ Assertions.assertEquals(columns[i].getDatatype(),
columninfo.getDatatype());
+ }
+
+ tableIdentity = "DEFAULT.T";
+ KylinException notExistException = null;
+ tables.clear();
+ try {
+ tables = internalTableService.getTableList(PROJECT, isFuzzy,
needDetails, tableIdentity);
+ } catch (KylinException e) {
+ notExistException = e;
+ }
+ Assertions.assertTrue(tables.isEmpty());
+ Assertions.assertTrue(
+ null != notExistException &&
notExistException.getErrorCode().getCodeString().equals("KE-010007014"));
}
}
diff --git
a/src/datasource-service/src/main/java/org/apache/kylin/job/service/InternalTableLoadingService.java
b/src/datasource-service/src/main/java/org/apache/kylin/job/service/InternalTableLoadingService.java
index 354d186fdd..8d10c4b3a6 100644
---
a/src/datasource-service/src/main/java/org/apache/kylin/job/service/InternalTableLoadingService.java
+++
b/src/datasource-service/src/main/java/org/apache/kylin/job/service/InternalTableLoadingService.java
@@ -77,8 +77,8 @@ public class InternalTableLoadingService extends BasicService
{
if (isIncremental &&
(Objects.isNull(internalTable.getTablePartition())
||
Objects.isNull(internalTable.getTablePartition().getPartitionColumns())
||
internalTable.getTablePartition().getPartitionColumns().length == 0)) {
- throw new KylinException(INTERNAL_TABLE_ERROR,
- "Incremental build is not supported for unPartitioned
table");
+ String errorMsg = String.format(Locale.ROOT,
MsgPicker.getMsg().getInternalTableUnpartitioned());
+ throw new KylinException(INTERNAL_TABLE_ERROR, errorMsg);
}
// check refresh time exceed loaded range
InternalTablePartition tablePartition =
internalTable.getTablePartition();
diff --git
a/src/datasource-service/src/main/java/org/apache/kylin/rest/response/InternalTableDescResponse.java
b/src/datasource-service/src/main/java/org/apache/kylin/rest/response/InternalTableDescResponse.java
index b4e9c4fe8b..e26bcab665 100644
---
a/src/datasource-service/src/main/java/org/apache/kylin/rest/response/InternalTableDescResponse.java
+++
b/src/datasource-service/src/main/java/org/apache/kylin/rest/response/InternalTableDescResponse.java
@@ -18,8 +18,14 @@
package org.apache.kylin.rest.response;
+import java.util.Arrays;
+import java.util.List;
import java.util.Map;
+import org.apache.commons.lang3.ArrayUtils;
+import org.apache.kylin.metadata.model.ColumnDesc;
+import org.apache.kylin.metadata.table.InternalTableDesc;
+
import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.Data;
@@ -57,4 +63,30 @@ public class InternalTableDescResponse {
@JsonProperty("tbl_properties")
private Map<String, String> tblProperties;
+ @JsonProperty("columns_info")
+ private List<ColumnDesc> columns;
+
+ public static InternalTableDescResponse
convertToResponse(InternalTableDesc internalTableDesc,
+ boolean needDetails) {
+ InternalTableDescResponse response = new InternalTableDescResponse();
+ response.setTableName(internalTableDesc.getName());
+ response.setUuid(internalTableDesc.getUuid());
+ response.setDatabaseName(internalTableDesc.getDatabase());
+ response.setRowCount(internalTableDesc.getRowCount());
+ response.setStorageSize(internalTableDesc.getStorageSize());
+ response.setHitCount(internalTableDesc.getHitCount());
+
+ String[] partitionColumns = internalTableDesc.getPartitionColumns();
+ String partitionColumn = ArrayUtils.isNotEmpty(partitionColumns) ?
partitionColumns[0] : null;
+ response.setTimePartitionCol(partitionColumn);
+
+ response.setUpdateTime(internalTableDesc.getLastModified());
+
response.setDatePartitionFormat(internalTableDesc.getDatePartitionFormat());
+ response.setTblProperties(internalTableDesc.getTblProperties());
+
+ if (needDetails) {
+ response.setColumns(Arrays.asList(internalTableDesc.getColumns()));
+ }
+ return response;
+ }
}
diff --git
a/src/datasource-service/src/main/java/org/apache/kylin/rest/service/InternalTableService.java
b/src/datasource-service/src/main/java/org/apache/kylin/rest/service/InternalTableService.java
index ea81cc175d..cc6913b8c3 100644
---
a/src/datasource-service/src/main/java/org/apache/kylin/rest/service/InternalTableService.java
+++
b/src/datasource-service/src/main/java/org/apache/kylin/rest/service/InternalTableService.java
@@ -116,7 +116,9 @@ public class InternalTableService extends BasicService {
throw new KylinException(TABLE_NOT_EXIST, errorMsg);
}
if (originTable.isHasInternal()) {
- throw new KylinException(INTERNAL_TABLE_ERROR, "Table is
already an internal table");
+ String errorMsg = String.format(Locale.ROOT,
MsgPicker.getMsg().getSameInternalTableNameExist(),
+ originTable.getName());
+ throw new KylinException(INTERNAL_TABLE_ERROR, errorMsg);
}
checkParameters(partitionCols, originTable, datePartitionFormat);
InternalTableDesc internalTable = new
InternalTableDesc(originTable);
@@ -155,11 +157,11 @@ public class InternalTableService extends BasicService {
Optional<ColumnDesc> dateCol =
partitionColList.stream().filter(col -> col.getTypeName().equals("date"))
.findFirst();
if (StringUtils.isEmpty(datePartitionFormat) &&
dateCol.isPresent()) {
- throw new KylinException(EMPTY_PARAMETER,
"date_partition_format can not be null, please check again");
+ throw new KylinException(EMPTY_PARAMETER,
+ String.format(Locale.ROOT,
MsgPicker.getMsg().getInternalTableNullPartitionFormat()));
}
if (!StringUtils.isEmpty(datePartitionFormat) &&
!dateCol.isPresent()) {
- throw new KylinException(EMPTY_PARAMETER,
- "couldn't find date_col present in partition_cols,
please check again");
+ throw new KylinException(EMPTY_PARAMETER,
MsgPicker.getMsg().getInternalTableNoDataCol());
}
checkIfFormatMatchCol(dateCol, originTable, datePartitionFormat);
}
@@ -229,7 +231,8 @@ public class InternalTableService extends BasicService {
throw new KylinException(INTERNAL_TABLE_NOT_EXIST, errorMsg);
}
if (internalTable.getRowCount() > 0L) {
- throw new KylinException(INTERNAL_TABLE_ERROR, "Non-empty
internal table can not be updated");
+ String errorMsg = String.format(Locale.ROOT,
MsgPicker.getMsg().getInternalTableEmpty(), dbTblName);
+ throw new KylinException(INTERNAL_TABLE_ERROR, errorMsg);
}
checkParameters(partitionCols, originTable, datePartitionFormat);
if (partitionCols != null && partitionCols.length != 0) {
@@ -255,7 +258,8 @@ public class InternalTableService extends BasicService {
Path location = new Path(path);
fs.mkdirs(location);
} catch (IOException e) {
- throw new KylinException(INTERNAL_TABLE_ERROR, "Failed to create
internal table location", e);
+ String errorMsg = String.format(Locale.ROOT,
MsgPicker.getMsg().getInternalTablePath());
+ throw new KylinException(INTERNAL_TABLE_ERROR, errorMsg);
}
}
@@ -397,28 +401,27 @@ public class InternalTableService extends BasicService {
startDate, endDate, yarnQueue);
}
- public List<InternalTableDescResponse> getTableList(String project) {
+ public List<InternalTableDescResponse> getTableList(String project,
boolean isFuzzy, boolean needDetails,
+ String fuzzyKey) {
InternalTableManager internalTableManager =
getManager(InternalTableManager.class, project);
List<InternalTableDesc> tableList =
internalTableManager.listAllTables();
- List<InternalTableDescResponse> descList = Lists.newArrayList();
- tableList.forEach(internalTableDesc -> {
- InternalTableDescResponse internalTableDescResponse = new
InternalTableDescResponse();
-
internalTableDescResponse.setTableName(internalTableDesc.getName());
- internalTableDescResponse.setUuid(internalTableDesc.getUuid());
-
internalTableDescResponse.setDatabaseName(internalTableDesc.getDatabase());
-
internalTableDescResponse.setRowCount(internalTableDesc.getRowCount());
-
internalTableDescResponse.setStorageSize(internalTableDesc.getStorageSize());
-
internalTableDescResponse.setHitCount(internalTableDesc.getHitCount());
- String[] partitionColumns =
internalTableDesc.getPartitionColumns();
- String partitionColumn = (partitionColumns == null ||
partitionColumns.length == 0) ? null
- : internalTableDesc.getPartitionColumns()[0];
- internalTableDescResponse.setTimePartitionCol(partitionColumn);
-
internalTableDescResponse.setUpdateTime(internalTableDesc.getLastModified());
-
internalTableDescResponse.setDatePartitionFormat(internalTableDesc.getDatePartitionFormat());
-
internalTableDescResponse.setTblProperties(internalTableDesc.getTblProperties());
- descList.add(internalTableDescResponse);
- });
- return descList;
+ if (StringUtils.isNotBlank(fuzzyKey)) {
+ if (isFuzzy) {
+ String regex = fuzzyKey.replace(".", ".*");
+ tableList = tableList.stream().filter(table ->
table.getIdentity().matches("(?i).*" + regex + ".*"))
+ .collect(Collectors.toList());
+ } else {
+ tableList = tableList.stream()
+ .filter(table ->
StringUtils.equalsIgnoreCase(table.getIdentity(), fuzzyKey))
+ .collect(Collectors.toList());
+ }
+ if (tableList.isEmpty()) {
+ throw new KylinException(INTERNAL_TABLE_NOT_EXIST,
+ String.format(Locale.ROOT,
MsgPicker.getMsg().getInternalTableNotFound(), fuzzyKey));
+ }
+ }
+ return tableList.stream().map(table ->
InternalTableDescResponse.convertToResponse(table, needDetails))
+ .collect(Collectors.toList());
}
public List<InternalTablePartitionDetail> getTableDetail(String project,
String databaseName, String tableName) {
diff --git
a/src/kylin-it/src/test/java/org/apache/kylin/query/QueryGroupingSetsTest.java
b/src/kylin-it/src/test/java/org/apache/kylin/query/QueryGroupingSetsTest.java
index 402c4165fe..02c5dbdd2f 100644
---
a/src/kylin-it/src/test/java/org/apache/kylin/query/QueryGroupingSetsTest.java
+++
b/src/kylin-it/src/test/java/org/apache/kylin/query/QueryGroupingSetsTest.java
@@ -23,6 +23,7 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
+import java.util.Objects;
import org.apache.calcite.rel.RelNode;
import org.apache.kylin.common.KylinConfig;
@@ -112,6 +113,8 @@ public class QueryGroupingSetsTest extends SemiAutoTestBase
{
@After
public void teardown() throws Exception {
queryHistoryDAO.deleteAllQueryHistory();
+ ReflectionTestUtils.invokeMethod(
+
Objects.requireNonNull(ReflectionTestUtils.getField(queryService,
"slowQueryDetector")), "interrupt");
super.tearDown();
QueryHistoryAccelerateScheduler.shutdown();
}
diff --git
a/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/InternalTableController.java
b/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/InternalTableController.java
index 6f9bbd25e0..d514bfc6e9 100644
---
a/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/InternalTableController.java
+++
b/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/InternalTableController.java
@@ -67,8 +67,8 @@ public class InternalTableController extends NBasicController
{
public EnvelopeResponse<String> createInternalTable(@RequestParam(value =
"project") String project,
@PathVariable(value = "database") String database, //
@PathVariable(value = "table") String table, @RequestBody
InternalTableRequest request) throws Exception {
- checkProjectName(project);
- if (StringUtils.isEmpty(table) || StringUtils.isEmpty(database)) {
+ project = checkProjectName(project);
+ if (StringUtils.isBlank(table) || StringUtils.isBlank(database)) {
throw new KylinException(EMPTY_PARAMETER, "Table or database can
not be null, please check again.");
}
internalTableService.createInternalTable(project, table, database,
request.getPartitionCols(),
@@ -82,7 +82,7 @@ public class InternalTableController extends NBasicController
{
public EnvelopeResponse<String> dropInternalTable(@RequestParam(value =
"project") String project,
@PathVariable(value = "database") String database, //
@PathVariable(value = "table") String table) throws Exception {
- checkProjectName(project);
+ project = checkProjectName(project);
String dbTblName = database + "." + table;
internalTableService.dropInternalTable(project, dbTblName);
return new EnvelopeResponse<>(KylinException.CODE_SUCCESS, "", "");
@@ -94,8 +94,8 @@ public class InternalTableController extends NBasicController
{
public EnvelopeResponse<String> truncateInternalTable(@RequestParam(value
= "project") String project,
@RequestParam(value = "database") String database,
@RequestParam(value = "table") String table)
throws Exception {
- checkProjectName(project);
- if (StringUtils.isEmpty(StringUtils.trim(table)) ||
StringUtils.isEmpty(StringUtils.trim(database))) {
+ project = checkProjectName(project);
+ if (StringUtils.isBlank(table) || StringUtils.isBlank(database)) {
throw new KylinException(INVALID_TABLE_NAME,
MsgPicker.getMsg().getTableOrDatabaseNameCannotEmpty());
}
String tableIdentity = database + "." + table;
@@ -109,8 +109,8 @@ public class InternalTableController extends
NBasicController {
public EnvelopeResponse<String> dropPartitions(@RequestParam(value =
"project") String project,
@RequestParam(value = "database") String database,
@RequestParam(value = "table") String table,
@RequestParam(value = "partitions") String[] partitionValues)
throws Exception {
- checkProjectName(project);
- if (StringUtils.isEmpty(StringUtils.trim(table)) ||
StringUtils.isEmpty(StringUtils.trim(database))) {
+ project = checkProjectName(project);
+ if (StringUtils.isBlank(table) || StringUtils.isBlank(database)) {
throw new KylinException(INVALID_TABLE_NAME,
MsgPicker.getMsg().getTableOrDatabaseNameCannotEmpty());
}
// If partitionValues is null, all files will be cleared
@@ -127,8 +127,8 @@ public class InternalTableController extends
NBasicController {
@PathVariable(value = "project") String project,
@PathVariable(value = "database") String database, //
@PathVariable(value = "table") String table, @RequestBody
InternalTableBuildRequest request)
throws Exception {
- checkProjectName(project);
- if (StringUtils.isEmpty(table) || StringUtils.isEmpty(database)) {
+ project = checkProjectName(project);
+ if (StringUtils.isBlank(table) || StringUtils.isBlank(database)) {
throw new KylinException(INVALID_TABLE_NAME,
MsgPicker.getMsg().getTableNameCannotEmpty());
}
InternalTableLoadingJobResponse response =
internalTableService.loadIntoInternalTable(project, table, database,
@@ -143,8 +143,8 @@ public class InternalTableController extends
NBasicController {
public EnvelopeResponse<String> updateTable(@RequestParam(value =
"project") String project,
@PathVariable(value = "database") String database, //
@PathVariable(value = "table") String table, @RequestBody
InternalTableRequest request) throws Exception {
- checkProjectName(project);
- if (StringUtils.isEmpty(table) || StringUtils.isEmpty(database)) {
+ project = checkProjectName(project);
+ if (StringUtils.isBlank(table) || StringUtils.isBlank(database)) {
throw new KylinException(EMPTY_PARAMETER, "Table or database can
not be null, please check again.");
}
if (table.isEmpty()) {
@@ -160,10 +160,24 @@ public class InternalTableController extends
NBasicController {
@ResponseBody
public EnvelopeResponse<DataResult<List<InternalTableDescResponse>>>
getTableList(
@RequestParam(value = "project") String project,
+ @RequestParam(value = "need_details", required = false,
defaultValue = "false") boolean needDetails,
+ @RequestParam(value = "is_fuzzy", required = false, defaultValue =
"false") boolean isFuzzy,
+ @RequestParam(value = "database", required = false) String
database,
+ @RequestParam(value = "table", required = false) String table,
@RequestParam(value = "page_offset", required = false,
defaultValue = "0") Integer offset,
@RequestParam(value = "page_size", required = false, defaultValue
= "10") Integer limit) {
- checkProjectName(project);
- val rep = internalTableService.getTableList(project);
+ project = checkProjectName(project);
+
+ String fuzzyKey = "";
+ if (StringUtils.isNotBlank(database) && StringUtils.isNotBlank(table))
{
+ fuzzyKey = StringUtils.trim(database) + "." +
StringUtils.trim(table);
+ } else if (StringUtils.isBlank(database)) {
+ fuzzyKey = StringUtils.trim(table);
+ } else if (StringUtils.isBlank(table)) {
+ fuzzyKey = StringUtils.trim(database);
+ }
+
+ val rep = internalTableService.getTableList(project, isFuzzy,
needDetails, fuzzyKey);
return new EnvelopeResponse<>(KylinException.CODE_SUCCESS,
DataResult.get(rep, offset, limit), "");
}
@@ -175,8 +189,9 @@ public class InternalTableController extends
NBasicController {
@PathVariable(value = "table") String table,
@RequestParam(value = "page_offset", required = false,
defaultValue = "0") Integer offset,
@RequestParam(value = "page_size", required = false, defaultValue
= "10") Integer limit) {
- checkProjectName(project);
+ project = checkProjectName(project);
val rep = internalTableService.getTableDetail(project, database,
table);
return new EnvelopeResponse<>(KylinException.CODE_SUCCESS,
DataResult.get(rep, offset, limit), "");
}
+
}
diff --git
a/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/open/OpenInternalTableController.java
b/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/open/OpenInternalTableController.java
index 20c0927381..c54d7ed853 100644
---
a/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/open/OpenInternalTableController.java
+++
b/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/open/OpenInternalTableController.java
@@ -67,7 +67,7 @@ public class OpenInternalTableController extends
NBasicController {
@PathVariable(value = "database") String database, //
@PathVariable(value = "table") String table, @RequestBody
InternalTableRequest request) throws Exception {
String projectName = checkProjectName(project);
- if (StringUtils.isEmpty(StringUtils.trim(table)) ||
StringUtils.isEmpty(StringUtils.trim(database))) {
+ if (StringUtils.isBlank(table) || StringUtils.isBlank(database)) {
throw new KylinException(EMPTY_PARAMETER,
MsgPicker.getMsg().getTableOrDatabaseNameCannotEmpty());
}
return internalTableController.createInternalTable(projectName,
database, table, request);
@@ -78,6 +78,10 @@ public class OpenInternalTableController extends
NBasicController {
@ResponseBody
public EnvelopeResponse<DataResult<List<InternalTableDescResponse>>>
getTableList(
@RequestParam(value = "project") String project,
+ @RequestParam(value = "need_details", required = false,
defaultValue = "false") boolean needDetails,
+ @RequestParam(value = "is_fuzzy", required = false, defaultValue =
"false") boolean isFuzzy,
+ @RequestParam(value = "database", required = false) String
database,
+ @RequestParam(value = "table", required = false) String table,
@RequestParam(value = "page_offset", required = false,
defaultValue = "0") Integer offset,
@RequestParam(value = "page_size", required = false, defaultValue
= "10") Integer limit) {
String projectName = checkProjectName(project);
@@ -85,10 +89,12 @@ public class OpenInternalTableController extends
NBasicController {
checkNonNegativeIntegerArg("page_offset", offset);
}
if (null != limit) {
- checkNonNegativeIntegerArg("page_offset", limit);
+ checkNonNegativeIntegerArg("page_size", limit);
+ }
+ if (StringUtils.isBlank(table) && StringUtils.isBlank(database) &&
isFuzzy) {
+ throw new KylinException(EMPTY_PARAMETER,
MsgPicker.getMsg().getTableOrDatabaseNameCannotEmpty());
}
- checkNonNegativeIntegerArg("page_size", limit);
- return internalTableController.getTableList(projectName, offset,
limit);
+ return internalTableController.getTableList(projectName, isFuzzy,
needDetails, database, table, offset, limit);
}
@ApiOperation(value = "get_table_detail", tags = { "AI" })
@@ -100,7 +106,7 @@ public class OpenInternalTableController extends
NBasicController {
@RequestParam(value = "page_offset", required = false,
defaultValue = "0") Integer offset,
@RequestParam(value = "page_size", required = false, defaultValue
= "10") Integer limit) {
String projectName = checkProjectName(project);
- if (StringUtils.isEmpty(StringUtils.trim(table)) ||
StringUtils.isEmpty(StringUtils.trim(database))) {
+ if (StringUtils.isBlank(table) || StringUtils.isBlank(database)) {
throw new KylinException(EMPTY_PARAMETER,
MsgPicker.getMsg().getTableOrDatabaseNameCannotEmpty());
}
if (null != offset) {
@@ -119,7 +125,7 @@ public class OpenInternalTableController extends
NBasicController {
@PathVariable(value = "database") String database, //
@PathVariable(value = "table") String table, @RequestBody
InternalTableRequest request) throws Exception {
String projectName = checkProjectName(project);
- if (StringUtils.isEmpty(StringUtils.trim(table)) ||
StringUtils.isEmpty(StringUtils.trim(database))) {
+ if (StringUtils.isBlank(table) || StringUtils.isBlank(database)) {
throw new KylinException(EMPTY_PARAMETER,
MsgPicker.getMsg().getTableOrDatabaseNameCannotEmpty());
}
return internalTableController.updateTable(projectName, database,
table, request);
@@ -133,7 +139,7 @@ public class OpenInternalTableController extends
NBasicController {
@PathVariable(value = "table") String table, @RequestBody
InternalTableBuildRequest request)
throws Exception {
String projectName = checkProjectName(project);
- if (StringUtils.isEmpty(StringUtils.trim(table)) ||
StringUtils.isEmpty(StringUtils.trim(database))) {
+ if (StringUtils.isBlank(table) || StringUtils.isBlank(database)) {
throw new KylinException(EMPTY_PARAMETER,
MsgPicker.getMsg().getTableOrDatabaseNameCannotEmpty());
}
return internalTableController.loadIntoInternalTable(projectName,
database, table, request);
@@ -146,7 +152,7 @@ public class OpenInternalTableController extends
NBasicController {
@RequestParam(value = "project") String project,
@RequestParam(value = "database") String database,
@RequestParam(value = "table") String table) throws Exception {
String projectName = checkProjectName(project);
- if (StringUtils.isEmpty(StringUtils.trim(table)) ||
StringUtils.isEmpty(StringUtils.trim(database))) {
+ if (StringUtils.isBlank(table) || StringUtils.isBlank(database)) {
throw new KylinException(INVALID_TABLE_NAME,
MsgPicker.getMsg().getTableOrDatabaseNameCannotEmpty());
}
return internalTableController.truncateInternalTable(projectName,
database, table);
@@ -160,7 +166,7 @@ public class OpenInternalTableController extends
NBasicController {
@RequestParam(value = "table") String table, @RequestParam(value =
"partitions") String[] partitionValues)
throws Exception {
String projectName = checkProjectName(project);
- if (StringUtils.isEmpty(StringUtils.trim(table)) ||
StringUtils.isEmpty(StringUtils.trim(database))) {
+ if (StringUtils.isBlank(table) || StringUtils.isBlank(database)) {
throw new KylinException(INVALID_TABLE_NAME,
MsgPicker.getMsg().getTableOrDatabaseNameCannotEmpty());
}
if (null == partitionValues || partitionValues.length == 0) {
@@ -177,7 +183,7 @@ public class OpenInternalTableController extends
NBasicController {
@PathVariable(value = "database") String database, //
@PathVariable(value = "table") String table) throws Exception {
String projectName = checkProjectName(project);
- if (StringUtils.isEmpty(StringUtils.trim(table)) ||
StringUtils.isEmpty(StringUtils.trim(database))) {
+ if (StringUtils.isBlank(table) || StringUtils.isBlank(database)) {
throw new KylinException(INVALID_TABLE_NAME,
MsgPicker.getMsg().getTableOrDatabaseNameCannotEmpty());
}
return internalTableController.dropInternalTable(projectName,
database, table);
