This is an automated email from the ASF dual-hosted git repository.
pfzhan pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
The following commit(s) were added to refs/heads/kylin5 by this push:
new 6477527d9f KYLIN-5980 reformat internal table truncate and drop
partition rest api
6477527d9f is described below
commit 6477527d9f3924ce3ff7ae7d6ba892658bff89bf
Author: huangsheng <[email protected]>
AuthorDate: Wed Sep 11 13:55:29 2024 +0800
KYLIN-5980 reformat internal table truncate and drop partition rest api
---
.../rest/service/InternalTableServiceTest.java | 4 ----
.../kylin/rest/service/InternalTableService.java | 3 +--
.../rest/controller/InternalTableController.java | 25 +++++++++++-----------
.../open/OpenInternalTableController.java | 8 +++----
.../engine/spark/job/InternalTableLoadJob.java | 3 +--
5 files changed, 18 insertions(+), 25 deletions(-)
diff --git
a/src/data-loading-service/src/test/java/org/apache/kylin/rest/service/InternalTableServiceTest.java
b/src/data-loading-service/src/test/java/org/apache/kylin/rest/service/InternalTableServiceTest.java
index 2a892ce18c..4abac68968 100644
---
a/src/data-loading-service/src/test/java/org/apache/kylin/rest/service/InternalTableServiceTest.java
+++
b/src/data-loading-service/src/test/java/org/apache/kylin/rest/service/InternalTableServiceTest.java
@@ -366,10 +366,6 @@ public class InternalTableServiceTest extends
AbstractTestCase {
internalTableService.truncateInternalTable(PROJECT, TABLE_INDENTITY);
Assertions.assertFalse(internalTableFolder.exists());
- // double truncate
- response = internalTableService.truncateInternalTable(PROJECT,
TABLE_INDENTITY);
- Assertions.assertTrue(response.getJobs().isEmpty());
-
// test truncate nonexistent internal tables
Assertions.assertThrows(KylinException.class,
() -> internalTableService.truncateInternalTable(PROJECT,
TABLE_INDENTITY + "2"));
diff --git
a/src/datasource-service/src/main/java/org/apache/kylin/rest/service/InternalTableService.java
b/src/datasource-service/src/main/java/org/apache/kylin/rest/service/InternalTableService.java
index 9775a9dee5..6c389eb0a7 100644
---
a/src/datasource-service/src/main/java/org/apache/kylin/rest/service/InternalTableService.java
+++
b/src/datasource-service/src/main/java/org/apache/kylin/rest/service/InternalTableService.java
@@ -301,7 +301,7 @@ public class InternalTableService extends BasicService {
// 1. delete data in file system
// 2. clear partition values in internal table meta
- public InternalTableLoadingJobResponse truncateInternalTable(String
project, String tableIdentity)
+ public void truncateInternalTable(String project, String tableIdentity)
throws Exception {
aclEvaluate.checkProjectWritePermission(project);
InternalTableManager internalTableManager =
getManager(InternalTableManager.class, project);
@@ -339,7 +339,6 @@ public class InternalTableService extends BasicService {
}, project);
logger.info("Successfully truncate internal table {} in {} ms",
tableIdentity,
System.currentTimeMillis() - start);
- return InternalTableLoadingJobResponse.of(new ArrayList<>(), "");
}
// 1. delete partition data in file system
diff --git
a/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/InternalTableController.java
b/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/InternalTableController.java
index 592a507c54..6f9bbd25e0 100644
---
a/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/InternalTableController.java
+++
b/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/InternalTableController.java
@@ -91,31 +91,32 @@ public class InternalTableController extends
NBasicController {
@ApiOperation(value = "truncate_internal_table", tags = { "AI" })
@DeleteMapping(value = "/truncate_internal_table", produces = {
HTTP_VND_APACHE_KYLIN_JSON })
@ResponseBody
- public EnvelopeResponse<InternalTableLoadingJobResponse>
truncateInternalTable(
- @RequestParam(value = "project") String project,
@RequestParam(value = "table") String table)
+ public EnvelopeResponse<String> truncateInternalTable(@RequestParam(value
= "project") String project,
+ @RequestParam(value = "database") String database,
@RequestParam(value = "table") String table)
throws Exception {
checkProjectName(project);
- if (StringUtils.isEmpty(table)) {
- throw new KylinException(INVALID_TABLE_NAME,
MsgPicker.getMsg().getTableNameCannotEmpty());
+ if (StringUtils.isEmpty(StringUtils.trim(table)) ||
StringUtils.isEmpty(StringUtils.trim(database))) {
+ throw new KylinException(INVALID_TABLE_NAME,
MsgPicker.getMsg().getTableOrDatabaseNameCannotEmpty());
}
- InternalTableLoadingJobResponse response =
internalTableService.truncateInternalTable(project, table);
- return new EnvelopeResponse<>(KylinException.CODE_SUCCESS, response,
"");
+ String tableIdentity = database + "." + table;
+ internalTableService.truncateInternalTable(project, tableIdentity);
+ return new EnvelopeResponse<>(KylinException.CODE_SUCCESS, null, "");
}
@ApiOperation(value = "drop_table_partitions", tags = { "AI" })
@DeleteMapping(value = "/partitions", produces = {
HTTP_VND_APACHE_KYLIN_JSON })
@ResponseBody
- public EnvelopeResponse<String> dropPartitions(
- @RequestParam(value = "project") String project,
@RequestParam(value = "table") String table,
+ public EnvelopeResponse<String> dropPartitions(@RequestParam(value =
"project") String project,
+ @RequestParam(value = "database") String database,
@RequestParam(value = "table") String table,
@RequestParam(value = "partitions") String[] partitionValues)
throws Exception {
checkProjectName(project);
- if (StringUtils.isEmpty(table)) {
- throw new KylinException(INVALID_TABLE_NAME,
MsgPicker.getMsg().getTableNameCannotEmpty());
+ if (StringUtils.isEmpty(StringUtils.trim(table)) ||
StringUtils.isEmpty(StringUtils.trim(database))) {
+ throw new KylinException(INVALID_TABLE_NAME,
MsgPicker.getMsg().getTableOrDatabaseNameCannotEmpty());
}
// If partitionValues is null, all files will be cleared
// otherwise only files in the specified partition will be cleared.
- internalTableService.dropPartitionsOnDeltaTable(project, table,
- partitionValues, null);
+ String tableIdentity = database + "." + table;
+ internalTableService.dropPartitionsOnDeltaTable(project,
tableIdentity, partitionValues, null);
return new EnvelopeResponse<>(KylinException.CODE_SUCCESS, null, "");
}
diff --git
a/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/open/OpenInternalTableController.java
b/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/open/OpenInternalTableController.java
index 88a46f774d..20c0927381 100644
---
a/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/open/OpenInternalTableController.java
+++
b/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/open/OpenInternalTableController.java
@@ -142,15 +142,14 @@ public class OpenInternalTableController extends
NBasicController {
@ApiOperation(value = "truncate_internal_table", tags = { "AI" })
@DeleteMapping(value = "/truncate_internal_table")
@ResponseBody
- public EnvelopeResponse<InternalTableLoadingJobResponse>
truncateInternalTable(
+ public EnvelopeResponse<String> truncateInternalTable(
@RequestParam(value = "project") String project,
@RequestParam(value = "database") String database,
@RequestParam(value = "table") String table) throws Exception {
String projectName = checkProjectName(project);
if (StringUtils.isEmpty(StringUtils.trim(table)) ||
StringUtils.isEmpty(StringUtils.trim(database))) {
throw new KylinException(INVALID_TABLE_NAME,
MsgPicker.getMsg().getTableOrDatabaseNameCannotEmpty());
}
- String tableIdentity = database + "." + table;
- return internalTableController.truncateInternalTable(projectName,
tableIdentity);
+ return internalTableController.truncateInternalTable(projectName,
database, table);
}
@ApiOperation(value = "drop_table_partitions", tags = { "AI" })
@@ -168,8 +167,7 @@ public class OpenInternalTableController extends
NBasicController {
throw new KylinException(EMPTY_PARAMETER,
String.format(Locale.ROOT,
MsgPicker.getMsg().getParameterEmpty(), "partitions"));
}
- String tableIdentity = database + "." + table;
- return internalTableController.dropPartitions(projectName,
tableIdentity, partitionValues);
+ return internalTableController.dropPartitions(projectName, database,
table, partitionValues);
}
@ApiOperation(value = "drop_internal_table", tags = { "AI" })
diff --git
a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/InternalTableLoadJob.java
b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/InternalTableLoadJob.java
index 2c6c93c6b6..42003d370d 100644
---
a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/InternalTableLoadJob.java
+++
b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/InternalTableLoadJob.java
@@ -25,7 +25,6 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
-import java.util.Locale;
import java.util.stream.Collectors;
import org.apache.commons.lang3.StringUtils;
@@ -171,7 +170,7 @@ public class InternalTableLoadJob extends SparkApplication {
String partitionValue = row.getString(0);
long partitionStorageSize = row.getLong(1);
long fileSize = row.getLong(2);
- val subPath = partitionCol.toLowerCase(Locale.ROOT) + "=" +
partitionValue;
+ val subPath = partitionCol + "=" + partitionValue;
val partitionsPath = new Path(internalTable.getLocation(),
subPath);
partitionValues.add(partitionValue);
detail.setSizeInBytes(partitionStorageSize);
