This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit becc65b6c9be26d2865198aa6d4b4847b5c031ff
Author: Yinghao Lin <39019287+yhca...@users.noreply.github.com>
AuthorDate: Thu Jun 8 23:45:12 2023 +0800
KYLIN-5714 job event log diag dump for model or project config
---
.../org/apache/kylin/tool/JobDiagInfoTool.java | 26 +++++++++++++----
.../org/apache/kylin/tool/JobDiagInfoToolTest.java | 33 ++++++++++++++++++++++
2 files changed, 53 insertions(+), 6 deletions(-)
diff --git a/src/tool/src/main/java/org/apache/kylin/tool/JobDiagInfoTool.java
b/src/tool/src/main/java/org/apache/kylin/tool/JobDiagInfoTool.java
index 1c9c47489f..c0a529864e 100644
--- a/src/tool/src/main/java/org/apache/kylin/tool/JobDiagInfoTool.java
+++ b/src/tool/src/main/java/org/apache/kylin/tool/JobDiagInfoTool.java
@@ -30,27 +30,28 @@ import java.util.Locale;
import java.util.Map;
import java.util.stream.Collectors;
-import org.apache.kylin.guava30.shaded.common.collect.Sets;
import org.apache.commons.cli.Option;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.exception.KylinRuntimeException;
+import org.apache.kylin.common.util.OptionBuilder;
import org.apache.kylin.common.util.OptionsHelper;
+import org.apache.kylin.guava30.shaded.common.annotations.VisibleForTesting;
+import org.apache.kylin.guava30.shaded.common.collect.Sets;
import org.apache.kylin.job.execution.AbstractExecutable;
import org.apache.kylin.job.execution.DefaultExecutable;
import org.apache.kylin.job.execution.JobTypeEnum;
import org.apache.kylin.job.execution.NExecutableManager;
-import org.apache.kylin.metadata.project.ProjectInstance;
-import org.apache.kylin.common.util.OptionBuilder;
+import org.apache.kylin.metadata.cube.model.IndexPlan;
+import org.apache.kylin.metadata.cube.model.NIndexPlanManager;
import org.apache.kylin.metadata.project.NProjectManager;
+import org.apache.kylin.metadata.project.ProjectInstance;
import org.apache.kylin.tool.snapshot.SnapshotSourceTableStatsTool;
import org.apache.kylin.tool.util.DiagnosticFilesChecker;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.kylin.guava30.shaded.common.annotations.VisibleForTesting;
-
import lombok.val;
public class JobDiagInfoTool extends AbstractInfoExtractorTool {
@@ -238,7 +239,8 @@ public class JobDiagInfoTool extends
AbstractInfoExtractorTool {
if (job instanceof DefaultExecutable) {
recordTaskStartTime(JOB_EVENTLOGS);
val appIds = NExecutableManager.getInstance(getKylinConfig(),
project).getYarnApplicationJobs(jobId);
- Map<String, String> sparkConf =
getKylinConfig().getSparkConfigOverride();
+ KylinConfig config =
getConfigForModelOrProjectLevel(job.getTargetModelId(), project);
+ Map<String, String> sparkConf =
config.getSparkConfigOverride();
KylinLogTool.extractJobEventLogs(exportDir, appIds, sparkConf);
recordTaskExecutorTimeToFile(JOB_EVENTLOGS, recordTime);
}
@@ -268,4 +270,16 @@ public class JobDiagInfoTool extends
AbstractInfoExtractorTool {
}
return null;
}
+
+ protected KylinConfig getConfigForModelOrProjectLevel(String modelId,
String project) {
+ KylinConfig config = null;
+ IndexPlan indexPlan = NIndexPlanManager.getInstance(getKylinConfig(),
project).getIndexPlan(modelId);
+ if (indexPlan != null) {
+ config = indexPlan.getConfig();
+ }
+ if (config == null) {
+ config = NProjectManager.getProjectConfig(project);
+ }
+ return config;
+ }
}
diff --git
a/src/tool/src/test/java/org/apache/kylin/tool/JobDiagInfoToolTest.java
b/src/tool/src/test/java/org/apache/kylin/tool/JobDiagInfoToolTest.java
index 24176dec52..fcfe41942b 100644
--- a/src/tool/src/test/java/org/apache/kylin/tool/JobDiagInfoToolTest.java
+++ b/src/tool/src/test/java/org/apache/kylin/tool/JobDiagInfoToolTest.java
@@ -20,6 +20,7 @@ package org.apache.kylin.tool;
import java.io.File;
import java.io.IOException;
import java.util.HashSet;
+import java.util.LinkedHashMap;
import java.util.Objects;
import java.util.concurrent.Executors;
import java.util.concurrent.LinkedBlockingQueue;
@@ -37,6 +38,7 @@ import org.apache.kylin.common.util.ZipFileUtils;
import org.apache.kylin.common.util.NLocalFileMetadataTestCase;
import org.apache.kylin.job.execution.DefaultExecutable;
import org.apache.kylin.job.execution.JobTypeEnum;
+import org.apache.kylin.metadata.cube.model.NIndexPlanManager;
import org.apache.kylin.metadata.project.NProjectManager;
import org.apache.kylin.tool.constant.SensitiveConfigKeysConstant;
import org.apache.kylin.tool.obf.KylinConfObfuscatorTest;
@@ -234,4 +236,35 @@ public class JobDiagInfoToolTest extends
NLocalFileMetadataTestCase {
}
}
}
+
+ @Test
+ public void testGetDifferentConfigLevel() {
+ final String PROJECT = "default";
+ final String KYLIN_SPARK_EVENTLOG_DIR =
"kylin.engine.spark-conf.spark.eventLog.dir";
+ final String SPARK_EVENTLOG_DIR = "spark.eventLog.dir";
+ JobDiagInfoTool tool = new JobDiagInfoTool();
+
+ // Update model config
+ NIndexPlanManager indexPlanManager =
NIndexPlanManager.getInstance(KylinConfig.getInstanceFromEnv(), PROJECT);
+
indexPlanManager.updateIndexPlan("82fa7671-a935-45f5-8779-85703601f49a",
copyForWrite -> {
+ LinkedHashMap<String, String> overrideProps = new
LinkedHashMap<>();
+ overrideProps.put(KYLIN_SPARK_EVENTLOG_DIR, "/path/to/dir1");
+ copyForWrite.setOverrideProps(overrideProps);
+ });
+ // Update project config
+ NProjectManager projectManager =
NProjectManager.getInstance(KylinConfig.getInstanceFromEnv());
+ projectManager.updateProject(PROJECT, copyForWrite -> {
+ LinkedHashMap<String, String> overrideProps = new
LinkedHashMap<>();
+ overrideProps.put(KYLIN_SPARK_EVENTLOG_DIR, "/path/to/dir2");
+ copyForWrite.setOverrideKylinProps(overrideProps);
+ });
+
+ // For jobs related to model: index build, index refresh etc.
+ KylinConfig modelConfig =
tool.getConfigForModelOrProjectLevel("82fa7671-a935-45f5-8779-85703601f49a",
PROJECT);
+ Assert.assertEquals("/path/to/dir1",
modelConfig.getSparkConfigOverride().get(SPARK_EVENTLOG_DIR));
+
+ // For jobs without model: snapshot build, sampling etc.
+ KylinConfig projectConfig =
tool.getConfigForModelOrProjectLevel("SSB.LINEORDER", PROJECT);
+ Assert.assertEquals("/path/to/dir2",
projectConfig.getSparkConfigOverride().get(SPARK_EVENTLOG_DIR));
+ }
}
