This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 443adfc16293bbe5dc2576897fa04d96f9dd33be
Author: jlf <longfei.ji...@kyligence.io>
AuthorDate: Mon Jun 5 17:17:22 2023 +0800
KYLIN-5708 add LOG_TIME_PATTERN_WITH_TRACE_ID in getJobLogPattern
---
.../org/apache/kylin/rest/interceptor/KEFilter.java | 6 ++++--
.../java/org/apache/kylin/rest/util/HttpUtil.java | 4 +++-
.../org/apache/kylin/common/constant/Constants.java | 1 +
.../main/java/org/apache/kylin/tool/KylinLogTool.java | 19 ++++++++++++++-----
.../java/org/apache/kylin/tool/KylinLogToolTest.java | 15 +++++++++++++--
5 files changed, 35 insertions(+), 10 deletions(-)
diff --git
a/src/common-service/src/main/java/org/apache/kylin/rest/interceptor/KEFilter.java
b/src/common-service/src/main/java/org/apache/kylin/rest/interceptor/KEFilter.java
index 071a4ab619..266fef735a 100644
---
a/src/common-service/src/main/java/org/apache/kylin/rest/interceptor/KEFilter.java
+++
b/src/common-service/src/main/java/org/apache/kylin/rest/interceptor/KEFilter.java
@@ -18,6 +18,8 @@
package org.apache.kylin.rest.interceptor;
+import static org.apache.kylin.common.constant.Constants.TRACE_ID;
+
import java.io.IOException;
import javax.servlet.FilterChain;
@@ -63,14 +65,14 @@ public class KEFilter extends OncePerRequestFilter {
// Set traceId for KE
String traceId = RandomUtil.randomUUIDStr();
- ThreadContext.put("traceId", String.format("traceId: %s ", traceId));
+ ThreadContext.put(TRACE_ID, String.format(TRACE_ID + ": %s ",
traceId));
filterChain.doFilter(request, response);
// clean ThreadContext
ThreadContext.clearAll();
// set `traceId` attribute for accesslog
- request.setAttribute("traceId", traceId);
+ request.setAttribute(TRACE_ID, traceId);
}
}
diff --git
a/src/common-service/src/main/java/org/apache/kylin/rest/util/HttpUtil.java
b/src/common-service/src/main/java/org/apache/kylin/rest/util/HttpUtil.java
index 29001b6b36..c073687604 100644
--- a/src/common-service/src/main/java/org/apache/kylin/rest/util/HttpUtil.java
+++ b/src/common-service/src/main/java/org/apache/kylin/rest/util/HttpUtil.java
@@ -18,6 +18,8 @@
package org.apache.kylin.rest.util;
+import static org.apache.kylin.common.constant.Constants.TRACE_ID;
+
import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.util.JsonUtil;
import org.apache.kylin.rest.response.ErrorResponse;
@@ -73,7 +75,7 @@ public class HttpUtil {
sb.append("RemoteAddr: ").append(request.getRemoteAddr()).append("\n");
sb.append("RemoteUser: ").append(request.getRemoteUser()).append("\n");
sb.append("Session:
").append(formatSession(request.getSession(false))).append("\n");
- Object attr = request.getAttribute("traceId");
+ Object attr = request.getAttribute(TRACE_ID);
if (attr != null) {
sb.append("TraceId: ").append(attr).append("\n");
}
diff --git
a/src/core-common/src/main/java/org/apache/kylin/common/constant/Constants.java
b/src/core-common/src/main/java/org/apache/kylin/common/constant/Constants.java
index 2002960311..a49d012fa2 100644
---
a/src/core-common/src/main/java/org/apache/kylin/common/constant/Constants.java
+++
b/src/core-common/src/main/java/org/apache/kylin/common/constant/Constants.java
@@ -50,6 +50,7 @@ public class Constants {
public static final String KYLIN_SOURCE_JDBC_DRIVER_KEY =
"kylin.source.jdbc.driver";
public static final String UNLIMITED = "Unlimited";
+ public static final String TRACE_ID = "traceId";
public static final String SNAPSHOT_AUTO_REFRESH = "snapshot_auto_refresh";
public static final String VIEW_MAPPING = "view_mapping";
diff --git a/src/tool/src/main/java/org/apache/kylin/tool/KylinLogTool.java
b/src/tool/src/main/java/org/apache/kylin/tool/KylinLogTool.java
index f237ac5b7b..6139876daf 100644
--- a/src/tool/src/main/java/org/apache/kylin/tool/KylinLogTool.java
+++ b/src/tool/src/main/java/org/apache/kylin/tool/KylinLogTool.java
@@ -18,6 +18,7 @@
package org.apache.kylin.tool;
import static org.apache.kylin.common.constant.Constants.KE_LICENSE;
+import static org.apache.kylin.common.constant.Constants.TRACE_ID;
import java.io.BufferedReader;
import java.io.BufferedWriter;
@@ -79,12 +80,18 @@ public class KylinLogTool {
public static final String SECOND_DATE_FORMAT = "yyyy-MM-dd'T'HH:mm:ss";
- private static final String LOG_TIME_PATTERN =
"^([0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2})";
+ private static final String TIME_PATTERN =
"([0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2})";
+
+ private static final String LOG_TIME_PATTERN = "^" + TIME_PATTERN;
+
+ private static final String UUID_PATTERN =
"([0-9a-z]{8}-[0-9a-z]{4}-[0-9a-z]{4}-[0-9a-z]{4}-[0-9a-z]{12})";
+
+ private static final String LOG_TIME_PATTERN_WITH_TRACE_ID = "^(" +
TRACE_ID + ": " + UUID_PATTERN + " )?" + TIME_PATTERN;
// 2019-11-11 09:30:04,628 INFO
[FetchJobWorker(project:test_fact)-p-94-t-94] threadpool.NDefaultScheduler :
start check project test_fact
private static final String LOG_PATTERN =
"([0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2},[0-9]{3}) ([^ ]*)[
]+\\[(.*)\\] ([^: ]*) :([\\n\\r. ]*)";
- private static final String QUERY_LOG_PATTERN = "Query
([0-9a-z]{8}-[0-9a-z]{4}-[0-9a-z]{4}-[0-9a-z]{4}-[0-9a-z]{12})";
+ private static final String QUERY_LOG_PATTERN = "Query " + UUID_PATTERN;
private static final int EXTRA_LINES = 100;
@@ -106,17 +113,19 @@ public class KylinLogTool {
public static String getJobLogPattern(String jobId) {
Preconditions.checkArgument(StringUtils.isNotEmpty(jobId));
return String.format(Locale.ROOT,
"%s(.*JobWorker.*jobid:%s.*)|%s.*%s", LOG_TIME_PATTERN, jobId.substring(0, 8),
- LOG_TIME_PATTERN, jobId);
+ LOG_TIME_PATTERN_WITH_TRACE_ID, jobId);
}
// group(1) is first ([0-9]{4}-[0-9]{2}-[0-9]{2}
[0-9]{2}:[0-9]{2}:[0-9]{2})
// group(2) is (.*JobWorker.*jobid:%s.*)
- // group(3) is second ([0-9]{4}-[0-9]{2}-[0-9]{2}
[0-9]{2}:[0-9]{2}:[0-9]{2})
+ // group(3) is first 'traceId:
([0-9a-z]{8}-[0-9a-z]{4}-[0-9a-z]{4}-[0-9a-z]{4}-[0-9a-z]{12}) '
+ // group(4) is first
'([0-9a-z]{8}-[0-9a-z]{4}-[0-9a-z]{4}-[0-9a-z]{4}-[0-9a-z]{12})'
+ // group(5) is second ([0-9]{4}-[0-9]{2}-[0-9]{2}
[0-9]{2}:[0-9]{2}:[0-9]{2})
@VisibleForTesting
public static String getJobTimeString(Matcher matcher) {
String result = matcher.group(1);
if (StringUtils.isEmpty(result)) {
- result = matcher.group(3);
+ result = matcher.group(5);
}
return result;
}
diff --git a/src/tool/src/test/java/org/apache/kylin/tool/KylinLogToolTest.java
b/src/tool/src/test/java/org/apache/kylin/tool/KylinLogToolTest.java
index 40eb1c6837..50f53366a9 100644
--- a/src/tool/src/test/java/org/apache/kylin/tool/KylinLogToolTest.java
+++ b/src/tool/src/test/java/org/apache/kylin/tool/KylinLogToolTest.java
@@ -543,8 +543,8 @@ public class KylinLogToolTest extends
NLocalFileMetadataTestCase {
String jobId = "d0f45b72-db2f-407b-9d6f-7cfe6f6624e8";
String patternString = KylinLogTool.getJobLogPattern(jobId);
String log;
- Assert.assertEquals(
-
"^([0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2})(.*JobWorker.*jobid:d0f45b72.*)|^([0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}).*d0f45b72-db2f-407b-9d6f-7cfe6f6624e8",
+
Assert.assertEquals("^([0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2})(.*JobWorker.*jobid:d0f45b72.*)"
+ + "|^(traceId:
([0-9a-z]{8}-[0-9a-z]{4}-[0-9a-z]{4}-[0-9a-z]{4}-[0-9a-z]{12})
)?([0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}).*d0f45b72-db2f-407b-9d6f-7cfe6f6624e8",
patternString);
Pattern pattern = Pattern.compile(patternString);
log = "2019-09-02T02:38:16,868 INFO
[FetchJobWorker(project:a_test)-p-9-t-9] runners.FetcherRunner : fetcher
schedule d0f45b72-db2f-407b-9d6f-7cfe6f6624e8";
@@ -555,6 +555,17 @@ public class KylinLogToolTest extends
NLocalFileMetadataTestCase {
matcher = pattern.matcher(log);
Assert.assertTrue(matcher.find());
Assert.assertEquals("2019-09-02T02:40:07",
KylinLogTool.getJobTimeString(matcher));
+ log = "traceId: ae0c0172-54b0-973f-de6c-3aab18289649
2019-09-02T02:38:16,868 INFO [FetchJobWorker(project:a_test)-p-9-t-9]
runners.FetcherRunner : fetcher schedule d0f45b72-db2f-407b-9d6f-7cfe6f6624e8";
+ matcher = pattern.matcher(log);
+ Assert.assertTrue(matcher.find());
+ Assert.assertEquals("2019-09-02T02:38:16",
KylinLogTool.getJobTimeString(matcher));
+ log = "traceId: 4b8324bb-6467-d0b6-8087-9c71c28cc443
2023-05-31T13:51:57,945 INFO [test2] [http-nio-17971-exec-5]
handler.AbstractJobHandler : Job
JobParam(jobId=d0f45b72-db2f-407b-9d6f-7cfe6f6624e8, "
+ + "targetSegments=[edbd2366-f8b3-0f29-39c1-dcf03895a295],
targetLayouts=[], owner=ADMIN, model=4b0a4399-0903-18ec-5d7f-52eb6c96df31,
project=test2, jobTypeEnum=IN"
+ + "C_BUILD, ignoredSnapshotTables=null, targetPartitions=[],
targetBuckets=[], priority=3, yarnQueue=null, tag=null, condition={}, "
+ + "processLayouts=[LayoutEntity{id=160001},
LayoutEntity{id=170001}], deleteLayouts=null,
secondStorageDeleteLayoutIds=null) creates job
NSparkCubingJob{id=d0f45b72-db2f-407b-9d6f-7cfe6f6624e8, name=INC_BUILD,
state=READY}";
+ matcher = pattern.matcher(log);
+ Assert.assertTrue(matcher.find());
+ Assert.assertEquals("2023-05-31T13:51:57",
KylinLogTool.getJobTimeString(matcher));
}
@Test
