This is an automated email from the ASF dual-hosted git repository.
richardstartin pushed a commit to branch tracing-spi
in repository https://gitbox.apache.org/repos/asf/pinot.git
commit 4aaa19eef2a7016e4e2b725a7aa357fc51f491d5
Author: richardstartin <rich...@startree.ai>
AuthorDate: Fri Apr 1 16:21:43 2022 +0100
add tracing SPI
---
.../apache/pinot/core/operator/BaseOperator.java | 23 ++---
.../query/executor/ServerQueryExecutorV1Impl.java | 3 +-
.../pinot/core/util/trace/DefaultTracer.java | 100 +++++++++++++++++++++
.../pinot/core/util/trace/TraceContextTest.java | 3 +-
.../apache/pinot/spi/trace/ExecutionRecording.java | 71 +++++++++++++++
.../org/apache/pinot/spi/trace/FilterType.java | 24 +++++
.../apache/pinot/spi/trace/OperatorExecution.java | 22 +++++
.../java/org/apache/pinot/spi/trace/Phase.java | 27 ++++++
.../java/org/apache/pinot/spi/trace/Scope.java | 25 ++++++
.../java/org/apache/pinot/spi/trace/Tracer.java | 30 +++++++
.../java/org/apache/pinot/spi/trace/Tracing.java | 70 +++++++++++++++
11 files changed, 382 insertions(+), 16 deletions(-)
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/operator/BaseOperator.java
b/pinot-core/src/main/java/org/apache/pinot/core/operator/BaseOperator.java
index e99d5c59a1..369f936e95 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/operator/BaseOperator.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/BaseOperator.java
@@ -20,36 +20,31 @@ package org.apache.pinot.core.operator;
import org.apache.pinot.core.common.Block;
import org.apache.pinot.core.common.Operator;
-import org.apache.pinot.core.util.trace.TraceContext;
import org.apache.pinot.spi.exception.EarlyTerminationException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import org.apache.pinot.spi.trace.ExecutionRecording;
+import org.apache.pinot.spi.trace.OperatorExecution;
+import org.apache.pinot.spi.trace.Tracing;
/**
* Any other Pinot Operators should extend BaseOperator
*/
public abstract class BaseOperator<T extends Block> implements Operator<T> {
- private static final Logger LOGGER =
LoggerFactory.getLogger(BaseOperator.class);
@Override
public final T nextBlock() {
if (Thread.interrupted()) {
throw new EarlyTerminationException();
}
- if (TraceContext.traceEnabled()) {
- long start = System.currentTimeMillis();
- T nextBlock = getNextBlock();
- long end = System.currentTimeMillis();
- String operatorName = getOperatorName();
- LOGGER.trace("Time spent in {}: {}", operatorName, (end - start));
- TraceContext.logTime(operatorName, (end - start));
- return nextBlock;
- } else {
- return getNextBlock();
+ try (OperatorExecution execution =
Tracing.getTracer().startOperatorExecution(getClass())) {
+ return getNextBlock(execution);
}
}
// Make it protected because we should always call nextBlock()
protected abstract T getNextBlock();
+
+ protected T getNextBlock(ExecutionRecording recording) {
+ return getNextBlock();
+ }
}
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/query/executor/ServerQueryExecutorV1Impl.java
b/pinot-core/src/main/java/org/apache/pinot/core/query/executor/ServerQueryExecutorV1Impl.java
index 6dcf6a9d95..af5465d199 100644
---
a/pinot-core/src/main/java/org/apache/pinot/core/query/executor/ServerQueryExecutorV1Impl.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/query/executor/ServerQueryExecutorV1Impl.java
@@ -66,6 +66,7 @@ import org.apache.pinot.segment.spi.MutableSegment;
import org.apache.pinot.segment.spi.SegmentMetadata;
import org.apache.pinot.spi.env.PinotConfiguration;
import org.apache.pinot.spi.exception.BadQueryRequestException;
+import org.apache.pinot.spi.trace.Tracing;
import org.apache.pinot.spi.utils.CommonConstants;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -195,7 +196,7 @@ public class ServerQueryExecutorV1Impl implements
QueryExecutor {
boolean enableTrace = queryRequest.isEnableTrace();
if (enableTrace) {
- TraceContext.register(requestId);
+ Tracing.getTracer().register(requestId);
}
DataTable dataTable = null;
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/util/trace/DefaultTracer.java
b/pinot-core/src/main/java/org/apache/pinot/core/util/trace/DefaultTracer.java
new file mode 100644
index 0000000000..91522f2591
--- /dev/null
+++
b/pinot-core/src/main/java/org/apache/pinot/core/util/trace/DefaultTracer.java
@@ -0,0 +1,100 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.core.util.trace;
+
+import org.apache.pinot.spi.data.FieldSpec;
+import org.apache.pinot.spi.trace.FilterType;
+import org.apache.pinot.spi.trace.OperatorExecution;
+import org.apache.pinot.spi.trace.Phase;
+import org.apache.pinot.spi.trace.Tracer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+public class DefaultTracer implements Tracer {
+
+ private static final Logger LOGGER =
LoggerFactory.getLogger(DefaultTracer.class);
+
+ private static class NoOpExecution implements OperatorExecution {
+
+ @Override
+ public void setDocsScanned(long docsScanned) {
+ }
+
+ @Override
+ public void setBytesProcessed(long bytesScanned) {
+ }
+
+ @Override
+ public void setFilterType(FilterType filterType, String predicateType) {
+ }
+
+ @Override
+ public void setPhase(Phase phase) {
+ }
+
+ @Override
+ public void setDataTypes(FieldSpec.DataType inputDataType,
FieldSpec.DataType outputDataType) {
+ }
+
+ @Override
+ public void setDocIdRange(int firstDocId, int lastDocId) {
+ }
+
+ @Override
+ public void setColumnCardinality(int cardinality) {
+ }
+
+ @Override
+ public void close() {
+ }
+ }
+
+ private static final NoOpExecution NO_OP_SPAN = new NoOpExecution();
+
+ private static final class MillisExecution extends NoOpExecution {
+
+ private final long _startTimeMillis = System.currentTimeMillis();
+ private final Class<?> _operator;
+
+ public MillisExecution(Class<?> operator) {
+ _operator = operator;
+ }
+
+ @Override
+ public void close() {
+ String operatorName = _operator.getSimpleName();
+ long duration = System.currentTimeMillis() - _startTimeMillis;
+ if (LOGGER.isTraceEnabled()) {
+ LOGGER.trace("Time spent in {}: {}", operatorName, duration);
+ }
+ TraceContext.logTime(operatorName, duration);
+ }
+ }
+
+ @Override
+ public void register(long requestId) {
+ TraceContext.register(requestId);
+ }
+
+ @Override
+ public OperatorExecution startOperatorExecution(Class<?> operatorClass) {
+ return TraceContext.traceEnabled() ? new MillisExecution(operatorClass) :
NO_OP_SPAN;
+ }
+}
diff --git
a/pinot-core/src/test/java/org/apache/pinot/core/util/trace/TraceContextTest.java
b/pinot-core/src/test/java/org/apache/pinot/core/util/trace/TraceContextTest.java
index 5ebdf7c7de..ba4a4acb0c 100644
---
a/pinot-core/src/test/java/org/apache/pinot/core/util/trace/TraceContextTest.java
+++
b/pinot-core/src/test/java/org/apache/pinot/core/util/trace/TraceContextTest.java
@@ -26,6 +26,7 @@ import java.util.Set;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
+import org.apache.pinot.spi.trace.Tracing;
import org.apache.pinot.spi.utils.JsonUtils;
import org.testng.Assert;
import org.testng.annotations.Test;
@@ -73,7 +74,7 @@ public class TraceContextTest {
private void testSingleRequest(ExecutorService executorService, final long
requestId)
throws Exception {
Set<String> expectedTraces = new HashSet<>(NUM_CHILDREN_PER_REQUEST + 1);
- TraceContext.register(requestId);
+ Tracing.getTracer().register(requestId);
String key = Integer.toString(RANDOM.nextInt());
int value = RANDOM.nextInt();
expectedTraces.add(getTraceString(key, value));
diff --git
a/pinot-spi/src/main/java/org/apache/pinot/spi/trace/ExecutionRecording.java
b/pinot-spi/src/main/java/org/apache/pinot/spi/trace/ExecutionRecording.java
new file mode 100644
index 0000000000..dd08aaa1ed
--- /dev/null
+++ b/pinot-spi/src/main/java/org/apache/pinot/spi/trace/ExecutionRecording.java
@@ -0,0 +1,71 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.spi.trace;
+
+import org.apache.pinot.spi.data.FieldSpec;
+
+
+public interface ExecutionRecording {
+
+ /**
+ * Sets the number of docs scanned by the operator.
+ * @param docsScanned how many docs were scanned.
+ */
+ void setDocsScanned(long docsScanned);
+
+ /**
+ * Sets the number of bytes scanned by the operator if this is possible to
compute.
+ * @param bytesScanned the number of bytes scanned
+ */
+ void setBytesProcessed(long bytesScanned);
+
+ /**
+ * If the operator is a filter, determines the filter type (scan or index)
and the predicate type
+ * @param filterType SCAN or INDEX
+ * @param predicateType e.g. BETWEEN, REGEXP_LIKE
+ */
+ void setFilterType(FilterType filterType, String predicateType);
+
+ /**
+ * The phase of the query
+ * @param phase the phase
+ */
+ void setPhase(Phase phase);
+
+ /**
+ * Records whether type transformation took place during the operator's
invocation and what the types were
+ * @param inputDataType the input data type
+ * @param outputDataType the output data type
+ */
+ void setDataTypes(FieldSpec.DataType inputDataType, FieldSpec.DataType
outputDataType);
+
+ /**
+ * Records the range of docIds during the operator invocation. This is
useful for implicating a range of records
+ * in a slow operator invocation.
+ * @param firstDocId the first docId in the block
+ * @param lastDocId the last docId in the block
+ */
+ void setDocIdRange(int firstDocId, int lastDocId);
+
+ /**
+ * If known, record the cardinality of the column within the segment this
operator invoked on
+ * @param cardinality the number of distinct values
+ */
+ void setColumnCardinality(int cardinality);
+}
diff --git a/pinot-spi/src/main/java/org/apache/pinot/spi/trace/FilterType.java
b/pinot-spi/src/main/java/org/apache/pinot/spi/trace/FilterType.java
new file mode 100644
index 0000000000..89eaee8611
--- /dev/null
+++ b/pinot-spi/src/main/java/org/apache/pinot/spi/trace/FilterType.java
@@ -0,0 +1,24 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.spi.trace;
+
+public enum FilterType {
+ INDEX,
+ SCAN
+}
diff --git
a/pinot-spi/src/main/java/org/apache/pinot/spi/trace/OperatorExecution.java
b/pinot-spi/src/main/java/org/apache/pinot/spi/trace/OperatorExecution.java
new file mode 100644
index 0000000000..ec2eafd135
--- /dev/null
+++ b/pinot-spi/src/main/java/org/apache/pinot/spi/trace/OperatorExecution.java
@@ -0,0 +1,22 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.spi.trace;
+
+public interface OperatorExecution extends Scope, ExecutionRecording {
+}
diff --git a/pinot-spi/src/main/java/org/apache/pinot/spi/trace/Phase.java
b/pinot-spi/src/main/java/org/apache/pinot/spi/trace/Phase.java
new file mode 100644
index 0000000000..c7f3d6808c
--- /dev/null
+++ b/pinot-spi/src/main/java/org/apache/pinot/spi/trace/Phase.java
@@ -0,0 +1,27 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.spi.trace;
+
+public enum Phase {
+ EXTRACT,
+ FILTER,
+ TRANSFORM,
+ PROJECT,
+ AGGREGATE
+}
diff --git a/pinot-spi/src/main/java/org/apache/pinot/spi/trace/Scope.java
b/pinot-spi/src/main/java/org/apache/pinot/spi/trace/Scope.java
new file mode 100644
index 0000000000..4ff56d0b6c
--- /dev/null
+++ b/pinot-spi/src/main/java/org/apache/pinot/spi/trace/Scope.java
@@ -0,0 +1,25 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.spi.trace;
+
+public interface Scope extends AutoCloseable {
+
+ @Override
+ void close();
+}
diff --git a/pinot-spi/src/main/java/org/apache/pinot/spi/trace/Tracer.java
b/pinot-spi/src/main/java/org/apache/pinot/spi/trace/Tracer.java
new file mode 100644
index 0000000000..92a77b4985
--- /dev/null
+++ b/pinot-spi/src/main/java/org/apache/pinot/spi/trace/Tracer.java
@@ -0,0 +1,30 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.spi.trace;
+
+public interface Tracer {
+
+ /**
+ * Registers the requestId on the current thread. This means the request
will be traced.
+ * @param requestId the requestId
+ */
+ void register(long requestId);
+
+ OperatorExecution startOperatorExecution(Class<?> clazz);
+}
diff --git a/pinot-spi/src/main/java/org/apache/pinot/spi/trace/Tracing.java
b/pinot-spi/src/main/java/org/apache/pinot/spi/trace/Tracing.java
new file mode 100644
index 0000000000..9ae8b30708
--- /dev/null
+++ b/pinot-spi/src/main/java/org/apache/pinot/spi/trace/Tracing.java
@@ -0,0 +1,70 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.spi.trace;
+
+import java.lang.invoke.MethodHandles;
+import java.lang.invoke.MethodType;
+import java.util.concurrent.atomic.AtomicReference;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+public class Tracing {
+
+ private Tracing() {
+ }
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(Tracing.class);
+
+ private static final AtomicReference<Tracer> REGISTRATION = new
AtomicReference<>();
+
+ /**
+ * User once registration point to allow customization of tracing behaviour.
Registration will be successful
+ * if this was the first attempt to register and registration happened
before first use of the tracer.
+ * @param tracer the tracer implementation
+ * @return true if the registration was successful.
+ */
+ public static boolean register(Tracer tracer) {
+ return REGISTRATION.compareAndSet(null, tracer);
+ }
+
+ private static final class Holder {
+ static final Tracer TRACER = REGISTRATION.get() == null ?
createDefaultTracer() : REGISTRATION.get();
+ }
+
+ /**
+ * @return the registered tracer.
+ */
+ public static Tracer getTracer() {
+ return Holder.TRACER;
+ }
+
+ private static Tracer createDefaultTracer() {
+ // create the default tracer via method handles if no override is
registered
+ String defaultImplementationClassName =
"org.apache.pinot.core.util.trace.DefaultTracer";
+ try {
+ Class<?> clazz = Class.forName(defaultImplementationClassName, false,
Tracing.class.getClassLoader());
+ return (Tracer) MethodHandles.publicLookup()
+ .findConstructor(clazz, MethodType.methodType(void.class)).invoke();
+ } catch (Throwable missing) {
+ LOGGER.error("could not construct MethodHandle for {}",
defaultImplementationClassName, missing);
+ return null;
+ }
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For additional commands, e-mail: commits-h...@pinot.apache.org
