Jackie-Jiang commented on code in PR #16728:
URL: https://github.com/apache/pinot/pull/16728#discussion_r2373807796
##########
pinot-core/src/main/java/org/apache/pinot/core/accounting/QueryAggregator.java:
##########
@@ -58,83 +55,23 @@ enum TriggeringLevel {
Normal, HeapMemoryAlarmingVerbose, CPUTimeBasedKilling,
HeapMemoryCritical, HeapMemoryPanic
}
- // For one time concurrent update of stats. This is to provide stats
collection for parts that are not
- // performance sensitive and query_id is not known beforehand (e.g. broker
inbound netty thread)
- private final ConcurrentHashMap<String, Long>
_concurrentTaskCPUStatsAggregator = new ConcurrentHashMap<>();
- private final ConcurrentHashMap<String, Long>
_concurrentTaskMemStatsAggregator = new ConcurrentHashMap<>();
-
- // for stats aggregation of finished (worker) threads when the runner is
still running.
- private final HashMap<String, Long> _finishedTaskCPUStatsAggregator = new
HashMap<>();
- private final HashMap<String, Long> _finishedTaskMemStatsAggregator = new
HashMap<>();
-
- private final boolean _isThreadCPUSamplingEnabled;
- private final boolean _isThreadMemorySamplingEnabled;
+ // Tracks the CPU and memory usage for queries not tracked by any thread.
+ // E.g. request ser/de where thread execution context cannot be set up
beforehand; tasks already finished.
+ private final ConcurrentHashMap<String, LongLongMutablePair>
_untrackedCpuMemUsage = new ConcurrentHashMap<>();
- private final Set<String> _inactiveQuery;
- private final PinotConfiguration _config;
+ // Tracks the id of the inactive queries, which is used to clean up entries
in _untrackedMemCpuUsage.
+ private final Set<String> _inactiveQueries = new HashSet<>();
- private final InstanceType _instanceType;
private final String _instanceId;
+ private final InstanceType _instanceType;
+ private final boolean _cpuSamplingEnabled;
+ private final boolean _memorySamplingEnabled;
+ private final AtomicReference<QueryMonitorConfig> _queryMonitorConfig;
- // max heap usage, Xmx
- private final long _maxHeapSize = ResourceUsageUtils.getMaxHeapSize();
-
- // don't kill a query if its memory footprint is below some ratio of
_maxHeapSize
- private final long _minMemoryFootprintForKill;
-
- // kill all queries if heap usage exceeds this
- private final long _panicLevel;
-
- // kill the most expensive query if heap usage exceeds this
- private final long _criticalLevel;
-
- // if after gc the heap usage is still above this, kill the most expensive
query
- // use this to prevent heap size oscillation and repeatedly triggering gc
- private final long _criticalLevelAfterGC;
-
- // trigger gc if consecutively kill more than some number of queries
- // set this to 0 to always trigger gc before killing a query to give gc a
second chance
- // as would minimize the chance of false positive killing in some usecases
- // should consider use -XX:+ExplicitGCInvokesConcurrent to avoid STW for
some gc algorithms
- private final int _gcBackoffCount;
-
- // start to sample more frequently if heap usage exceeds this
- private final long _alarmingLevel;
-
- // normal sleep time
- private final int _normalSleepTime;
-
- // wait for gc to complete, according to system.gc() javadoc, when control
returns from the method call,
- // the Java Virtual Machine has made a best effort to reclaim space from all
discarded objects.
- // Therefore, we default this to 0.
- // Tested with Shenandoah GC and G1GC, with -XX:+ExplicitGCInvokesConcurrent
- private final int _gcWaitTime;
-
- // alarming sleep time denominator, should be > 1 to sample more frequent at
alarming level
- private final int _alarmingSleepTimeDenominator;
-
- // alarming sleep time
- private final int _alarmingSleepTime;
-
- // the framework would not commit to kill any query if this is disabled
- private final boolean _oomKillQueryEnabled;
-
- // if we want to publish the heap usage
- private final boolean _publishHeapUsageMetric;
-
- // if we want kill query based on CPU time
- private final boolean _isCPUTimeBasedKillingEnabled;
-
- // CPU time based killing threshold
- private final long _cpuTimeBasedKillingThresholdNS;
-
- private final boolean _isQueryKilledMetricEnabled;
-
- private long _usedBytes;
+ private volatile long _usedBytes;
private int _sleepTime;
- private int _numQueriesKilledConsecutively = 0;
- protected Map<String, AggregatedStats> _aggregatedUsagePerActiveQuery;
- private TriggeringLevel _triggeringLevel;
+ private Map<String, QueryResourceTrackerImpl> _queryResourceUsages;
Review Comment:
Done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
