jasperjiaguo commented on code in PR #11942:
URL: https://github.com/apache/pinot/pull/11942#discussion_r1382289868
##########
pinot-core/src/test/java/org/apache/pinot/core/accounting/ResourceManagerAccountingTest.java:
##########
@@ -223,6 +238,145 @@ public void testWorkerThreadInterruption()
Assert.fail("Expected EarlyTerminationException to be thrown");
}
+ /**
+ * Test instrumentation during {@link DataTable} creation
+ */
+ @Test
+ public void testGetDataTableOOMSelect()
+ throws Exception {
+
+ // generate random rows
+ String[] columnNames = {
+ "int", "long", "float", "double", "big_decimal", "string", "bytes",
"int_array", "long_array", "float_array",
+ "double_array", "string_array"
+ };
+ DataSchema.ColumnDataType[] columnDataTypes = {
+ DataSchema.ColumnDataType.INT, DataSchema.ColumnDataType.LONG,
DataSchema.ColumnDataType.FLOAT,
+ DataSchema.ColumnDataType.DOUBLE,
DataSchema.ColumnDataType.BIG_DECIMAL, DataSchema.ColumnDataType.STRING,
+ DataSchema.ColumnDataType.BYTES, DataSchema.ColumnDataType.INT_ARRAY,
DataSchema.ColumnDataType.LONG_ARRAY,
+ DataSchema.ColumnDataType.FLOAT_ARRAY,
DataSchema.ColumnDataType.DOUBLE_ARRAY,
+ DataSchema.ColumnDataType.STRING_ARRAY
+ };
+ DataSchema dataSchema = new DataSchema(columnNames, columnDataTypes);
+ List<Object[]> rows = DataBlockTestUtils.getRandomRows(dataSchema,
NUM_ROWS, 0);
+
+ // set up logging and configs
+
LogManager.getLogger(PerQueryCPUMemResourceUsageAccountant.class).setLevel(Level.DEBUG);
+
LogManager.getLogger(ThreadResourceUsageProvider.class).setLevel(Level.DEBUG);
+ ThreadResourceUsageProvider.setThreadMemoryMeasurementEnabled(true);
+ HashMap<String, Object> configs = new HashMap<>();
+ ServerMetrics.register(Mockito.mock(ServerMetrics.class));
+
configs.put(CommonConstants.Accounting.CONFIG_OF_ALARMING_LEVEL_HEAP_USAGE_RATIO,
0.00f);
+
configs.put(CommonConstants.Accounting.CONFIG_OF_CRITICAL_LEVEL_HEAP_USAGE_RATIO,
0.00f);
+ configs.put(CommonConstants.Accounting.CONFIG_OF_FACTORY_NAME,
+ "org.apache.pinot.core.accounting.PerQueryCPUMemAccountantFactory");
+
configs.put(CommonConstants.Accounting.CONFIG_OF_ENABLE_THREAD_MEMORY_SAMPLING,
true);
+
configs.put(CommonConstants.Accounting.CONFIG_OF_ENABLE_THREAD_CPU_SAMPLING,
false);
+
configs.put(CommonConstants.Accounting.CONFIG_OF_OOM_PROTECTION_KILLING_QUERY,
true);
+ PinotConfiguration config = getConfig(1, 2, configs);
+ ResourceManager rm = getResourceManager(1, 2, 1, 1, configs);
+ // init accountant and start watcher task
+ Tracing.ThreadAccountantOps.initializeThreadAccountant(config,
"testSelect");
+
+ ListenableFuture<?> future = rm.getQueryRunners().submit(() -> {
+ Tracing.ThreadAccountantOps.setupRunner("testSelectQueryId");
+ try {
+ int numIterations = 0;
+ List<DataTable> dataTables = new ArrayList<>();
+ while (numIterations < 100) {
+ // build data table with sampling
+ dataTables.add(SelectionOperatorUtils.getDataTableFromRows(rows,
dataSchema, false));
Review Comment:
Could we add DataTable.toBytes() as well here? Meanwhile, the testing would
probably fit better to the real use case if we call
`SelectionOperatorUtils.getDataTableFromRows(rows, dataSchema, false)` only
once in each thread and submit multiple runner threads. Similar for the groupby
case.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For additional commands, e-mail: commits-h...@pinot.apache.org
