morningman commented on code in PR #68842:
URL: https://github.com/apache/doris/pull/68842#discussion_r4233945449
##########
fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonCacheSizeEstimator.java:
##########
@@ -98,10 +133,62 @@ private static long estimateFileStoreTable(FileStoreTable
table, Set<Object> vis
((DelegatedFileStoreTable) table).wrapped(), visited));
}
bytes = add(bytes, estimateCompleteOnce(table.schema(), visited));
+ bytes = add(bytes, estimateStoreGrowth(table.schema()));
+ bytes = add(bytes, estimateTableOwnedFileIO(table.fileIO(), visited));
bytes = add(bytes, estimatePath(table.location(), visited));
return add(bytes,
estimateCatalogEnvironment(table.catalogEnvironment(), visited));
}
+ /** Reserves the store and index graph this table builds after admission;
no IO, no store access. */
+ private static long estimateStoreGrowth(TableSchema schema) {
+ long bytes = add(STORE_GROWTH_BASE_BYTES,
+ multiply(schema.options().size(), STORE_GROWTH_OPTION_BYTES));
+ long keys = (long) schema.partitionKeys().size() +
schema.primaryKeys().size() + schema.bucketKeys().size();
+ bytes = add(bytes, multiply(keys, STORE_GROWTH_KEY_BYTES));
+ for (String primaryKey : schema.primaryKeys()) {
+ bytes = add(bytes, JvmSizeUtils.stringSize(primaryKey));
+ }
+ return add(bytes, estimateFieldGrowth(schema.fields(),
STORE_GROWTH_FIELD_BYTES));
+ }
+
+ private static long estimateFieldGrowth(List<DataField> fields, long
perFieldBytes) {
+ long bytes = multiply(fields.size(), perFieldBytes);
+ for (DataField field : fields) {
+ bytes = add(bytes, estimateTypeGrowth(field.type()));
+ }
+ return bytes;
+ }
+
+ private static long estimateTypeGrowth(DataType type) {
+ if (type instanceof RowType) {
+ return estimateFieldGrowth(((RowType) type).getFields(),
+ STORE_GROWTH_FIELD_BYTES +
STORE_GROWTH_NESTED_FIELD_BYTES);
+ }
+ if (type instanceof ArrayType) {
+ return estimateChildTypeGrowth(((ArrayType)
type).getElementType());
+ }
+ if (type instanceof MultisetType) {
+ return estimateChildTypeGrowth(((MultisetType)
type).getElementType());
+ }
+ if (type instanceof VectorType) {
+ return estimateChildTypeGrowth(((VectorType)
type).getElementType());
+ }
+ if (type instanceof MapType) {
+ MapType mapType = (MapType) type;
+ return add(estimateChildTypeGrowth(mapType.getKeyType()),
+ estimateChildTypeGrowth(mapType.getValueType()));
+ }
+ return 0L;
+ }
+
+ private static long estimateChildTypeGrowth(DataType child) {
+ return add(STORE_GROWTH_TYPE_NODE_BYTES, estimateTypeGrowth(child));
+ }
+
+ private static long estimateTableOwnedFileIO(FileIO fileIO, Set<Object>
visited) {
+ return fileIO instanceof RESTTokenFileIO && visited.add(fileIO) ?
REST_TOKEN_FILE_IO_BYTES : 0L;
+ }
Review Comment:
Agreed, fixed in ce25717d42c. A REST catalog without data tokens creates a
`ResolvingFileIO` per table (`RESTCatalog.fileIOFromOptions`), and
`HiveCatalog.fileIO(Path)` resolves one per table location; both build their
backend maps after admission. Paimon offers no IO-free way to tell an owned
FileIO from one the catalog shares, so every concrete table now carries the
fixed 16 KB FileIO allowance, as on branch-4.x. Charging a shared FileIO to
each table only makes the estimate conservative.
`everyTableCarriesAnAllowanceForItsFileIo` builds the same table on
`LocalFileIO`, `ResolvingFileIO`, `HadoopFileIO` and `RESTTokenFileIO` and
checks that each weight carries the allowance, and that the allowance covers a
`ResolvingFileIO` once it has built its backend. Charging only
`RESTTokenFileIO` again makes it fail.
##########
fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonCacheSizeEstimator.java:
##########
@@ -98,10 +133,62 @@ private static long estimateFileStoreTable(FileStoreTable
table, Set<Object> vis
((DelegatedFileStoreTable) table).wrapped(), visited));
}
bytes = add(bytes, estimateCompleteOnce(table.schema(), visited));
+ bytes = add(bytes, estimateStoreGrowth(table.schema()));
+ bytes = add(bytes, estimateTableOwnedFileIO(table.fileIO(), visited));
bytes = add(bytes, estimatePath(table.location(), visited));
return add(bytes,
estimateCatalogEnvironment(table.catalogEnvironment(), visited));
}
+ /** Reserves the store and index graph this table builds after admission;
no IO, no store access. */
+ private static long estimateStoreGrowth(TableSchema schema) {
+ long bytes = add(STORE_GROWTH_BASE_BYTES,
+ multiply(schema.options().size(), STORE_GROWTH_OPTION_BYTES));
+ long keys = (long) schema.partitionKeys().size() +
schema.primaryKeys().size() + schema.bucketKeys().size();
+ bytes = add(bytes, multiply(keys, STORE_GROWTH_KEY_BYTES));
+ for (String primaryKey : schema.primaryKeys()) {
+ bytes = add(bytes, JvmSizeUtils.stringSize(primaryKey));
+ }
+ return add(bytes, estimateFieldGrowth(schema.fields(),
STORE_GROWTH_FIELD_BYTES));
+ }
Review Comment:
Agreed, fixed in ce25717d42c. I measured it on Paimon 1.4.2 with a 300-field
primary-key table, one sequence group over every field and
`fields.default-aggregate-function=last_non_null_value`. After a scan it
retains about 460 bytes per field, about 150 more than the same table under
deduplicate or aggregation, while the store reserve allowed 320. Partial-update
tables now reserve another 256 bytes per field.
The grown-graph test now leaves the FileIO allowance out of the comparison,
since that allowance pays for the FileIO and not the store, and it includes
this table and an aggregation one. Without the new reserve the partial-update
table is admitted at 139,256 bytes against a grown graph of 142,608, and the
test fails.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
