morningman commented on code in PR #68842:
URL: https://github.com/apache/doris/pull/68842#discussion_r4236815553
##########
fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonCacheSizeEstimator.java:
##########
@@ -98,10 +133,62 @@ private static long estimateFileStoreTable(FileStoreTable
table, Set<Object> vis
((DelegatedFileStoreTable) table).wrapped(), visited));
}
bytes = add(bytes, estimateCompleteOnce(table.schema(), visited));
+ bytes = add(bytes, estimateStoreGrowth(table.schema()));
+ bytes = add(bytes, estimateTableOwnedFileIO(table.fileIO(), visited));
bytes = add(bytes, estimatePath(table.location(), visited));
return add(bytes,
estimateCatalogEnvironment(table.catalogEnvironment(), visited));
}
+ /** Reserves the store and index graph this table builds after admission;
no IO, no store access. */
+ private static long estimateStoreGrowth(TableSchema schema) {
+ long bytes = add(STORE_GROWTH_BASE_BYTES,
+ multiply(schema.options().size(), STORE_GROWTH_OPTION_BYTES));
+ long keys = (long) schema.partitionKeys().size() +
schema.primaryKeys().size() + schema.bucketKeys().size();
+ bytes = add(bytes, multiply(keys, STORE_GROWTH_KEY_BYTES));
+ for (String primaryKey : schema.primaryKeys()) {
+ bytes = add(bytes, JvmSizeUtils.stringSize(primaryKey));
+ }
+ return add(bytes, estimateFieldGrowth(schema.fields(),
STORE_GROWTH_FIELD_BYTES));
+ }
+
+ private static long estimateFieldGrowth(List<DataField> fields, long
perFieldBytes) {
+ long bytes = multiply(fields.size(), perFieldBytes);
+ for (DataField field : fields) {
+ bytes = add(bytes, estimateTypeGrowth(field.type()));
+ }
+ return bytes;
+ }
+
+ private static long estimateTypeGrowth(DataType type) {
+ if (type instanceof RowType) {
+ return estimateFieldGrowth(((RowType) type).getFields(),
+ STORE_GROWTH_FIELD_BYTES +
STORE_GROWTH_NESTED_FIELD_BYTES);
+ }
+ if (type instanceof ArrayType) {
+ return estimateChildTypeGrowth(((ArrayType)
type).getElementType());
+ }
+ if (type instanceof MultisetType) {
+ return estimateChildTypeGrowth(((MultisetType)
type).getElementType());
+ }
+ if (type instanceof VectorType) {
+ return estimateChildTypeGrowth(((VectorType)
type).getElementType());
+ }
+ if (type instanceof MapType) {
+ MapType mapType = (MapType) type;
+ return add(estimateChildTypeGrowth(mapType.getKeyType()),
+ estimateChildTypeGrowth(mapType.getValueType()));
+ }
+ return 0L;
+ }
+
+ private static long estimateChildTypeGrowth(DataType child) {
+ return add(STORE_GROWTH_TYPE_NODE_BYTES, estimateTypeGrowth(child));
+ }
+
+ private static long estimateTableOwnedFileIO(FileIO fileIO, Set<Object>
visited) {
+ return fileIO instanceof RESTTokenFileIO && visited.add(fileIO) ?
REST_TOKEN_FILE_IO_BYTES : 0L;
+ }
Review Comment:
Thanks, the mechanism is right: `ResolvingFileIO.configure` copies
`context.options()` into a fresh `Options` for every table, and
`RESTCatalog.options()` is that same map, so each table carries about 40 bytes
per catalog option (a map node and its bucket; the strings are shared with the
catalog).
We are keeping the fixed allowance, as branch-4.x does. 16 KiB covers about
400 copied entries, and a Paimon catalog's options (the Doris `paimon.*`
properties plus a REST server's `/config` defaults and overrides) are normally
a few dozen. Reaching 1,024 takes a configuration that the operator or the
catalog's own REST server would have to choose, and the overrun stays
proportional to that count. Scaling the allowance with the option count would
change branch-4.x's estimate as well, so it belongs in a separate PR for both
branches if a deployment ever comes close.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
