imay commented on a change in pull request #2305: Add a variable to
specifically limit the memory usage of the load part in the insert operation
URL: https://github.com/apache/incubator-doris/pull/2305#discussion_r351321463
##########
File path: fe/src/main/java/org/apache/doris/qe/SessionVariable.java
##########
@@ -474,88 +397,140 @@ public TQueryOptions toThrift() {
tResult.setBatch_size(batchSize);
tResult.setDisable_stream_preaggregations(disableStreamPreaggregations);
+ tResult.setLoad_mem_limit(loadMemLimit);
return tResult;
}
@Override
public void write(DataOutput out) throws IOException {
- out.writeInt(codegenLevel);
- out.writeInt(netBufferLength);
- out.writeInt(sqlSafeUpdates);
- Text.writeString(out, timeZone);
- out.writeInt(netReadTimeout);
- out.writeInt(netWriteTimeout);
- out.writeInt(waitTimeout);
- out.writeInt(interactiveTimeout);
- out.writeInt(queryCacheType);
- out.writeInt(autoIncrementIncrement);
- out.writeInt(maxAllowedPacket);
- out.writeLong(sqlSelectLimit);
- out.writeBoolean(sqlAutoIsNull);
- Text.writeString(out, collationDatabase);
- Text.writeString(out, collationConnection);
- Text.writeString(out, charsetServer);
- Text.writeString(out, charsetResults);
- Text.writeString(out, charsetConnection);
- Text.writeString(out, charsetClient);
- Text.writeString(out, txIsolation);
- out.writeBoolean(autoCommit);
- Text.writeString(out, resourceGroup);
- out.writeLong(sqlMode);
- out.writeBoolean(isReportSucc);
- out.writeInt(queryTimeoutS);
- out.writeLong(maxExecMemByte);
- Text.writeString(out, collationServer);
- out.writeInt(batchSize);
- out.writeBoolean(disableStreamPreaggregations);
- out.writeInt(parallelExecInstanceNum);
- out.writeInt(exchangeInstanceParallel);
+ JSONObject root = new JSONObject();
+ try {
+ for (Field field : SessionVariable.class.getDeclaredFields()) {
+ VarAttr attr = field.getAnnotation(VarAttr.class);
+ if (attr == null) {
+ continue;
+ }
+ switch (field.getType().getSimpleName()) {
+ case "boolean":
+ root.put(attr.name(), (Boolean) field.get(this));
+ break;
+ case "int":
+ root.put(attr.name(), (Integer) field.get(this));
+ break;
+ case "long":
+ root.put(attr.name(), (Long) field.get(this));
+ break;
+ case "float":
+ root.put(attr.name(), (Float) field.get(this));
+ break;
+ case "double":
+ root.put(attr.name(), (Double) field.get(this));
+ break;
+ case "String":
+ root.put(attr.name(), (String) field.get(this));
+ break;
+ default:
+ // Unsupported type variable.
+ throw new IOException("invalid type: " +
field.getType().getSimpleName());
+ }
+ }
+ } catch (Exception e) {
+ throw new IOException("failed to write session variable: " +
e.getMessage());
+ }
+ Text.writeString(out, root.toString());
}
@Override
public void readFields(DataInput in) throws IOException {
- codegenLevel = in.readInt();
- netBufferLength = in.readInt();
- sqlSafeUpdates = in.readInt();
- timeZone = Text.readString(in);
- netReadTimeout = in.readInt();
- netWriteTimeout = in.readInt();
- waitTimeout = in.readInt();
- interactiveTimeout = in.readInt();
- queryCacheType = in.readInt();
- autoIncrementIncrement = in.readInt();
- maxAllowedPacket = in.readInt();
- sqlSelectLimit = in.readLong();
- sqlAutoIsNull = in.readBoolean();
- collationDatabase = Text.readString(in);
- collationConnection = Text.readString(in);
- charsetServer = Text.readString(in);
- charsetResults = Text.readString(in);
- charsetConnection = Text.readString(in);
- charsetClient = Text.readString(in);
- txIsolation = Text.readString(in);
- autoCommit = in.readBoolean();
- resourceGroup = Text.readString(in);
- if (Catalog.getCurrentCatalogJournalVersion() >=
FeMetaVersion.VERSION_65) {
- sqlMode = in.readLong();
+ if (Catalog.getCurrentCatalogJournalVersion() <
FeMetaVersion.VERSION_67) {
+ codegenLevel = in.readInt();
+ netBufferLength = in.readInt();
+ sqlSafeUpdates = in.readInt();
+ timeZone = Text.readString(in);
+ netReadTimeout = in.readInt();
+ netWriteTimeout = in.readInt();
+ waitTimeout = in.readInt();
+ interactiveTimeout = in.readInt();
+ queryCacheType = in.readInt();
+ autoIncrementIncrement = in.readInt();
+ maxAllowedPacket = in.readInt();
+ sqlSelectLimit = in.readLong();
+ sqlAutoIsNull = in.readBoolean();
+ collationDatabase = Text.readString(in);
+ collationConnection = Text.readString(in);
+ charsetServer = Text.readString(in);
+ charsetResults = Text.readString(in);
+ charsetConnection = Text.readString(in);
+ charsetClient = Text.readString(in);
+ txIsolation = Text.readString(in);
+ autoCommit = in.readBoolean();
+ resourceGroup = Text.readString(in);
+ if (Catalog.getCurrentCatalogJournalVersion() >=
FeMetaVersion.VERSION_65) {
+ sqlMode = in.readLong();
+ } else {
+ // read old version SQL mode
+ Text.readString(in);
+ sqlMode = 0L;
+ }
+ isReportSucc = in.readBoolean();
+ queryTimeoutS = in.readInt();
+ maxExecMemByte = in.readLong();
+ if (Catalog.getCurrentCatalogJournalVersion() >=
FeMetaVersion.VERSION_37) {
+ collationServer = Text.readString(in);
+ }
+ if (Catalog.getCurrentCatalogJournalVersion() >=
FeMetaVersion.VERSION_38) {
+ batchSize = in.readInt();
+ disableStreamPreaggregations = in.readBoolean();
+ parallelExecInstanceNum = in.readInt();
+ }
+ if (Catalog.getCurrentCatalogJournalVersion() >=
FeMetaVersion.VERSION_62) {
+ exchangeInstanceParallel = in.readInt();
+ }
} else {
- // read old version SQL mode
- Text.readString(in);
- sqlMode = 0L;
+ readFromJson(in);
}
- isReportSucc = in.readBoolean();
- queryTimeoutS = in.readInt();
- maxExecMemByte = in.readLong();
- if (Catalog.getCurrentCatalogJournalVersion() >=
FeMetaVersion.VERSION_37) {
- collationServer = Text.readString(in);
- }
- if (Catalog.getCurrentCatalogJournalVersion() >=
FeMetaVersion.VERSION_38) {
- batchSize = in.readInt();
- disableStreamPreaggregations = in.readBoolean();
- parallelExecInstanceNum = in.readInt();
- }
- if (Catalog.getCurrentCatalogJournalVersion() >=
FeMetaVersion.VERSION_62) {
- exchangeInstanceParallel = in.readInt();
+ }
+
+ private void readFromJson(DataInput in) throws IOException {
Review comment:
I think some json library has already implement this things, like Google
Gson.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
