This is an automated email from the ASF dual-hosted git repository.
arafat2198 pushed a commit to branch HDDS-13177
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/HDDS-13177 by this push:
new 4d7891777a5 Added new proto message and introduced a light weight
OmMultiPartUpload light weight object
4d7891777a5 is described below
commit 4d7891777a57042b7094cb726d15fb3ad979c5f2
Author: arafat <[email protected]>
AuthorDate: Wed Aug 13 13:39:50 2025 +0530
Added new proto message and introduced a light weight OmMultiPartUpload
light weight object
---
.../ozone/om/helpers/OmMultipartKeyInfoLight.java | 184 +++++++++++++++++++++
.../src/main/proto/OmClientProtocol.proto | 20 +++
.../ozone/recon/api/types/ReconBasicOmKeyInfo.java | 27 +++
.../recon/tasks/MultipartInfoInsightHandler.java | 28 ++--
4 files changed, 242 insertions(+), 17 deletions(-)
diff --git
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmMultipartKeyInfoLight.java
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmMultipartKeyInfoLight.java
new file mode 100644
index 00000000000..082ef3255c5
--- /dev/null
+++
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmMultipartKeyInfoLight.java
@@ -0,0 +1,184 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.om.helpers;
+
+import org.apache.hadoop.hdds.client.ECReplicationConfig;
+import org.apache.hadoop.hdds.client.ReplicationConfig;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
+import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.MultipartKeyInfoLight;
+import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.PartKeyInfoLight;
+
+import java.util.Map;
+import java.util.TreeMap;
+import java.util.stream.Collectors;
+
+/**
+ * This class is the light-weight version of OmMultipartKeyInfo.
+ * It is used by Recon to reduce payload and deserialization cost.
+ */
+public class OmMultipartKeyInfoLight {
+
+ private final String uploadID;
+ private final long creationTime;
+ private final ReplicationConfig replicationConfig;
+ private final Map<Integer, PartKeyInfoLight> partKeyInfoMap;
+ private final long objectID;
+ private final long updateID;
+ private final long parentID;
+
+ public OmMultipartKeyInfoLight(Builder b) {
+ this.uploadID = b.uploadID;
+ this.creationTime = b.creationTime;
+ this.replicationConfig = b.replicationConfig;
+ this.partKeyInfoMap = b.partKeyInfoMap;
+ this.objectID = b.objectID;
+ this.updateID = b.updateID;
+ this.parentID = b.parentID;
+ }
+
+ public String getUploadID() {
+ return uploadID;
+ }
+
+ public long getCreationTime() {
+ return creationTime;
+ }
+
+ public ReplicationConfig getReplicationConfig() {
+ return replicationConfig;
+ }
+
+ public Map<Integer, PartKeyInfoLight> getPartKeyInfoMap() {
+ return partKeyInfoMap;
+ }
+
+ public long getObjectID() {
+ return objectID;
+ }
+
+ public long getUpdateID() {
+ return updateID;
+ }
+
+ public long getParentID() {
+ return parentID;
+ }
+
+ public static OmMultipartKeyInfoLight getFromProtobuf(
+ MultipartKeyInfoLight multipartKeyInfoLight) {
+ return new Builder()
+ .setUploadID(multipartKeyInfoLight.getUploadID())
+ .setCreationTime(multipartKeyInfoLight.getCreationTime())
+ .setReplicationConfig(
+ ReplicationConfig.fromProto(
+ multipartKeyInfoLight.getType(),
+ multipartKeyInfoLight.getFactor(),
+ multipartKeyInfoLight.getEcReplicationConfig()
+ ))
+ .setPartKeyInfoMap(
+ multipartKeyInfoLight.getPartKeyInfoListList().stream()
+ .collect(Collectors.toMap(
+ PartKeyInfoLight::getPartNumber,
+ partKeyInfo -> partKeyInfo,
+ (v1, v2) -> v1,
+ TreeMap::new
+ )))
+ .setObjectID(multipartKeyInfoLight.getObjectID())
+ .setUpdateID(multipartKeyInfoLight.getUpdateID())
+ .setParentID(multipartKeyInfoLight.getParentID())
+ .build();
+ }
+
+ public MultipartKeyInfoLight getProtobuf() {
+ MultipartKeyInfoLight.Builder builder = MultipartKeyInfoLight.newBuilder()
+ .setUploadID(uploadID)
+ .setCreationTime(creationTime)
+ .setObjectID(objectID)
+ .setUpdateID(updateID)
+ .setParentID(parentID);
+
+ ReplicationConfig repConfig = getReplicationConfig();
+ builder.setType(repConfig.getReplicationType());
+
+ if (repConfig.getReplicationType() == HddsProtos.ReplicationType.RATIS) {
+ builder.setFactor(ReplicationConfig.getLegacyFactor(repConfig));
+ } else if (repConfig.getReplicationType() ==
+ HddsProtos.ReplicationType.EC) {
+ builder.setEcReplicationConfig(
+ ((ECReplicationConfig) repConfig).toProto());
+ }
+
+ builder.addAllPartKeyInfoList(partKeyInfoMap.values());
+
+ return builder.build();
+ }
+
+ /**
+ * Builder of OmMultipartKeyInfoLight.
+ */
+ public static class Builder {
+ private String uploadID;
+ private long creationTime;
+ private ReplicationConfig replicationConfig;
+ private Map<Integer, PartKeyInfoLight> partKeyInfoMap;
+ private long objectID;
+ private long updateID;
+ private long parentID;
+
+ public Builder setUploadID(String uploadID) {
+ this.uploadID = uploadID;
+ return this;
+ }
+
+ public Builder setCreationTime(long creationTime) {
+ this.creationTime = creationTime;
+ return this;
+ }
+
+ public Builder setReplicationConfig(ReplicationConfig replicationConfig) {
+ this.replicationConfig = replicationConfig;
+ return this;
+ }
+
+ public Builder setPartKeyInfoMap(Map<Integer, PartKeyInfoLight>
partKeyInfoMap) {
+ this.partKeyInfoMap = partKeyInfoMap;
+ return this;
+ }
+
+ public Builder setObjectID(long objectID) {
+ this.objectID = objectID;
+ return this;
+ }
+
+ public Builder setUpdateID(long updateID) {
+ this.updateID = updateID;
+ return this;
+ }
+
+ public Builder setParentID(long parentID) {
+ this.parentID = parentID;
+ return this;
+ }
+
+ public OmMultipartKeyInfoLight build() {
+ return new OmMultipartKeyInfoLight(this);
+ }
+ }
+}
+
diff --git
a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
index ee9535ac393..0ac8ff82fe6 100644
--- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
+++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
@@ -1703,6 +1703,26 @@ message PartKeyInfo {
required KeyInfo partKeyInfo = 3;
}
+// Lightweight version of PartKeyInfo that uses KeyInfoProtoLight.
+message PartKeyInfoLight {
+ required string partName = 1;
+ required uint32 partNumber = 2;
+ required KeyInfoProtoLight partKeyInfo = 3;
+}
+
+// Lightweight version of MultipartKeyInfo.
+message MultipartKeyInfoLight {
+ required string uploadID = 1;
+ required uint64 creationTime = 2;
+ required hadoop.hdds.ReplicationType type = 3;
+ optional hadoop.hdds.ReplicationFactor factor = 4;
+ repeated PartKeyInfoLight partKeyInfoList = 5;
+ optional uint64 objectID = 6;
+ optional uint64 updateID = 7;
+ optional uint64 parentID = 8;
+ optional hadoop.hdds.ECReplicationConfig ecReplicationConfig = 9;
+}
+
message MultipartCommitUploadPartRequest {
required KeyArgs keyArgs = 1;
required uint64 clientID = 2;
diff --git
a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/ReconBasicOmKeyInfo.java
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/ReconBasicOmKeyInfo.java
index df62f72c845..c4273baad28 100644
---
a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/ReconBasicOmKeyInfo.java
+++
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/ReconBasicOmKeyInfo.java
@@ -24,9 +24,14 @@
import org.apache.hadoop.hdds.utils.db.Codec;
import org.apache.hadoop.hdds.utils.db.DelegatedCodec;
import org.apache.hadoop.hdds.utils.db.Proto2Codec;
+import org.apache.hadoop.ozone.om.helpers.OmMultipartKeyInfoLight;
import org.apache.hadoop.ozone.om.helpers.QuotaUtil;
import org.apache.hadoop.ozone.om.helpers.WithParentObjectId;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
+import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.PartKeyInfoLight;
+
+import java.util.ArrayList;
+import java.util.List;
/**
* Lightweight OmKeyInfo class.
@@ -279,6 +284,28 @@ public static ReconBasicOmKeyInfo
getFromProtobuf(OzoneManagerProtocolProtos.Key
return builder.build();
}
+ public static ReconBasicOmKeyInfo getFromPartKeyInfoLight(PartKeyInfoLight
partKeyInfoLight) {
+ if (partKeyInfoLight == null) {
+ return null;
+ }
+ return getFromProtobuf(partKeyInfoLight.getPartKeyInfo());
+ }
+
+ public static List<ReconBasicOmKeyInfo>
convertMultipartLightToBasic(OmMultipartKeyInfoLight omMultipartKeyInfoLight) {
+ if (omMultipartKeyInfoLight == null) {
+ return new ArrayList<>();
+ }
+
+ List<ReconBasicOmKeyInfo> basicKeyInfos = new ArrayList<>();
+ for (PartKeyInfoLight partKeyInfoLight :
omMultipartKeyInfoLight.getPartKeyInfoMap().values()) {
+ ReconBasicOmKeyInfo basicKeyInfo =
getFromPartKeyInfoLight(partKeyInfoLight);
+ if (basicKeyInfo != null) {
+ basicKeyInfos.add(basicKeyInfo);
+ }
+ }
+ return basicKeyInfos;
+ }
+
public OzoneManagerProtocolProtos.KeyInfoProtoLight toProtobuf() {
throw new UnsupportedOperationException("This method is not supported.");
}
diff --git
a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/MultipartInfoInsightHandler.java
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/MultipartInfoInsightHandler.java
index 3976919ab4b..a231fd53b90 100644
---
a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/MultipartInfoInsightHandler.java
+++
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/MultipartInfoInsightHandler.java
@@ -21,8 +21,7 @@
import org.apache.commons.lang3.tuple.Triple;
import org.apache.hadoop.hdds.utils.db.Table;
import org.apache.hadoop.hdds.utils.db.TableIterator;
-import org.apache.hadoop.ozone.om.helpers.OmMultipartKeyInfo;
-import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.PartKeyInfo;
+import org.apache.hadoop.ozone.om.helpers.OmMultipartKeyInfoLight;
import org.apache.hadoop.ozone.recon.api.types.ReconBasicOmKeyInfo;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -45,12 +44,11 @@ public void handlePutEvent(OMDBUpdateEvent<String, Object>
event, String tableNa
Map<String, Long> unReplicatedSizeMap, Map<String, Long>
replicatedSizeMap) {
if (event.getValue() != null) {
- OmMultipartKeyInfo multipartKeyInfo = (OmMultipartKeyInfo)
event.getValue();
+ OmMultipartKeyInfoLight multipartKeyInfo = (OmMultipartKeyInfoLight)
event.getValue();
objectCountMap.computeIfPresent(getTableCountKeyFromTable(tableName),
(k, count) -> count + 1L);
- for (PartKeyInfo partKeyInfo : multipartKeyInfo.getPartKeyInfoMap()) {
- ReconBasicOmKeyInfo omKeyInfo =
ReconBasicOmKeyInfo.getFromProtobuf(partKeyInfo.getPartKeyInfo());
+ for (ReconBasicOmKeyInfo omKeyInfo :
ReconBasicOmKeyInfo.convertMultipartLightToBasic(multipartKeyInfo)) {
unReplicatedSizeMap.computeIfPresent(getUnReplicatedSizeKeyFromTable(tableName),
(k, size) -> size + omKeyInfo.getDataSize());
replicatedSizeMap.computeIfPresent(getReplicatedSizeKeyFromTable(tableName),
@@ -70,12 +68,11 @@ public void handleDeleteEvent(OMDBUpdateEvent<String,
Object> event, String tabl
Map<String, Long> objectCountMap, Map<String, Long> unReplicatedSizeMap,
Map<String, Long> replicatedSizeMap) {
if (event.getValue() != null) {
- OmMultipartKeyInfo multipartKeyInfo = (OmMultipartKeyInfo)
event.getValue();
+ OmMultipartKeyInfoLight multipartKeyInfo = (OmMultipartKeyInfoLight)
event.getValue();
objectCountMap.computeIfPresent(getTableCountKeyFromTable(tableName),
(k, count) -> count > 0 ? count - 1L : 0L);
- for (PartKeyInfo partKeyInfo : multipartKeyInfo.getPartKeyInfoMap()) {
- ReconBasicOmKeyInfo omKeyInfo =
ReconBasicOmKeyInfo.getFromProtobuf(partKeyInfo.getPartKeyInfo());
+ for (ReconBasicOmKeyInfo omKeyInfo :
ReconBasicOmKeyInfo.convertMultipartLightToBasic(multipartKeyInfo)) {
unReplicatedSizeMap.computeIfPresent(getUnReplicatedSizeKeyFromTable(tableName),
(k, size) -> {
long newSize = size > omKeyInfo.getDataSize() ? size -
omKeyInfo.getDataSize() : 0L;
@@ -116,12 +113,11 @@ public void handleUpdateEvent(OMDBUpdateEvent<String,
Object> event, String tabl
// In Update event the count for the multipart info table will not
change. So we
// don't need to update the count.
- OmMultipartKeyInfo oldMultipartKeyInfo = (OmMultipartKeyInfo)
event.getOldValue();
- OmMultipartKeyInfo newMultipartKeyInfo = (OmMultipartKeyInfo)
event.getValue();
+ OmMultipartKeyInfoLight oldMultipartKeyInfo = (OmMultipartKeyInfoLight)
event.getOldValue();
+ OmMultipartKeyInfoLight newMultipartKeyInfo = (OmMultipartKeyInfoLight)
event.getValue();
// Calculate old sizes
- for (PartKeyInfo partKeyInfo : oldMultipartKeyInfo.getPartKeyInfoMap()) {
- ReconBasicOmKeyInfo omKeyInfo =
ReconBasicOmKeyInfo.getFromProtobuf(partKeyInfo.getPartKeyInfo());
+ for (ReconBasicOmKeyInfo omKeyInfo :
ReconBasicOmKeyInfo.convertMultipartLightToBasic(oldMultipartKeyInfo)) {
unReplicatedSizeMap.computeIfPresent(getUnReplicatedSizeKeyFromTable(tableName),
(k, size) -> size - omKeyInfo.getDataSize());
replicatedSizeMap.computeIfPresent(getReplicatedSizeKeyFromTable(tableName),
@@ -129,8 +125,7 @@ public void handleUpdateEvent(OMDBUpdateEvent<String,
Object> event, String tabl
}
// Calculate new sizes
- for (PartKeyInfo partKeyInfo : newMultipartKeyInfo.getPartKeyInfoMap()) {
- ReconBasicOmKeyInfo omKeyInfo =
ReconBasicOmKeyInfo.getFromProtobuf(partKeyInfo.getPartKeyInfo());
+ for (ReconBasicOmKeyInfo omKeyInfo :
ReconBasicOmKeyInfo.convertMultipartLightToBasic(newMultipartKeyInfo)) {
unReplicatedSizeMap.computeIfPresent(getUnReplicatedSizeKeyFromTable(tableName),
(k, size) -> size + omKeyInfo.getDataSize());
replicatedSizeMap.computeIfPresent(getReplicatedSizeKeyFromTable(tableName),
@@ -158,9 +153,8 @@ public Triple<Long, Long, Long> getTableSizeAndCount(
while (iterator.hasNext()) {
Table.KeyValue<String, ?> kv = iterator.next();
if (kv != null && kv.getValue() != null) {
- OmMultipartKeyInfo multipartKeyInfo = (OmMultipartKeyInfo)
kv.getValue();
- for (PartKeyInfo partKeyInfo : multipartKeyInfo.getPartKeyInfoMap())
{
- ReconBasicOmKeyInfo omKeyInfo =
ReconBasicOmKeyInfo.getFromProtobuf(partKeyInfo.getPartKeyInfo());
+ OmMultipartKeyInfoLight multipartKeyInfo = (OmMultipartKeyInfoLight)
kv.getValue();
+ for (ReconBasicOmKeyInfo omKeyInfo :
ReconBasicOmKeyInfo.convertMultipartLightToBasic(multipartKeyInfo)) {
unReplicatedSize += omKeyInfo.getDataSize();
replicatedSize += omKeyInfo.getReplicatedSize();
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
