This is an automated email from the ASF dual-hosted git repository. quantranhong1999 pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/james-project.git
commit a229eeaead19207f7493ace2a54fd34d90b35042 Author: Benoit TELLIER <[email protected]> AuthorDate: Fri Sep 4 18:01:07 2026 +0200 JAMES-4225 Setting to control entropy in BlobIds --- docs/modules/servers/partials/configure/jvm.adoc | 29 +++++++ .../sample-configuration/jvm.properties | 8 ++ .../org/apache/james/blob/api/BlobIdEntropy.java | 88 +++++++++++++++++++ .../apache/james/blob/api/BlobIdEntropyTest.java | 98 ++++++++++++++++++++++ .../deduplication/DeDuplicationBlobStore.scala | 9 +- 5 files changed, 229 insertions(+), 3 deletions(-) diff --git a/docs/modules/servers/partials/configure/jvm.adoc b/docs/modules/servers/partials/configure/jvm.adoc index e2418d40af..eda2806d8f 100644 --- a/docs/modules/servers/partials/configure/jvm.adoc +++ b/docs/modules/servers/partials/configure/jvm.adoc @@ -99,6 +99,35 @@ lines at most. Lowering it only affects previews computed afterwards. Those already stored keep their length until the message is reindexed, so the space comes back progressively rather than at once. +== Change the entropy of the blobId + +By default a blobId carries 256 bits of entropy: the full SHA-256 of the content it addresses for +deduplicated blobs, and as many random bits for the randomly generated ones. The property +`james.blobid.entropy` shortens them, in bits. + +Ex in `jvm.properties` +---- +james.blobid.entropy=128 +---- + +Optional. Integer, a multiple of 8 within [128, 256]. Defaults to 256. + +Shorter ids cost less everywhere an id is stored. The object key itself, but above all the Cassandra +columns referencing it, which hold one id per message rather than one per blob: going from 256 to 128 +bits takes a body blob id from 50 to 28 characters. Truncated ids are also left unpadded, which is where +the last two characters go. + +128 bits is the sensible alternative to the default. The birthday bound puts a collision at +`n^2/2^129`, ie. 1.5e-19 for ten billion blobs, twenty orders of magnitude below the silent error rate of +the storage underneath; and truncating a cryptographic hash to its leading bits is standard practice +(NIST SP 800-107, FIPS 180-4). Values below 128 bits are rejected: a collision in a deduplicated store +means a message silently inheriting the body of another. + +WARNING: This is an install time setting, not one to flip on a live deployment. Lowering it loses +nothing, since ids are stored alongside the messages and existing blobs stay readable, but content +already stored under a longer id will not deduplicate against its shorter counterpart until it is +rewritten. + == Improve listing support for MinIO Due to blobs being stored in folder, adding `/` in blobs name emulates folder and avoids blobs to be all stored in a diff --git a/server/apps/distributed-app/sample-configuration/jvm.properties b/server/apps/distributed-app/sample-configuration/jvm.properties index ce4087873b..4a6fd73b98 100644 --- a/server/apps/distributed-app/sample-configuration/jvm.properties +++ b/server/apps/distributed-app/sample-configuration/jvm.properties @@ -101,6 +101,14 @@ jmx.remote.x.mlet.allow.getMBeansFromURL=false # messageFastViewProjection table. Only applies to previews computed afterwards. # james.jmap.preview.length=128 +# Bits of entropy carried by a blobId: the SHA-256 is truncated to that many leading bits, and randomly +# generated ids draw that many. A multiple of 8 within [128, 256], defaults to 256. +# 128 shortens a body blobId from 50 to 28 chars, in the object key and in every Cassandra column +# referencing it, for a collision probability of 1.5e-19 at ten billion blobs. +# Install time setting: changing it on a live deployment stops new writes from deduplicating against +# blobs already stored under a longer id. +# james.blobid.entropy=128 + # Count of octet from which hashing shall be done out of the IO threads in deduplicating blob store # james.deduplicating.blobstore.thread.switch.threshold=32768 # Count of octet from which streams are buffered to files and not to memory diff --git a/server/blob/blob-api/src/main/java/org/apache/james/blob/api/BlobIdEntropy.java b/server/blob/blob-api/src/main/java/org/apache/james/blob/api/BlobIdEntropy.java new file mode 100644 index 0000000000..294ee7200f --- /dev/null +++ b/server/blob/blob-api/src/main/java/org/apache/james/blob/api/BlobIdEntropy.java @@ -0,0 +1,88 @@ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ + +package org.apache.james.blob.api; + +import java.security.SecureRandom; +import java.util.Arrays; +import java.util.Optional; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; + +/** + * How many bits of entropy a blob id carries, as set by the {@code james.blobid.entropy} system property. + * + * <p>Defaults to {@value #DEFAULT_ENTROPY_BITS} bits, the full SHA-256 output, so that ids of existing + * deployments are left untouched. {@code 128} is the sensible alternative: the birthday bound puts a + * collision at {@code n^2/2^129}, ie. 1.5e-19 for ten billion blobs, and truncating a cryptographic hash + * to its leading bits is standard practice (NIST SP 800-107, FIPS 180-4).</p> + */ +public class BlobIdEntropy { + public static final String ENTROPY_BITS_PROPERTY = "james.blobid.entropy"; + public static final int DEFAULT_ENTROPY_BITS = 256; + private static final int MIN_ENTROPY_BITS = 128; + private static final int BITS_PER_BYTE = 8; + + private static final SecureRandom SECURE_RANDOM = new SecureRandom(); + private static final int ENTROPY_BITS = parse(System.getProperty(ENTROPY_BITS_PROPERTY)); + + @VisibleForTesting + static int parse(String value) { + return Optional.ofNullable(value) + .map(String::trim) + .filter(trimmed -> !trimmed.isEmpty()) + .map(BlobIdEntropy::parseBits) + .orElse(DEFAULT_ENTROPY_BITS); + } + + private static int parseBits(String value) { + try { + int bits = Integer.parseInt(value); + Preconditions.checkArgument(bits % BITS_PER_BYTE == 0, + "'%s' must be a multiple of %s, got %s", ENTROPY_BITS_PROPERTY, BITS_PER_BYTE, bits); + Preconditions.checkArgument(bits >= MIN_ENTROPY_BITS && bits <= DEFAULT_ENTROPY_BITS, + "'%s' must be within [%s, %s], got %s", ENTROPY_BITS_PROPERTY, MIN_ENTROPY_BITS, DEFAULT_ENTROPY_BITS, bits); + return bits; + } catch (NumberFormatException e) { + throw new IllegalArgumentException("Invalid '" + ENTROPY_BITS_PROPERTY + "' value: '" + value + "'. Expected a bit count, eg. 128 or 256", e); + } + } + + public static int entropyBits() { + return ENTROPY_BITS; + } + + public static int entropyBytes() { + return ENTROPY_BITS / BITS_PER_BYTE; + } + + public static byte[] randomBytes() { + byte[] bytes = new byte[entropyBytes()]; + SECURE_RANDOM.nextBytes(bytes); + return bytes; + } + + public static byte[] truncate(byte[] hash) { + if (hash.length <= entropyBytes()) { + return hash; + } + return Arrays.copyOf(hash, entropyBytes()); + } +} diff --git a/server/blob/blob-api/src/test/java/org/apache/james/blob/api/BlobIdEntropyTest.java b/server/blob/blob-api/src/test/java/org/apache/james/blob/api/BlobIdEntropyTest.java new file mode 100644 index 0000000000..e573c4d64a --- /dev/null +++ b/server/blob/blob-api/src/test/java/org/apache/james/blob/api/BlobIdEntropyTest.java @@ -0,0 +1,98 @@ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ + +package org.apache.james.blob.api; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import org.junit.jupiter.api.Test; + +class BlobIdEntropyTest { + @Test + void parseShouldReturnDefaultWhenNotSet() { + assertThat(BlobIdEntropy.parse(null)).isEqualTo(BlobIdEntropy.DEFAULT_ENTROPY_BITS); + } + + @Test + void parseShouldReturnDefaultWhenBlank() { + assertThat(BlobIdEntropy.parse(" ")).isEqualTo(BlobIdEntropy.DEFAULT_ENTROPY_BITS); + } + + @Test + void parseShouldAcceptTruncatedValue() { + assertThat(BlobIdEntropy.parse("128")).isEqualTo(128); + } + + @Test + void parseShouldRejectNonNumericValue() { + assertThatThrownBy(() -> BlobIdEntropy.parse("many")) + .isInstanceOf(IllegalArgumentException.class); + } + + @Test + void parseShouldRejectValueThatIsNotAByteCount() { + assertThatThrownBy(() -> BlobIdEntropy.parse("130")) + .isInstanceOf(IllegalArgumentException.class); + } + + @Test + void parseShouldRejectValueBelowTheSafetyFloor() { + assertThatThrownBy(() -> BlobIdEntropy.parse("96")) + .isInstanceOf(IllegalArgumentException.class); + } + + @Test + void parseShouldRejectValueAboveTheHashLength() { + assertThatThrownBy(() -> BlobIdEntropy.parse("512")) + .isInstanceOf(IllegalArgumentException.class); + } + + @Test + void randomBytesShouldHonourEntropyLength() { + assertThat(BlobIdEntropy.randomBytes()).hasSize(BlobIdEntropy.entropyBytes()); + } + + @Test + void randomBytesShouldNotRepeatItself() { + assertThat(BlobIdEntropy.randomBytes()).isNotEqualTo(BlobIdEntropy.randomBytes()); + } + + @Test + void truncateShouldKeepLeadingBytes() { + byte[] hash = new byte[BlobIdEntropy.entropyBytes() + 4]; + for (int i = 0; i < hash.length; i++) { + hash[i] = (byte) i; + } + + byte[] truncated = BlobIdEntropy.truncate(hash); + + assertThat(truncated).hasSize(BlobIdEntropy.entropyBytes()); + for (int i = 0; i < truncated.length; i++) { + assertThat(truncated[i]).isEqualTo((byte) i); + } + } + + @Test + void truncateShouldLeaveShorterHashesUntouched() { + byte[] hash = new byte[] {1, 2, 3}; + + assertThat(BlobIdEntropy.truncate(hash)).isEqualTo(hash); + } +} diff --git a/server/blob/blob-storage-strategy/src/main/scala/org/apache/james/server/blob/deduplication/DeDuplicationBlobStore.scala b/server/blob/blob-storage-strategy/src/main/scala/org/apache/james/server/blob/deduplication/DeDuplicationBlobStore.scala index 25651d63c9..bec9312852 100644 --- a/server/blob/blob-storage-strategy/src/main/scala/org/apache/james/server/blob/deduplication/DeDuplicationBlobStore.scala +++ b/server/blob/blob-storage-strategy/src/main/scala/org/apache/james/server/blob/deduplication/DeDuplicationBlobStore.scala @@ -26,7 +26,7 @@ import jakarta.inject.{Inject, Named} import org.apache.commons.io.IOUtils import org.apache.james.blob.api.BlobStore.BlobIdProvider import org.apache.james.blob.api.BlobStoreDAO.{ByteSourceBlob, BytesBlob, InputStreamBlob} -import org.apache.james.blob.api.{BlobId, BlobStore, BlobStoreDAO, BucketName} +import org.apache.james.blob.api.{BlobId, BlobIdEntropy, BlobStore, BlobStoreDAO, BucketName} import org.apache.james.server.blob.deduplication.DeDuplicationBlobStore.THREAD_SWITCH_THRESHOLD import org.reactivestreams.Publisher import reactor.core.publisher.{Flux, Mono} @@ -68,6 +68,9 @@ class DeDuplicationBlobStore @Inject()(blobStoreDAO: BlobStoreDAO, private val HASH_BLOB_ID_ENCODING_TYPE_PROPERTY = "james.blob.id.hash.encoding" private val HASH_BLOB_ID_ENCODING_DEFAULT = BaseEncoding.base64Url private val baseEncoding = Option(System.getProperty(HASH_BLOB_ID_ENCODING_TYPE_PROPERTY)).map(DeDuplicationBlobStore.baseEncodingFrom).getOrElse(HASH_BLOB_ID_ENCODING_DEFAULT) + // Truncated ids exist to be short: padding them back up would give away part of what was saved. + // Left untouched at full entropy so that ids of existing deployments are preserved. + private val blobIdEncoding = if (BlobIdEntropy.entropyBits() == BlobIdEntropy.DEFAULT_ENTROPY_BITS) baseEncoding else baseEncoding.omitPadding() override def save(bucketName: BucketName, data: Array[Byte], storagePolicy: BlobStore.StoragePolicy): Publisher[BlobId] = { save(bucketName, data, withBlobIdFromArray, storagePolicy) @@ -142,8 +145,8 @@ class DeDuplicationBlobStore @Inject()(blobStoreDAO: BlobStoreDAO, } private def base64(hashCode: HashCode) = { - val bytes = hashCode.asBytes - baseEncoding.encode(bytes) + val bytes = BlobIdEntropy.truncate(hashCode.asBytes) + blobIdEncoding.encode(bytes) } override def save(bucketName: BucketName, --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
