This is an automated email from the ASF dual-hosted git repository.
krathbun pushed a commit to branch 2.1
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/2.1 by this push:
new 11a9bd9c84 Fix ErasureCodeConfigurer bug (#5994)
11a9bd9c84 is described below
commit 11a9bd9c8489e470b5ce956424d61ddd1f39ad09
Author: Kevin Rathbun <[email protected]>
AuthorDate: Tue Dec 2 11:02:20 2025 -0500
Fix ErasureCodeConfigurer bug (#5994)
Using the ErasureCodeConfigurer with a default table.file.ec=inherit
caused an error when attempting to compact files greater than the size
threshold.
Now correctly inherits the EC from DFS when the size threshold is exceeded.
closes #5960
Co-authored-by: Keith Turner <[email protected]>
---
.../admin/compaction/ErasureCodeConfigurer.java | 11 ++++++++++-
.../accumulo/test/compaction/ErasureCodeIT.java | 22 ++++++++++++----------
2 files changed, 22 insertions(+), 11 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/client/admin/compaction/ErasureCodeConfigurer.java
b/core/src/main/java/org/apache/accumulo/core/client/admin/compaction/ErasureCodeConfigurer.java
index a6b31564fb..a6fa0d80e7 100644
---
a/core/src/main/java/org/apache/accumulo/core/client/admin/compaction/ErasureCodeConfigurer.java
+++
b/core/src/main/java/org/apache/accumulo/core/client/admin/compaction/ErasureCodeConfigurer.java
@@ -85,9 +85,18 @@ public class ErasureCodeConfigurer extends
CompressionConfigurer {
long inputsSum =
params.getInputFiles().stream().mapToLong(CompactableFile::getEstimatedSize).sum();
if (inputsSum >= this.ecSize) {
- overs.put(Property.TABLE_ENABLE_ERASURE_CODES.getKey(), "enable");
+ var tableConfig =
params.getEnvironment().getConfiguration(params.getTableId());
+ var tableECPolicy =
tableConfig.get(Property.TABLE_ERASURE_CODE_POLICY.getKey());
if (ecPolicyName != null) {
+ overs.put(Property.TABLE_ENABLE_ERASURE_CODES.getKey(), "enable");
overs.put(Property.TABLE_ERASURE_CODE_POLICY.getKey(), ecPolicyName);
+ } else if (tableECPolicy == null || tableECPolicy.isEmpty()) {
+ // Assume EC is configured at the DFS directory level, because no
policy was set in
+ // plugin or table config
+ overs.put(Property.TABLE_ENABLE_ERASURE_CODES.getKey(), "inherit");
+ } else {
+ // enable and use the policy set on the table
+ overs.put(Property.TABLE_ENABLE_ERASURE_CODES.getKey(), "enable");
}
} else {
overs.put(Property.TABLE_ENABLE_ERASURE_CODES.getKey(), "disable");
diff --git
a/test/src/main/java/org/apache/accumulo/test/compaction/ErasureCodeIT.java
b/test/src/main/java/org/apache/accumulo/test/compaction/ErasureCodeIT.java
index 66ef84d02c..669371d0ad 100644
--- a/test/src/main/java/org/apache/accumulo/test/compaction/ErasureCodeIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/compaction/ErasureCodeIT.java
@@ -108,6 +108,7 @@ public class ErasureCodeIT extends ConfigurableMacBase {
var table3 = names[2];
try (AccumuloClient c =
Accumulo.newClient().from(getClientProperties()).build()) {
+ var ctx = ((ClientContext) c);
var policy1 = "XOR-2-1-1024k";
var policy2 = "RS-3-2-1024k";
var dfs = getCluster().getMiniDfs().getFileSystem();
@@ -122,12 +123,12 @@ public class ErasureCodeIT extends ConfigurableMacBase {
Property.TABLE_ENABLE_ERASURE_CODES.getKey(), "enable");
c.tableOperations().create(table1, new
NewTableConfiguration().setProperties(options));
- var options2 = Map.of(Property.TABLE_ERASURE_CODE_POLICY.getKey(),
policy1,
- Property.TABLE_ENABLE_ERASURE_CODES.getKey(), "inherit");
- c.tableOperations().create(table2, new
NewTableConfiguration().setProperties(options2));
+ // default should be to inherit
+ c.tableOperations().create(table2);
+ assertEquals("inherit",
Property.TABLE_ENABLE_ERASURE_CODES.getDefaultValue());
- var options3 = Map.of(Property.TABLE_ENABLE_ERASURE_CODES.getKey(),
"disable");
- c.tableOperations().create(table3, new
NewTableConfiguration().setProperties(options3));
+ var options2 = Map.of(Property.TABLE_ENABLE_ERASURE_CODES.getKey(),
"disable");
+ c.tableOperations().create(table3, new
NewTableConfiguration().setProperties(options2));
SecureRandom random = new SecureRandom();
@@ -149,10 +150,10 @@ public class ErasureCodeIT extends ConfigurableMacBase {
}
c.tableOperations().flush(table1, null, null, true);
c.tableOperations().flush(table2, null, null, true);
+ // table2 will inherit this EC policy from dfs
+ dfs.setErasureCodingPolicy(getTableDir(ctx, table2), policy1);
c.tableOperations().flush(table3, null, null, true);
- var ctx = ((ClientContext) c);
-
assertEquals(List.of(policy1), getECPolicies(dfs, ctx, table1));
assertEquals(List.of("none"), getECPolicies(dfs, ctx, table2));
assertEquals(List.of("none"), getECPolicies(dfs, ctx, table3));
@@ -182,7 +183,7 @@ public class ErasureCodeIT extends ConfigurableMacBase {
Map.of(ErasureCodeConfigurer.ERASURE_CODE_SIZE, "1M")))
.setWait(true);
c.tableOperations().compact(table2, cconfig);
- assertEquals(List.of("none"), getECPolicies(dfs, ctx, table1));
+ assertEquals(List.of("none"), getECPolicies(dfs, ctx, table2));
// set a different policy for this compaction than what is configured on
the table
cconfig = new CompactionConfig()
@@ -215,10 +216,11 @@ public class ErasureCodeIT extends ConfigurableMacBase {
c.tableOperations().flush(table2, null, null, true);
assertEquals(List.of("none", policy1), getECPolicies(dfs, ctx, table1));
- assertEquals(List.of("none", "none"), getECPolicies(dfs, ctx, table2));
+ assertEquals(List.of("none", policy1), getECPolicies(dfs, ctx, table2));
// set the table dir erasure coding policy for all tables
dfs.setErasureCodingPolicy(getTableDir(ctx, table1), policy2);
+ // should change table2 from policy1 -> policy2
dfs.setErasureCodingPolicy(getTableDir(ctx, table2), policy2);
dfs.setErasureCodingPolicy(getTableDir(ctx, table3), policy2);
// compact all the tables and see how setting an EC policy on the table
dir influenced the
@@ -243,7 +245,7 @@ public class ErasureCodeIT extends ConfigurableMacBase {
c.tableOperations().compact(table3, new
CompactionConfig().setWait(true));
// the table settings specify policy1 so that should win
assertEquals(List.of(policy1), getECPolicies(dfs, ctx, table1));
- // the table settings specify to use the dfs dir settings so that should
win and iit should
+ // the table settings specify to use the dfs dir settings so that should
win and it should
// replicate
assertEquals(List.of("none"), getECPolicies(dfs, ctx, table2));
// the table setting specify to use replication and so do the directory
settings
