This is an automated email from the ASF dual-hosted git repository.
krathbun pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit 6e19d8315f7f9e5fdd3bf6839c5515619f27eff8
Merge: 4d440c52d1 e8a7f7875b
Author: Kevin Rathbun <kevinrr...@gmail.com>
AuthorDate: Mon May 12 11:04:40 2025 -0400
Merge branch '2.1'
.../apache/accumulo/core/client/rfile/RFile.java | 25 +-
.../core/iterators/IteratorEnvironment.java | 20 +-
.../iteratorsImpl/ClientIteratorEnvironment.java | 4 +-
.../org/apache/accumulo/test/IteratorEnvIT.java | 257 +++++++++++++++------
4 files changed, 218 insertions(+), 88 deletions(-)
diff --cc core/src/main/java/org/apache/accumulo/core/client/rfile/RFile.java
index 897e87213e,cc21b50643..008620eb3f
--- a/core/src/main/java/org/apache/accumulo/core/client/rfile/RFile.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/rfile/RFile.java
@@@ -23,9 -23,9 +23,10 @@@ import java.io.OutputStream
import java.util.Collection;
import java.util.Map;
import java.util.Map.Entry;
+import java.util.Objects;
import java.util.function.Predicate;
+ import org.apache.accumulo.core.client.PluginEnvironment;
import org.apache.accumulo.core.client.Scanner;
import org.apache.accumulo.core.client.admin.TableOperations;
import org.apache.accumulo.core.client.sample.SamplerConfiguration;
@@@ -37,10 -37,10 +38,12 @@@ import org.apache.accumulo.core.conf.Pr
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.LoadPlan;
import org.apache.accumulo.core.data.Range;
+ import org.apache.accumulo.core.data.TableId;
+ import org.apache.accumulo.core.iterators.IteratorEnvironment;
import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.util.RowRangeUtil;
import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
/**
diff --cc
core/src/main/java/org/apache/accumulo/core/iterators/IteratorEnvironment.java
index 8e56163c80,7872c8c878..7e6834f807
---
a/core/src/main/java/org/apache/accumulo/core/iterators/IteratorEnvironment.java
+++
b/core/src/main/java/org/apache/accumulo/core/iterators/IteratorEnvironment.java
@@@ -27,9 -33,25 +27,9 @@@ import org.apache.accumulo.core.securit
public interface IteratorEnvironment {
- /**
- * @deprecated since 2.0.0. This is a legacy method used for internal
backwards compatibility.
- */
- @Deprecated(since = "2.0.0")
- SortedKeyValueIterator<Key,Value> reserveMapFileReader(String mapFileName)
throws IOException;
-
- /**
- * @deprecated since 2.0.0. This method was using an unstable, non-public
type. Use
- * {@link #getPluginEnv()}
- */
- @Deprecated(since = "2.0.0")
- default AccumuloConfiguration getConfig() {
- return new
ConfigurationCopy(getPluginEnv().getConfiguration(getTableId()));
- }
-
/**
- * Return the executed scope of the Iterator. Value will be one of the
following:
- * {@link IteratorScope#scan}, {@link IteratorScope#minc}, {@link
IteratorScope#majc}
+ * @return the executed scope of the Iterator. Value will be one of the
following:
+ * {@link IteratorScope#scan}, {@link IteratorScope#minc}, {@link
IteratorScope#majc}
*/
IteratorScope getIteratorScope();
@@@ -39,9 -61,16 +39,10 @@@
*/
boolean isFullMajorCompaction();
- /**
- * @deprecated since 2.0.0. This was an experimental feature and was never
tested or documented.
- */
- @Deprecated(since = "2.0.0")
- void registerSideChannel(SortedKeyValueIterator<Key,Value> iter);
-
/**
- * Return the Scan Authorizations used in this Iterator. Will throw
UnsupportedOperationException
- * if {@link #getIteratorScope()} != {@link IteratorScope#scan}.
+ * @return the Scan Authorizations used in this Iterator.
+ * @throws UnsupportedOperationException if {@link #getIteratorScope()} !=
+ * {@link IteratorScope#scan}.
*/
Authorizations getAuthorizations();
diff --cc
core/src/main/java/org/apache/accumulo/core/iteratorsImpl/ClientIteratorEnvironment.java
index b67014d72b,d0e25957a4..417a38ffcd
---
a/core/src/main/java/org/apache/accumulo/core/iteratorsImpl/ClientIteratorEnvironment.java
+++
b/core/src/main/java/org/apache/accumulo/core/iteratorsImpl/ClientIteratorEnvironment.java
@@@ -197,12 -221,7 +197,12 @@@ public class ClientIteratorEnvironment
@Override
public TableId getTableId() {
- return this.tableId.orElseThrow();
+ return this.tableId.orElse(null);
}
+ @Override
+ public boolean isRunningLowOnMemory() {
+ return false;
+ }
+
}
diff --cc test/src/main/java/org/apache/accumulo/test/IteratorEnvIT.java
index 00b5224ddc,ad298d2d8c..bd1850b3ad
--- a/test/src/main/java/org/apache/accumulo/test/IteratorEnvIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/IteratorEnvIT.java
@@@ -42,7 -54,8 +54,7 @@@ import org.apache.accumulo.core.iterato
import org.apache.accumulo.core.iterators.IteratorEnvironment;
import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope;
import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
- import org.apache.accumulo.core.iterators.WrappingIterator;
+ import org.apache.accumulo.core.security.Authorizations;
-import org.apache.accumulo.core.spi.common.ServiceEnvironment;
import org.apache.accumulo.harness.AccumuloClusterHarness;
import org.apache.accumulo.miniclusterImpl.MiniAccumuloConfigImpl;
import org.apache.hadoop.conf.Configuration;
@@@ -146,33 -158,55 +157,46 @@@ public class IteratorEnvIT extends Accu
*/
private static void testEnv(IteratorScope scope, Map<String,String> opts,
IteratorEnvironment env) {
- TableId expectedTableId = TableId.of(opts.get("expected.table.id"));
+ String expTableIdString = opts.get(EXPECTED_TABLE_ID_OPT);
+ TableId expectedTableId = expTableIdString == null ? null :
TableId.of(expTableIdString);
+
+ if (scope != IteratorScope.majc) {
+ assertThrows(IllegalStateException.class, env::isUserCompaction,
+ "Test failed - Expected to throw IllegalStateException when
checking compaction");
+ assertThrows(IllegalStateException.class, env::isFullMajorCompaction,
+ "Test failed - Expected to throw IllegalStateException when
checking compaction");
+ } else {
+ assertDoesNotThrow(env::isUserCompaction,
+ "Test failed - Expected not to throw exception when checking
compaction");
+ assertDoesNotThrow(env::isFullMajorCompaction,
+ "Test failed - Expected not to throw exception when checking
compaction");
+ }
- // verify getServiceEnv() and getPluginEnv() are the same objects,
- // so further checks only need to use getPluginEnv()
- @SuppressWarnings("deprecation")
- ServiceEnvironment serviceEnv = env.getServiceEnv();
PluginEnvironment pluginEnv = env.getPluginEnv();
- assertEquals(serviceEnv, pluginEnv, "Test failed -
assertSame(getServiceEnv(),getPluginEnv())");
- // verify property exists on the table config (deprecated and new),
+ // verify property exists on the table config,
// with and without custom prefix, but not in the system config
- @SuppressWarnings("deprecation")
- String accTableConf = env.getConfig().get(CUSTOM_PROP_KEY);
- assertEquals(CUSTOM_PROP_VAL, accTableConf,
- "Test failed - Expected table property not found in getConfig().");
var tableConf = pluginEnv.getConfiguration(env.getTableId());
- if (!"value1".equals(tableConf.get("table.custom.iterator.env.test"))) {
- throw new RuntimeException("Test failed - Expected table property not
found in table conf.");
- }
- if (!"value1".equals(tableConf.getTableCustom("iterator.env.test"))) {
- throw new RuntimeException("Test failed - Expected table property not
found in table conf.");
- }
+ assertEquals(CUSTOM_PROP_VAL, tableConf.get(CUSTOM_PROP_KEY),
+ "Test failed - Expected table property not found in table conf.");
+ assertEquals(CUSTOM_PROP_VAL,
tableConf.getTableCustom(CUSTOM_PROP_KEY_SUFFIX),
+ "Test failed - Expected table property not found in table conf.");
var systemConf = pluginEnv.getConfiguration();
- if (systemConf.get("table.custom.iterator.env.test") != null) {
- throw new RuntimeException("Test failed - Unexpected table property
found in system conf.");
- }
+ assertNull(systemConf.get(CUSTOM_PROP_KEY),
+ "Test failed - Unexpected table property found in system conf.");
// check other environment settings
- if (!scope.equals(env.getIteratorScope())) {
- throw new RuntimeException("Test failed - Error getting iterator
scope");
- }
- if (env.isSamplingEnabled()) {
- throw new RuntimeException("Test failed - isSamplingEnabled returned
true, expected false");
- }
- if (!expectedTableId.equals(env.getTableId())) {
- throw new RuntimeException("Test failed - Error getting Table ID");
+ assertEquals(scope, env.getIteratorScope(), "Test failed - Error getting
iterator scope");
+ assertFalse(env.isSamplingEnabled(),
+ "Test failed - isSamplingEnabled returned true, expected false");
+ assertEquals(expectedTableId, env.getTableId(), "Test failed - Error
getting Table ID");
+ assertNull(env.getSamplerConfiguration());
+ assertThrows(Exception.class, env::cloneWithSamplingEnabled);
+ // per javadoc, getAuthorizations will throw UOE if scope != scan
+ if (env.getIteratorScope() == IteratorScope.scan) {
+ assertNotNull(env.getAuthorizations());
+ } else {
+ assertThrows(UnsupportedOperationException.class,
env::getAuthorizations);
}
}
@@@ -229,7 -270,93 +260,93 @@@
client.tableOperations().attachIterator(tableName, cfg,
EnumSet.of(IteratorScope.minc));
- client.tableOperations().flush(tableName);
+ client.tableOperations().flush(tableName, null, null, true);
+
+ try (Scanner scan = client.createScanner(tableName)) {
+ validateScanner(scan);
+ }
+ }
+
+ private void testRFileScan(Class<? extends
SortedKeyValueIterator<Key,Value>> iteratorClass)
+ throws Exception {
+ TreeMap<Key,Value> data = createTestData();
+ var fs = FileSystem.getLocal(new Configuration());
+ String rFilePath = createRFile(fs, data);
+
+ IteratorSetting cfg = new IteratorSetting(1, iteratorClass);
+
+ try (Scanner scan = RFile.newScanner().from(rFilePath).withFileSystem(fs)
+ .withTableProperties(getTableConfig().getProperties()).build()) {
+ scan.addScanIterator(cfg);
+ validateScanner(scan);
+ }
+ }
+
+ public void testOfflineScan(String tableName,
+ Class<? extends SortedKeyValueIterator<Key,Value>> iteratorClass)
throws Exception {
+ writeData(tableName);
+
+ TableId tableId =
TableId.of(client.tableOperations().tableIdMap().get(tableName));
+ client.tableOperations().offline(tableName, true);
+
+ IteratorSetting cfg = new IteratorSetting(1, iteratorClass);
+ cfg.addOption(EXPECTED_TABLE_ID_OPT, tableId.canonical());
+
+ try (OfflineScanner scan =
+ new OfflineScanner((ClientContext) client, tableId,
Authorizations.EMPTY)) {
+ scan.addScanIterator(cfg);
+ validateScanner(scan);
+ }
+ }
+
+ public void testClientSideScan(String tableName,
+ Class<? extends SortedKeyValueIterator<Key,Value>> iteratorClass)
throws Exception {
+ writeData(tableName);
+
+ IteratorSetting cfg = new IteratorSetting(1, iteratorClass);
+ cfg.addOption(EXPECTED_TABLE_ID_OPT,
client.tableOperations().tableIdMap().get(tableName));
+
+ try (Scanner scan = client.createScanner(tableName);
+ var clientIterScan = new ClientSideIteratorScanner(scan)) {
+ clientIterScan.addScanIterator(cfg);
+ validateScanner(clientIterScan);
+ }
+ }
+
+ private TreeMap<Key,Value> createTestData() {
+ TreeMap<Key,Value> testData = new TreeMap<>();
+
+ // Write data that we do not expect to be filtered out
+ testData.put(new Key("row1", GOOD_COL_FAM, "cq1"), new Value("val1"));
+ testData.put(new Key("row2", GOOD_COL_FAM, "cq1"), new Value("val2"));
+ testData.put(new Key("row3", GOOD_COL_FAM, "cq1"), new Value("val3"));
+ // Write data that we expect to be filtered out
+ testData.put(new Key("row4", BAD_COL_FAM, "badcq"), new Value("val1"));
+ testData.put(new Key("row5", BAD_COL_FAM, "badcq"), new Value("val2"));
+ testData.put(new Key("row6", BAD_COL_FAM, "badcq"), new Value("val3"));
+
+ return testData;
+ }
+
+ private void validateScanner(Scanner scan) {
+ // Ensure the bad cf was filtered out to ensure init() and testEnv() were
called
+ int numElts = 0;
+ for (var e : scan) {
+ numElts++;
+ assertEquals(GOOD_COL_FAM, e.getKey().getColumnFamily().toString());
+ }
+ assertEquals(3, numElts);
+ }
+
+ private String createRFile(FileSystem fs, TreeMap<Key,Value> data) throws
Exception {
- File testFile = new File(tempDir, "test.rf");
++ File testFile = tempDir.toPath().resolve("test.rf").toFile();
+ String filePath = testFile.getAbsolutePath();
+
+ try (RFileWriter writer =
RFile.newWriter().to(filePath).withFileSystem(fs).build()) {
+ writer.append(data.entrySet());
+ }
+
+ return filePath;
}
private NewTableConfiguration getTableConfig() {
