This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit 3a874c2972c568975b523fc09846cbb4be450ff5
Merge: d2b15204c5 4de6087f27
Author: Keith Turner <ktur...@apache.org>
AuthorDate: Mon Oct 16 11:41:55 2023 -0400
Merge branch 'main' into elasticity
LICENSE | 11 +-
assemble/src/main/resources/LICENSE | 25 +
.../core/clientImpl/TableOperationsImpl.java | 2 +-
.../org/apache/accumulo/core/data/Mutation.java | 4 +-
.../accumulo/core/file/BloomFilterLayer.java | 8 +-
.../apache/accumulo/core/file/FileSKVIterator.java | 6 +-
.../core/file/rfile/MultiIndexIterator.java | 5 +-
.../org/apache/accumulo/core/file/rfile/RFile.java | 61 +-
.../iteratorsImpl/system/SequenceFileIterator.java | 5 +-
.../core/file/rfile/AbstractRFileTest.java | 8 +-
.../core/file/rfile/MultiThreadedRFileTest.java | 8 +-
.../apache/accumulo/core/file/rfile/RFileTest.java | 14 +-
.../miniclusterImpl/MiniAccumuloClusterImpl.java | 2 +-
pom.xml | 6 +-
.../org/apache/accumulo/server/util/FileUtil.java | 21 +-
.../apache/accumulo/server/util/FileInfoTest.java | 12 +-
.../java/org/apache/accumulo/manager/Manager.java | 5 +-
.../manager/ManagerClientServiceHandler.java | 2 +-
.../manager/tableOps/split/UpdateTabletsTest.java | 3 +-
.../src/main/appended-resources/META-INF/LICENSE | 11 +-
.../accumulo/monitor/resources/css/screen.css | 3 +-
.../external/bootstrap/css/bootstrap-icons.css | 2078 ++++++++++++++++++++
.../external/bootstrap/fonts/arrow-repeat.svg | 4 -
.../external/bootstrap/fonts/bootstrap-icons.woff | Bin 0 -> 176200 bytes
.../external/bootstrap/fonts/bootstrap-icons.woff2 | Bin 0 -> 130608 bytes
.../bootstrap/fonts/three-dots-vertical.svg | 3 -
.../apache/accumulo/monitor/resources/js/navbar.js | 1 +
.../accumulo/monitor/resources/js/systemAlert.js | 172 ++
.../apache/accumulo/monitor/templates/default.ftl | 3 +
.../org/apache/accumulo/monitor/templates/ec.ftl | 4 +-
.../apache/accumulo/monitor/templates/navbar.ftl | 5 +-
.../accumulo/monitor/templates/systemAlert.ftl | 31 +
.../org/apache/accumulo/tserver/log/DfsLogger.java | 2 +-
.../org/apache/accumulo/tserver/log/LogSorter.java | 4 +-
.../accumulo/tserver/logger/LogFileValue.java | 2 +-
.../apache/accumulo/tserver/logger/LogReader.java | 2 +-
.../accumulo/shell/commands/GetAuthsCommand.java | 4 +-
test/pom.xml | 4 +-
.../org/apache/accumulo/test/ComprehensiveIT.java | 1252 ++++++++++++
.../test/compaction/CompactionExecutorIT.java | 2 +-
.../accumulo/test/conf/PropStoreConfigIT.java | 2 -
.../accumulo/test/conf/util/ZooPropEditorIT.java | 2 -
.../apache/accumulo/test/functional/BulkIT.java | 4 -
.../accumulo/test/functional/LocalityGroupIT.java | 218 ++
.../accumulo/test/functional/ReadWriteIT.java | 227 +--
.../accumulo/test/functional/ScanIteratorIT.java | 3 -
.../test/shell/ShellCreateNamespaceIT.java | 2 -
.../accumulo/test/shell/ShellCreateTableIT.java | 2 -
.../org/apache/accumulo/test/shell/ShellIT.java | 2 -
.../apache/accumulo/test/shell/ShellServerIT.java | 2 -
50 files changed, 3907 insertions(+), 352 deletions(-)
diff --cc
server/base/src/main/java/org/apache/accumulo/server/util/FileUtil.java
index 8baa673d84,a600dff94b..78a541ca6e
--- a/server/base/src/main/java/org/apache/accumulo/server/util/FileUtil.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/FileUtil.java
@@@ -18,16 -18,36 +18,15 @@@
*/
package org.apache.accumulo.server.util;
-import static org.apache.accumulo.core.util.LazySingletons.RANDOM;
-
-import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
import java.util.HashMap;
-import java.util.List;
import java.util.Map;
import java.util.Set;
-import java.util.SortedMap;
-import java.util.TreeMap;
-import java.util.stream.Collectors;
-import org.apache.accumulo.core.conf.Property;
--import org.apache.accumulo.core.data.Key;
-import org.apache.accumulo.core.data.PartialKey;
-import org.apache.accumulo.core.data.Range;
-import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.file.FileOperations;
import org.apache.accumulo.core.file.FileSKVIterator;
-import org.apache.accumulo.core.file.FileSKVWriter;
-import org.apache.accumulo.core.file.rfile.RFile;
-import org.apache.accumulo.core.file.rfile.RFileOperations;
-import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
-import org.apache.accumulo.core.iteratorsImpl.system.MultiIterator;
-import org.apache.accumulo.core.metadata.ReferencedTabletFile;
import org.apache.accumulo.core.metadata.TabletFile;
-import org.apache.accumulo.core.metadata.UnreferencedTabletFile;
import org.apache.accumulo.server.ServerContext;
import org.apache.accumulo.server.conf.TableConfiguration;
import org.apache.accumulo.server.fs.VolumeManager;
diff --cc server/manager/src/main/java/org/apache/accumulo/manager/Manager.java
index c960e786c9,009a407dd0..21183349f5
--- a/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java
+++ b/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java
@@@ -19,7 -19,7 +19,8 @@@
package org.apache.accumulo.manager;
import static
com.google.common.util.concurrent.Uninterruptibles.sleepUninterruptibly;
+ import static java.nio.charset.StandardCharsets.UTF_8;
+import static java.util.Collections.emptyMap;
import static java.util.Collections.emptySortedMap;
import static java.util.concurrent.TimeUnit.HOURS;
import static java.util.concurrent.TimeUnit.MILLISECONDS;
diff --cc
server/manager/src/test/java/org/apache/accumulo/manager/tableOps/split/UpdateTabletsTest.java
index 68f1a85424,0000000000..313fc24891
mode 100644,000000..100644
---
a/server/manager/src/test/java/org/apache/accumulo/manager/tableOps/split/UpdateTabletsTest.java
+++
b/server/manager/src/test/java/org/apache/accumulo/manager/tableOps/split/UpdateTabletsTest.java
@@@ -1,119 -1,0 +1,118 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.manager.tableOps.split;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+import java.util.Map;
+import java.util.Set;
+
- import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.TableId;
+import org.apache.accumulo.core.dataImpl.KeyExtent;
+import org.apache.accumulo.core.metadata.ReferencedTabletFile;
+import org.apache.accumulo.core.metadata.StoredTabletFile;
+import org.apache.accumulo.core.metadata.schema.DataFileValue;
+import org.apache.accumulo.core.metadata.schema.TabletMetadata;
+import org.apache.accumulo.server.util.FileUtil;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+import org.easymock.EasyMock;
+import org.junit.jupiter.api.Test;
+
+public class UpdateTabletsTest {
+
+ StoredTabletFile newSTF(int fileNum) {
+ return new ReferencedTabletFile(new Path(
+ "hdfs://localhost:8020/accumulo/tables/2a/default_tablet/F00000" +
fileNum + ".rf"))
+ .insert();
+ }
+
+ FileUtil.FileInfo newFileInfo(String start, String end) {
- return new FileUtil.FileInfo(new Key(start), new Key(end));
++ return new FileUtil.FileInfo(new Text(start), new Text(end));
+ }
+
+ // When a tablet splits its files are partitioned among the new children
tablets. This test
+ // exercises the partitioning code.
+ @Test
+ public void testFileParitioning() {
+
+ var file1 = newSTF(1);
+ var file2 = newSTF(2);
+ var file3 = newSTF(3);
+ var file4 = newSTF(4);
+
+ var tabletFiles =
+ Map.of(file1, new DataFileValue(1000, 100, 20), file2, new
DataFileValue(2000, 200, 50),
+ file3, new DataFileValue(4000, 400), file4, new
DataFileValue(4000, 400));
+
+ var ke1 = new KeyExtent(TableId.of("1"), new Text("m"), null);
+ var ke2 = new KeyExtent(TableId.of("1"), new Text("r"), new Text("m"));
+ var ke3 = new KeyExtent(TableId.of("1"), new Text("v"), new Text("r"));
+ var ke4 = new KeyExtent(TableId.of("1"), null, new Text("v"));
+
+ var firstAndLastKeys = Map.of(file2, newFileInfo("m", "r"), file3,
newFileInfo("g", "x"), file4,
+ newFileInfo("s", "v"));
+
+ var ke1Expected = Map.of(file1, new DataFileValue(250, 25, 20), file2,
+ new DataFileValue(1000, 100, 50), file3, new DataFileValue(1000,
100));
+ var ke2Expected = Map.of(file1, new DataFileValue(250, 25, 20), file2,
+ new DataFileValue(1000, 100, 50), file3, new DataFileValue(1000,
100));
+ var ke3Expected = Map.of(file1, new DataFileValue(250, 25, 20), file3,
+ new DataFileValue(1000, 100), file4, new DataFileValue(4000, 400));
+ var ke4Expected =
+ Map.of(file1, new DataFileValue(250, 25, 20), file3, new
DataFileValue(1000, 100));
+
+ var expected = Map.of(ke1, ke1Expected, ke2, ke2Expected, ke3,
ke3Expected, ke4, ke4Expected);
+
+ Set<KeyExtent> newExtents = Set.of(ke1, ke2, ke3, ke4);
+
+ TabletMetadata tabletMeta = EasyMock.createMock(TabletMetadata.class);
+
EasyMock.expect(tabletMeta.getFilesMap()).andReturn(tabletFiles).anyTimes();
+ EasyMock.replay(tabletMeta);
+
+ Map<KeyExtent,Map<StoredTabletFile,DataFileValue>> results =
+ UpdateTablets.getNewTabletFiles(newExtents, tabletMeta,
firstAndLastKeys::get);
+
+ assertEquals(expected.keySet(), results.keySet());
+ expected.forEach(((extent, files) -> {
+ assertEquals(files, results.get(extent));
+ }));
+
+ // Test a tablet with no files going to it
+
+ var tabletFiles2 = Map.of(file2, tabletFiles.get(file2), file4,
tabletFiles.get(file4));
+ ke1Expected = Map.of(file2, new DataFileValue(1000, 100, 50));
+ ke2Expected = Map.of(file2, new DataFileValue(1000, 100, 50));
+ ke3Expected = Map.of(file4, new DataFileValue(4000, 400));
+ ke4Expected = Map.of();
+ expected = Map.of(ke1, ke1Expected, ke2, ke2Expected, ke3, ke3Expected,
ke4, ke4Expected);
+
+ tabletMeta = EasyMock.createMock(TabletMetadata.class);
+
EasyMock.expect(tabletMeta.getFilesMap()).andReturn(tabletFiles2).anyTimes();
+ EasyMock.replay(tabletMeta);
+
+ Map<KeyExtent,Map<StoredTabletFile,DataFileValue>> results2 =
+ UpdateTablets.getNewTabletFiles(newExtents, tabletMeta,
firstAndLastKeys::get);
+ assertEquals(expected.keySet(), results2.keySet());
+ expected.forEach(((extent, files) -> {
+ assertEquals(files, results2.get(extent));
+ }));
+
+ }
+}
