This is an automated email from the ASF dual-hosted git repository.
cshannon pushed a commit to branch no-chop-merge
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit 7fbb262b77a7dcee86af169db27e90cbd28a6742
Merge: 04b614c35f 0214c6f99f
Author: Christopher L. Shannon (cshannon) <christopher.l.shan...@gmail.com>
AuthorDate: Fri Jun 23 09:19:30 2023 -0400
Merge branch 'main' into no-chop-merge
.../core/clientImpl/bulk/BulkSerialize.java | 3 +-
.../apache/accumulo/core/lock/ServiceLockData.java | 11 +-
.../accumulo/core/metadata/AbstractTabletFile.java | 7 -
.../core/metadata/ReferencedTabletFile.java | 156 ++++++++++++++++-----
.../accumulo/core/metadata/StoredTabletFile.java | 18 +--
.../accumulo/core/metadata/TabletDirectory.java | 65 ---------
.../core/metadata/UnreferencedTabletFile.java | 10 +-
.../accumulo/core/metadata/ValidationUtil.java | 1 +
.../schema/ExternalCompactionFinalState.java | 8 +-
.../schema/ExternalCompactionMetadata.java | 9 +-
.../core/metadata/schema/RootTabletMetadata.java | 8 +-
.../spi/compaction/DefaultCompactionPlanner.java | 6 +-
.../spi/scan/ConfigurableScanServerSelector.java | 7 +-
.../apache/accumulo/core/util/ComparablePair.java | 6 +-
.../{ComparablePair.java => GsonSingleton.java} | 30 ++--
.../java/org/apache/accumulo/core/util/Pair.java | 4 +-
.../metadata/schema/ReferencedTabletFileTest.java | 6 +-
pom.xml | 2 +-
.../accumulo/server/metadata/RootGcCandidates.java | 8 +-
.../accumulo/manager/tableOps/TraceRepo.java | 5 +-
.../util/logging/AccumuloMonitorAppender.java | 6 +-
.../accumulo/test/CountNameNodeOpsBulkIT.java | 5 +-
22 files changed, 201 insertions(+), 180 deletions(-)
diff --cc
core/src/main/java/org/apache/accumulo/core/metadata/AbstractTabletFile.java
index c6d1ded80d,146c2b2467..51b8f6166f
---
a/core/src/main/java/org/apache/accumulo/core/metadata/AbstractTabletFile.java
+++
b/core/src/main/java/org/apache/accumulo/core/metadata/AbstractTabletFile.java
@@@ -32,21 -31,12 +32,14 @@@ import org.apache.hadoop.fs.Path
public abstract class AbstractTabletFile<T extends AbstractTabletFile<T>>
implements TabletFile, Comparable<T> {
- private final String fileName; // C0004.rf
protected final Path path;
+ protected final Range range;
- protected AbstractTabletFile(Path path) {
+ protected AbstractTabletFile(Path path, Range range) {
this.path = Objects.requireNonNull(path);
- this.fileName = path.getName();
+ this.range = Objects.requireNonNull(range);
}
- @Override
- public String getFileName() {
- return fileName;
- }
-
@Override
public Path getPath() {
return path;
diff --cc
core/src/main/java/org/apache/accumulo/core/metadata/ReferencedTabletFile.java
index b1cefc5f45,ff1bfe7e3a..3edfd1d41f
---
a/core/src/main/java/org/apache/accumulo/core/metadata/ReferencedTabletFile.java
+++
b/core/src/main/java/org/apache/accumulo/core/metadata/ReferencedTabletFile.java
@@@ -20,11 -20,11 +20,13 @@@ package org.apache.accumulo.core.metada
import static org.apache.accumulo.core.Constants.HDFS_TABLES_DIR;
+ import java.net.URI;
+import java.util.Comparator;
import java.util.Objects;
+import org.apache.accumulo.core.data.Range;
import org.apache.accumulo.core.data.TableId;
+ import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.slf4j.Logger;
@@@ -43,48 -43,122 +45,130 @@@ import com.google.common.base.Precondit
* in Upgrader9to10.upgradeRelativePaths()
*/
public class ReferencedTabletFile extends
AbstractTabletFile<ReferencedTabletFile> {
- // parts of an absolute URI, like
"hdfs://1.2.3.4/accumulo/tables/2a/t-0003/C0004.rf"
- private final TabletDirectory tabletDir; //
hdfs://1.2.3.4/accumulo/tables/2a/t-0003
- private final String normalizedPath;
+
+ public static class FileParts {
+
+ // parts of an absolute URI, like
"hdfs://1.2.3.4/accumulo/tables/2a/t-0003/C0004.rf"
+ // volume: hdfs://1.2.3.4/accumulo
+ // tableId: 2a
+ // tabletDir: t-0003
+ // fileName: C0004.rf
+ // normalizedPath: hdfs://1.2.3.4/accumulo/tables/2a/t-0003/C0004.rf
+ private final String volume;
+ private final TableId tableId;
+ private final String tabletDir;
+ private final String fileName;
+ private final String normalizedPath;
+
+ public FileParts(String volume, TableId tableId, String tabletDir, String
fileName,
+ String normalizedPath) {
+ this.volume = volume;
+ this.tableId = tableId;
+ this.tabletDir = tabletDir;
+ this.fileName = fileName;
+ this.normalizedPath = normalizedPath;
+ }
+
+ public String getVolume() {
+ return volume;
+ }
+
+ public TableId getTableId() {
+ return tableId;
+ }
+
+ public String getTabletDir() {
+ return tabletDir;
+ }
+
+ public String getFileName() {
+ return fileName;
+ }
+
+ public String getNormalizedPath() {
+ return normalizedPath;
+ }
+
+ }
+
+ private static String constructErrorMsg(Path filePath) {
+ return "Missing or invalid part of tablet file metadata entry: " +
filePath;
+ }
+
+ public static FileParts parsePath(Path filePath) {
+ // File name construct: <volume>/<tablePath>/<tableId>/<tablet>/<file>
+ // Example:
hdfs://namenode:9020/accumulo/tables/1/default_tablet/F00001.rf
+ final URI uri = filePath.toUri();
+
+ // validate that this is a fully qualified uri
+ Preconditions.checkArgument(uri.getScheme() != null,
constructErrorMsg(filePath));
+
+ final String path = uri.getPath(); // ex:
/accumulo/tables/1/default_tablet/F00001.rf
+ final String[] parts = path.split("/");
+ final int numParts = parts.length; // should contain tables, 1,
default_tablet, F00001.rf
+
+ if (numParts < 4) {
+ throw new IllegalArgumentException(constructErrorMsg(filePath));
+ }
+
+ final String fileName = parts[numParts - 1];
+ final String tabletDirectory = parts[numParts - 2];
+ final TableId tableId = TableId.of(parts[numParts - 3]);
+ final String tablesPath = parts[numParts - 4];
+
+ // determine where file path starts, the rest is the volume
+ final String computedFilePath =
+ HDFS_TABLES_DIR + "/" + tableId.canonical() + "/" + tabletDirectory +
"/" + fileName;
+ final String uriString = uri.toString();
+ int idx = uriString.lastIndexOf(computedFilePath);
+
+ if (idx == -1) {
+ throw new IllegalArgumentException(constructErrorMsg(filePath));
+ }
+
+ // The volume is the beginning portion of the uri up to the start
+ // of the file path.
+ final String volume = uriString.substring(0, idx);
+
+ if (StringUtils.isBlank(fileName) || StringUtils.isBlank(tabletDirectory)
+ || StringUtils.isBlank(tablesPath) || StringUtils.isBlank(volume)) {
+ throw new IllegalArgumentException(constructErrorMsg(filePath));
+ }
+ ValidationUtil.validateFileName(fileName);
+ Preconditions.checkArgument(tablesPath.equals(HDFS_TABLES_DIR_NAME),
+ "tables directory name is not " + HDFS_TABLES_DIR_NAME + ", is " +
tablesPath);
+
+ final String normalizedPath = volume + computedFilePath;
+
+ if (!normalizedPath.equals(uriString)) {
+ throw new RuntimeException("Error parsing file path, " + normalizedPath
+ " != " + uriString);
+ }
+
+ return new FileParts(volume, tableId, tabletDirectory, fileName,
normalizedPath);
+
+ }
+
+ private final FileParts parts;
private static final Logger log =
LoggerFactory.getLogger(ReferencedTabletFile.class);
+ private static final String HDFS_TABLES_DIR_NAME =
HDFS_TABLES_DIR.substring(1);
+ private static final Comparator<ReferencedTabletFile> comparator =
+ Comparator.comparing(ReferencedTabletFile::getNormalizedPathStr)
+ .thenComparing(ReferencedTabletFile::getRange);
+
+ public ReferencedTabletFile(Path metaPath) {
+ this(metaPath, new Range());
+ }
+
/**
* Construct new tablet file using a Path. Used in the case where we had to
use Path object to
* qualify an absolute path or create a new file.
*/
- public ReferencedTabletFile(Path metaPath) {
- super(Objects.requireNonNull(metaPath));
+ public ReferencedTabletFile(Path metaPath, Range range) {
+ super(Objects.requireNonNull(metaPath), range);
- String errorMsg = "Missing or invalid part of tablet file metadata entry:
" + metaPath;
log.trace("Parsing TabletFile from {}", metaPath);
-
- // Validate characters in file name
- ValidationUtil.validateFileName(path.getName());
-
- // use Path object to step backwards from the filename through all the
parts
- Path tabletDirPath = Objects.requireNonNull(metaPath.getParent(),
errorMsg);
-
- Path tableIdPath = Objects.requireNonNull(tabletDirPath.getParent(),
errorMsg);
- var id = tableIdPath.getName();
-
- Path tablePath = Objects.requireNonNull(tableIdPath.getParent(),
errorMsg);
- String tpString = "/" + tablePath.getName();
- Preconditions.checkArgument(tpString.equals(HDFS_TABLES_DIR), errorMsg);
-
- Path volumePath = Objects.requireNonNull(tablePath.getParent(), errorMsg);
- Preconditions.checkArgument(volumePath.toUri().getScheme() != null,
errorMsg);
- var volume = volumePath.toString();
-
- this.tabletDir = new TabletDirectory(volume, TableId.of(id),
tabletDirPath.getName());
- this.normalizedPath = tabletDir.getNormalizedPath() + "/" + getFileName();
+ parts = parsePath(metaPath);
}
public String getVolume() {
@@@ -141,14 -220,14 +230,15 @@@
public boolean equals(Object obj) {
if (obj instanceof ReferencedTabletFile) {
ReferencedTabletFile that = (ReferencedTabletFile) obj;
- return normalizedPath.equals(that.normalizedPath) &&
range.equals(that.range);
- return parts.getNormalizedPath().equals(that.parts.getNormalizedPath());
++ return parts.getNormalizedPath().equals(that.parts.getNormalizedPath())
++ && range.equals(that.range);
}
return false;
}
@Override
public int hashCode() {
- return Objects.hash(normalizedPath, range);
- return parts.getNormalizedPath().hashCode();
++ return Objects.hash(parts.getNormalizedPath(), range);
}
@Override
diff --cc
core/src/main/java/org/apache/accumulo/core/metadata/StoredTabletFile.java
index 68f84e5a5b,3bf5e701d5..feb3fd03e9
--- a/core/src/main/java/org/apache/accumulo/core/metadata/StoredTabletFile.java
+++ b/core/src/main/java/org/apache/accumulo/core/metadata/StoredTabletFile.java
@@@ -19,9 -19,7 +19,8 @@@
package org.apache.accumulo.core.metadata;
import java.util.Objects;
- import java.util.function.Supplier;
+import org.apache.accumulo.core.data.Range;
import org.apache.accumulo.core.data.TableId;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
@@@ -48,11 -44,9 +45,11 @@@ public class StoredTabletFile extends A
* the entry can be deleted.
*/
public StoredTabletFile(String metadataEntry) {
- super(new Path(metadataEntry));
+ // TODO: Future version of metadataEntry will contains the path
+ // and the range so we will need to parse the string here
+ super(new Path(metadataEntry), new Range());
this.metadataEntry = metadataEntry;
- this.referencedTabletFile = Suppliers.memoize(() ->
ReferencedTabletFile.of(getPath()));
+ this.referencedTabletFile = ReferencedTabletFile.of(getPath());
}
/**
diff --cc
core/src/main/java/org/apache/accumulo/core/metadata/UnreferencedTabletFile.java
index 4b39023fc1,4f6b16a946..3d5c1d40f8
---
a/core/src/main/java/org/apache/accumulo/core/metadata/UnreferencedTabletFile.java
+++
b/core/src/main/java/org/apache/accumulo/core/metadata/UnreferencedTabletFile.java
@@@ -41,13 -40,17 +41,21 @@@ import org.apache.hadoop.fs.Path
*/
public class UnreferencedTabletFile extends
AbstractTabletFile<UnreferencedTabletFile> {
+ private final String fileName; // C0004.rf
+
public UnreferencedTabletFile(FileSystem fs, Path path) {
-
super(Objects.requireNonNull(fs).makeQualified(Objects.requireNonNull(path)));
+ this(fs, path, new Range());
+ }
+
+ public UnreferencedTabletFile(FileSystem fs, Path path, Range range) {
+
super(Objects.requireNonNull(fs).makeQualified(Objects.requireNonNull(path)),
range);
- ValidationUtil.validateFileName(path.getName());
+ this.fileName = path.getName();
+ ValidationUtil.validateFileName(fileName);
+ }
+
+ @Override
+ public String getFileName() {
+ return fileName;
}
@Override
