This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 6c5e5683eb Create TabletDirectory classes (#2688)
6c5e5683eb is described below
commit 6c5e5683eb8e267aba60e9e93af3ea642cfb9ddb
Author: Mike Miller <mmil...@apache.org>
AuthorDate: Wed May 18 15:19:09 2022 +0000
Create TabletDirectory classes (#2688)
* Create TabletDirectory and RelativeTabletDirectory
* Move Reference class and improve it so other classes can be children
* Add validate method to StoredTabletFile
* Use TabletDirectory in TabletFile
* Refactor some GC code
---
.../apache/accumulo/core/metadata/Reference.java | 41 +++++++++++++++++
.../core/metadata/RelativeTabletDirectory.java | 31 +++++++++++++
.../accumulo/core/metadata/StoredTabletFile.java | 13 ++++++
.../accumulo/core/metadata/TabletDirectory.java | 51 ++++++++++++++++++++++
.../apache/accumulo/core/metadata/TabletFile.java | 26 +++++------
.../main/java/org/apache/accumulo/gc/GCRun.java | 8 ++--
.../accumulo/gc/GarbageCollectionAlgorithm.java | 29 ++++++------
.../accumulo/gc/GarbageCollectionEnvironment.java | 13 +-----
.../apache/accumulo/gc/GarbageCollectionTest.java | 7 +--
9 files changed, 172 insertions(+), 47 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/metadata/Reference.java
b/core/src/main/java/org/apache/accumulo/core/metadata/Reference.java
new file mode 100644
index 0000000000..0254220040
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/core/metadata/Reference.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.core.metadata;
+
+import org.apache.accumulo.core.data.TableId;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema;
+
+/**
+ * A reference to a tablet file or directory.
+ */
+public class Reference {
+ // parts of an absolute URI, like "hdfs://1.2.3.4/accumulo/tables/2a/t-0003"
+ public final TableId tableId; // 2a
+ public final String tabletDir; // t-0003
+
+ // the exact string that is stored in the metadata
+ public final String metadataEntry;
+
+ public Reference(TableId tableId, String metadataEntry) {
+
MetadataSchema.TabletsSection.ServerColumnFamily.validateDirCol(tableId.canonical());
+ this.tableId = tableId;
+ this.metadataEntry = metadataEntry;
+ this.tabletDir = metadataEntry;
+ }
+}
diff --git
a/core/src/main/java/org/apache/accumulo/core/metadata/RelativeTabletDirectory.java
b/core/src/main/java/org/apache/accumulo/core/metadata/RelativeTabletDirectory.java
new file mode 100644
index 0000000000..ae5d53cbc5
--- /dev/null
+++
b/core/src/main/java/org/apache/accumulo/core/metadata/RelativeTabletDirectory.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.core.metadata;
+
+import org.apache.accumulo.core.data.TableId;
+
+/**
+ * Part of the Tablet File path that is definitely a directory.
+ */
+public class RelativeTabletDirectory extends Reference {
+
+ public RelativeTabletDirectory(TableId tableId, String dirName) {
+ super(tableId, dirName);
+ }
+}
diff --git
a/core/src/main/java/org/apache/accumulo/core/metadata/StoredTabletFile.java
b/core/src/main/java/org/apache/accumulo/core/metadata/StoredTabletFile.java
index 390c7e8532..8dd904c84c 100644
--- a/core/src/main/java/org/apache/accumulo/core/metadata/StoredTabletFile.java
+++ b/core/src/main/java/org/apache/accumulo/core/metadata/StoredTabletFile.java
@@ -59,4 +59,17 @@ public class StoredTabletFile extends TabletFile {
public Text getMetaUpdateDeleteText() {
return new Text(getMetaUpdateDelete());
}
+
+ /**
+ * Validate that the provided reference matches what is in the metadata
table.
+ *
+ * @param reference
+ * the relative path to check against
+ */
+ public void validate(String reference) {
+ if (!metadataEntry.equals(reference)) {
+ throw new IllegalStateException("The reference " + reference
+ + " does not match what was in the metadata: " + metadataEntry);
+ }
+ }
}
diff --git
a/core/src/main/java/org/apache/accumulo/core/metadata/TabletDirectory.java
b/core/src/main/java/org/apache/accumulo/core/metadata/TabletDirectory.java
new file mode 100644
index 0000000000..18180c1b46
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/core/metadata/TabletDirectory.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.core.metadata;
+
+import org.apache.accumulo.core.data.TableId;
+
+/**
+ * The Tablet directory that may exist in the metadata table.
+ */
+public class TabletDirectory extends RelativeTabletDirectory {
+ // parts of an absolute URI, like "hdfs://1.2.3.4/accumulo/tables/2a/t-0003"
+ private final String volume; // hdfs://1.2.3.4/accumulo
+
+ public TabletDirectory(String volume, TableId tableId, String tabletDir) {
+ super(tableId, tabletDir);
+ this.volume = volume;
+ }
+
+ public String getVolume() {
+ return volume;
+ }
+
+ public TableId getTableId() {
+ return tableId;
+ }
+
+ public String getTabletDir() {
+ return tabletDir;
+ }
+
+ @Override
+ public String toString() {
+ return tabletDir;
+ }
+}
diff --git
a/core/src/main/java/org/apache/accumulo/core/metadata/TabletFile.java
b/core/src/main/java/org/apache/accumulo/core/metadata/TabletFile.java
index 0274921740..71bdfe6efc 100644
--- a/core/src/main/java/org/apache/accumulo/core/metadata/TabletFile.java
+++ b/core/src/main/java/org/apache/accumulo/core/metadata/TabletFile.java
@@ -18,9 +18,10 @@
*/
package org.apache.accumulo.core.metadata;
+import static org.apache.accumulo.core.Constants.HDFS_TABLES_DIR;
+
import java.util.Objects;
-import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.data.TableId;
import
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily;
import org.apache.hadoop.fs.Path;
@@ -40,9 +41,7 @@ import com.google.common.base.Preconditions;
*/
public class TabletFile implements Comparable<TabletFile> {
// parts of an absolute URI, like
"hdfs://1.2.3.4/accumulo/tables/2a/t-0003/C0004.rf"
- private final String volume; // hdfs://1.2.3.4/accumulo
- private final TableId tableId; // 2a
- private final String tabletDir; // t-0003
+ private final TabletDirectory tabletDir; //
hdfs://1.2.3.4/accumulo/tables/2a/t-0003
private final String fileName; // C0004.rf
protected final Path metaPath;
private final String normalizedPath;
@@ -60,35 +59,32 @@ public class TabletFile implements Comparable<TabletFile> {
ServerColumnFamily.validateDirCol(fileName);
Path tabletDirPath = Objects.requireNonNull(metaPath.getParent(),
errorMsg);
- this.tabletDir = tabletDirPath.getName();
- ServerColumnFamily.validateDirCol(tabletDir);
Path tableIdPath = Objects.requireNonNull(tabletDirPath.getParent(),
errorMsg);
- this.tableId = TableId.of(tableIdPath.getName());
- ServerColumnFamily.validateDirCol(tableId.canonical());
+ var id = tableIdPath.getName();
Path tablePath = Objects.requireNonNull(tableIdPath.getParent(), errorMsg);
String tpString = "/" + tablePath.getName();
- Preconditions.checkArgument(tpString.equals(Constants.HDFS_TABLES_DIR),
errorMsg);
+ Preconditions.checkArgument(tpString.equals(HDFS_TABLES_DIR), errorMsg);
Path volumePath = Objects.requireNonNull(tablePath.getParent(), errorMsg);
Preconditions.checkArgument(volumePath.toUri().getScheme() != null,
errorMsg);
- this.volume = volumePath.toString();
+ var volume = volumePath.toString();
- this.normalizedPath = volume + Constants.HDFS_TABLES_DIR + "/" +
tableId.canonical() + "/"
- + tabletDir + "/" + fileName;
+ this.tabletDir = new TabletDirectory(volume, TableId.of(id),
tabletDirPath.getName());
+ this.normalizedPath = volume + HDFS_TABLES_DIR + "/" + id + "/" +
tabletDir + "/" + fileName;
}
public String getVolume() {
- return volume;
+ return tabletDir.getVolume();
}
public TableId getTableId() {
- return tableId;
+ return tabletDir.getTableId();
}
public String getTabletDir() {
- return tabletDir;
+ return tabletDir.getTabletDir();
}
public String getFileName() {
diff --git a/server/gc/src/main/java/org/apache/accumulo/gc/GCRun.java
b/server/gc/src/main/java/org/apache/accumulo/gc/GCRun.java
index b864bb6cca..02d376a956 100644
--- a/server/gc/src/main/java/org/apache/accumulo/gc/GCRun.java
+++ b/server/gc/src/main/java/org/apache/accumulo/gc/GCRun.java
@@ -46,6 +46,8 @@ import org.apache.accumulo.core.conf.AccumuloConfiguration;
import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.data.TableId;
import org.apache.accumulo.core.manager.state.tables.TableState;
+import org.apache.accumulo.core.metadata.Reference;
+import org.apache.accumulo.core.metadata.RelativeTabletDirectory;
import org.apache.accumulo.core.metadata.RootTable;
import org.apache.accumulo.core.metadata.TabletFileUtil;
import org.apache.accumulo.core.metadata.schema.Ample;
@@ -146,10 +148,10 @@ public class GCRun implements
GarbageCollectionEnvironment {
return tabletStream.flatMap(tm -> {
Stream<Reference> refs = Stream.concat(tm.getFiles().stream(),
tm.getScans().stream())
- .map(f -> new Reference(tm.getTableId(), f.getMetaUpdateDelete(),
false));
+ .map(f -> new Reference(tm.getTableId(), f.getMetaUpdateDelete()));
if (tm.getDirName() != null) {
- refs =
- Stream.concat(refs, Stream.of(new Reference(tm.getTableId(),
tm.getDirName(), true)));
+ refs = Stream.concat(refs,
+ Stream.of(new RelativeTabletDirectory(tm.getTableId(),
tm.getDirName())));
}
return refs;
});
diff --git
a/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectionAlgorithm.java
b/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectionAlgorithm.java
index 2f4b5c85cc..762c597675 100644
---
a/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectionAlgorithm.java
+++
b/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectionAlgorithm.java
@@ -34,9 +34,10 @@ import java.util.stream.Stream;
import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.client.TableNotFoundException;
import org.apache.accumulo.core.data.TableId;
+import org.apache.accumulo.core.metadata.Reference;
+import org.apache.accumulo.core.metadata.RelativeTabletDirectory;
import
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily;
import org.apache.accumulo.core.trace.TraceUtil;
-import org.apache.accumulo.gc.GarbageCollectionEnvironment.Reference;
import org.apache.accumulo.server.replication.proto.Replication.Status;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -106,6 +107,7 @@ public class GarbageCollectionAlgorithm {
throw new IllegalArgumentException(path);
}
+ log.trace("{} -> {} expectedLen = {}", path, relPath, expectedLen);
return relPath;
}
@@ -132,36 +134,35 @@ public class GarbageCollectionAlgorithm {
while (iter.hasNext()) {
Reference ref = iter.next();
- if (ref.isDir) {
- String tableID = ref.id.toString();
- String dirName = ref.ref;
- ServerColumnFamily.validateDirCol(dirName);
+ if (ref instanceof RelativeTabletDirectory) {
+ var dirReference = (RelativeTabletDirectory) ref;
+ ServerColumnFamily.validateDirCol(dirReference.tabletDir);
- String dir = "/" + tableID + "/" + dirName;
+ String dir = "/" + dirReference.tableId + "/" + dirReference.tabletDir;
dir = makeRelative(dir, 2);
if (candidateMap.remove(dir) != null)
log.debug("Candidate was still in use: {}", dir);
} else {
-
- String reference = ref.ref;
+ String reference = ref.metadataEntry;
if (reference.startsWith("/")) {
- reference = "/" + ref.id + reference;
+ log.debug("Candidate {} has a relative path, prepend tableId {}",
reference, ref.tableId);
+ reference = "/" + ref.tableId + ref.metadataEntry;
} else if (!reference.contains(":") && !reference.startsWith("../")) {
throw new RuntimeException("Bad file reference " + reference);
}
- reference = makeRelative(reference, 3);
+ String relativePath = makeRelative(reference, 3);
// WARNING: This line is EXTREMELY IMPORTANT.
// You MUST REMOVE candidates that are still in use
- if (candidateMap.remove(reference) != null)
- log.debug("Candidate was still in use: {}", reference);
+ if (candidateMap.remove(relativePath) != null)
+ log.debug("Candidate was still in use: {}", relativePath);
- String dir = reference.substring(0, reference.lastIndexOf('/'));
+ String dir = relativePath.substring(0, relativePath.lastIndexOf('/'));
if (candidateMap.remove(dir) != null)
- log.debug("Candidate was still in use: {}", reference);
+ log.debug("Candidate was still in use: {}", relativePath);
}
}
}
diff --git
a/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectionEnvironment.java
b/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectionEnvironment.java
index 38b6ac5456..6d943c1555 100644
---
a/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectionEnvironment.java
+++
b/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectionEnvironment.java
@@ -32,6 +32,7 @@ import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.TableId;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.Reference;
import org.apache.accumulo.core.metadata.RootTable;
import
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily;
import
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ScanFileColumnFamily;
@@ -65,18 +66,6 @@ public interface GarbageCollectionEnvironment {
*/
Stream<String> getBlipPaths() throws TableNotFoundException;
- static class Reference {
- public final TableId id;
- public final String ref;
- public final boolean isDir;
-
- Reference(TableId id, String ref, boolean isDir) {
- this.id = id;
- this.ref = ref;
- this.isDir = isDir;
- }
- }
-
/**
* Fetches the references to files, {@link DataFileColumnFamily#NAME} or
* {@link ScanFileColumnFamily#NAME}, from tablets
diff --git
a/server/gc/src/test/java/org/apache/accumulo/gc/GarbageCollectionTest.java
b/server/gc/src/test/java/org/apache/accumulo/gc/GarbageCollectionTest.java
index 88f29ad707..03a0a445ef 100644
--- a/server/gc/src/test/java/org/apache/accumulo/gc/GarbageCollectionTest.java
+++ b/server/gc/src/test/java/org/apache/accumulo/gc/GarbageCollectionTest.java
@@ -37,6 +37,8 @@ import java.util.stream.Stream;
import org.apache.accumulo.core.client.TableNotFoundException;
import org.apache.accumulo.core.data.TableId;
+import org.apache.accumulo.core.metadata.Reference;
+import org.apache.accumulo.core.metadata.RelativeTabletDirectory;
import org.apache.accumulo.server.replication.proto.Replication.Status;
import org.junit.jupiter.api.Test;
@@ -93,8 +95,7 @@ public class GarbageCollectionTest {
}
public void addFileReference(String tableId, String endRow, String file) {
- references.put(tableId + ":" + endRow + ":" + file,
- new Reference(TableId.of(tableId), file, false));
+ references.put(tableId + ":" + endRow + ":" + file, new
Reference(TableId.of(tableId), file));
}
public void removeFileReference(String tableId, String endRow, String
file) {
@@ -102,7 +103,7 @@ public class GarbageCollectionTest {
}
public void addDirReference(String tableId, String endRow, String dir) {
- references.put(tableId + ":" + endRow, new
Reference(TableId.of(tableId), dir, true));
+ references.put(tableId + ":" + endRow, new
RelativeTabletDirectory(TableId.of(tableId), dir));
}
public void removeDirReference(String tableId, String endRow) {
