[
https://issues.apache.org/jira/browse/HADOOP-18193?focusedWorklogId=757520&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-757520
]
ASF GitHub Bot logged work on HADOOP-18193:
-------------------------------------------
Author: ASF GitHub Bot
Created on: 15/Apr/22 22:05
Start Date: 15/Apr/22 22:05
Worklog Time Spent: 10m
Work Description: mccormickt12 commented on code in PR #4181:
URL: https://github.com/apache/hadoop/pull/4181#discussion_r851530313
##########
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestNestedMountPoint.java:
##########
@@ -0,0 +1,345 @@
+package org.apache.hadoop.fs.viewfs;
+
+import java.net.URI;
+import java.util.List;
+import java.util.function.Function;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FsConstants;
+import org.apache.hadoop.fs.Path;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+
+public class TestNestedMountPoint {
+ private InodeTree inodeTree;
+ private Configuration conf;
+ private String mtName;
+ private URI fsUri;
+
+ static class TestNestMountPointFileSystem {
+ public URI getUri() {
+ return uri;
+ }
+
+ private URI uri;
+
+ TestNestMountPointFileSystem(URI uri) {
+ this.uri = uri;
+ }
+ }
+
+ static class TestNestMountPointInternalFileSystem extends
TestNestMountPointFileSystem {
+ TestNestMountPointInternalFileSystem(URI uri) {
+ super(uri);
+ }
+ }
+
+ private static final URI LINKFALLBACK_TARGET = URI.create("hdfs://nn00");
+ private static final URI NN1_TARGET = URI.create("hdfs://nn01/a/b");
+ private static final URI NN2_TARGET = URI.create("hdfs://nn02/a/b/e");
+ private static final URI NN3_TARGET = URI.create("hdfs://nn03/a/b/c/d");
+ private static final URI NN4_TARGET = URI.create("hdfs://nn04/a/b/c/d/e");
+ private static final URI NN5_TARGET = URI.create("hdfs://nn05/b/c/d/e");
+ private static final URI NN6_TARGET = URI.create("hdfs://nn06/b/c/d/e/f");
+
+ @Before
+ public void setUp() throws Exception {
+ conf = new Configuration();
+ mtName = TestNestedMountPoint.class.getName();
+ ConfigUtil.setIsNestedMountPointSupported(conf, true);
+ ConfigUtil.addLink(conf, mtName, "/a/b", NN1_TARGET);
+ ConfigUtil.addLink(conf, mtName, "/a/b/e", NN2_TARGET);
Review Comment:
each dirlink, only has one nested mount point. Could could add one dirlink
that has multiple nested mount points under it.
##########
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestNestedMountPoint.java:
##########
@@ -0,0 +1,345 @@
+package org.apache.hadoop.fs.viewfs;
+
+import java.net.URI;
+import java.util.List;
+import java.util.function.Function;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FsConstants;
+import org.apache.hadoop.fs.Path;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+
+public class TestNestedMountPoint {
+ private InodeTree inodeTree;
+ private Configuration conf;
+ private String mtName;
+ private URI fsUri;
+
+ static class TestNestMountPointFileSystem {
+ public URI getUri() {
+ return uri;
+ }
+
+ private URI uri;
+
+ TestNestMountPointFileSystem(URI uri) {
+ this.uri = uri;
+ }
+ }
+
+ static class TestNestMountPointInternalFileSystem extends
TestNestMountPointFileSystem {
+ TestNestMountPointInternalFileSystem(URI uri) {
+ super(uri);
+ }
+ }
+
+ private static final URI LINKFALLBACK_TARGET = URI.create("hdfs://nn00");
+ private static final URI NN1_TARGET = URI.create("hdfs://nn01/a/b");
+ private static final URI NN2_TARGET = URI.create("hdfs://nn02/a/b/e");
+ private static final URI NN3_TARGET = URI.create("hdfs://nn03/a/b/c/d");
+ private static final URI NN4_TARGET = URI.create("hdfs://nn04/a/b/c/d/e");
+ private static final URI NN5_TARGET = URI.create("hdfs://nn05/b/c/d/e");
+ private static final URI NN6_TARGET = URI.create("hdfs://nn06/b/c/d/e/f");
+
+ @Before
+ public void setUp() throws Exception {
+ conf = new Configuration();
+ mtName = TestNestedMountPoint.class.getName();
+ ConfigUtil.setIsNestedMountPointSupported(conf, true);
+ ConfigUtil.addLink(conf, mtName, "/a/b", NN1_TARGET);
+ ConfigUtil.addLink(conf, mtName, "/a/b/e", NN2_TARGET);
+ ConfigUtil.addLink(conf, mtName, "/a/b/c/d", NN3_TARGET);
+ ConfigUtil.addLink(conf, mtName, "/a/b/c/d/e", NN4_TARGET);
+ ConfigUtil.addLink(conf, mtName, "/b/c/d/e", NN5_TARGET);
+ ConfigUtil.addLink(conf, mtName, "/b/c/d/e/f", NN6_TARGET);
+ ConfigUtil.addLinkFallback(conf, mtName, LINKFALLBACK_TARGET);
+
+ fsUri = new URI(FsConstants.VIEWFS_SCHEME, mtName, "/", null, null);
+
+ inodeTree = new
InodeTree<TestNestedMountPoint.TestNestMountPointFileSystem>(conf,
+ mtName, fsUri, false) {
+ @Override
+ protected Function<URI,
TestNestedMountPoint.TestNestMountPointFileSystem> initAndGetTargetFs() {
+ return new Function<URI, TestNestMountPointFileSystem>() {
+ @Override
+ public TestNestedMountPoint.TestNestMountPointFileSystem apply(URI
uri) {
+ return new TestNestMountPointFileSystem(uri);
+ }
+ };
+ }
+
+ // For intenral dir fs
+ @Override
+ protected TestNestedMountPoint.TestNestMountPointInternalFileSystem
getTargetFileSystem(
+ final INodeDir<TestNestedMountPoint.TestNestMountPointFileSystem>
dir) {
+ return new TestNestMountPointInternalFileSystem(fsUri);
+ }
+
+ @Override
+ protected TestNestedMountPoint.TestNestMountPointInternalFileSystem
getTargetFileSystem(
+ final String settings, final URI[] mergeFsURIList) {
+ return new TestNestMountPointInternalFileSystem(null);
+ }
+ };
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ inodeTree = null;
+ }
+
+ @Test
+ public void testPathResolveToLink() throws Exception {
+ // /a/b/c/d/e/f resolves to /a/b/c/d/e and /f
+ InodeTree.ResolveResult resolveResult = inodeTree.resolve("/a/b/c/d/e/f",
true);
+ Assert.assertEquals(InodeTree.ResultKind.EXTERNAL_DIR, resolveResult.kind);
+ Assert.assertEquals("/a/b/c/d/e", resolveResult.resolvedPath);
+ Assert.assertEquals(new Path("/f"), resolveResult.remainingPath);
+ Assert.assertTrue(resolveResult.targetFileSystem instanceof
TestNestMountPointFileSystem);
+ Assert.assertEquals(NN4_TARGET, ((TestNestMountPointFileSystem)
resolveResult.targetFileSystem).getUri());
+ Assert.assertTrue(resolveResult.isLastInternalDirLink());
+
+ // /a/b/c/d/e resolves to /a/b/c/d/e and /
+ InodeTree.ResolveResult resolveResult2 = inodeTree.resolve("/a/b/c/d/e",
true);
+ Assert.assertEquals(InodeTree.ResultKind.EXTERNAL_DIR,
resolveResult2.kind);
+ Assert.assertEquals("/a/b/c/d/e", resolveResult2.resolvedPath);
+ Assert.assertEquals(new Path("/"), resolveResult2.remainingPath);
+ Assert.assertTrue(resolveResult2.targetFileSystem instanceof
TestNestMountPointFileSystem);
+ Assert.assertEquals(NN4_TARGET, ((TestNestMountPointFileSystem)
resolveResult2.targetFileSystem).getUri());
+ Assert.assertTrue(resolveResult2.isLastInternalDirLink());
+
+ // /a/b/c/d/e/f/g/h/i resolves to /a/b/c/d/e and /f/g/h/i
+ InodeTree.ResolveResult resolveResult3 =
inodeTree.resolve("/a/b/c/d/e/f/g/h/i", true);
+ Assert.assertEquals(InodeTree.ResultKind.EXTERNAL_DIR,
resolveResult3.kind);
+ Assert.assertEquals("/a/b/c/d/e", resolveResult3.resolvedPath);
+ Assert.assertEquals(new Path("/f/g/h/i"), resolveResult3.remainingPath);
+ Assert.assertTrue(resolveResult3.targetFileSystem instanceof
TestNestMountPointFileSystem);
+ Assert.assertEquals(NN4_TARGET, ((TestNestMountPointFileSystem)
resolveResult3.targetFileSystem).getUri());
+ Assert.assertTrue(resolveResult3.isLastInternalDirLink());
+ }
+
+ @Test
+ public void testPathResolveToLink_NotResolveLastComponent() throws Exception
{
+ // /a/b/c/d/e/f resolves to /a/b/c/d/e and /f
+ InodeTree.ResolveResult resolveResult = inodeTree.resolve("/a/b/c/d/e/f",
false);
+ Assert.assertEquals(InodeTree.ResultKind.EXTERNAL_DIR, resolveResult.kind);
+ Assert.assertEquals("/a/b/c/d/e", resolveResult.resolvedPath);
+ Assert.assertEquals(new Path("/f"), resolveResult.remainingPath);
+ Assert.assertTrue(resolveResult.targetFileSystem instanceof
TestNestMountPointFileSystem);
+ Assert.assertEquals(NN4_TARGET, ((TestNestMountPointFileSystem)
resolveResult.targetFileSystem).getUri());
+ Assert.assertTrue(resolveResult.isLastInternalDirLink());
+
+ // /a/b/c/d/e resolves to /a/b/c/d and /e
+ InodeTree.ResolveResult resolveResult2 = inodeTree.resolve("/a/b/c/d/e",
false);
+ Assert.assertEquals(InodeTree.ResultKind.EXTERNAL_DIR,
resolveResult2.kind);
+ Assert.assertEquals("/a/b/c/d", resolveResult2.resolvedPath);
+ Assert.assertEquals(new Path("/e"), resolveResult2.remainingPath);
+ Assert.assertTrue(resolveResult2.targetFileSystem instanceof
TestNestMountPointFileSystem);
+ Assert.assertEquals(NN3_TARGET, ((TestNestMountPointFileSystem)
resolveResult2.targetFileSystem).getUri());
+ Assert.assertTrue(resolveResult2.isLastInternalDirLink());
+
+ // /a/b/c/d/e/f/g/h/i resolves to /a/b/c/d/e and /f/g/h/i
+ InodeTree.ResolveResult resolveResult3 =
inodeTree.resolve("/a/b/c/d/e/f/g/h/i", false);
+ Assert.assertEquals(InodeTree.ResultKind.EXTERNAL_DIR,
resolveResult3.kind);
+ Assert.assertEquals("/a/b/c/d/e", resolveResult3.resolvedPath);
+ Assert.assertEquals(new Path("/f/g/h/i"), resolveResult3.remainingPath);
+ Assert.assertTrue(resolveResult3.targetFileSystem instanceof
TestNestMountPointFileSystem);
+ Assert.assertEquals(NN4_TARGET, ((TestNestMountPointFileSystem)
resolveResult3.targetFileSystem).getUri());
+ Assert.assertTrue(resolveResult3.isLastInternalDirLink());
+ }
+
+ @Test
+ public void testPathResolveToDirLink() throws Exception {
+ // /a/b/c/d/f resolves to /a/b/c/d, /f
+ InodeTree.ResolveResult resolveResult = inodeTree.resolve("/a/b/c/d/f",
true);
+ Assert.assertEquals(InodeTree.ResultKind.EXTERNAL_DIR, resolveResult.kind);
+ Assert.assertEquals("/a/b/c/d", resolveResult.resolvedPath);
+ Assert.assertEquals(new Path("/f"), resolveResult.remainingPath);
+ Assert.assertTrue(resolveResult.targetFileSystem instanceof
TestNestMountPointFileSystem);
+ Assert.assertEquals(NN3_TARGET, ((TestNestMountPointFileSystem)
resolveResult.targetFileSystem).getUri());
+ Assert.assertTrue(resolveResult.isLastInternalDirLink());
+
+ // /a/b/c/d resolves to /a/b/c/d and /
+ InodeTree.ResolveResult resolveResult2 = inodeTree.resolve("/a/b/c/d",
true);
+ Assert.assertEquals(InodeTree.ResultKind.EXTERNAL_DIR,
resolveResult2.kind);
+ Assert.assertEquals("/a/b/c/d", resolveResult2.resolvedPath);
+ Assert.assertEquals(new Path("/"), resolveResult2.remainingPath);
+ Assert.assertTrue(resolveResult2.targetFileSystem instanceof
TestNestMountPointFileSystem);
+ Assert.assertEquals(NN3_TARGET, ((TestNestMountPointFileSystem)
resolveResult2.targetFileSystem).getUri());
+ Assert.assertTrue(resolveResult2.isLastInternalDirLink());
+
+ // /a/b/c/d/f/g/h/i resolves to /a/b/c/d and /f/g/h/i
+ InodeTree.ResolveResult resolveResult3 =
inodeTree.resolve("/a/b/c/d/f/g/h/i", true);
+ Assert.assertEquals(InodeTree.ResultKind.EXTERNAL_DIR,
resolveResult3.kind);
+ Assert.assertEquals("/a/b/c/d", resolveResult3.resolvedPath);
+ Assert.assertEquals(new Path("/f/g/h/i"), resolveResult3.remainingPath);
+ Assert.assertTrue(resolveResult3.targetFileSystem instanceof
TestNestMountPointFileSystem);
+ Assert.assertEquals(NN3_TARGET, ((TestNestMountPointFileSystem)
resolveResult3.targetFileSystem).getUri());
+ Assert.assertTrue(resolveResult3.isLastInternalDirLink());
+ }
+
+ @Test
+ public void testPathResolveToDirLink_NotResolveLastComponent() throws
Exception {
+ // /a/b/c/d/f resolves to /a/b/c/d, /f
+ InodeTree.ResolveResult resolveResult = inodeTree.resolve("/a/b/c/d/f",
false);
+ Assert.assertEquals(InodeTree.ResultKind.EXTERNAL_DIR, resolveResult.kind);
+ Assert.assertEquals("/a/b/c/d", resolveResult.resolvedPath);
+ Assert.assertEquals(new Path("/f"), resolveResult.remainingPath);
+ Assert.assertTrue(resolveResult.targetFileSystem instanceof
TestNestMountPointFileSystem);
+ Assert.assertEquals(NN3_TARGET, ((TestNestMountPointFileSystem)
resolveResult.targetFileSystem).getUri());
+ Assert.assertTrue(resolveResult.isLastInternalDirLink());
+
+ // /a/b/c/d resolves to /a/b and /c/d
+ InodeTree.ResolveResult resolveResult2 = inodeTree.resolve("/a/b/c/d",
false);
+ Assert.assertEquals(InodeTree.ResultKind.EXTERNAL_DIR,
resolveResult2.kind);
+ Assert.assertEquals("/a/b", resolveResult2.resolvedPath);
+ Assert.assertEquals(new Path("/c/d"), resolveResult2.remainingPath);
+ Assert.assertTrue(resolveResult2.targetFileSystem instanceof
TestNestMountPointFileSystem);
+ Assert.assertEquals(NN1_TARGET, ((TestNestMountPointFileSystem)
resolveResult2.targetFileSystem).getUri());
+ Assert.assertTrue(resolveResult2.isLastInternalDirLink());
+
+ // /a/b/c/d/f/g/h/i resolves to /a/b/c/d and /f/g/h/i
+ InodeTree.ResolveResult resolveResult3 =
inodeTree.resolve("/a/b/c/d/f/g/h/i", false);
+ Assert.assertEquals(InodeTree.ResultKind.EXTERNAL_DIR,
resolveResult3.kind);
+ Assert.assertEquals("/a/b/c/d", resolveResult3.resolvedPath);
+ Assert.assertEquals(new Path("/f/g/h/i"), resolveResult3.remainingPath);
+ Assert.assertTrue(resolveResult3.targetFileSystem instanceof
TestNestMountPointFileSystem);
+ Assert.assertEquals(NN3_TARGET, ((TestNestMountPointFileSystem)
resolveResult3.targetFileSystem).getUri());
+ Assert.assertTrue(resolveResult3.isLastInternalDirLink());
+ }
+
+ @Test
+ public void testMultiNestedMountPointsPathResolveToDirLink() throws
Exception {
+ // /a/b/f resolves to /a/b and /f
Review Comment:
Here you have a test that tests a top level dirlink correctly picks the
correct link instead of a nested mount point.
Since essentially unlimited nesting of these dirlinks is supported, can you
add another test that tests that a nested mount point also handles this
scenario correctly
Issue Time Tracking
-------------------
Worklog Id: (was: 757520)
Time Spent: 20m (was: 10m)
> Support nested mount points in INodeTree
> ----------------------------------------
>
> Key: HADOOP-18193
> URL: https://issues.apache.org/jira/browse/HADOOP-18193
> Project: Hadoop Common
> Issue Type: Improvement
> Components: viewfs
> Affects Versions: 2.10.0
> Reporter: Lei Yang
> Priority: Major
> Labels: pull-request-available
> Time Spent: 20m
> Remaining Estimate: 0h
>
> Defining following client mount table config is not supported in INodeTree
> and will throw FileAlreadyExistsException
> fs.viewfs.mounttable.link./foo/bar=hdfs://nn1/foo/bar
> fs.viewfs.mounttable.link./foo=hdfs://nn02/foo
>
> INodeTree has 2 methods that need change to support nested mount points.
> createLink(..): build INodeTree during fs init.
> resolve(..): resolve path in INodeTree with viewfs apis.
>
> ViewFileSystem and ViewFs referes INodeTree.resolve(..) to resolve path to
> specific mount point. No changes are expected in both classes. However, we
> need to support existing use cases and make sure no regression are caused.
>
> AC:
> # INodeTree.createlink should support creating nested mount
> points.(INodeTree is constructed during fs init)
> # INodeTree.resolve should support resolve path based on nested mount
> points. (INodeTree.resolve is used in viewfs apis)
> # No regression in existing ViewFileSystem and ViewFs apis.
> # Ensure some important apis are not broken with nested mount points.
> (Rename, getContentSummary, listStatus...)
--
This message was sent by Atlassian Jira
(v8.20.1#820001)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
