[IGNITE-349]: corrections by review results.
Project: http://git-wip-us.apache.org/repos/asf/incubator-ignite/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ignite/commit/0a62a799 Tree: http://git-wip-us.apache.org/repos/asf/incubator-ignite/tree/0a62a799 Diff: http://git-wip-us.apache.org/repos/asf/incubator-ignite/diff/0a62a799 Branch: refs/heads/ignite-349 Commit: 0a62a799de9067c27eaf97782bb17d6bbe46d15a Parents: 3015458 Author: iveselovskiy <iveselovs...@gridgain.com> Authored: Tue Mar 3 15:56:33 2015 +0300 Committer: iveselovskiy <iveselovs...@gridgain.com> Committed: Tue Mar 3 15:56:33 2015 +0300 ---------------------------------------------------------------------- .../java/org/apache/ignite/igfs/IgfsConst.java | 28 ---- .../ignite/internal/processors/igfs/IgfsEx.java | 10 ++ .../visor/node/VisorIgfsConfiguration.java | 5 +- .../hadoop/IgfsHadoopFileSystemWrapper.java | 14 +- .../hadoop/SecondaryFileSystemProvider.java | 152 ------------------- .../igfs/hadoop/v1/IgfsHadoopFileSystem.java | 12 +- .../igfs/hadoop/v2/IgfsHadoopFileSystem.java | 12 +- .../hadoop/SecondaryFileSystemProvider.java | 112 ++++++++++++++ 8 files changed, 148 insertions(+), 197 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/0a62a799/modules/core/src/main/java/org/apache/ignite/igfs/IgfsConst.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/igfs/IgfsConst.java b/modules/core/src/main/java/org/apache/ignite/igfs/IgfsConst.java deleted file mode 100644 index 859415b..0000000 --- a/modules/core/src/main/java/org/apache/ignite/igfs/IgfsConst.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ignite.igfs; - -/** - * Constants related to Igfs secondary filesystem configuration. - */ -public class IgfsConst { - /** Property name for path to Hadoop configuration. */ - public static final String SECONDARY_FS_CONFIG_PATH = "SECONDARY_FS_CONFIG_PATH"; - /** Property name for URI of file system. */ - public static final String SECONDARY_FS_URI = "SECONDARY_FS_URI"; -} http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/0a62a799/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsEx.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsEx.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsEx.java index e067e78..f247fba 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsEx.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsEx.java @@ -30,6 +30,16 @@ import java.net.*; */ public interface IgfsEx extends IgniteFs { /** + * Property name for path to Hadoop configuration. + */ + String SECONDARY_FS_CONFIG_PATH = "SECONDARY_FS_CONFIG_PATH"; + + /** + * Property name for URI of file system. + */ + String SECONDARY_FS_URI = "SECONDARY_FS_URI"; + + /** * Stops IGFS cleaning all used resources. */ public void stop(); http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/0a62a799/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorIgfsConfiguration.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorIgfsConfiguration.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorIgfsConfiguration.java index 3ec9d87..b8fa57c 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorIgfsConfiguration.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorIgfsConfiguration.java @@ -19,6 +19,7 @@ package org.apache.ignite.internal.visor.node; import org.apache.ignite.configuration.*; import org.apache.ignite.igfs.*; +import org.apache.ignite.internal.processors.igfs.*; import org.apache.ignite.internal.util.typedef.internal.*; import org.jetbrains.annotations.*; @@ -137,8 +138,8 @@ public class VisorIgfsConfiguration implements Serializable { if (secFs != null) { Map<String, String> props = secFs.properties(); - cfg.secondaryHadoopFileSysUri = props.get(IgfsConst.SECONDARY_FS_URI); - cfg.secondaryHadoopFileSysCfgPath = props.get(IgfsConst.SECONDARY_FS_CONFIG_PATH); + cfg.secondaryHadoopFileSysUri = props.get(IgfsEx.SECONDARY_FS_URI); + cfg.secondaryHadoopFileSysCfgPath = props.get(IgfsEx.SECONDARY_FS_CONFIG_PATH); } cfg.dfltMode = igfs.getDefaultMode(); http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/0a62a799/modules/hadoop/src/main/java/org/apache/ignite/igfs/hadoop/IgfsHadoopFileSystemWrapper.java ---------------------------------------------------------------------- diff --git a/modules/hadoop/src/main/java/org/apache/ignite/igfs/hadoop/IgfsHadoopFileSystemWrapper.java b/modules/hadoop/src/main/java/org/apache/ignite/igfs/hadoop/IgfsHadoopFileSystemWrapper.java index 2577516..fee4681 100644 --- a/modules/hadoop/src/main/java/org/apache/ignite/igfs/hadoop/IgfsHadoopFileSystemWrapper.java +++ b/modules/hadoop/src/main/java/org/apache/ignite/igfs/hadoop/IgfsHadoopFileSystemWrapper.java @@ -24,6 +24,7 @@ import org.apache.hadoop.ipc.*; import org.apache.ignite.*; import org.apache.ignite.igfs.*; import org.apache.ignite.internal.igfs.hadoop.*; +import org.apache.ignite.internal.processors.hadoop.*; import org.apache.ignite.internal.processors.igfs.*; import org.apache.ignite.internal.util.typedef.*; import org.jetbrains.annotations.*; @@ -63,10 +64,17 @@ public class IgfsHadoopFileSystemWrapper implements Igfs, AutoCloseable { public IgfsHadoopFileSystemWrapper(@Nullable String uri, @Nullable String cfgPath) throws IgniteCheckedException { try { SecondaryFileSystemProvider secondaryFileSystemProvider = - new SecondaryFileSystemProvider(uri, cfgPath, false); // TODO: may be disableCaching == true there? + new SecondaryFileSystemProvider(uri, cfgPath); - fileSys = secondaryFileSystemProvider.getFileSystem(); - secondaryFileSystemProvider.exportProperties(props); + fileSys = secondaryFileSystemProvider.createFileSystem(); + + uri = secondaryFileSystemProvider.secondaryFsUri().toString(); + + if (!uri.endsWith("/")) + uri += "/"; + + props.put(IgfsEx.SECONDARY_FS_CONFIG_PATH, cfgPath); + props.put(IgfsEx.SECONDARY_FS_URI, uri); } catch (IOException e) { throw new IgniteCheckedException(e); http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/0a62a799/modules/hadoop/src/main/java/org/apache/ignite/igfs/hadoop/SecondaryFileSystemProvider.java ---------------------------------------------------------------------- diff --git a/modules/hadoop/src/main/java/org/apache/ignite/igfs/hadoop/SecondaryFileSystemProvider.java b/modules/hadoop/src/main/java/org/apache/ignite/igfs/hadoop/SecondaryFileSystemProvider.java deleted file mode 100644 index 81a12ea..0000000 --- a/modules/hadoop/src/main/java/org/apache/ignite/igfs/hadoop/SecondaryFileSystemProvider.java +++ /dev/null @@ -1,152 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ignite.igfs.hadoop; - -import com.sun.istack.NotNull; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.AbstractFileSystem; -import org.apache.hadoop.fs.FileSystem; -import org.apache.ignite.igfs.IgfsConst; -import org.apache.ignite.internal.util.typedef.internal.U; -import org.jetbrains.annotations.Nullable; - -import java.io.IOException; -import java.net.URI; -import java.net.URISyntaxException; -import java.net.URL; -import java.util.Map; - -/** - * Encapsulates logic of secondary filesystem creation. - */ -public class SecondaryFileSystemProvider { - - private @NotNull final Configuration cfg = new Configuration(); - private @NotNull final URI secondaryFsUri; - private @Nullable final String secConfPath; - private final boolean disableCaching; - - /** - * Creates new provider with given config parameters. - * The configuration URL is optional. The filesystem URI must be specified either explicitly or in the - * configuration provided. - * - * @param secUri the secondary Fs URI (optional). If not given explicitly, it must be specified - * as "fs.defaultFS" property in the provided configuration. - * @param secConfPath the secondary Fs config URL (optional) - * @param disableCaching 'true' to disable caching in the created Hadoop filesystem. Used to avoid double caching. - * @throws IOException - */ - public SecondaryFileSystemProvider(@Nullable String secUri, @Nullable String secConfPath, - boolean disableCaching) throws IOException { - if (secUri == null && secConfPath == null) { - throw new IllegalArgumentException("Failed to get secondary file system: neither " + - "secondaryUri nor secondaryConfigPath given."); - } - - this.disableCaching = disableCaching; - - if (secConfPath != null) { - URL url = U.resolveIgniteUrl(secConfPath); - if (url == null) { - // If secConfPath is given, it should be resolvable: - throw new IllegalArgumentException("Failed to resolve secondary Fs " + - "configuration path [" + secConfPath + "]"); - } - cfg.addResource(url); - } - - this.secConfPath = secConfPath; - - try { - // if secondary fs URI is not given explicitly, try to take it from the configuration: - secondaryFsUri = (secUri == null) ? FileSystem.getDefaultUri(cfg) : new URI(secUri); - } catch (URISyntaxException use) { - throw new IOException("Failed to resolve secondary file system URI: " + secUri); - } - - if (secondaryFsUri == null) { - throw new IllegalArgumentException("Failed to get secondary file system URI: it is neither given " + - "explicitly nor specified in the configuration ["+secConfPath+"]"); - } - } - - /** - * @return {@link org.apache.hadoop.fs.FileSystem} instance for this secondary Fs. - * @throws IOException - */ - public @NotNull FileSystem getFileSystem() throws IOException { - if (disableCaching) { - disableCaching(); - } - - FileSystem fileSys = FileSystem.get(secondaryFsUri, cfg); - - return fileSys; - } - - /** - * Exports properties of the filesystem to the given map. - * @param props - */ - public void exportProperties(@NotNull Map<String,String> props) { - props.put(IgfsConst.SECONDARY_FS_CONFIG_PATH, secConfPath); - props.put(IgfsConst.SECONDARY_FS_URI, getSecondaryFsUriStr()); - } - - /** - * @return {@link org.apache.hadoop.fs.AbstractFileSystem} instance for this secondary Fs. - * @throws IOException - */ - public @NotNull AbstractFileSystem getAbstractFileSystem() throws IOException { - if (disableCaching) { - disableCaching(); - } - - AbstractFileSystem secondaryFs = AbstractFileSystem.get(secondaryFsUri, cfg); - - return secondaryFs; - } - - private void disableCaching() { - String prop = String.format("fs.%s.impl.disable.cache", secondaryFsUri.getScheme()); - - cfg.setBoolean(prop, true); - } - - /** - * @return the secondary fs URI, never null. - */ - public @NotNull URI getSecondaryFsUri() { - return secondaryFsUri; - } - - /** - * - * @return "slashified" secondary Fs URI in String form, never null. - */ - private @NotNull String getSecondaryFsUriStr() { - String uri = secondaryFsUri.toString(); - - if (!uri.endsWith("/")) - uri += "/"; - - return uri; - } - -} http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/0a62a799/modules/hadoop/src/main/java/org/apache/ignite/igfs/hadoop/v1/IgfsHadoopFileSystem.java ---------------------------------------------------------------------- diff --git a/modules/hadoop/src/main/java/org/apache/ignite/igfs/hadoop/v1/IgfsHadoopFileSystem.java b/modules/hadoop/src/main/java/org/apache/ignite/igfs/hadoop/v1/IgfsHadoopFileSystem.java index cdfe1a1..1f7b87e 100644 --- a/modules/hadoop/src/main/java/org/apache/ignite/igfs/hadoop/v1/IgfsHadoopFileSystem.java +++ b/modules/hadoop/src/main/java/org/apache/ignite/igfs/hadoop/v1/IgfsHadoopFileSystem.java @@ -26,9 +26,9 @@ import org.apache.hadoop.mapreduce.*; import org.apache.hadoop.util.*; import org.apache.ignite.*; import org.apache.ignite.igfs.*; -import org.apache.ignite.igfs.hadoop.*; import org.apache.ignite.internal.igfs.common.*; import org.apache.ignite.internal.igfs.hadoop.*; +import org.apache.ignite.internal.processors.hadoop.*; import org.apache.ignite.internal.processors.igfs.*; import org.apache.ignite.internal.util.typedef.*; import org.apache.ignite.internal.util.typedef.internal.*; @@ -282,15 +282,15 @@ public class IgfsHadoopFileSystem extends FileSystem { if (initSecondary) { Map<String, String> props = paths.properties(); - String secUri = props.get(IgfsConst.SECONDARY_FS_URI); - String secConfPath = props.get(IgfsConst.SECONDARY_FS_CONFIG_PATH); + String secUri = props.get(IgfsEx.SECONDARY_FS_URI); + String secConfPath = props.get(IgfsEx.SECONDARY_FS_CONFIG_PATH); try { SecondaryFileSystemProvider secondaryFileSystemProvider = - new SecondaryFileSystemProvider(secUri, secConfPath, true); + new SecondaryFileSystemProvider(secUri, secConfPath); - secondaryFs = secondaryFileSystemProvider.getFileSystem(); - secondaryUri = secondaryFileSystemProvider.getSecondaryFsUri(); + secondaryFs = secondaryFileSystemProvider.createFileSystem(); + secondaryUri = secondaryFileSystemProvider.secondaryFsUri(); } catch (IOException e) { if (!mgmt) http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/0a62a799/modules/hadoop/src/main/java/org/apache/ignite/igfs/hadoop/v2/IgfsHadoopFileSystem.java ---------------------------------------------------------------------- diff --git a/modules/hadoop/src/main/java/org/apache/ignite/igfs/hadoop/v2/IgfsHadoopFileSystem.java b/modules/hadoop/src/main/java/org/apache/ignite/igfs/hadoop/v2/IgfsHadoopFileSystem.java index 691705b..cb35bec 100644 --- a/modules/hadoop/src/main/java/org/apache/ignite/igfs/hadoop/v2/IgfsHadoopFileSystem.java +++ b/modules/hadoop/src/main/java/org/apache/ignite/igfs/hadoop/v2/IgfsHadoopFileSystem.java @@ -26,9 +26,9 @@ import org.apache.hadoop.mapreduce.*; import org.apache.hadoop.util.*; import org.apache.ignite.*; import org.apache.ignite.igfs.*; -import org.apache.ignite.igfs.hadoop.*; import org.apache.ignite.internal.igfs.common.*; import org.apache.ignite.internal.igfs.hadoop.*; +import org.apache.ignite.internal.processors.hadoop.*; import org.apache.ignite.internal.processors.igfs.*; import org.apache.ignite.internal.util.typedef.*; import org.apache.ignite.internal.util.typedef.internal.*; @@ -277,15 +277,15 @@ public class IgfsHadoopFileSystem extends AbstractFileSystem implements Closeabl if (initSecondary) { Map<String, String> props = paths.properties(); - String secUri = props.get(IgfsConst.SECONDARY_FS_URI); - String secConfPath = props.get(IgfsConst.SECONDARY_FS_CONFIG_PATH); + String secUri = props.get(IgfsEx.SECONDARY_FS_URI); + String secConfPath = props.get(IgfsEx.SECONDARY_FS_CONFIG_PATH); try { SecondaryFileSystemProvider secondaryFileSystemProvider = - new SecondaryFileSystemProvider(secUri, secConfPath, true); + new SecondaryFileSystemProvider(secUri, secConfPath); - secondaryFs = secondaryFileSystemProvider.getAbstractFileSystem(); - secondaryUri = secondaryFileSystemProvider.getSecondaryFsUri(); + secondaryFs = secondaryFileSystemProvider.createAbstractFileSystem(); + secondaryUri = secondaryFileSystemProvider.secondaryFsUri(); } catch (IOException e) { throw new IOException("Failed to connect to the secondary file system: " + secUri, e); http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/0a62a799/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/SecondaryFileSystemProvider.java ---------------------------------------------------------------------- diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/SecondaryFileSystemProvider.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/SecondaryFileSystemProvider.java new file mode 100644 index 0000000..a63242c --- /dev/null +++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/SecondaryFileSystemProvider.java @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.hadoop; + +import org.apache.hadoop.conf.*; +import org.apache.hadoop.fs.*; +import org.apache.ignite.internal.util.typedef.internal.*; +import org.jetbrains.annotations.*; + +import java.io.*; +import java.net.*; + +/** + * Encapsulates logic of secondary filesystem creation. + */ +public class SecondaryFileSystemProvider { + /** + * Configuration of the secondary filesystem, never null. + */ + private final Configuration cfg = new Configuration(); + + /** + * The secondary filesystem URI, never null. + */ + private final URI secondaryFsUri; + + /** + * Creates new provider with given config parameters. The configuration URL is optional. The filesystem URI must be + * specified either explicitly or in the configuration provided. + * + * @param secUri the secondary Fs URI (optional). If not given explicitly, it must be specified as "fs.defaultFS" + * property in the provided configuration. + * @param secConfPath the secondary Fs config URL (optional) + * @throws IOException + */ + public SecondaryFileSystemProvider(final @Nullable String secUri, + final @Nullable String secConfPath) throws IOException { + if (secConfPath != null) { + URL url = U.resolveIgniteUrl(secConfPath); + if (url == null) { + // If secConfPath is given, it should be resolvable: + throw new IllegalArgumentException("Failed to resolve secondary Fs " + + "configuration path [" + secConfPath + "]"); + } + cfg.addResource(url); + } + + // if secondary fs URI is not given explicitly, try to get it from the configuration: + if (secUri == null) { + secondaryFsUri = FileSystem.getDefaultUri(cfg); + } + else { + try { + secondaryFsUri = new URI(secUri); + } + catch (URISyntaxException use) { + throw new IOException("Failed to resolve secondary file system URI: " + secUri); + } + } + + if (secondaryFsUri == null) { + throw new IllegalArgumentException("Failed to get secondary file system URI: it is neither given " + + "explicitly nor specified in the configuration: " + secConfPath); + } + + // Disable caching: + String prop = String.format("fs.%s.impl.disable.cache", secondaryFsUri.getScheme()); + cfg.setBoolean(prop, true); + } + + /** + * @return {@link org.apache.hadoop.fs.FileSystem} instance for this secondary Fs. + * @throws IOException + */ + public FileSystem createFileSystem() throws IOException { + FileSystem fileSys = FileSystem.get(secondaryFsUri, cfg); + + return fileSys; + } + + /** + * @return {@link org.apache.hadoop.fs.AbstractFileSystem} instance for this secondary Fs. + * @throws IOException + */ + public AbstractFileSystem createAbstractFileSystem() throws IOException { + AbstractFileSystem secondaryFs = AbstractFileSystem.get(secondaryFsUri, cfg); + + return secondaryFs; + } + + /** + * @return the secondary fs URI, never null. + */ + public URI secondaryFsUri() { + return secondaryFsUri; + } +}
