steveloughran commented on a change in pull request #2971: URL: https://github.com/apache/hadoop/pull/2971#discussion_r819741101
########## File path: hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/impl/StoreOperations.java ########## @@ -0,0 +1,351 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.mapreduce.lib.output.committer.manifest.impl; + +import java.io.Closeable; +import java.io.FileNotFoundException; +import java.io.IOException; + +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.RemoteIterator; +import org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.AbstractManifestData; +import org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.FileOrDirEntry; +import org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.TaskManifest; +import org.apache.hadoop.util.JsonSerialization; + +import static java.util.Objects.requireNonNull; + +/** + * FileSystem operations which are needed to generate the task manifest. + * The specific choice of which implementation to use is configurable. + * + * If a store provides a custom file commit operation then it must + * offer a subclass of this which returns true in + * {@link #storeSupportsResilientCommit()} + * and then implement + * {@link #commitFile(FileOrDirEntry)}. + * + */ +public abstract class StoreOperations implements Closeable { + + /** + * Bind to the filesystem. + * This is called by the manifest committer after the operations + * have been instantiated. + * @param fileSystem target FS + * @param path actual path under FS. + * @throws IOException if there are binding problems. + */ + public void bindToFileSystem(FileSystem fileSystem, Path path) throws IOException { + + } + + /** + * Forward to {@link FileSystem#getFileStatus(Path)}. + * @param path path + * @return status + * @throws IOException failure. + */ + public abstract FileStatus getFileStatus(Path path) throws IOException; + + /** + * Is a path a file? Used during directory creation. + * The is a copy & paste of FileSystem.isFile(); + * {@code StoreOperationsThroughFileSystem} calls into + * the FS direct so that stores which optimize their probes + * can save on IO. + * @param path path to probe + * @return true if the path exists and resolves to a file + * @throws IOException failure other than FileNotFoundException + */ + public boolean isFile(Path path) throws IOException { + try { + return getFileStatus(path).isFile(); + } catch (FileNotFoundException e) { + return false; + } + } + + /** + * Forward to {@link FileSystem#delete(Path, boolean)}. + * If it returns without an error: there is nothing at + * the end of the path. + * @param path path + * @param recursive recursive delete. + * @return true if the path was deleted. + * @throws IOException failure. + */ + public abstract boolean delete(Path path, boolean recursive) + throws IOException; + + /** + * Forward to {@link FileSystem#mkdirs(Path)}. + * Usual "what does 'false' mean" ambiguity. + * @param path path + * @return true if the directory was created. + * @throws IOException failure. + */ + public abstract boolean mkdirs(Path path) throws IOException; + + /** + * Forward to {@link FileSystem#rename(Path, Path)}. + * Usual "what does 'false' mean" ambiguity. + * @param source source file + * @param dest destination path -which must not exist. + * @return the return value of the rename + * @throws IOException failure. + */ + public abstract boolean renameFile(Path source, Path dest) + throws IOException; + + /** + * Rename a dir; defaults to invoking + * Forward to {@link #renameFile(Path, Path)}. + * Usual "what does 'false' mean?" ambiguity. + * @param source source file + * @param dest destination path -which must not exist. + * @return true if the directory was created. + * @throws IOException failure. + */ + public boolean renameDir(Path source, Path dest) + throws IOException { + return renameFile(source, dest); + } + + /** + * List the directory. + * @param path path to list. + * @return an iterator over the results. + * @throws IOException any immediate failure. + */ + public abstract RemoteIterator<FileStatus> listStatusIterator(Path path) + throws IOException; + + /** + * Load a task manifest from the store. + * with a real FS, this is done with + * {@link TaskManifest#load(JsonSerialization, FileSystem, Path, FileStatus)} + * + * @param serializer serializer. + * @param st status with the path and other data. + * @return the manifest + * @throws IOException failure to load/parse + */ + public abstract TaskManifest loadTaskManifest( + JsonSerialization<TaskManifest> serializer, + FileStatus st) throws IOException; + + /** + * Save a task manifest by create(); there's no attempt at Review comment: `fs.create(path, overwrite)`. clarified -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
