sprint-1 - File system example renaming.
Project: http://git-wip-us.apache.org/repos/asf/incubator-ignite/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ignite/commit/252ccdff Tree: http://git-wip-us.apache.org/repos/asf/incubator-ignite/tree/252ccdff Diff: http://git-wip-us.apache.org/repos/asf/incubator-ignite/diff/252ccdff Branch: refs/heads/ignite-51 Commit: 252ccdff7a6c181c8953f95efa0b5ef7c6ec40a1 Parents: 068931b Author: Dmitiry Setrakyan <dsetrak...@gridgain.com> Authored: Sun Feb 8 13:59:24 2015 -0800 Committer: Dmitiry Setrakyan <dsetrak...@gridgain.com> Committed: Sun Feb 8 13:59:24 2015 -0800 ---------------------------------------------------------------------- .../ignite/examples/fs/IgniteFsExample.java | 278 ------------------- .../examples/fs/IgniteFsMapReduceExample.java | 249 ----------------- .../ignite/examples/fs/IgniteFsNodeStartup.java | 41 --- .../org/apache/ignite/examples/fs/package.html | 24 -- .../examples/ignitefs/IgniteFsExample.java | 278 +++++++++++++++++++ .../ignitefs/IgniteFsMapReduceExample.java | 249 +++++++++++++++++ .../examples/ignitefs/IgniteFsNodeStartup.java | 41 +++ .../ignite/examples/ignitefs/package.html | 24 ++ .../examples/IgniteFsExamplesSelfTest.java | 2 +- 9 files changed, 593 insertions(+), 593 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/252ccdff/examples/src/main/java/org/apache/ignite/examples/fs/IgniteFsExample.java ---------------------------------------------------------------------- diff --git a/examples/src/main/java/org/apache/ignite/examples/fs/IgniteFsExample.java b/examples/src/main/java/org/apache/ignite/examples/fs/IgniteFsExample.java deleted file mode 100644 index 9a017dc..0000000 --- a/examples/src/main/java/org/apache/ignite/examples/fs/IgniteFsExample.java +++ /dev/null @@ -1,278 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ignite.examples.fs; - -import org.apache.ignite.*; -import org.apache.ignite.ignitefs.*; -import org.jetbrains.annotations.*; - -import java.io.*; -import java.util.*; - -/** - * Example that shows usage of {@link IgniteFs} API. It starts a node with {@code IgniteFs} - * configured and performs several file system operations (create, write, append, read and delete - * files, create, list and delete directories). - * <p> - * Remote nodes should always be started with configuration file which includes - * IgniteFs: {@code 'ignite.sh examples/config/filesystem/example-ignitefs.xml'}. - * <p> - * Alternatively you can run {@link IgniteFsNodeStartup} in another JVM which will start - * node with {@code examples/config/filesystem/example-ignitefs.xml} configuration. - */ -public final class IgniteFsExample { - /** - * Executes example. - * - * @param args Command line arguments, none required. - * @throws Exception If example execution failed. - */ - public static void main(String[] args) throws Exception { - Ignite ignite = Ignition.start("examples/config/filesystem/example-ignitefs.xml"); - - System.out.println(); - System.out.println(">>> Ignitefs example started."); - - try { - // Get an instance of Ignite File System. - org.apache.ignite.IgniteFs fs = ignite.fileSystem("ignitefs"); - - // Working directory path. - IgniteFsPath workDir = new IgniteFsPath("/examples/fs"); - - // Cleanup working directory. - delete(fs, workDir); - - // Create empty working directory. - mkdirs(fs, workDir); - - // Print information for working directory. - printInfo(fs, workDir); - - // File path. - IgniteFsPath filePath = new IgniteFsPath(workDir, "file.txt"); - - // Create file. - create(fs, filePath, new byte[] {1, 2, 3}); - - // Print information for file. - printInfo(fs, filePath); - - // Append more data to previously created file. - append(fs, filePath, new byte[] {4, 5}); - - // Print information for file. - printInfo(fs, filePath); - - // Read data from file. - read(fs, filePath); - - // Delete file. - delete(fs, filePath); - - // Print information for file. - printInfo(fs, filePath); - - // Create several files. - for (int i = 0; i < 5; i++) - create(fs, new IgniteFsPath(workDir, "file-" + i + ".txt"), null); - - list(fs, workDir); - } - finally { - Ignition.stop(false); - } - } - - /** - * Deletes file or directory. If directory - * is not empty, it's deleted recursively. - * - * @param fs IgniteFs. - * @param path File or directory path. - * @throws IgniteException In case of error. - */ - private static void delete(org.apache.ignite.IgniteFs fs, IgniteFsPath path) throws IgniteException { - assert fs != null; - assert path != null; - - if (fs.exists(path)) { - boolean isFile = fs.info(path).isFile(); - - try { - fs.delete(path, true); - - System.out.println(); - System.out.println(">>> Deleted " + (isFile ? "file" : "directory") + ": " + path); - } - catch (IgniteFsException e) { - System.out.println(); - System.out.println(">>> Failed to delete " + (isFile ? "file" : "directory") + " [path=" + path + - ", msg=" + e.getMessage() + ']'); - } - } - else { - System.out.println(); - System.out.println(">>> Won't delete file or directory (doesn't exist): " + path); - } - } - - /** - * Creates directories. - * - * @param fs IgniteFs. - * @param path Directory path. - * @throws IgniteException In case of error. - */ - private static void mkdirs(org.apache.ignite.IgniteFs fs, IgniteFsPath path) throws IgniteException { - assert fs != null; - assert path != null; - - try { - fs.mkdirs(path); - - System.out.println(); - System.out.println(">>> Created directory: " + path); - } - catch (IgniteFsException e) { - System.out.println(); - System.out.println(">>> Failed to create a directory [path=" + path + ", msg=" + e.getMessage() + ']'); - } - - System.out.println(); - } - - /** - * Creates file and writes provided data to it. - * - * @param fs IgniteFs. - * @param path File path. - * @param data Data. - * @throws IgniteException If file can't be created. - * @throws IOException If data can't be written. - */ - private static void create(org.apache.ignite.IgniteFs fs, IgniteFsPath path, @Nullable byte[] data) - throws IgniteException, IOException { - assert fs != null; - assert path != null; - - try (OutputStream out = fs.create(path, true)) { - System.out.println(); - System.out.println(">>> Created file: " + path); - - if (data != null) { - out.write(data); - - System.out.println(); - System.out.println(">>> Wrote data to file: " + path); - } - } - - System.out.println(); - } - - /** - * Opens file and appends provided data to it. - * - * @param fs IgniteFs. - * @param path File path. - * @param data Data. - * @throws IgniteException If file can't be created. - * @throws IOException If data can't be written. - */ - private static void append(org.apache.ignite.IgniteFs fs, IgniteFsPath path, byte[] data) throws IgniteException, IOException { - assert fs != null; - assert path != null; - assert data != null; - assert fs.info(path).isFile(); - - try (OutputStream out = fs.append(path, true)) { - System.out.println(); - System.out.println(">>> Opened file: " + path); - - out.write(data); - } - - System.out.println(); - System.out.println(">>> Appended data to file: " + path); - } - - /** - * Opens file and reads it to byte array. - * - * @param fs IgniteFs. - * @param path File path. - * @throws IgniteException If file can't be opened. - * @throws IOException If data can't be read. - */ - private static void read(org.apache.ignite.IgniteFs fs, IgniteFsPath path) throws IgniteException, IOException { - assert fs != null; - assert path != null; - assert fs.info(path).isFile(); - - byte[] data = new byte[(int)fs.info(path).length()]; - - try (IgniteFsInputStream in = fs.open(path)) { - in.read(data); - } - - System.out.println(); - System.out.println(">>> Read data from " + path + ": " + Arrays.toString(data)); - } - - /** - * Lists files in directory. - * - * @param fs IgniteFs. - * @param path Directory path. - * @throws IgniteException In case of error. - */ - private static void list(org.apache.ignite.IgniteFs fs, IgniteFsPath path) throws IgniteException { - assert fs != null; - assert path != null; - assert fs.info(path).isDirectory(); - - Collection<IgniteFsPath> files = fs.listPaths(path); - - if (files.isEmpty()) { - System.out.println(); - System.out.println(">>> No files in directory: " + path); - } - else { - System.out.println(); - System.out.println(">>> List of files in directory: " + path); - - for (IgniteFsPath f : files) - System.out.println(">>> " + f.name()); - } - - System.out.println(); - } - - /** - * Prints information for file or directory. - * - * @param fs IgniteFs. - * @param path File or directory path. - * @throws IgniteException In case of error. - */ - private static void printInfo(org.apache.ignite.IgniteFs fs, IgniteFsPath path) throws IgniteException { - System.out.println(); - System.out.println("Information for " + path + ": " + fs.info(path)); - } -} http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/252ccdff/examples/src/main/java/org/apache/ignite/examples/fs/IgniteFsMapReduceExample.java ---------------------------------------------------------------------- diff --git a/examples/src/main/java/org/apache/ignite/examples/fs/IgniteFsMapReduceExample.java b/examples/src/main/java/org/apache/ignite/examples/fs/IgniteFsMapReduceExample.java deleted file mode 100644 index 07fec09..0000000 --- a/examples/src/main/java/org/apache/ignite/examples/fs/IgniteFsMapReduceExample.java +++ /dev/null @@ -1,249 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ignite.examples.fs; - -import org.apache.ignite.*; -import org.apache.ignite.compute.*; -import org.apache.ignite.ignitefs.*; -import org.apache.ignite.ignitefs.mapreduce.*; -import org.apache.ignite.ignitefs.mapreduce.records.*; - -import java.io.*; -import java.util.*; - -/** - * Example that shows how to use {@link IgniteFsTask} to find lines matching particular pattern in the file in pretty - * the same way as {@code grep} command does. - * <p> - * Remote nodes should always be started with configuration file which includes - * IgniteFs: {@code 'ignite.sh examples/config/filesystem/example-ignitefs.xml'}. - * <p> - * Alternatively you can run {@link IgniteFsNodeStartup} in another JVM which will start - * node with {@code examples/config/filesystem/example-ignitefs.xml} configuration. - */ -public class IgniteFsMapReduceExample { - /** - * Executes example. - * - * @param args Command line arguments. First argument is file name, second argument is regex to look for. - * @throws Exception If failed. - */ - public static void main(String[] args) throws Exception { - if (args.length == 0) - System.out.println("Please provide file name and regular expression."); - else if (args.length == 1) - System.out.println("Please provide regular expression."); - else { - try (Ignite ignite = Ignition.start("examples/config/filesystem/example-ignitefs.xml")) { - System.out.println(); - System.out.println(">>> IgniteFs map reduce example started."); - - // Prepare arguments. - String fileName = args[0]; - - File file = new File(fileName); - - String regexStr = args[1]; - - // Get an instance of Ignite File System. - IgniteFs fs = ignite.fileSystem("ignitefs"); - - // Working directory path. - IgniteFsPath workDir = new IgniteFsPath("/examples/fs"); - - // Write file to GGFS. - IgniteFsPath fsPath = new IgniteFsPath(workDir, file.getName()); - - writeFile(fs, fsPath, file); - - Collection<Line> lines = fs.execute(new GrepTask(), IgniteFsNewLineRecordResolver.NEW_LINE, - Collections.singleton(fsPath), regexStr); - - if (lines.isEmpty()) { - System.out.println(); - System.out.println("No lines were found."); - } - else { - for (Line line : lines) - print(line.fileLine()); - } - } - } - } - - /** - * Write file to the Ignite file system. - * - * @param fs Ignite file system. - * @param fsPath Ignite file system path. - * @param file File to write. - * @throws Exception In case of exception. - */ - private static void writeFile(IgniteFs fs, IgniteFsPath fsPath, File file) throws Exception { - System.out.println(); - System.out.println("Copying file to IgniteFs: " + file); - - try ( - IgniteFsOutputStream os = fs.create(fsPath, true); - FileInputStream fis = new FileInputStream(file) - ) { - byte[] buf = new byte[2048]; - - int read = fis.read(buf); - - while (read != -1) { - os.write(buf, 0, read); - - read = fis.read(buf); - } - } - } - - /** - * Print particular string. - * - * @param str String. - */ - private static void print(String str) { - System.out.println(">>> " + str); - } - - /** - * Grep task. - */ - private static class GrepTask extends IgniteFsTask<String, Collection<Line>> { - /** {@inheritDoc} */ - @Override public IgniteFsJob createJob(IgniteFsPath path, IgniteFsFileRange range, - IgniteFsTaskArgs<String> args) { - return new GrepJob(args.userArgument()); - } - - /** {@inheritDoc} */ - @Override public Collection<Line> reduce(List<ComputeJobResult> results) { - Collection<Line> lines = new TreeSet<>(new Comparator<Line>() { - @Override public int compare(Line line1, Line line2) { - return line1.rangePosition() < line2.rangePosition() ? -1 : - line1.rangePosition() > line2.rangePosition() ? 1 : line1.lineIndex() - line2.lineIndex(); - } - }); - - for (ComputeJobResult res : results) { - if (res.getException() != null) - throw res.getException(); - - Collection<Line> line = res.getData(); - - if (line != null) - lines.addAll(line); - } - - return lines; - } - } - - /** - * Grep job. - */ - private static class GrepJob extends IgniteFsInputStreamJobAdapter { - /** Regex string. */ - private final String regex; - - /** - * Constructor. - * - * @param regex Regex string. - */ - private GrepJob(String regex) { - this.regex = regex; - } - - /** {@inheritDoc} */ - @Override public Object execute(IgniteFs igniteFs, IgniteFsRangeInputStream in) throws IgniteException, IOException { - Collection<Line> res = null; - - long start = in.startOffset(); - - try (BufferedReader br = new BufferedReader(new InputStreamReader(in))) { - int ctr = 0; - - String line = br.readLine(); - - while (line != null) { - if (line.matches(".*" + regex + ".*")) { - if (res == null) - res = new HashSet<>(); - - res.add(new Line(start, ctr++, line)); - } - - line = br.readLine(); - } - } - - return res; - } - } - - /** - * Single file line with it's position. - */ - private static class Line { - /** Line start position in the file. */ - private long rangePos; - - /** Matching line index within the range. */ - private final int lineIdx; - - /** File line. */ - private String line; - - /** - * Constructor. - * - * @param rangePos Range position. - * @param lineIdx Matching line index within the range. - * @param line File line. - */ - private Line(long rangePos, int lineIdx, String line) { - this.rangePos = rangePos; - this.lineIdx = lineIdx; - this.line = line; - } - - /** - * @return Range position. - */ - public long rangePosition() { - return rangePos; - } - - /** - * @return Matching line index within the range. - */ - public int lineIndex() { - return lineIdx; - } - - /** - * @return File line. - */ - public String fileLine() { - return line; - } - } -} http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/252ccdff/examples/src/main/java/org/apache/ignite/examples/fs/IgniteFsNodeStartup.java ---------------------------------------------------------------------- diff --git a/examples/src/main/java/org/apache/ignite/examples/fs/IgniteFsNodeStartup.java b/examples/src/main/java/org/apache/ignite/examples/fs/IgniteFsNodeStartup.java deleted file mode 100644 index 0e7d19f..0000000 --- a/examples/src/main/java/org/apache/ignite/examples/fs/IgniteFsNodeStartup.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ignite.examples.fs; - -import org.apache.ignite.*; - -/** - * Starts up an empty node with IgniteFs configuration. - * You can also start a stand-alone Ignite instance by passing the path - * to configuration file to {@code 'ignite.{sh|bat}'} script, like so: - * {@code 'ignite.sh examples/config/filesystem/example-ignitefs.xml'}. - * <p> - * The difference is that running this class from IDE adds all example classes to classpath - * but running from command line doesn't. - */ -public class IgniteFsNodeStartup { - /** - * Start up an empty node with specified cache configuration. - * - * @param args Command line arguments, none required. - * @throws IgniteException If example execution failed. - */ - public static void main(String[] args) throws IgniteException { - Ignition.start("examples/config/filesystem/example-ignitefs.xml"); - } -} http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/252ccdff/examples/src/main/java/org/apache/ignite/examples/fs/package.html ---------------------------------------------------------------------- diff --git a/examples/src/main/java/org/apache/ignite/examples/fs/package.html b/examples/src/main/java/org/apache/ignite/examples/fs/package.html deleted file mode 100644 index 2414651..0000000 --- a/examples/src/main/java/org/apache/ignite/examples/fs/package.html +++ /dev/null @@ -1,24 +0,0 @@ -<!-- - Licensed to the Apache Software Foundation (ASF) under one or more - contributor license agreements. See the NOTICE file distributed with - this work for additional information regarding copyright ownership. - The ASF licenses this file to You under the Apache License, Version 2.0 - (the "License"); you may not use this file except in compliance with - the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. ---> - -<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd";> -<html> -<body> - <!-- Package description. --> - Demonstrates using of Ignite File System. -</body> -</html> http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/252ccdff/examples/src/main/java/org/apache/ignite/examples/ignitefs/IgniteFsExample.java ---------------------------------------------------------------------- diff --git a/examples/src/main/java/org/apache/ignite/examples/ignitefs/IgniteFsExample.java b/examples/src/main/java/org/apache/ignite/examples/ignitefs/IgniteFsExample.java new file mode 100644 index 0000000..49aa00f --- /dev/null +++ b/examples/src/main/java/org/apache/ignite/examples/ignitefs/IgniteFsExample.java @@ -0,0 +1,278 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.examples.ignitefs; + +import org.apache.ignite.*; +import org.apache.ignite.ignitefs.*; +import org.jetbrains.annotations.*; + +import java.io.*; +import java.util.*; + +/** + * Example that shows usage of {@link IgniteFs} API. It starts a node with {@code IgniteFs} + * configured and performs several file system operations (create, write, append, read and delete + * files, create, list and delete directories). + * <p> + * Remote nodes should always be started with configuration file which includes + * IgniteFs: {@code 'ignite.sh examples/config/filesystem/example-ignitefs.xml'}. + * <p> + * Alternatively you can run {@link IgniteFsNodeStartup} in another JVM which will start + * node with {@code examples/config/filesystem/example-ignitefs.xml} configuration. + */ +public final class IgniteFsExample { + /** + * Executes example. + * + * @param args Command line arguments, none required. + * @throws Exception If example execution failed. + */ + public static void main(String[] args) throws Exception { + Ignite ignite = Ignition.start("examples/config/filesystem/example-ignitefs.xml"); + + System.out.println(); + System.out.println(">>> IgniteFs example started."); + + try { + // Get an instance of Ignite File System. + org.apache.ignite.IgniteFs fs = ignite.fileSystem("ignitefs"); + + // Working directory path. + IgniteFsPath workDir = new IgniteFsPath("/examples/fs"); + + // Cleanup working directory. + delete(fs, workDir); + + // Create empty working directory. + mkdirs(fs, workDir); + + // Print information for working directory. + printInfo(fs, workDir); + + // File path. + IgniteFsPath filePath = new IgniteFsPath(workDir, "file.txt"); + + // Create file. + create(fs, filePath, new byte[] {1, 2, 3}); + + // Print information for file. + printInfo(fs, filePath); + + // Append more data to previously created file. + append(fs, filePath, new byte[] {4, 5}); + + // Print information for file. + printInfo(fs, filePath); + + // Read data from file. + read(fs, filePath); + + // Delete file. + delete(fs, filePath); + + // Print information for file. + printInfo(fs, filePath); + + // Create several files. + for (int i = 0; i < 5; i++) + create(fs, new IgniteFsPath(workDir, "file-" + i + ".txt"), null); + + list(fs, workDir); + } + finally { + Ignition.stop(false); + } + } + + /** + * Deletes file or directory. If directory + * is not empty, it's deleted recursively. + * + * @param fs IgniteFs. + * @param path File or directory path. + * @throws IgniteException In case of error. + */ + private static void delete(org.apache.ignite.IgniteFs fs, IgniteFsPath path) throws IgniteException { + assert fs != null; + assert path != null; + + if (fs.exists(path)) { + boolean isFile = fs.info(path).isFile(); + + try { + fs.delete(path, true); + + System.out.println(); + System.out.println(">>> Deleted " + (isFile ? "file" : "directory") + ": " + path); + } + catch (IgniteFsException e) { + System.out.println(); + System.out.println(">>> Failed to delete " + (isFile ? "file" : "directory") + " [path=" + path + + ", msg=" + e.getMessage() + ']'); + } + } + else { + System.out.println(); + System.out.println(">>> Won't delete file or directory (doesn't exist): " + path); + } + } + + /** + * Creates directories. + * + * @param fs IgniteFs. + * @param path Directory path. + * @throws IgniteException In case of error. + */ + private static void mkdirs(org.apache.ignite.IgniteFs fs, IgniteFsPath path) throws IgniteException { + assert fs != null; + assert path != null; + + try { + fs.mkdirs(path); + + System.out.println(); + System.out.println(">>> Created directory: " + path); + } + catch (IgniteFsException e) { + System.out.println(); + System.out.println(">>> Failed to create a directory [path=" + path + ", msg=" + e.getMessage() + ']'); + } + + System.out.println(); + } + + /** + * Creates file and writes provided data to it. + * + * @param fs IgniteFs. + * @param path File path. + * @param data Data. + * @throws IgniteException If file can't be created. + * @throws IOException If data can't be written. + */ + private static void create(org.apache.ignite.IgniteFs fs, IgniteFsPath path, @Nullable byte[] data) + throws IgniteException, IOException { + assert fs != null; + assert path != null; + + try (OutputStream out = fs.create(path, true)) { + System.out.println(); + System.out.println(">>> Created file: " + path); + + if (data != null) { + out.write(data); + + System.out.println(); + System.out.println(">>> Wrote data to file: " + path); + } + } + + System.out.println(); + } + + /** + * Opens file and appends provided data to it. + * + * @param fs IgniteFs. + * @param path File path. + * @param data Data. + * @throws IgniteException If file can't be created. + * @throws IOException If data can't be written. + */ + private static void append(org.apache.ignite.IgniteFs fs, IgniteFsPath path, byte[] data) throws IgniteException, IOException { + assert fs != null; + assert path != null; + assert data != null; + assert fs.info(path).isFile(); + + try (OutputStream out = fs.append(path, true)) { + System.out.println(); + System.out.println(">>> Opened file: " + path); + + out.write(data); + } + + System.out.println(); + System.out.println(">>> Appended data to file: " + path); + } + + /** + * Opens file and reads it to byte array. + * + * @param fs IgniteFs. + * @param path File path. + * @throws IgniteException If file can't be opened. + * @throws IOException If data can't be read. + */ + private static void read(org.apache.ignite.IgniteFs fs, IgniteFsPath path) throws IgniteException, IOException { + assert fs != null; + assert path != null; + assert fs.info(path).isFile(); + + byte[] data = new byte[(int)fs.info(path).length()]; + + try (IgniteFsInputStream in = fs.open(path)) { + in.read(data); + } + + System.out.println(); + System.out.println(">>> Read data from " + path + ": " + Arrays.toString(data)); + } + + /** + * Lists files in directory. + * + * @param fs IgniteFs. + * @param path Directory path. + * @throws IgniteException In case of error. + */ + private static void list(org.apache.ignite.IgniteFs fs, IgniteFsPath path) throws IgniteException { + assert fs != null; + assert path != null; + assert fs.info(path).isDirectory(); + + Collection<IgniteFsPath> files = fs.listPaths(path); + + if (files.isEmpty()) { + System.out.println(); + System.out.println(">>> No files in directory: " + path); + } + else { + System.out.println(); + System.out.println(">>> List of files in directory: " + path); + + for (IgniteFsPath f : files) + System.out.println(">>> " + f.name()); + } + + System.out.println(); + } + + /** + * Prints information for file or directory. + * + * @param fs IgniteFs. + * @param path File or directory path. + * @throws IgniteException In case of error. + */ + private static void printInfo(org.apache.ignite.IgniteFs fs, IgniteFsPath path) throws IgniteException { + System.out.println(); + System.out.println("Information for " + path + ": " + fs.info(path)); + } +} http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/252ccdff/examples/src/main/java/org/apache/ignite/examples/ignitefs/IgniteFsMapReduceExample.java ---------------------------------------------------------------------- diff --git a/examples/src/main/java/org/apache/ignite/examples/ignitefs/IgniteFsMapReduceExample.java b/examples/src/main/java/org/apache/ignite/examples/ignitefs/IgniteFsMapReduceExample.java new file mode 100644 index 0000000..a19f2d0 --- /dev/null +++ b/examples/src/main/java/org/apache/ignite/examples/ignitefs/IgniteFsMapReduceExample.java @@ -0,0 +1,249 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.examples.ignitefs; + +import org.apache.ignite.*; +import org.apache.ignite.compute.*; +import org.apache.ignite.ignitefs.*; +import org.apache.ignite.ignitefs.mapreduce.*; +import org.apache.ignite.ignitefs.mapreduce.records.*; + +import java.io.*; +import java.util.*; + +/** + * Example that shows how to use {@link IgniteFsTask} to find lines matching particular pattern in the file in pretty + * the same way as {@code grep} command does. + * <p> + * Remote nodes should always be started with configuration file which includes + * IgniteFs: {@code 'ignite.sh examples/config/filesystem/example-ignitefs.xml'}. + * <p> + * Alternatively you can run {@link IgniteFsNodeStartup} in another JVM which will start + * node with {@code examples/config/filesystem/example-ignitefs.xml} configuration. + */ +public class IgniteFsMapReduceExample { + /** + * Executes example. + * + * @param args Command line arguments. First argument is file name, second argument is regex to look for. + * @throws Exception If failed. + */ + public static void main(String[] args) throws Exception { + if (args.length == 0) + System.out.println("Please provide file name and regular expression."); + else if (args.length == 1) + System.out.println("Please provide regular expression."); + else { + try (Ignite ignite = Ignition.start("examples/config/filesystem/example-ignitefs.xml")) { + System.out.println(); + System.out.println(">>> IgniteFs map reduce example started."); + + // Prepare arguments. + String fileName = args[0]; + + File file = new File(fileName); + + String regexStr = args[1]; + + // Get an instance of Ignite File System. + IgniteFs fs = ignite.fileSystem("ignitefs"); + + // Working directory path. + IgniteFsPath workDir = new IgniteFsPath("/examples/fs"); + + // Write file to GGFS. + IgniteFsPath fsPath = new IgniteFsPath(workDir, file.getName()); + + writeFile(fs, fsPath, file); + + Collection<Line> lines = fs.execute(new GrepTask(), IgniteFsNewLineRecordResolver.NEW_LINE, + Collections.singleton(fsPath), regexStr); + + if (lines.isEmpty()) { + System.out.println(); + System.out.println("No lines were found."); + } + else { + for (Line line : lines) + print(line.fileLine()); + } + } + } + } + + /** + * Write file to the Ignite file system. + * + * @param fs Ignite file system. + * @param fsPath Ignite file system path. + * @param file File to write. + * @throws Exception In case of exception. + */ + private static void writeFile(IgniteFs fs, IgniteFsPath fsPath, File file) throws Exception { + System.out.println(); + System.out.println("Copying file to IgniteFs: " + file); + + try ( + IgniteFsOutputStream os = fs.create(fsPath, true); + FileInputStream fis = new FileInputStream(file) + ) { + byte[] buf = new byte[2048]; + + int read = fis.read(buf); + + while (read != -1) { + os.write(buf, 0, read); + + read = fis.read(buf); + } + } + } + + /** + * Print particular string. + * + * @param str String. + */ + private static void print(String str) { + System.out.println(">>> " + str); + } + + /** + * Grep task. + */ + private static class GrepTask extends IgniteFsTask<String, Collection<Line>> { + /** {@inheritDoc} */ + @Override public IgniteFsJob createJob(IgniteFsPath path, IgniteFsFileRange range, + IgniteFsTaskArgs<String> args) { + return new GrepJob(args.userArgument()); + } + + /** {@inheritDoc} */ + @Override public Collection<Line> reduce(List<ComputeJobResult> results) { + Collection<Line> lines = new TreeSet<>(new Comparator<Line>() { + @Override public int compare(Line line1, Line line2) { + return line1.rangePosition() < line2.rangePosition() ? -1 : + line1.rangePosition() > line2.rangePosition() ? 1 : line1.lineIndex() - line2.lineIndex(); + } + }); + + for (ComputeJobResult res : results) { + if (res.getException() != null) + throw res.getException(); + + Collection<Line> line = res.getData(); + + if (line != null) + lines.addAll(line); + } + + return lines; + } + } + + /** + * Grep job. + */ + private static class GrepJob extends IgniteFsInputStreamJobAdapter { + /** Regex string. */ + private final String regex; + + /** + * Constructor. + * + * @param regex Regex string. + */ + private GrepJob(String regex) { + this.regex = regex; + } + + /** {@inheritDoc} */ + @Override public Object execute(IgniteFs igniteFs, IgniteFsRangeInputStream in) throws IgniteException, IOException { + Collection<Line> res = null; + + long start = in.startOffset(); + + try (BufferedReader br = new BufferedReader(new InputStreamReader(in))) { + int ctr = 0; + + String line = br.readLine(); + + while (line != null) { + if (line.matches(".*" + regex + ".*")) { + if (res == null) + res = new HashSet<>(); + + res.add(new Line(start, ctr++, line)); + } + + line = br.readLine(); + } + } + + return res; + } + } + + /** + * Single file line with it's position. + */ + private static class Line { + /** Line start position in the file. */ + private long rangePos; + + /** Matching line index within the range. */ + private final int lineIdx; + + /** File line. */ + private String line; + + /** + * Constructor. + * + * @param rangePos Range position. + * @param lineIdx Matching line index within the range. + * @param line File line. + */ + private Line(long rangePos, int lineIdx, String line) { + this.rangePos = rangePos; + this.lineIdx = lineIdx; + this.line = line; + } + + /** + * @return Range position. + */ + public long rangePosition() { + return rangePos; + } + + /** + * @return Matching line index within the range. + */ + public int lineIndex() { + return lineIdx; + } + + /** + * @return File line. + */ + public String fileLine() { + return line; + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/252ccdff/examples/src/main/java/org/apache/ignite/examples/ignitefs/IgniteFsNodeStartup.java ---------------------------------------------------------------------- diff --git a/examples/src/main/java/org/apache/ignite/examples/ignitefs/IgniteFsNodeStartup.java b/examples/src/main/java/org/apache/ignite/examples/ignitefs/IgniteFsNodeStartup.java new file mode 100644 index 0000000..d331077 --- /dev/null +++ b/examples/src/main/java/org/apache/ignite/examples/ignitefs/IgniteFsNodeStartup.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.examples.ignitefs; + +import org.apache.ignite.*; + +/** + * Starts up an empty node with IgniteFs configuration. + * You can also start a stand-alone Ignite instance by passing the path + * to configuration file to {@code 'ignite.{sh|bat}'} script, like so: + * {@code 'ignite.sh examples/config/filesystem/example-ignitefs.xml'}. + * <p> + * The difference is that running this class from IDE adds all example classes to classpath + * but running from command line doesn't. + */ +public class IgniteFsNodeStartup { + /** + * Start up an empty node with specified cache configuration. + * + * @param args Command line arguments, none required. + * @throws IgniteException If example execution failed. + */ + public static void main(String[] args) throws IgniteException { + Ignition.start("examples/config/filesystem/example-ignitefs.xml"); + } +} http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/252ccdff/examples/src/main/java/org/apache/ignite/examples/ignitefs/package.html ---------------------------------------------------------------------- diff --git a/examples/src/main/java/org/apache/ignite/examples/ignitefs/package.html b/examples/src/main/java/org/apache/ignite/examples/ignitefs/package.html new file mode 100644 index 0000000..1257c60 --- /dev/null +++ b/examples/src/main/java/org/apache/ignite/examples/ignitefs/package.html @@ -0,0 +1,24 @@ +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd";> +<html> +<body> + <!-- Package description. --> + Demonstrates usage of Ignite File System. +</body> +</html> http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/252ccdff/examples/src/test/java/org/apache/ignite/examples/IgniteFsExamplesSelfTest.java ---------------------------------------------------------------------- diff --git a/examples/src/test/java/org/apache/ignite/examples/IgniteFsExamplesSelfTest.java b/examples/src/test/java/org/apache/ignite/examples/IgniteFsExamplesSelfTest.java index 790faa5..4566496 100644 --- a/examples/src/test/java/org/apache/ignite/examples/IgniteFsExamplesSelfTest.java +++ b/examples/src/test/java/org/apache/ignite/examples/IgniteFsExamplesSelfTest.java @@ -17,7 +17,7 @@ package org.apache.ignite.examples; -import org.apache.ignite.examples.fs.*; +import org.apache.ignite.examples.ignitefs.*; import org.apache.ignite.internal.util.typedef.internal.*; import org.apache.ignite.testframework.junits.common.*;
