IGNITE-386: Squashed changes.
Project: http://git-wip-us.apache.org/repos/asf/incubator-ignite/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ignite/commit/6423cf02 Tree: http://git-wip-us.apache.org/repos/asf/incubator-ignite/tree/6423cf02 Diff: http://git-wip-us.apache.org/repos/asf/incubator-ignite/diff/6423cf02 Branch: refs/heads/ignite-141 Commit: 6423cf027e9639b5260bd0cfd6a5e8eb82cd7acd Parents: 162760b Author: vozerov-gridgain <voze...@gridgain.com> Authored: Thu Mar 5 10:15:16 2015 +0300 Committer: vozerov-gridgain <voze...@gridgain.com> Committed: Thu Mar 5 10:15:22 2015 +0300 ---------------------------------------------------------------------- bin/setup-hadoop.bat | 2 +- bin/setup-hadoop.sh | 2 +- config/hadoop/default-config.xml | 10 +- docs/core-site.ignite.xml | 6 +- examples/config/filesystem/core-site.xml | 4 +- examples/config/filesystem/example-igfs.xml | 4 +- .../ignite/examples/igfs/IgfsExample.java | 18 +- .../examples/igfs/IgfsMapReduceExample.java | 6 +- .../src/main/java/org/apache/ignite/Ignite.java | 13 +- .../org/apache/ignite/IgniteFileSystem.java | 457 ++++ .../main/java/org/apache/ignite/IgniteFs.java | 346 --- .../configuration/FileSystemConfiguration.java | 805 ++++++ .../configuration/HadoopConfiguration.java | 173 ++ .../ignite/configuration/IgfsConfiguration.java | 807 ------ .../configuration/IgniteConfiguration.java | 19 +- .../main/java/org/apache/ignite/igfs/Igfs.java | 216 -- .../apache/ignite/igfs/IgfsBlockLocation.java | 2 +- .../java/org/apache/ignite/igfs/IgfsFile.java | 2 +- .../org/apache/ignite/igfs/IgfsInputStream.java | 6 +- .../org/apache/ignite/igfs/IgfsMetrics.java | 2 +- .../java/org/apache/ignite/igfs/IgfsMode.java | 8 +- .../java/org/apache/ignite/igfs/IgfsReader.java | 38 - .../mapreduce/IgfsInputStreamJobAdapter.java | 4 +- .../apache/ignite/igfs/mapreduce/IgfsJob.java | 4 +- .../igfs/mapreduce/IgfsRecordResolver.java | 2 +- .../apache/ignite/igfs/mapreduce/IgfsTask.java | 4 +- .../ignite/igfs/mapreduce/IgfsTaskArgs.java | 2 +- .../IgfsByteDelimiterRecordResolver.java | 3 +- .../records/IgfsFixedLengthRecordResolver.java | 2 +- .../igfs/secondary/IgfsSecondaryFileSystem.java | 201 ++ ...fsSecondaryFileSystemPositionedReadable.java | 38 + .../apache/ignite/igfs/secondary/package.html | 24 + .../ignite/internal/GridKernalContext.java | 2 +- .../ignite/internal/GridKernalContextImpl.java | 8 +- .../ignite/internal/IgniteComponentType.java | 4 +- .../org/apache/ignite/internal/IgniteEx.java | 4 +- .../apache/ignite/internal/IgniteKernal.java | 14 +- .../org/apache/ignite/internal/IgnitionEx.java | 8 +- .../processors/cache/GridCacheAdapter.java | 4 +- .../processors/cache/GridCacheProcessor.java | 4 +- .../processors/cache/GridCacheUtils.java | 4 +- .../internal/processors/hadoop/GridHadoop.java | 86 - .../hadoop/GridHadoopConfiguration.java | 172 -- .../processors/hadoop/GridHadoopCounter.java | 44 - .../hadoop/GridHadoopCounterWriter.java | 36 - .../processors/hadoop/GridHadoopCounters.java | 49 - .../processors/hadoop/GridHadoopFileBlock.java | 162 -- .../processors/hadoop/GridHadoopInputSplit.java | 54 - .../processors/hadoop/GridHadoopJob.java | 102 - .../processors/hadoop/GridHadoopJobId.java | 103 - .../processors/hadoop/GridHadoopJobInfo.java | 83 - .../processors/hadoop/GridHadoopJobPhase.java | 38 - .../hadoop/GridHadoopJobProperty.java | 138 - .../processors/hadoop/GridHadoopJobStatus.java | 207 -- .../hadoop/GridHadoopMapReducePlan.java | 80 - .../hadoop/GridHadoopMapReducePlanner.java | 40 - .../hadoop/GridHadoopPartitioner.java | 33 - .../hadoop/GridHadoopSerialization.java | 54 - .../processors/hadoop/GridHadoopTask.java | 72 - .../hadoop/GridHadoopTaskContext.java | 189 -- .../processors/hadoop/GridHadoopTaskInfo.java | 153 -- .../processors/hadoop/GridHadoopTaskInput.java | 55 - .../processors/hadoop/GridHadoopTaskOutput.java | 40 - .../processors/hadoop/GridHadoopTaskType.java | 56 - .../internal/processors/hadoop/Hadoop.java | 88 + .../processors/hadoop/HadoopFileBlock.java | 162 ++ .../processors/hadoop/HadoopInputSplit.java | 54 + .../internal/processors/hadoop/HadoopJob.java | 102 + .../internal/processors/hadoop/HadoopJobId.java | 103 + .../processors/hadoop/HadoopJobInfo.java | 83 + .../processors/hadoop/HadoopJobPhase.java | 38 + .../processors/hadoop/HadoopJobProperty.java | 138 + .../processors/hadoop/HadoopJobStatus.java | 207 ++ .../processors/hadoop/HadoopMapReducePlan.java | 80 + .../hadoop/HadoopMapReducePlanner.java | 40 + .../processors/hadoop/HadoopNoopProcessor.java | 76 + .../processors/hadoop/HadoopPartitioner.java | 33 + .../hadoop/HadoopProcessorAdapter.java | 96 + .../processors/hadoop/HadoopSerialization.java | 54 + .../internal/processors/hadoop/HadoopTask.java | 72 + .../processors/hadoop/HadoopTaskContext.java | 190 ++ .../processors/hadoop/HadoopTaskInfo.java | 153 ++ .../processors/hadoop/HadoopTaskInput.java | 55 + .../processors/hadoop/HadoopTaskOutput.java | 40 + .../processors/hadoop/HadoopTaskType.java | 56 + .../hadoop/IgniteHadoopNoopProcessor.java | 74 - .../hadoop/IgniteHadoopProcessorAdapter.java | 94 - .../hadoop/counter/HadoopCounter.java | 44 + .../hadoop/counter/HadoopCounterWriter.java | 37 + .../hadoop/counter/HadoopCounters.java | 49 + .../internal/processors/igfs/IgfsAsyncImpl.java | 9 +- .../internal/processors/igfs/IgfsContext.java | 6 +- .../processors/igfs/IgfsDataManager.java | 5 +- .../ignite/internal/processors/igfs/IgfsEx.java | 28 +- .../internal/processors/igfs/IgfsFileInfo.java | 4 +- .../internal/processors/igfs/IgfsImpl.java | 42 +- .../processors/igfs/IgfsInputStreamAdapter.java | 4 +- .../processors/igfs/IgfsInputStreamImpl.java | 5 +- .../internal/processors/igfs/IgfsJobImpl.java | 2 +- .../processors/igfs/IgfsMetaManager.java | 27 +- .../processors/igfs/IgfsNoopProcessor.java | 4 +- .../processors/igfs/IgfsOutputStreamImpl.java | 2 +- .../internal/processors/igfs/IgfsProcessor.java | 26 +- .../processors/igfs/IgfsProcessorAdapter.java | 4 +- .../igfs/IgfsSecondaryFileSystemImpl.java | 121 + .../IgfsSecondaryInputStreamDescriptor.java | 8 +- .../processors/igfs/IgfsServerManager.java | 4 +- .../ignite/internal/visor/igfs/VisorIgfs.java | 4 +- .../visor/igfs/VisorIgfsProfilerClearTask.java | 2 +- .../visor/node/VisorGridConfiguration.java | 2 +- .../visor/node/VisorIgfsConfiguration.java | 9 +- .../visor/node/VisorNodeDataCollectorJob.java | 2 +- .../internal/visor/util/VisorTaskUtils.java | 4 +- .../hadoop/core-site-loopback-secondary.xml | 4 +- .../test/config/hadoop/core-site-loopback.xml | 4 +- .../test/config/hadoop/core-site-secondary.xml | 4 +- .../core/src/test/config/hadoop/core-site.xml | 4 +- modules/core/src/test/config/igfs-loopback.xml | 4 +- modules/core/src/test/config/igfs-shmem.xml | 4 +- .../ignite/igfs/IgfsEventsAbstractSelfTest.java | 10 +- .../igfs/IgfsFragmentizerAbstractSelfTest.java | 4 +- .../ignite/igfs/IgfsFragmentizerSelfTest.java | 8 +- .../igfs/IgfsFragmentizerTopologySelfTest.java | 2 +- ...heIgfsPerBlockLruEvictionPolicySelfTest.java | 485 ---- .../processors/igfs/IgfsAbstractSelfTest.java | 85 +- ...sCachePerBlockLruEvictionPolicySelfTest.java | 485 ++++ .../processors/igfs/IgfsCacheSelfTest.java | 4 +- .../igfs/IgfsDataManagerSelfTest.java | 5 +- .../igfs/IgfsDualAbstractSelfTest.java | 12 +- .../igfs/IgfsMetaManagerSelfTest.java | 4 +- .../processors/igfs/IgfsMetricsSelfTest.java | 22 +- .../processors/igfs/IgfsModesSelfTest.java | 14 +- .../processors/igfs/IgfsProcessorSelfTest.java | 10 +- .../igfs/IgfsProcessorValidationSelfTest.java | 50 +- ...IpcEndpointRegistrationAbstractSelfTest.java | 10 +- ...dpointRegistrationOnLinuxAndMacSelfTest.java | 4 +- ...pcEndpointRegistrationOnWindowsSelfTest.java | 2 +- .../processors/igfs/IgfsSizeSelfTest.java | 4 +- .../processors/igfs/IgfsStreamsSelfTest.java | 14 +- .../processors/igfs/IgfsTaskSelfTest.java | 10 +- .../IgfsAbstractRecordResolverSelfTest.java | 6 +- .../ipc/shmem/IpcSharedMemoryNodeStartup.java | 4 +- .../ignite/testframework/junits/IgniteMock.java | 4 +- .../ignite/testsuites/IgniteIgfsTestSuite.java | 2 +- .../client/hadoop/GridHadoopClientProtocol.java | 334 --- .../GridHadoopClientProtocolProvider.java | 137 - .../counter/GridHadoopClientCounterGroup.java | 121 - .../counter/GridHadoopClientCounters.java | 217 -- .../apache/ignite/client/hadoop/package.html | 24 - .../fs/IgniteHadoopFileSystemCounterWriter.java | 93 + .../fs/IgniteHadoopIgfsSecondaryFileSystem.java | 414 +++ .../org/apache/ignite/hadoop/fs/package.html | 24 + .../hadoop/fs/v1/IgniteHadoopFileSystem.java | 1237 +++++++++ .../org/apache/ignite/hadoop/fs/v1/package.html | 24 + .../hadoop/fs/v2/IgniteHadoopFileSystem.java | 982 ++++++++ .../org/apache/ignite/hadoop/fs/v2/package.html | 24 + .../IgniteHadoopClientProtocolProvider.java | 138 + .../mapreduce/IgniteHadoopMapReducePlanner.java | 435 ++++ .../apache/ignite/hadoop/mapreduce/package.html | 24 + .../java/org/apache/ignite/hadoop/package.html | 24 + .../hadoop/IgfsHadoopFileSystemWrapper.java | 414 --- .../igfs/hadoop/IgfsHadoopParameters.java | 94 - .../org/apache/ignite/igfs/hadoop/package.html | 24 - .../igfs/hadoop/v1/IgfsHadoopFileSystem.java | 1234 --------- .../apache/ignite/igfs/hadoop/v1/package.html | 24 - .../igfs/hadoop/v2/IgfsHadoopFileSystem.java | 982 -------- .../apache/ignite/igfs/hadoop/v2/package.html | 24 - .../java/org/apache/ignite/igfs/package.html | 24 - .../ignite/internal/igfs/hadoop/IgfsHadoop.java | 198 -- .../IgfsHadoopCommunicationException.java | 57 - .../igfs/hadoop/IgfsHadoopEndpoint.java | 210 -- .../internal/igfs/hadoop/IgfsHadoopEx.java | 88 - .../igfs/hadoop/IgfsHadoopFSProperties.java | 88 - .../internal/igfs/hadoop/IgfsHadoopFuture.java | 94 - .../internal/igfs/hadoop/IgfsHadoopInProc.java | 409 --- .../igfs/hadoop/IgfsHadoopInputStream.java | 626 ----- .../internal/igfs/hadoop/IgfsHadoopIo.java | 76 - .../internal/igfs/hadoop/IgfsHadoopIpcIo.java | 599 ----- .../igfs/hadoop/IgfsHadoopIpcIoListener.java | 36 - .../igfs/hadoop/IgfsHadoopJclLogger.java | 112 - .../internal/igfs/hadoop/IgfsHadoopOutProc.java | 466 ---- .../igfs/hadoop/IgfsHadoopOutputStream.java | 201 -- .../igfs/hadoop/IgfsHadoopProxyInputStream.java | 335 --- .../hadoop/IgfsHadoopProxyOutputStream.java | 165 -- .../internal/igfs/hadoop/IgfsHadoopReader.java | 104 - .../igfs/hadoop/IgfsHadoopStreamDelegate.java | 96 - .../hadoop/IgfsHadoopStreamEventListener.java | 39 - .../internal/igfs/hadoop/IgfsHadoopUtils.java | 131 - .../internal/igfs/hadoop/IgfsHadoopWrapper.java | 511 ---- .../ignite/internal/igfs/hadoop/package.html | 24 - .../apache/ignite/internal/igfs/package.html | 24 - .../hadoop/GridHadoopClassLoader.java | 552 ---- .../processors/hadoop/GridHadoopComponent.java | 61 - .../processors/hadoop/GridHadoopContext.java | 196 -- .../hadoop/GridHadoopDefaultJobInfo.java | 163 -- .../processors/hadoop/GridHadoopImpl.java | 132 - .../processors/hadoop/GridHadoopSetup.java | 505 ---- .../GridHadoopTaskCancelledException.java | 35 - .../processors/hadoop/GridHadoopUtils.java | 308 --- .../processors/hadoop/HadoopClassLoader.java | 552 ++++ .../processors/hadoop/HadoopComponent.java | 61 + .../processors/hadoop/HadoopContext.java | 197 ++ .../processors/hadoop/HadoopDefaultJobInfo.java | 163 ++ .../internal/processors/hadoop/HadoopImpl.java | 134 + .../hadoop/HadoopMapReduceCounterGroup.java | 121 + .../hadoop/HadoopMapReduceCounters.java | 216 ++ .../processors/hadoop/HadoopProcessor.java | 227 ++ .../internal/processors/hadoop/HadoopSetup.java | 505 ++++ .../hadoop/HadoopTaskCancelledException.java | 35 + .../internal/processors/hadoop/HadoopUtils.java | 308 +++ .../hadoop/IgniteHadoopProcessor.java | 225 -- .../counter/GridHadoopCounterAdapter.java | 128 - .../hadoop/counter/GridHadoopCountersImpl.java | 198 -- .../counter/GridHadoopFSCounterWriter.java | 91 - .../hadoop/counter/GridHadoopLongCounter.java | 92 - .../counter/GridHadoopPerformanceCounter.java | 279 --- .../hadoop/counter/HadoopCounterAdapter.java | 127 + .../hadoop/counter/HadoopCountersImpl.java | 197 ++ .../hadoop/counter/HadoopLongCounter.java | 90 + .../counter/HadoopPerformanceCounter.java | 279 +++ .../fs/GridHadoopDistributedFileSystem.java | 91 - .../hadoop/fs/GridHadoopFileSystemsUtils.java | 57 - .../hadoop/fs/GridHadoopLocalFileSystemV1.java | 39 - .../hadoop/fs/GridHadoopLocalFileSystemV2.java | 86 - .../hadoop/fs/GridHadoopRawLocalFileSystem.java | 304 --- .../hadoop/fs/HadoopDistributedFileSystem.java | 91 + .../hadoop/fs/HadoopFileSystemsUtils.java | 57 + .../hadoop/fs/HadoopLocalFileSystemV1.java | 39 + .../hadoop/fs/HadoopLocalFileSystemV2.java | 86 + .../processors/hadoop/fs/HadoopParameters.java | 94 + .../hadoop/fs/HadoopRawLocalFileSystem.java | 304 +++ .../processors/hadoop/igfs/HadoopIgfs.java | 198 ++ .../igfs/HadoopIgfsCommunicationException.java | 57 + .../hadoop/igfs/HadoopIgfsEndpoint.java | 210 ++ .../processors/hadoop/igfs/HadoopIgfsEx.java | 88 + .../hadoop/igfs/HadoopIgfsFuture.java | 94 + .../hadoop/igfs/HadoopIgfsInProc.java | 409 +++ .../hadoop/igfs/HadoopIgfsInputStream.java | 626 +++++ .../processors/hadoop/igfs/HadoopIgfsIo.java | 76 + .../processors/hadoop/igfs/HadoopIgfsIpcIo.java | 599 +++++ .../hadoop/igfs/HadoopIgfsIpcIoListener.java | 36 + .../hadoop/igfs/HadoopIgfsJclLogger.java | 115 + .../hadoop/igfs/HadoopIgfsOutProc.java | 466 ++++ .../hadoop/igfs/HadoopIgfsOutputStream.java | 201 ++ .../hadoop/igfs/HadoopIgfsProperties.java | 88 + .../hadoop/igfs/HadoopIgfsProxyInputStream.java | 335 +++ .../igfs/HadoopIgfsProxyOutputStream.java | 165 ++ ...fsSecondaryFileSystemPositionedReadable.java | 104 + .../hadoop/igfs/HadoopIgfsStreamDelegate.java | 96 + .../igfs/HadoopIgfsStreamEventListener.java | 39 + .../processors/hadoop/igfs/HadoopIgfsUtils.java | 131 + .../hadoop/igfs/HadoopIgfsWrapper.java | 511 ++++ .../jobtracker/GridHadoopJobMetadata.java | 305 --- .../hadoop/jobtracker/GridHadoopJobTracker.java | 1625 ------------ .../hadoop/jobtracker/HadoopJobMetadata.java | 306 +++ .../hadoop/jobtracker/HadoopJobTracker.java | 1626 ++++++++++++ .../hadoop/message/GridHadoopMessage.java | 27 - .../hadoop/message/HadoopMessage.java | 27 + .../planner/GridHadoopDefaultMapReducePlan.java | 107 - .../GridHadoopDefaultMapReducePlanner.java | 434 ---- .../planner/HadoopDefaultMapReducePlan.java | 107 + .../GridHadoopProtocolJobCountersTask.java | 45 - .../proto/GridHadoopProtocolJobStatusTask.java | 81 - .../proto/GridHadoopProtocolKillJobTask.java | 46 - .../proto/GridHadoopProtocolNextTaskIdTask.java | 35 - .../proto/GridHadoopProtocolSubmitJobTask.java | 57 - .../proto/GridHadoopProtocolTaskAdapter.java | 113 - .../proto/GridHadoopProtocolTaskArguments.java | 81 - .../hadoop/proto/HadoopClientProtocol.java | 333 +++ .../proto/HadoopProtocolJobCountersTask.java | 46 + .../proto/HadoopProtocolJobStatusTask.java | 81 + .../hadoop/proto/HadoopProtocolKillJobTask.java | 46 + .../proto/HadoopProtocolNextTaskIdTask.java | 35 + .../proto/HadoopProtocolSubmitJobTask.java | 57 + .../hadoop/proto/HadoopProtocolTaskAdapter.java | 113 + .../proto/HadoopProtocolTaskArguments.java | 81 + .../hadoop/shuffle/GridHadoopShuffle.java | 256 -- .../hadoop/shuffle/GridHadoopShuffleAck.java | 91 - .../hadoop/shuffle/GridHadoopShuffleJob.java | 593 ----- .../shuffle/GridHadoopShuffleMessage.java | 242 -- .../hadoop/shuffle/HadoopShuffle.java | 256 ++ .../hadoop/shuffle/HadoopShuffleAck.java | 91 + .../hadoop/shuffle/HadoopShuffleJob.java | 593 +++++ .../hadoop/shuffle/HadoopShuffleMessage.java | 241 ++ .../GridHadoopConcurrentHashMultimap.java | 611 ----- .../collections/GridHadoopHashMultimap.java | 174 -- .../collections/GridHadoopHashMultimapBase.java | 208 -- .../shuffle/collections/GridHadoopMultimap.java | 112 - .../collections/GridHadoopMultimapBase.java | 368 --- .../shuffle/collections/GridHadoopSkipList.java | 726 ------ .../HadoopConcurrentHashMultimap.java | 611 +++++ .../shuffle/collections/HadoopHashMultimap.java | 174 ++ .../collections/HadoopHashMultimapBase.java | 208 ++ .../shuffle/collections/HadoopMultimap.java | 112 + .../shuffle/collections/HadoopMultimapBase.java | 368 +++ .../shuffle/collections/HadoopSkipList.java | 726 ++++++ .../shuffle/streams/GridHadoopDataInStream.java | 170 -- .../streams/GridHadoopDataOutStream.java | 131 - .../streams/GridHadoopOffheapBuffer.java | 122 - .../shuffle/streams/HadoopDataInStream.java | 170 ++ .../shuffle/streams/HadoopDataOutStream.java | 131 + .../shuffle/streams/HadoopOffheapBuffer.java | 122 + .../GridHadoopEmbeddedTaskExecutor.java | 146 -- .../taskexecutor/GridHadoopExecutorService.java | 232 -- .../taskexecutor/GridHadoopRunnableTask.java | 268 -- .../GridHadoopTaskExecutorAdapter.java | 57 - .../taskexecutor/GridHadoopTaskState.java | 38 - .../taskexecutor/GridHadoopTaskStatus.java | 114 - .../HadoopEmbeddedTaskExecutor.java | 146 ++ .../taskexecutor/HadoopExecutorService.java | 231 ++ .../hadoop/taskexecutor/HadoopRunnableTask.java | 268 ++ .../taskexecutor/HadoopTaskExecutorAdapter.java | 57 + .../hadoop/taskexecutor/HadoopTaskState.java | 38 + .../hadoop/taskexecutor/HadoopTaskStatus.java | 114 + .../GridHadoopExternalTaskExecutor.java | 960 ------- .../GridHadoopExternalTaskMetadata.java | 68 - .../GridHadoopJobInfoUpdateRequest.java | 109 - .../GridHadoopPrepareForJobRequest.java | 126 - .../external/GridHadoopProcessDescriptor.java | 150 -- .../external/GridHadoopProcessStartedAck.java | 46 - .../GridHadoopTaskExecutionRequest.java | 110 - .../external/GridHadoopTaskFinishedMessage.java | 92 - .../external/HadoopExternalTaskExecutor.java | 960 +++++++ .../external/HadoopExternalTaskMetadata.java | 68 + .../external/HadoopJobInfoUpdateRequest.java | 109 + .../external/HadoopPrepareForJobRequest.java | 126 + .../external/HadoopProcessDescriptor.java | 150 ++ .../external/HadoopProcessStartedAck.java | 46 + .../external/HadoopTaskExecutionRequest.java | 110 + .../external/HadoopTaskFinishedMessage.java | 92 + .../child/GridHadoopChildProcessRunner.java | 440 ---- .../child/GridHadoopExternalProcessStarter.java | 296 --- .../child/HadoopChildProcessRunner.java | 440 ++++ .../child/HadoopExternalProcessStarter.java | 296 +++ .../GridHadoopAbstractCommunicationClient.java | 96 - .../GridHadoopCommunicationClient.java | 72 - .../GridHadoopExternalCommunication.java | 1431 ----------- .../GridHadoopHandshakeTimeoutException.java | 42 - .../GridHadoopIpcToNioAdapter.java | 239 -- .../GridHadoopMarshallerFilter.java | 84 - .../GridHadoopMessageListener.java | 39 - .../GridHadoopTcpNioCommunicationClient.java | 99 - .../HadoopAbstractCommunicationClient.java | 96 + .../HadoopCommunicationClient.java | 72 + .../HadoopExternalCommunication.java | 1431 +++++++++++ .../HadoopHandshakeTimeoutException.java | 42 + .../communication/HadoopIpcToNioAdapter.java | 239 ++ .../communication/HadoopMarshallerFilter.java | 84 + .../communication/HadoopMessageListener.java | 39 + .../HadoopTcpNioCommunicationClient.java | 99 + .../hadoop/v1/GridHadoopV1CleanupTask.java | 62 - .../hadoop/v1/GridHadoopV1Counter.java | 105 - .../hadoop/v1/GridHadoopV1MapTask.java | 111 - .../hadoop/v1/GridHadoopV1OutputCollector.java | 130 - .../hadoop/v1/GridHadoopV1Partitioner.java | 44 - .../hadoop/v1/GridHadoopV1ReduceTask.java | 92 - .../hadoop/v1/GridHadoopV1Reporter.java | 79 - .../hadoop/v1/GridHadoopV1SetupTask.java | 56 - .../hadoop/v1/GridHadoopV1Splitter.java | 97 - .../processors/hadoop/v1/GridHadoopV1Task.java | 95 - .../hadoop/v1/HadoopV1CleanupTask.java | 62 + .../processors/hadoop/v1/HadoopV1Counter.java | 105 + .../processors/hadoop/v1/HadoopV1MapTask.java | 111 + .../hadoop/v1/HadoopV1OutputCollector.java | 130 + .../hadoop/v1/HadoopV1Partitioner.java | 44 + .../hadoop/v1/HadoopV1ReduceTask.java | 92 + .../processors/hadoop/v1/HadoopV1Reporter.java | 79 + .../processors/hadoop/v1/HadoopV1SetupTask.java | 56 + .../processors/hadoop/v1/HadoopV1Splitter.java | 97 + .../processors/hadoop/v1/HadoopV1Task.java | 95 + .../hadoop/v2/GridHadoopExternalSplit.java | 87 - .../hadoop/v2/GridHadoopNativeCodeLoader.java | 74 - .../v2/GridHadoopSerializationWrapper.java | 133 - .../v2/GridHadoopShutdownHookManager.java | 96 - .../hadoop/v2/GridHadoopSplitWrapper.java | 118 - .../hadoop/v2/GridHadoopV2CleanupTask.java | 73 - .../hadoop/v2/GridHadoopV2Context.java | 230 -- .../hadoop/v2/GridHadoopV2Counter.java | 87 - .../processors/hadoop/v2/GridHadoopV2Job.java | 280 --- .../v2/GridHadoopV2JobResourceManager.java | 305 --- .../hadoop/v2/GridHadoopV2MapTask.java | 109 - .../hadoop/v2/GridHadoopV2Partitioner.java | 44 - .../hadoop/v2/GridHadoopV2ReduceTask.java | 88 - .../hadoop/v2/GridHadoopV2SetupTask.java | 66 - .../hadoop/v2/GridHadoopV2Splitter.java | 105 - .../processors/hadoop/v2/GridHadoopV2Task.java | 181 -- .../hadoop/v2/GridHadoopV2TaskContext.java | 443 ---- .../v2/GridHadoopWritableSerialization.java | 74 - .../hadoop/v2/HadoopExternalSplit.java | 87 + .../hadoop/v2/HadoopNativeCodeLoader.java | 74 + .../hadoop/v2/HadoopSerializationWrapper.java | 133 + .../hadoop/v2/HadoopShutdownHookManager.java | 96 + .../hadoop/v2/HadoopSplitWrapper.java | 118 + .../hadoop/v2/HadoopV2CleanupTask.java | 73 + .../processors/hadoop/v2/HadoopV2Context.java | 230 ++ .../processors/hadoop/v2/HadoopV2Counter.java | 87 + .../processors/hadoop/v2/HadoopV2Job.java | 280 +++ .../hadoop/v2/HadoopV2JobResourceManager.java | 305 +++ .../processors/hadoop/v2/HadoopV2MapTask.java | 109 + .../hadoop/v2/HadoopV2Partitioner.java | 44 + .../hadoop/v2/HadoopV2ReduceTask.java | 88 + .../processors/hadoop/v2/HadoopV2SetupTask.java | 66 + .../processors/hadoop/v2/HadoopV2Splitter.java | 105 + .../processors/hadoop/v2/HadoopV2Task.java | 181 ++ .../hadoop/v2/HadoopV2TaskContext.java | 444 ++++ .../hadoop/v2/HadoopWritableSerialization.java | 74 + ...op.mapreduce.protocol.ClientProtocolProvider | 2 +- ...ridHadoopClientProtocolEmbeddedSelfTest.java | 34 - .../GridHadoopClientProtocolSelfTest.java | 633 ----- .../HadoopClientProtocolEmbeddedSelfTest.java | 34 + .../hadoop/HadoopClientProtocolSelfTest.java | 635 +++++ .../HadoopIgfs20FileSystemAbstractSelfTest.java | 1967 +++++++++++++++ ...Igfs20FileSystemLoopbackPrimarySelfTest.java | 74 + ...oopIgfs20FileSystemShmemPrimarySelfTest.java | 74 + .../igfs/HadoopIgfsDualAbstractSelfTest.java | 305 +++ .../igfs/HadoopIgfsDualAsyncSelfTest.java | 32 + .../ignite/igfs/HadoopIgfsDualSyncSelfTest.java | 32 + ...oopSecondaryFileSystemConfigurationTest.java | 27 +- .../apache/ignite/igfs/IgfsEventsTestSuite.java | 54 +- .../IgfsHadoop20FileSystemAbstractSelfTest.java | 1967 --------------- ...doop20FileSystemLoopbackPrimarySelfTest.java | 74 - ...sHadoop20FileSystemShmemPrimarySelfTest.java | 74 - .../igfs/IgfsHadoopDualAbstractSelfTest.java | 304 --- .../igfs/IgfsHadoopDualAsyncSelfTest.java | 32 - .../ignite/igfs/IgfsHadoopDualSyncSelfTest.java | 32 - .../IgfsHadoopFileSystemAbstractSelfTest.java | 2366 ------------------ .../IgfsHadoopFileSystemClientSelfTest.java | 199 -- .../IgfsHadoopFileSystemHandshakeSelfTest.java | 311 --- .../IgfsHadoopFileSystemIpcCacheSelfTest.java | 207 -- .../IgfsHadoopFileSystemLoggerSelfTest.java | 287 --- ...IgfsHadoopFileSystemLoggerStateSelfTest.java | 325 --- ...adoopFileSystemLoopbackAbstractSelfTest.java | 46 - ...SystemLoopbackEmbeddedDualAsyncSelfTest.java | 33 - ...eSystemLoopbackEmbeddedDualSyncSelfTest.java | 33 - ...leSystemLoopbackEmbeddedPrimarySelfTest.java | 33 - ...SystemLoopbackEmbeddedSecondarySelfTest.java | 34 - ...SystemLoopbackExternalDualAsyncSelfTest.java | 33 - ...eSystemLoopbackExternalDualSyncSelfTest.java | 33 - ...leSystemLoopbackExternalPrimarySelfTest.java | 33 - ...SystemLoopbackExternalSecondarySelfTest.java | 34 - ...fsHadoopFileSystemSecondaryModeSelfTest.java | 319 --- ...fsHadoopFileSystemShmemAbstractSelfTest.java | 88 - ...ileSystemShmemEmbeddedDualAsyncSelfTest.java | 33 - ...FileSystemShmemEmbeddedDualSyncSelfTest.java | 33 - ...pFileSystemShmemEmbeddedPrimarySelfTest.java | 33 - ...ileSystemShmemEmbeddedSecondarySelfTest.java | 33 - ...ileSystemShmemExternalDualAsyncSelfTest.java | 33 - ...FileSystemShmemExternalDualSyncSelfTest.java | 33 - ...pFileSystemShmemExternalPrimarySelfTest.java | 33 - ...ileSystemShmemExternalSecondarySelfTest.java | 33 - .../igfs/IgfsNearOnlyMultiNodeSelfTest.java | 4 +- .../IgniteHadoopFileSystemAbstractSelfTest.java | 2366 ++++++++++++++++++ .../IgniteHadoopFileSystemClientSelfTest.java | 199 ++ ...IgniteHadoopFileSystemHandshakeSelfTest.java | 311 +++ .../IgniteHadoopFileSystemIpcCacheSelfTest.java | 207 ++ .../IgniteHadoopFileSystemLoggerSelfTest.java | 287 +++ ...niteHadoopFileSystemLoggerStateSelfTest.java | 325 +++ ...adoopFileSystemLoopbackAbstractSelfTest.java | 46 + ...SystemLoopbackEmbeddedDualAsyncSelfTest.java | 33 + ...eSystemLoopbackEmbeddedDualSyncSelfTest.java | 33 + ...leSystemLoopbackEmbeddedPrimarySelfTest.java | 33 + ...SystemLoopbackEmbeddedSecondarySelfTest.java | 34 + ...SystemLoopbackExternalDualAsyncSelfTest.java | 33 + ...eSystemLoopbackExternalDualSyncSelfTest.java | 33 + ...leSystemLoopbackExternalPrimarySelfTest.java | 33 + ...SystemLoopbackExternalSecondarySelfTest.java | 34 + ...teHadoopFileSystemSecondaryModeSelfTest.java | 319 +++ ...teHadoopFileSystemShmemAbstractSelfTest.java | 88 + ...ileSystemShmemEmbeddedDualAsyncSelfTest.java | 33 + ...FileSystemShmemEmbeddedDualSyncSelfTest.java | 33 + ...pFileSystemShmemEmbeddedPrimarySelfTest.java | 33 + ...ileSystemShmemEmbeddedSecondarySelfTest.java | 33 + ...ileSystemShmemExternalDualAsyncSelfTest.java | 33 + ...FileSystemShmemExternalDualSyncSelfTest.java | 33 + ...pFileSystemShmemExternalPrimarySelfTest.java | 33 + ...ileSystemShmemExternalSecondarySelfTest.java | 33 + .../hadoop/GridHadoopAbstractSelfTest.java | 222 -- .../hadoop/GridHadoopAbstractWordCountTest.java | 138 - .../hadoop/GridHadoopClassLoaderTest.java | 69 - .../hadoop/GridHadoopCommandLineTest.java | 440 ---- ...idHadoopDefaultMapReducePlannerSelfTest.java | 1005 -------- .../hadoop/GridHadoopFileSystemsTest.java | 177 -- .../hadoop/GridHadoopGroupingTest.java | 286 --- .../hadoop/GridHadoopJobTrackerSelfTest.java | 330 --- .../GridHadoopMapReduceEmbeddedSelfTest.java | 245 -- .../hadoop/GridHadoopMapReduceTest.java | 195 -- .../hadoop/GridHadoopPopularWordsTest.java | 294 --- .../GridHadoopSerializationWrapperSelfTest.java | 74 - .../processors/hadoop/GridHadoopSharedMap.java | 67 - .../hadoop/GridHadoopSortingExternalTest.java | 32 - .../hadoop/GridHadoopSortingTest.java | 281 --- .../hadoop/GridHadoopSplitWrapperSelfTest.java | 68 - .../processors/hadoop/GridHadoopStartup.java | 55 - .../hadoop/GridHadoopTaskExecutionSelfTest.java | 551 ---- .../hadoop/GridHadoopTasksAllVersionsTest.java | 259 -- .../hadoop/GridHadoopTasksV1Test.java | 57 - .../hadoop/GridHadoopTasksV2Test.java | 75 - .../GridHadoopTestRoundRobinMrPlanner.java | 66 - .../hadoop/GridHadoopTestTaskContext.java | 219 -- .../processors/hadoop/GridHadoopTestUtils.java | 102 - .../hadoop/GridHadoopV2JobSelfTest.java | 88 - .../hadoop/GridHadoopValidationSelfTest.java | 53 - .../hadoop/HadoopAbstractSelfTest.java | 222 ++ .../hadoop/HadoopAbstractWordCountTest.java | 138 + .../hadoop/HadoopClassLoaderTest.java | 69 + .../hadoop/HadoopCommandLineTest.java | 440 ++++ .../HadoopDefaultMapReducePlannerSelfTest.java | 1006 ++++++++ .../hadoop/HadoopFileSystemsTest.java | 177 ++ .../processors/hadoop/HadoopGroupingTest.java | 287 +++ .../hadoop/HadoopJobTrackerSelfTest.java | 331 +++ .../hadoop/HadoopMapReduceEmbeddedSelfTest.java | 246 ++ .../processors/hadoop/HadoopMapReduceTest.java | 197 ++ .../hadoop/HadoopPopularWordsTest.java | 294 +++ .../HadoopSerializationWrapperSelfTest.java | 74 + .../processors/hadoop/HadoopSharedMap.java | 67 + .../hadoop/HadoopSortingExternalTest.java | 34 + .../processors/hadoop/HadoopSortingTest.java | 282 +++ .../hadoop/HadoopSplitWrapperSelfTest.java | 68 + .../processors/hadoop/HadoopStartup.java | 54 + .../hadoop/HadoopTaskExecutionSelfTest.java | 551 ++++ .../hadoop/HadoopTasksAllVersionsTest.java | 259 ++ .../processors/hadoop/HadoopTasksV1Test.java | 57 + .../processors/hadoop/HadoopTasksV2Test.java | 75 + .../hadoop/HadoopTestRoundRobinMrPlanner.java | 66 + .../hadoop/HadoopTestTaskContext.java | 219 ++ .../processors/hadoop/HadoopTestUtils.java | 102 + .../processors/hadoop/HadoopV2JobSelfTest.java | 88 + .../hadoop/HadoopValidationSelfTest.java | 53 + .../hadoop/examples/GridHadoopWordCount1.java | 88 - .../examples/GridHadoopWordCount1Map.java | 62 - .../examples/GridHadoopWordCount1Reduce.java | 51 - .../hadoop/examples/GridHadoopWordCount2.java | 95 - .../examples/GridHadoopWordCount2Mapper.java | 72 - .../examples/GridHadoopWordCount2Reducer.java | 70 - .../hadoop/examples/HadoopWordCount1.java | 88 + .../hadoop/examples/HadoopWordCount1Map.java | 62 + .../hadoop/examples/HadoopWordCount1Reduce.java | 51 + .../hadoop/examples/HadoopWordCount2.java | 95 + .../hadoop/examples/HadoopWordCount2Mapper.java | 72 + .../examples/HadoopWordCount2Reducer.java | 70 + .../collections/GridHadoopAbstractMapTest.java | 152 -- ...ridHadoopConcurrentHashMultimapSelftest.java | 267 -- .../collections/GridHadoopHashMapSelfTest.java | 170 -- .../collections/GridHadoopSkipListSelfTest.java | 303 --- .../collections/HadoopAbstractMapTest.java | 154 ++ .../HadoopConcurrentHashMultimapSelftest.java | 267 ++ .../collections/HadoopHashMapSelfTest.java | 170 ++ .../collections/HadoopSkipListSelfTest.java | 303 +++ .../streams/GridHadoopDataStreamSelfTest.java | 151 -- .../streams/HadoopDataStreamSelfTest.java | 151 ++ .../GridHadoopExecutorServiceTest.java | 119 - .../taskexecutor/HadoopExecutorServiceTest.java | 119 + ...GridHadoopExternalTaskExecutionSelfTest.java | 210 -- .../HadoopExternalTaskExecutionSelfTest.java | 211 ++ ...GridHadoopExternalCommunicationSelfTest.java | 209 -- .../HadoopExternalCommunicationSelfTest.java | 209 ++ .../testsuites/IgniteHadoopTestSuite.java | 82 +- .../IgniteIgfsLinuxAndMacOSTestSuite.java | 22 +- .../org/apache/ignite/IgniteSpringBean.java | 4 +- .../scala/org/apache/ignite/visor/visor.scala | 4 +- pom.xml | 12 +- 561 files changed, 47565 insertions(+), 47291 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/6423cf02/bin/setup-hadoop.bat ---------------------------------------------------------------------- diff --git a/bin/setup-hadoop.bat b/bin/setup-hadoop.bat index c4c73b3..a11ef8c 100644 --- a/bin/setup-hadoop.bat +++ b/bin/setup-hadoop.bat @@ -23,6 +23,6 @@ if "%OS%" == "Windows_NT" setlocal -set MAIN_CLASS=org.apache.ignite.hadoop.GridHadoopSetup +set MAIN_CLASS=org.apache.ignite.internal.processors.hadoop.HadoopSetup call "%~dp0\ignite.bat" %* http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/6423cf02/bin/setup-hadoop.sh ---------------------------------------------------------------------- diff --git a/bin/setup-hadoop.sh b/bin/setup-hadoop.sh index 8969dfa..d66353f 100755 --- a/bin/setup-hadoop.sh +++ b/bin/setup-hadoop.sh @@ -54,7 +54,7 @@ setIgniteHome # # Set utility environment. # -export MAIN_CLASS=org.apache.ignite.internal.processors.hadoop.GridHadoopSetup +export MAIN_CLASS=org.apache.ignite.internal.processors.hadoop.HadoopSetup # # Start utility. http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/6423cf02/config/hadoop/default-config.xml ---------------------------------------------------------------------- diff --git a/config/hadoop/default-config.xml b/config/hadoop/default-config.xml index a264749..65a281e 100644 --- a/config/hadoop/default-config.xml +++ b/config/hadoop/default-config.xml @@ -52,7 +52,7 @@ <!-- Abstract IGFS file system configuration to be used as a template. --> - <bean id="igfsCfgBase" class="org.apache.ignite.configuration.IgfsConfiguration" abstract="true"> + <bean id="igfsCfgBase" class="org.apache.ignite.configuration.FileSystemConfiguration" abstract="true"> <!-- Must correlate with cache affinity mapper. --> <property name="blockSize" value="#{128 * 1024}"/> <property name="perNodeBatchSize" value="512"/> @@ -94,7 +94,7 @@ Apache Hadoop Accelerator configuration. --> <property name="hadoopConfiguration"> - <bean class="org.apache.ignite.internal.processors.hadoop.GridHadoopConfiguration"> + <bean class="org.apache.ignite.configuration.HadoopConfiguration"> <!-- Information about finished jobs will be kept for 30 seconds. --> <property name="finishedJobInfoTtl" value="30000"/> </bean> @@ -112,9 +112,9 @@ <!-- Configure one IGFS file system instance named "igfs" on this node. --> - <property name="igfsConfiguration"> + <property name="fileSystemConfiguration"> <list> - <bean class="org.apache.ignite.configuration.IgfsConfiguration" parent="igfsCfgBase"> + <bean class="org.apache.ignite.configuration.FileSystemConfiguration" parent="igfsCfgBase"> <property name="name" value="igfs"/> <!-- Caches with these names must be configured. --> @@ -135,7 +135,7 @@ <property name="defaultMode" value="PROXY"/> <property name="secondaryFileSystem"> - <bean class="org.apache.ignite.igfs.hadoop.IgfsHadoopFileSystemWrapper"> + <bean class="org.apache.ignite.hadoop.fs.IgniteHadoopIgfsSecondaryFileSystem"> <constructor-arg name="uri" value="hdfs://1.2.3.4:9000"/> <constructor-arg name="cfgPath" value="/opt/hadoop-server/etc/hadoop/core-site.xml"/> </bean> http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/6423cf02/docs/core-site.ignite.xml ---------------------------------------------------------------------- diff --git a/docs/core-site.ignite.xml b/docs/core-site.ignite.xml index ed11a79..8b8e634 100644 --- a/docs/core-site.ignite.xml +++ b/docs/core-site.ignite.xml @@ -48,7 +48,7 @@ --> <property> <name>fs.igfs.impl</name> - <value>org.apache.ignite.igfs.hadoop.v1.IgfsHadoopFileSystem</value> + <value>org.apache.ignite.hadoop.fs.v1.IgniteHadoopFileSystem</value> </property> <!-- @@ -56,7 +56,7 @@ --> <property> <name>fs.AbstractFileSystem.igfs.impl</name> - <value>org.apache.ignite.igfs.hadoop.v2.IgfsHadoopFileSystem</value> + <value>org.apache.ignite.hadoop.fs.v2.IgniteHadoopFileSystem</value> </property> <!-- @@ -73,7 +73,7 @@ <!-- <property> <name>ignite.counters.writer</name> - <value>org.apache.ignite.internal.processors.hadoop.counter.GridHadoopFSCounterWriter</value> + <value>org.apache.ignite.hadoop.fs.IgniteHadoopFileSystemCounterWriter</value> </property> --> http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/6423cf02/examples/config/filesystem/core-site.xml ---------------------------------------------------------------------- diff --git a/examples/config/filesystem/core-site.xml b/examples/config/filesystem/core-site.xml index 7c6cfaa..b6f0291 100644 --- a/examples/config/filesystem/core-site.xml +++ b/examples/config/filesystem/core-site.xml @@ -31,12 +31,12 @@ <property> <!-- FS driver class for the 'igfs://' URIs. --> <name>fs.igfs.impl</name> - <value>org.apache.ignite.igfs.hadoop.v1.IgfsHadoopFileSystem</value> + <value>org.apache.ignite.hadoop.fs.v1.IgniteHadoopFileSystem</value> </property> <property> <!-- FS driver class for the 'igfs://' URIs in Hadoop2.x --> <name>fs.AbstractFileSystem.igfs.impl</name> - <value>org.apache.ignite.igfs.hadoop.v2.IgfsHadoopFileSystem</value> + <value>org.apache.ignite.hadoop.fs.v2.IgniteHadoopFileSystem</value> </property> </configuration> http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/6423cf02/examples/config/filesystem/example-igfs.xml ---------------------------------------------------------------------- diff --git a/examples/config/filesystem/example-igfs.xml b/examples/config/filesystem/example-igfs.xml index 30cf51e..d8ccd34 100644 --- a/examples/config/filesystem/example-igfs.xml +++ b/examples/config/filesystem/example-igfs.xml @@ -63,9 +63,9 @@ </bean> </property> - <property name="igfsConfiguration"> + <property name="fileSystemConfiguration"> <list> - <bean class="org.apache.ignite.configuration.IgfsConfiguration"> + <bean class="org.apache.ignite.configuration.FileSystemConfiguration"> <property name="name" value="igfs"/> <property name="metaCacheName" value="igfs-meta"/> <property name="dataCacheName" value="igfs-data"/> http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/6423cf02/examples/src/main/java/org/apache/ignite/examples/igfs/IgfsExample.java ---------------------------------------------------------------------- diff --git a/examples/src/main/java/org/apache/ignite/examples/igfs/IgfsExample.java b/examples/src/main/java/org/apache/ignite/examples/igfs/IgfsExample.java index 56f3fa4..8bfd38d 100644 --- a/examples/src/main/java/org/apache/ignite/examples/igfs/IgfsExample.java +++ b/examples/src/main/java/org/apache/ignite/examples/igfs/IgfsExample.java @@ -25,7 +25,7 @@ import java.io.*; import java.util.*; /** - * Example that shows usage of {@link org.apache.ignite.IgniteFs} API. It starts a node with {@code IgniteFs} + * Example that shows usage of {@link org.apache.ignite.IgniteFileSystem} API. It starts a node with {@code IgniteFs} * configured and performs several file system operations (create, write, append, read and delete * files, create, list and delete directories). * <p> @@ -50,7 +50,7 @@ public final class IgfsExample { try { // Get an instance of Ignite File System. - IgniteFs fs = ignite.fileSystem("igfs"); + IgniteFileSystem fs = ignite.fileSystem("igfs"); // Working directory path. IgfsPath workDir = new IgfsPath("/examples/fs"); @@ -107,7 +107,7 @@ public final class IgfsExample { * @param path File or directory path. * @throws IgniteException In case of error. */ - private static void delete(IgniteFs fs, IgfsPath path) throws IgniteException { + private static void delete(IgniteFileSystem fs, IgfsPath path) throws IgniteException { assert fs != null; assert path != null; @@ -139,7 +139,7 @@ public final class IgfsExample { * @param path Directory path. * @throws IgniteException In case of error. */ - private static void mkdirs(IgniteFs fs, IgfsPath path) throws IgniteException { + private static void mkdirs(IgniteFileSystem fs, IgfsPath path) throws IgniteException { assert fs != null; assert path != null; @@ -166,7 +166,7 @@ public final class IgfsExample { * @throws IgniteException If file can't be created. * @throws IOException If data can't be written. */ - private static void create(IgniteFs fs, IgfsPath path, @Nullable byte[] data) + private static void create(IgniteFileSystem fs, IgfsPath path, @Nullable byte[] data) throws IgniteException, IOException { assert fs != null; assert path != null; @@ -195,7 +195,7 @@ public final class IgfsExample { * @throws IgniteException If file can't be created. * @throws IOException If data can't be written. */ - private static void append(IgniteFs fs, IgfsPath path, byte[] data) throws IgniteException, IOException { + private static void append(IgniteFileSystem fs, IgfsPath path, byte[] data) throws IgniteException, IOException { assert fs != null; assert path != null; assert data != null; @@ -220,7 +220,7 @@ public final class IgfsExample { * @throws IgniteException If file can't be opened. * @throws IOException If data can't be read. */ - private static void read(IgniteFs fs, IgfsPath path) throws IgniteException, IOException { + private static void read(IgniteFileSystem fs, IgfsPath path) throws IgniteException, IOException { assert fs != null; assert path != null; assert fs.info(path).isFile(); @@ -242,7 +242,7 @@ public final class IgfsExample { * @param path Directory path. * @throws IgniteException In case of error. */ - private static void list(IgniteFs fs, IgfsPath path) throws IgniteException { + private static void list(IgniteFileSystem fs, IgfsPath path) throws IgniteException { assert fs != null; assert path != null; assert fs.info(path).isDirectory(); @@ -271,7 +271,7 @@ public final class IgfsExample { * @param path File or directory path. * @throws IgniteException In case of error. */ - private static void printInfo(IgniteFs fs, IgfsPath path) throws IgniteException { + private static void printInfo(IgniteFileSystem fs, IgfsPath path) throws IgniteException { System.out.println(); System.out.println("Information for " + path + ": " + fs.info(path)); } http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/6423cf02/examples/src/main/java/org/apache/ignite/examples/igfs/IgfsMapReduceExample.java ---------------------------------------------------------------------- diff --git a/examples/src/main/java/org/apache/ignite/examples/igfs/IgfsMapReduceExample.java b/examples/src/main/java/org/apache/ignite/examples/igfs/IgfsMapReduceExample.java index 22a66a3..ed0abe4 100644 --- a/examples/src/main/java/org/apache/ignite/examples/igfs/IgfsMapReduceExample.java +++ b/examples/src/main/java/org/apache/ignite/examples/igfs/IgfsMapReduceExample.java @@ -61,7 +61,7 @@ public class IgfsMapReduceExample { String regexStr = args[1]; // Get an instance of Ignite File System. - IgniteFs fs = ignite.fileSystem("igfs"); + IgniteFileSystem fs = ignite.fileSystem("igfs"); // Working directory path. IgfsPath workDir = new IgfsPath("/examples/fs"); @@ -94,7 +94,7 @@ public class IgfsMapReduceExample { * @param file File to write. * @throws Exception In case of exception. */ - private static void writeFile(IgniteFs fs, IgfsPath fsPath, File file) throws Exception { + private static void writeFile(IgniteFileSystem fs, IgfsPath fsPath, File file) throws Exception { System.out.println(); System.out.println("Copying file to IGFS: " + file); @@ -173,7 +173,7 @@ public class IgfsMapReduceExample { } /** {@inheritDoc} */ - @Override public Object execute(IgniteFs igfs, IgfsRangeInputStream in) throws IgniteException, IOException { + @Override public Object execute(IgniteFileSystem igfs, IgfsRangeInputStream in) throws IgniteException, IOException { Collection<Line> res = null; long start = in.startOffset(); http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/6423cf02/modules/core/src/main/java/org/apache/ignite/Ignite.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/Ignite.java b/modules/core/src/main/java/org/apache/ignite/Ignite.java index 8851d8f..31b827e 100644 --- a/modules/core/src/main/java/org/apache/ignite/Ignite.java +++ b/modules/core/src/main/java/org/apache/ignite/Ignite.java @@ -43,7 +43,7 @@ import java.util.concurrent.*; * <ul> * <li>{@link org.apache.ignite.cache.GridCache} - functionality for in-memory distributed cache.</li> * <li>{@link IgniteDataLoader} - functionality for loading data large amounts of data into cache.</li> - * <li>{@link IgniteFs} - functionality for distributed Hadoop-compliant in-memory file system and map-reduce.</li> + * <li>{@link IgniteFileSystem} - functionality for distributed Hadoop-compliant in-memory file system and map-reduce.</li> * <li>{@link IgniteStreamer} - functionality for streaming events workflow with queries and indexes into rolling windows.</li> * <li>{@link IgniteScheduler} - functionality for scheduling jobs using UNIX Cron syntax.</li> * <li>{@link IgniteCompute} - functionality for executing tasks and closures on all grid nodes (inherited form {@link ClusterGroup}).</li> @@ -213,7 +213,7 @@ public interface Ignite extends AutoCloseable { public <K, V> IgniteDataLoader<K, V> dataLoader(@Nullable String cacheName); /** - * Gets an instance of IGFS - Ignite In-Memory File System, if one is not + * Gets an instance of IGFS (Ignite In-Memory File System). If one is not * configured then {@link IllegalArgumentException} will be thrown. * <p> * IGFS is fully compliant with Hadoop {@code FileSystem} APIs and can @@ -222,15 +222,16 @@ public interface Ignite extends AutoCloseable { * * @param name IGFS name. * @return IGFS instance. + * @throws IllegalArgumentException If IGFS with such name is not configured. */ - public IgniteFs fileSystem(String name); + public IgniteFileSystem fileSystem(String name); /** - * Gets all instances of the grid file systems. + * Gets all instances of IGFS (Ignite In-Memory File System). * - * @return Collection of grid file systems instances. + * @return Collection of IGFS instances. */ - public Collection<IgniteFs> fileSystems(); + public Collection<IgniteFileSystem> fileSystems(); /** * Gets an instance of streamer by name, if one does not exist then http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/6423cf02/modules/core/src/main/java/org/apache/ignite/IgniteFileSystem.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteFileSystem.java b/modules/core/src/main/java/org/apache/ignite/IgniteFileSystem.java new file mode 100644 index 0000000..d221ae2 --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/IgniteFileSystem.java @@ -0,0 +1,457 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite; + +import org.apache.ignite.configuration.*; +import org.apache.ignite.igfs.*; +import org.apache.ignite.igfs.mapreduce.*; +import org.apache.ignite.lang.*; +import org.jetbrains.annotations.*; + +import java.util.*; + +/** + * <b>IG</b>nite <b>F</b>ile <b>S</b>ystem API. It provides a typical file system "view" on a particular cache: + * <ul> + * <li>list directories or get information for a single path</li> + * <li>create/move/delete files or directories</li> + * <li>write/read data streams into/from files</li> + * </ul> + * The data of each file is split on separate data blocks and stored in the cache. + * You can access file's data with a standard Java streaming API. Moreover, for each part + * of the file you can calculate an affinity and process file's content on corresponding + * nodes to escape unnecessary networking. + * <p/> + * This API is fully thread-safe and you can use it from several threads. + * <h1 class="header">IGFS Configuration</h1> + * The simplest way to run a Ignite node with configured file system is to pass + * special configuration file included in Ignite distribution to {@code ignite.sh} or + * {@code ignite.bat} scripts, like this: {@code ignite.sh config/hadoop/default-config.xml} + * <p> + * {@code IGFS} can be started as a data node or as a client node. Data node is responsible for + * caching data, while client node is responsible for basic file system operations and accessing + * data nodes remotely. When used as Hadoop file system, clients nodes usually started together + * with {@code job-submitter} or {@code job-scheduler} processes, while data nodes are usually + * started together with Hadoop {@code task-tracker} processes. + * <h1 class="header">Integration With Hadoop</h1> + * In addition to direct file system API, {@code IGFS} can be integrated with {@code Hadoop} by + * plugging in as {@code Hadoop FileSystem}. Refer to + * {@code org.apache.ignite.hadoop.fs.v1.IgniteHadoopFileSystem} or + * {@code org.apache.ignite.hadoop.fs.v2.IgniteHadoopFileSystem} for more information. + * <p> + * <b>NOTE:</b> integration with Hadoop is available only in {@code In-Memory Accelerator For Hadoop} edition. + */ +public interface IgniteFileSystem extends IgniteAsyncSupport { + /** IGFS scheme name. */ + public static final String IGFS_SCHEME = "igfs"; + + /** + * Gets IGFS name. + * + * @return IGFS name, or {@code null} for default file system. + */ + public String name(); + + /** + * Gets IGFS configuration. + * + * @return IGFS configuration. + */ + public FileSystemConfiguration configuration(); + + /** + * Gets summary (total number of files, total number of directories and total length) + * for a given path. + * + * @param path Path to get information for. + * @return Summary object. + * @throws org.apache.ignite.igfs.IgfsFileNotFoundException If path is not found. + * @throws IgniteException If failed. + */ + public IgfsPathSummary summary(IgfsPath path) throws IgniteException; + + /** + * Opens a file for reading. + * + * @param path File path to read. + * @return File input stream to read data from. + * @throws IgniteException In case of error. + * @throws org.apache.ignite.igfs.IgfsFileNotFoundException If path doesn't exist. + */ + public IgfsInputStream open(IgfsPath path) throws IgniteException; + + /** + * Opens a file for reading. + * + * @param path File path to read. + * @param bufSize Read buffer size (bytes) or {@code zero} to use default value. + * @return File input stream to read data from. + * @throws IgniteException In case of error. + * @throws org.apache.ignite.igfs.IgfsFileNotFoundException If path doesn't exist. + */ + public IgfsInputStream open(IgfsPath path, int bufSize) throws IgniteException; + + /** + * Opens a file for reading. + * + * @param path File path to read. + * @param bufSize Read buffer size (bytes) or {@code zero} to use default value. + * @param seqReadsBeforePrefetch Amount of sequential reads before prefetch is started. + * @return File input stream to read data from. + * @throws IgniteException In case of error. + * @throws org.apache.ignite.igfs.IgfsFileNotFoundException If path doesn't exist. + */ + public IgfsInputStream open(IgfsPath path, int bufSize, int seqReadsBeforePrefetch) throws IgniteException; + + /** + * Creates a file and opens it for writing. + * + * @param path File path to create. + * @param overwrite Overwrite file if it already exists. Note: you cannot overwrite an existent directory. + * @return File output stream to write data to. + * @throws IgniteException In case of error. + */ + public IgfsOutputStream create(IgfsPath path, boolean overwrite) throws IgniteException; + + /** + * Creates a file and opens it for writing. + * + * @param path File path to create. + * @param bufSize Write buffer size (bytes) or {@code zero} to use default value. + * @param overwrite Overwrite file if it already exists. Note: you cannot overwrite an existent directory. + * @param replication Replication factor. + * @param blockSize Block size. + * @param props File properties to set. + * @return File output stream to write data to. + * @throws IgniteException In case of error. + */ + public IgfsOutputStream create(IgfsPath path, int bufSize, boolean overwrite, int replication, + long blockSize, @Nullable Map<String, String> props) throws IgniteException; + + /** + * Creates a file and opens it for writing. + * + * @param path File path to create. + * @param bufSize Write buffer size (bytes) or {@code zero} to use default value. + * @param overwrite Overwrite file if it already exists. Note: you cannot overwrite an existent directory. + * @param affKey Affinity key used to store file blocks. If not {@code null}, the whole file will be + * stored on node where {@code affKey} resides. + * @param replication Replication factor. + * @param blockSize Block size. + * @param props File properties to set. + * @return File output stream to write data to. + * @throws IgniteException In case of error. + */ + public IgfsOutputStream create(IgfsPath path, int bufSize, boolean overwrite, + @Nullable IgniteUuid affKey, int replication, long blockSize, @Nullable Map<String, String> props) + throws IgniteException; + + /** + * Opens an output stream to an existing file for appending data. + * + * @param path File path to append. + * @param create Create file if it doesn't exist yet. + * @return File output stream to append data to. + * @throws IgniteException In case of error. + * @throws org.apache.ignite.igfs.IgfsFileNotFoundException If path doesn't exist and create flag is {@code false}. + */ + public IgfsOutputStream append(IgfsPath path, boolean create) throws IgniteException; + + /** + * Opens an output stream to an existing file for appending data. + * + * @param path File path to append. + * @param bufSize Write buffer size (bytes) or {@code zero} to use default value. + * @param create Create file if it doesn't exist yet. + * @param props File properties to set only in case it file was just created. + * @return File output stream to append data to. + * @throws IgniteException In case of error. + * @throws org.apache.ignite.igfs.IgfsFileNotFoundException If path doesn't exist and create flag is {@code false}. + */ + public IgfsOutputStream append(IgfsPath path, int bufSize, boolean create, @Nullable Map<String, String> props) + throws IgniteException; + + /** + * Sets last access time and last modification time for a given path. If argument is {@code null}, + * corresponding time will not be changed. + * + * @param path Path to update. + * @param accessTime Optional last access time to set. Value {@code -1} does not update access time. + * @param modificationTime Optional last modification time to set. Value {@code -1} does not update + * modification time. + * @throws org.apache.ignite.igfs.IgfsFileNotFoundException If target was not found. + * @throws IgniteException If error occurred. + */ + public void setTimes(IgfsPath path, long accessTime, long modificationTime) throws IgniteException; + + /** + * Gets affinity block locations for data blocks of the file, i.e. the nodes, on which the blocks + * are stored. + * + * @param path File path to get affinity for. + * @param start Position in the file to start affinity resolution from. + * @param len Size of data in the file to resolve affinity for. + * @return Affinity block locations. + * @throws IgniteException In case of error. + * @throws org.apache.ignite.igfs.IgfsFileNotFoundException If path doesn't exist. + */ + public Collection<IgfsBlockLocation> affinity(IgfsPath path, long start, long len) throws IgniteException; + + /** + * Get affinity block locations for data blocks of the file. In case {@code maxLen} parameter is set and + * particular block location length is greater than this value, block locations will be split into smaller + * chunks. + * + * @param path File path to get affinity for. + * @param start Position in the file to start affinity resolution from. + * @param len Size of data in the file to resolve affinity for. + * @param maxLen Maximum length of a single returned block location length. + * @return Affinity block locations. + * @throws IgniteException In case of error. + * @throws org.apache.ignite.igfs.IgfsFileNotFoundException If path doesn't exist. + */ + public Collection<IgfsBlockLocation> affinity(IgfsPath path, long start, long len, long maxLen) + throws IgniteException; + + /** + * Gets metrics snapshot for this file system. + * + * @return Metrics. + * @throws IgniteException In case of error. + */ + public IgfsMetrics metrics() throws IgniteException; + + /** + * Resets metrics for this file system. + * + * @throws IgniteException In case of error. + */ + public void resetMetrics() throws IgniteException; + + /** + * Determines size of the file denoted by provided path. In case if path is a directory, then + * total size of all containing entries will be calculated recursively. + * + * @param path File system path. + * @return Total size. + * @throws IgniteException In case of error. + */ + public long size(IgfsPath path) throws IgniteException; + + /** + * Formats the file system removing all existing entries from it. + * <p> + * Supports asynchronous execution (see {@link IgniteAsyncSupport}). + * + * @throws IgniteException In case format has failed. + */ + @IgniteAsyncSupported + public void format() throws IgniteException; + + /** + * Executes IGFS task. + * <p> + * Supports asynchronous execution (see {@link IgniteAsyncSupport}). + * + * @param task Task to execute. + * @param rslvr Optional resolver to control split boundaries. + * @param paths Collection of paths to be processed within this task. + * @param arg Optional task argument. + * @return Task result. + * @throws IgniteException If execution failed. + */ + @IgniteAsyncSupported + public <T, R> R execute(IgfsTask<T, R> task, @Nullable IgfsRecordResolver rslvr, + Collection<IgfsPath> paths, @Nullable T arg) throws IgniteException; + + /** + * Executes IGFS task with overridden maximum range length (see + * {@link org.apache.ignite.configuration.FileSystemConfiguration#getMaximumTaskRangeLength()} for more information). + * <p> + * Supports asynchronous execution (see {@link IgniteAsyncSupport}). + * + * @param task Task to execute. + * @param rslvr Optional resolver to control split boundaries. + * @param paths Collection of paths to be processed within this task. + * @param skipNonExistentFiles Whether to skip non existent files. If set to {@code true} non-existent files will + * be ignored. Otherwise an exception will be thrown. + * @param maxRangeLen Optional maximum range length. If {@code 0}, then by default all consecutive + * IGFS blocks will be included. + * @param arg Optional task argument. + * @return Task result. + * @throws IgniteException If execution failed. + */ + @IgniteAsyncSupported + public <T, R> R execute(IgfsTask<T, R> task, @Nullable IgfsRecordResolver rslvr, + Collection<IgfsPath> paths, boolean skipNonExistentFiles, long maxRangeLen, @Nullable T arg) + throws IgniteException; + + /** + * Executes IGFS task. + * <p> + * Supports asynchronous execution (see {@link IgniteAsyncSupport}). + * + * @param taskCls Task class to execute. + * @param rslvr Optional resolver to control split boundaries. + * @param paths Collection of paths to be processed within this task. + * @param arg Optional task argument. + * @return Task result. + * @throws IgniteException If execution failed. + */ + @IgniteAsyncSupported + public <T, R> R execute(Class<? extends IgfsTask<T, R>> taskCls, + @Nullable IgfsRecordResolver rslvr, Collection<IgfsPath> paths, @Nullable T arg) throws IgniteException; + + /** + * Executes IGFS task with overridden maximum range length (see + * {@link org.apache.ignite.configuration.FileSystemConfiguration#getMaximumTaskRangeLength()} for more information). + * <p> + * Supports asynchronous execution (see {@link IgniteAsyncSupport}). + * + * @param taskCls Task class to execute. + * @param rslvr Optional resolver to control split boundaries. + * @param paths Collection of paths to be processed within this task. + * @param skipNonExistentFiles Whether to skip non existent files. If set to {@code true} non-existent files will + * be ignored. Otherwise an exception will be thrown. + * @param maxRangeLen Maximum range length. + * @param arg Optional task argument. + * @return Task result. + * @throws IgniteException If execution failed. + */ + @IgniteAsyncSupported + public <T, R> R execute(Class<? extends IgfsTask<T, R>> taskCls, + @Nullable IgfsRecordResolver rslvr, Collection<IgfsPath> paths, boolean skipNonExistentFiles, + long maxRangeLen, @Nullable T arg) throws IgniteException; + + /** + * Checks if the specified path exists in the file system. + * + * @param path Path to check for existence in the file system. + * @return {@code True} if such file exists, otherwise - {@code false}. + * @throws IgniteException In case of error. + */ + public boolean exists(IgfsPath path); + + /** + * Updates file information for the specified path. Existent properties, not listed in the passed collection, + * will not be affected. Other properties will be added or overwritten. Passed properties with {@code null} values + * will be removed from the stored properties or ignored if they don't exist in the file info. + * <p> + * When working in {@code DUAL_SYNC} or {@code DUAL_ASYNC} modes only the following properties will be propagated + * to the secondary file system: + * <ul> + * <li>{@code usrName} - file owner name;</li> + * <li>{@code grpName} - file owner group;</li> + * <li>{@code permission} - Unix-style string representing file permissions.</li> + * </ul> + * + * @param path File path to set properties for. + * @param props Properties to update. + * @return File information for specified path or {@code null} if such path does not exist. + * @throws IgniteException In case of error. + */ + public IgfsFile update(IgfsPath path, Map<String, String> props) throws IgniteException; + + /** + * Renames/moves a file. + * <p> + * You are free to rename/move data files as you wish, but directories can be only renamed. + * You cannot move the directory between different parent directories. + * <p> + * Examples: + * <ul> + * <li>"/work/file.txt" => "/home/project/Presentation Scenario.txt"</li> + * <li>"/work" => "/work-2012.bkp"</li> + * <li>"/work" => "<strike>/backups/work</strike>" - such operation is restricted for directories.</li> + * </ul> + * + * @param src Source file path to rename. + * @param dest Destination file path. If destination path is a directory, then source file will be placed + * into destination directory with original name. + * @throws IgniteException In case of error. + * @throws IgfsFileNotFoundException If source file doesn't exist. + */ + public void rename(IgfsPath src, IgfsPath dest) throws IgniteException; + + /** + * Deletes file. + * + * @param path File path to delete. + * @param recursive Delete non-empty directories recursively. + * @return {@code True} in case of success, {@code false} otherwise. + * @throws IgniteException In case of error. + */ + public boolean delete(IgfsPath path, boolean recursive) throws IgniteException; + + /** + * Creates directories under specified path. + * + * @param path Path of directories chain to create. + * @throws IgniteException In case of error. + */ + public void mkdirs(IgfsPath path) throws IgniteException; + + /** + * Creates directories under specified path with the specified properties. + * + * @param path Path of directories chain to create. + * @param props Metadata properties to set on created directories. + * @throws IgniteException In case of error. + */ + public void mkdirs(IgfsPath path, @Nullable Map<String, String> props) throws IgniteException; + + /** + * Lists file paths under the specified path. + * + * @param path Path to list files under. + * @return List of files under the specified path. + * @throws IgniteException In case of error. + * @throws IgfsFileNotFoundException If path doesn't exist. + */ + public Collection<IgfsPath> listPaths(IgfsPath path) throws IgniteException; + + /** + * Lists files under the specified path. + * + * @param path Path to list files under. + * @return List of files under the specified path. + * @throws IgniteException In case of error. + * @throws IgfsFileNotFoundException If path doesn't exist. + */ + public Collection<IgfsFile> listFiles(IgfsPath path) throws IgniteException; + + /** + * Gets file information for the specified path. + * + * @param path Path to get information for. + * @return File information for specified path or {@code null} if such path does not exist. + * @throws IgniteException In case of error. + */ + public IgfsFile info(IgfsPath path) throws IgniteException; + + /** + * Gets used space in bytes. + * + * @return Used space in bytes. + * @throws IgniteException In case of error. + */ + public long usedSpaceSize() throws IgniteException; + + /** {@inheritDoc} */ + @Override public IgniteFileSystem withAsync(); +} http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/6423cf02/modules/core/src/main/java/org/apache/ignite/IgniteFs.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteFs.java b/modules/core/src/main/java/org/apache/ignite/IgniteFs.java deleted file mode 100644 index dc11973..0000000 --- a/modules/core/src/main/java/org/apache/ignite/IgniteFs.java +++ /dev/null @@ -1,346 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ignite; - -import org.apache.ignite.configuration.*; -import org.apache.ignite.igfs.*; -import org.apache.ignite.igfs.mapreduce.*; -import org.apache.ignite.lang.*; -import org.jetbrains.annotations.*; - -import java.util.*; - -/** - * <b>IG</b>nite <b>F</b>ile <b>S</b>ystem API. It provides a typical file system "view" on a particular cache: - * <ul> - * <li>list directories or get information for a single path</li> - * <li>create/move/delete files or directories</li> - * <li>write/read data streams into/from files</li> - * </ul> - * The data of each file is split on separate data blocks and stored in the cache. - * You can access file's data with a standard Java streaming API. Moreover, for each part - * of the file you can calculate an affinity and process file's content on corresponding - * nodes to escape unnecessary networking. - * <p/> - * This API is fully thread-safe and you can use it from several threads. - * <h1 class="header">IGFS Configuration</h1> - * The simplest way to run a Ignite node with configured file system is to pass - * special configuration file included in Ignite distribution to {@code ignite.sh} or - * {@code ignite.bat} scripts, like this: {@code ignite.sh config/hadoop/default-config.xml} - * <p> - * {@code IGFS} can be started as a data node or as a client node. Data node is responsible for - * caching data, while client node is responsible for basic file system operations and accessing - * data nodes remotely. When used as Hadoop file system, clients nodes usually started together - * with {@code job-submitter} or {@code job-scheduler} processes, while data nodes are usually - * started together with Hadoop {@code task-tracker} processes. - * <h1 class="header">Integration With Hadoop</h1> - * In addition to direct file system API, {@code IGFS} can be integrated with {@code Hadoop} by - * plugging in as {@code Hadoop FileSystem}. Refer to - * {@code org.apache.ignite.igfs.hadoop.v1.IgfsHadoopFileSystem} or - * {@code org.apache.ignite.igfs.hadoop.v2.IgfsHadoopFileSystem} for more information. - * <p> - * <b>NOTE:</b> integration with Hadoop is available only in {@code In-Memory Accelerator For Hadoop} edition. - */ -public interface IgniteFs extends Igfs, IgniteAsyncSupport { - /** IGFS scheme name. */ - public static final String IGFS_SCHEME = "igfs"; - - /** File property: prefer writes to local node. */ - public static final String PROP_PREFER_LOCAL_WRITES = "locWrite"; - - /** - * Gets IGFS name. - * - * @return IGFS name, or {@code null} for default file system. - */ - public String name(); - - /** - * Gets IGFS configuration. - * - * @return IGFS configuration. - */ - public IgfsConfiguration configuration(); - - /** - * Gets summary (total number of files, total number of directories and total length) - * for a given path. - * - * @param path Path to get information for. - * @return Summary object. - * @throws org.apache.ignite.igfs.IgfsFileNotFoundException If path is not found. - * @throws IgniteException If failed. - */ - public IgfsPathSummary summary(IgfsPath path) throws IgniteException; - - /** - * Opens a file for reading. - * - * @param path File path to read. - * @return File input stream to read data from. - * @throws IgniteException In case of error. - * @throws org.apache.ignite.igfs.IgfsFileNotFoundException If path doesn't exist. - */ - public IgfsInputStream open(IgfsPath path) throws IgniteException; - - /** - * Opens a file for reading. - * - * @param path File path to read. - * @param bufSize Read buffer size (bytes) or {@code zero} to use default value. - * @return File input stream to read data from. - * @throws IgniteException In case of error. - * @throws org.apache.ignite.igfs.IgfsFileNotFoundException If path doesn't exist. - */ - @Override public IgfsInputStream open(IgfsPath path, int bufSize) throws IgniteException; - - /** - * Opens a file for reading. - * - * @param path File path to read. - * @param bufSize Read buffer size (bytes) or {@code zero} to use default value. - * @param seqReadsBeforePrefetch Amount of sequential reads before prefetch is started. - * @return File input stream to read data from. - * @throws IgniteException In case of error. - * @throws org.apache.ignite.igfs.IgfsFileNotFoundException If path doesn't exist. - */ - public IgfsInputStream open(IgfsPath path, int bufSize, int seqReadsBeforePrefetch) throws IgniteException; - - /** - * Creates a file and opens it for writing. - * - * @param path File path to create. - * @param overwrite Overwrite file if it already exists. Note: you cannot overwrite an existent directory. - * @return File output stream to write data to. - * @throws IgniteException In case of error. - */ - @Override public IgfsOutputStream create(IgfsPath path, boolean overwrite) throws IgniteException; - - /** - * Creates a file and opens it for writing. - * - * @param path File path to create. - * @param bufSize Write buffer size (bytes) or {@code zero} to use default value. - * @param overwrite Overwrite file if it already exists. Note: you cannot overwrite an existent directory. - * @param replication Replication factor. - * @param blockSize Block size. - * @param props File properties to set. - * @return File output stream to write data to. - * @throws IgniteException In case of error. - */ - @Override public IgfsOutputStream create(IgfsPath path, int bufSize, boolean overwrite, int replication, - long blockSize, @Nullable Map<String, String> props) throws IgniteException; - - /** - * Creates a file and opens it for writing. - * - * @param path File path to create. - * @param bufSize Write buffer size (bytes) or {@code zero} to use default value. - * @param overwrite Overwrite file if it already exists. Note: you cannot overwrite an existent directory. - * @param affKey Affinity key used to store file blocks. If not {@code null}, the whole file will be - * stored on node where {@code affKey} resides. - * @param replication Replication factor. - * @param blockSize Block size. - * @param props File properties to set. - * @return File output stream to write data to. - * @throws IgniteException In case of error. - */ - public IgfsOutputStream create(IgfsPath path, int bufSize, boolean overwrite, - @Nullable IgniteUuid affKey, int replication, long blockSize, @Nullable Map<String, String> props) - throws IgniteException; - - /** - * Opens an output stream to an existing file for appending data. - * - * @param path File path to append. - * @param create Create file if it doesn't exist yet. - * @return File output stream to append data to. - * @throws IgniteException In case of error. - * @throws org.apache.ignite.igfs.IgfsFileNotFoundException If path doesn't exist and create flag is {@code false}. - */ - public IgfsOutputStream append(IgfsPath path, boolean create) throws IgniteException; - - /** - * Opens an output stream to an existing file for appending data. - * - * @param path File path to append. - * @param bufSize Write buffer size (bytes) or {@code zero} to use default value. - * @param create Create file if it doesn't exist yet. - * @param props File properties to set only in case it file was just created. - * @return File output stream to append data to. - * @throws IgniteException In case of error. - * @throws org.apache.ignite.igfs.IgfsFileNotFoundException If path doesn't exist and create flag is {@code false}. - */ - @Override public IgfsOutputStream append(IgfsPath path, int bufSize, boolean create, - @Nullable Map<String, String> props) throws IgniteException; - - /** - * Sets last access time and last modification time for a given path. If argument is {@code null}, - * corresponding time will not be changed. - * - * @param path Path to update. - * @param accessTime Optional last access time to set. Value {@code -1} does not update access time. - * @param modificationTime Optional last modification time to set. Value {@code -1} does not update - * modification time. - * @throws org.apache.ignite.igfs.IgfsFileNotFoundException If target was not found. - * @throws IgniteException If error occurred. - */ - public void setTimes(IgfsPath path, long accessTime, long modificationTime) throws IgniteException; - - /** - * Gets affinity block locations for data blocks of the file, i.e. the nodes, on which the blocks - * are stored. - * - * @param path File path to get affinity for. - * @param start Position in the file to start affinity resolution from. - * @param len Size of data in the file to resolve affinity for. - * @return Affinity block locations. - * @throws IgniteException In case of error. - * @throws org.apache.ignite.igfs.IgfsFileNotFoundException If path doesn't exist. - */ - public Collection<IgfsBlockLocation> affinity(IgfsPath path, long start, long len) throws IgniteException; - - /** - * Get affinity block locations for data blocks of the file. In case {@code maxLen} parameter is set and - * particular block location length is greater than this value, block locations will be split into smaller - * chunks. - * - * @param path File path to get affinity for. - * @param start Position in the file to start affinity resolution from. - * @param len Size of data in the file to resolve affinity for. - * @param maxLen Maximum length of a single returned block location length. - * @return Affinity block locations. - * @throws IgniteException In case of error. - * @throws org.apache.ignite.igfs.IgfsFileNotFoundException If path doesn't exist. - */ - public Collection<IgfsBlockLocation> affinity(IgfsPath path, long start, long len, long maxLen) - throws IgniteException; - - /** - * Gets metrics snapshot for this file system. - * - * @return Metrics. - * @throws IgniteException In case of error. - */ - public IgfsMetrics metrics() throws IgniteException; - - /** - * Resets metrics for this file system. - * - * @throws IgniteException In case of error. - */ - public void resetMetrics() throws IgniteException; - - /** - * Determines size of the file denoted by provided path. In case if path is a directory, then - * total size of all containing entries will be calculated recursively. - * - * @param path File system path. - * @return Total size. - * @throws IgniteException In case of error. - */ - public long size(IgfsPath path) throws IgniteException; - - /** - * Formats the file system removing all existing entries from it. - * <p> - * Supports asynchronous execution (see {@link IgniteAsyncSupport}). - * - * @throws IgniteException In case format has failed. - */ - @IgniteAsyncSupported - public void format() throws IgniteException; - - /** - * Executes IGFS task. - * <p> - * Supports asynchronous execution (see {@link IgniteAsyncSupport}). - * - * @param task Task to execute. - * @param rslvr Optional resolver to control split boundaries. - * @param paths Collection of paths to be processed within this task. - * @param arg Optional task argument. - * @return Task result. - * @throws IgniteException If execution failed. - */ - @IgniteAsyncSupported - public <T, R> R execute(IgfsTask<T, R> task, @Nullable IgfsRecordResolver rslvr, - Collection<IgfsPath> paths, @Nullable T arg) throws IgniteException; - - /** - * Executes IGFS task with overridden maximum range length (see - * {@link org.apache.ignite.configuration.IgfsConfiguration#getMaximumTaskRangeLength()} for more information). - * <p> - * Supports asynchronous execution (see {@link IgniteAsyncSupport}). - * - * @param task Task to execute. - * @param rslvr Optional resolver to control split boundaries. - * @param paths Collection of paths to be processed within this task. - * @param skipNonExistentFiles Whether to skip non existent files. If set to {@code true} non-existent files will - * be ignored. Otherwise an exception will be thrown. - * @param maxRangeLen Optional maximum range length. If {@code 0}, then by default all consecutive - * IGFS blocks will be included. - * @param arg Optional task argument. - * @return Task result. - * @throws IgniteException If execution failed. - */ - @IgniteAsyncSupported - public <T, R> R execute(IgfsTask<T, R> task, @Nullable IgfsRecordResolver rslvr, - Collection<IgfsPath> paths, boolean skipNonExistentFiles, long maxRangeLen, @Nullable T arg) - throws IgniteException; - - /** - * Executes IGFS task. - * <p> - * Supports asynchronous execution (see {@link IgniteAsyncSupport}). - * - * @param taskCls Task class to execute. - * @param rslvr Optional resolver to control split boundaries. - * @param paths Collection of paths to be processed within this task. - * @param arg Optional task argument. - * @return Task result. - * @throws IgniteException If execution failed. - */ - @IgniteAsyncSupported - public <T, R> R execute(Class<? extends IgfsTask<T, R>> taskCls, - @Nullable IgfsRecordResolver rslvr, Collection<IgfsPath> paths, @Nullable T arg) throws IgniteException; - - /** - * Executes IGFS task with overridden maximum range length (see - * {@link org.apache.ignite.configuration.IgfsConfiguration#getMaximumTaskRangeLength()} for more information). - * <p> - * Supports asynchronous execution (see {@link IgniteAsyncSupport}). - * - * @param taskCls Task class to execute. - * @param rslvr Optional resolver to control split boundaries. - * @param paths Collection of paths to be processed within this task. - * @param skipNonExistentFiles Whether to skip non existent files. If set to {@code true} non-existent files will - * be ignored. Otherwise an exception will be thrown. - * @param maxRangeLen Maximum range length. - * @param arg Optional task argument. - * @return Task result. - * @throws IgniteException If execution failed. - */ - @IgniteAsyncSupported - public <T, R> R execute(Class<? extends IgfsTask<T, R>> taskCls, - @Nullable IgfsRecordResolver rslvr, Collection<IgfsPath> paths, boolean skipNonExistentFiles, - long maxRangeLen, @Nullable T arg) throws IgniteException; - - /** {@inheritDoc} */ - @Override public IgniteFs withAsync(); -}
