sprint-1 - Renaming and moving ignitefs classes.
Project: http://git-wip-us.apache.org/repos/asf/incubator-ignite/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ignite/commit/afa29526 Tree: http://git-wip-us.apache.org/repos/asf/incubator-ignite/tree/afa29526 Diff: http://git-wip-us.apache.org/repos/asf/incubator-ignite/diff/afa29526 Branch: refs/heads/ignite-188 Commit: afa29526589b01ea16d826182cb01fb64bacd5fb Parents: 049f9d4 Author: Dmitiry Setrakyan <dsetrak...@gridgain.com> Authored: Thu Feb 5 18:35:26 2015 -0800 Committer: Dmitiry Setrakyan <dsetrak...@gridgain.com> Committed: Thu Feb 5 18:35:27 2015 -0800 ---------------------------------------------------------------------- config/hadoop/default-config.xml | 6 +- docs/core-site.ignite.xml | 4 +- examples/config/filesystem/core-site.xml | 4 +- examples/config/filesystem/example-ignitefs.xml | 4 +- .../ignite/examples/fs/IgniteFsExample.java | 2 +- .../examples/fs/IgniteFsMapReduceExample.java | 6 +- .../src/main/java/org/apache/ignite/Ignite.java | 2 +- .../main/java/org/apache/ignite/IgniteFs.java | 9 +- .../CacheGgfsPerBlockLruEvictionPolicy.java | 2 +- .../configuration/IgniteConfiguration.java | 1 - .../configuration/IgniteFsConfiguration.java | 807 ++++++ .../org/apache/ignite/events/IgniteFsEvent.java | 2 +- .../apache/ignite/fs/IgniteFsBlockLocation.java | 63 - ...IgniteFsConcurrentModificationException.java | 36 - .../apache/ignite/fs/IgniteFsConfiguration.java | 806 ------ .../fs/IgniteFsCorruptedFileException.java | 50 - .../org/apache/ignite/fs/IgniteFsException.java | 57 - .../java/org/apache/ignite/fs/IgniteFsFile.java | 120 - .../fs/IgniteFsFileNotFoundException.java | 44 - .../apache/ignite/fs/IgniteFsFileSystem.java | 216 -- .../fs/IgniteFsGroupDataBlocksKeyMapper.java | 101 - .../apache/ignite/fs/IgniteFsInputStream.java | 80 - .../fs/IgniteFsInvalidHdfsVersionException.java | 42 - .../ignite/fs/IgniteFsInvalidPathException.java | 57 - .../org/apache/ignite/fs/IgniteFsMetrics.java | 159 -- .../java/org/apache/ignite/fs/IgniteFsMode.java | 80 - .../ignite/fs/IgniteFsOutOfSpaceException.java | 58 - .../apache/ignite/fs/IgniteFsOutputStream.java | 35 - .../fs/IgniteFsParentNotDirectoryException.java | 50 - .../java/org/apache/ignite/fs/IgniteFsPath.java | 262 -- .../fs/IgniteFsPathAlreadyExistsException.java | 50 - .../apache/ignite/fs/IgniteFsPathSummary.java | 138 - .../org/apache/ignite/fs/IgniteFsReader.java | 38 - .../ignite/fs/mapreduce/IgniteFsFileRange.java | 80 - .../IgniteFsInputStreamJobAdapter.java | 52 - .../apache/ignite/fs/mapreduce/IgniteFsJob.java | 69 - .../ignite/fs/mapreduce/IgniteFsJobAdapter.java | 28 - .../fs/mapreduce/IgniteFsRangeInputStream.java | 197 -- .../fs/mapreduce/IgniteFsRecordResolver.java | 56 - .../ignite/fs/mapreduce/IgniteFsTask.java | 172 -- .../ignite/fs/mapreduce/IgniteFsTaskArgs.java | 82 - .../mapreduce/IgniteFsTaskNoReduceAdapter.java | 42 - .../org/apache/ignite/fs/mapreduce/package.html | 23 - .../IgniteFsByteDelimiterRecordResolver.java | 347 --- .../IgniteFsFixedLengthRecordResolver.java | 86 - .../records/IgniteFsNewLineRecordResolver.java | 66 - .../IgniteFsStringDelimiterRecordResolver.java | 84 - .../ignite/fs/mapreduce/records/package.html | 24 - .../main/java/org/apache/ignite/fs/package.html | 24 - .../ignite/ignitefs/IgniteFsBlockLocation.java | 63 + ...IgniteFsConcurrentModificationException.java | 36 + .../IgniteFsCorruptedFileException.java | 50 + .../ignite/ignitefs/IgniteFsException.java | 57 + .../apache/ignite/ignitefs/IgniteFsFile.java | 120 + .../ignitefs/IgniteFsFileNotFoundException.java | 44 + .../ignite/ignitefs/IgniteFsFileSystem.java | 216 ++ .../IgniteFsGroupDataBlocksKeyMapper.java | 101 + .../ignite/ignitefs/IgniteFsInputStream.java | 80 + .../IgniteFsInvalidHdfsVersionException.java | 42 + .../ignitefs/IgniteFsInvalidPathException.java | 57 + .../apache/ignite/ignitefs/IgniteFsMetrics.java | 159 ++ .../apache/ignite/ignitefs/IgniteFsMode.java | 80 + .../ignitefs/IgniteFsOutOfSpaceException.java | 58 + .../ignite/ignitefs/IgniteFsOutputStream.java | 35 + .../IgniteFsParentNotDirectoryException.java | 50 + .../apache/ignite/ignitefs/IgniteFsPath.java | 262 ++ .../IgniteFsPathAlreadyExistsException.java | 50 + .../ignite/ignitefs/IgniteFsPathSummary.java | 138 + .../apache/ignite/ignitefs/IgniteFsReader.java | 38 + .../ignitefs/mapreduce/IgniteFsFileRange.java | 80 + .../IgniteFsInputStreamJobAdapter.java | 52 + .../ignite/ignitefs/mapreduce/IgniteFsJob.java | 69 + .../ignitefs/mapreduce/IgniteFsJobAdapter.java | 28 + .../mapreduce/IgniteFsRangeInputStream.java | 197 ++ .../mapreduce/IgniteFsRecordResolver.java | 56 + .../ignite/ignitefs/mapreduce/IgniteFsTask.java | 172 ++ .../ignitefs/mapreduce/IgniteFsTaskArgs.java | 82 + .../mapreduce/IgniteFsTaskNoReduceAdapter.java | 42 + .../ignite/ignitefs/mapreduce/package.html | 23 + .../IgniteFsByteDelimiterRecordResolver.java | 347 +++ .../IgniteFsFixedLengthRecordResolver.java | 86 + .../records/IgniteFsNewLineRecordResolver.java | 66 + .../IgniteFsStringDelimiterRecordResolver.java | 84 + .../ignitefs/mapreduce/records/package.html | 24 + .../org/apache/ignite/ignitefs/package.html | 24 + .../org/apache/ignite/internal/IgnitionEx.java | 1 - .../fs/common/GridGgfsControlResponse.java | 2 +- .../internal/fs/common/GridGgfsLogger.java | 2 +- .../internal/fs/common/GridGgfsMarshaller.java | 4 +- .../fs/common/GridGgfsPathControlRequest.java | 2 +- .../processors/cache/GridCacheAdapter.java | 1 - .../processors/cache/GridCacheProcessor.java | 1 - .../processors/cache/GridCacheUtils.java | 1 - .../processors/fs/GridGgfsAsyncImpl.java | 5 +- .../processors/fs/GridGgfsAttributes.java | 8 +- .../fs/GridGgfsBlockLocationImpl.java | 2 +- .../internal/processors/fs/GridGgfsContext.java | 2 +- .../processors/fs/GridGgfsDataManager.java | 3 +- .../fs/GridGgfsDirectoryNotEmptyException.java | 2 +- .../internal/processors/fs/GridGgfsEx.java | 2 +- .../processors/fs/GridGgfsFileInfo.java | 5 +- .../processors/fs/GridGgfsFileWorkerBatch.java | 2 +- .../internal/processors/fs/GridGgfsImpl.java | 11 +- .../fs/GridGgfsInputStreamAdapter.java | 2 +- .../processors/fs/GridGgfsInputStreamImpl.java | 2 +- .../processors/fs/GridGgfsIpcHandler.java | 2 +- .../internal/processors/fs/GridGgfsJobImpl.java | 4 +- .../processors/fs/GridGgfsMetaManager.java | 3 +- .../processors/fs/GridGgfsModeResolver.java | 2 +- .../internal/processors/fs/GridGgfsPaths.java | 2 +- .../GridGgfsSecondaryInputStreamDescriptor.java | 2 +- .../processors/fs/GridGgfsServerManager.java | 4 +- .../processors/fs/IgniteFsFileImpl.java | 2 +- .../processors/fs/IgniteFsMetricsAdapter.java | 2 +- .../processors/fs/IgniteFsNoopProcessor.java | 4 +- .../fs/IgniteFsOutputStreamAdapter.java | 2 +- .../processors/fs/IgniteFsOutputStreamImpl.java | 4 +- .../processors/fs/IgniteFsProcessor.java | 6 +- .../processors/fs/IgniteFsProcessorAdapter.java | 4 +- .../processors/fs/IgniteFsTaskArgsImpl.java | 4 +- .../internal/processors/job/GridJobWorker.java | 2 +- .../processors/task/GridTaskWorker.java | 2 +- .../ignite/internal/visor/ggfs/VisorGgfs.java | 2 +- .../internal/visor/ggfs/VisorGgfsMetrics.java | 4 +- .../internal/visor/ggfs/VisorGgfsProfiler.java | 2 +- .../visor/ggfs/VisorGgfsProfilerEntry.java | 2 +- .../visor/ggfs/VisorGgfsProfilerTask.java | 2 +- .../visor/node/VisorGgfsConfiguration.java | 3 +- .../internal/visor/util/VisorTaskUtils.java | 2 +- modules/core/src/test/config/ggfs-loopback.xml | 4 +- modules/core/src/test/config/ggfs-shmem.xml | 4 +- .../hadoop/core-site-loopback-secondary.xml | 4 +- .../test/config/hadoop/core-site-loopback.xml | 4 +- .../test/config/hadoop/core-site-secondary.xml | 4 +- .../core/src/test/config/hadoop/core-site.xml | 4 +- .../fs/GridGgfsEventsAbstractSelfTest.java | 849 ------- .../GridGgfsFragmentizerAbstractSelfTest.java | 168 -- .../ignite/fs/GridGgfsFragmentizerSelfTest.java | 268 -- .../GridGgfsFragmentizerTopologySelfTest.java | 49 - .../apache/ignite/fs/GridGgfsPathSelfTest.java | 162 -- .../ignite/fs/GridGgfsTestInputGenerator.java | 51 - .../GridGgfsEventsAbstractSelfTest.java | 849 +++++++ .../GridGgfsFragmentizerAbstractSelfTest.java | 168 ++ .../ignitefs/GridGgfsFragmentizerSelfTest.java | 268 ++ .../GridGgfsFragmentizerTopologySelfTest.java | 49 + .../ignite/ignitefs/GridGgfsPathSelfTest.java | 162 ++ .../ignitefs/GridGgfsTestInputGenerator.java | 51 + ...heGgfsPerBlockLruEvictionPolicySelfTest.java | 4 +- .../processors/fs/GridGgfsAbstractSelfTest.java | 4 +- .../fs/GridGgfsAttributesSelfTest.java | 4 +- .../processors/fs/GridGgfsCacheSelfTest.java | 2 +- .../fs/GridGgfsDataManagerSelfTest.java | 2 +- .../fs/GridGgfsDualAbstractSelfTest.java | 4 +- .../fs/GridGgfsDualAsyncSelfTest.java | 2 +- .../processors/fs/GridGgfsDualSyncSelfTest.java | 2 +- ...GgfsGroupDataBlockKeyMapperHashSelfTest.java | 4 +- .../fs/GridGgfsMetaManagerSelfTest.java | 2 +- .../processors/fs/GridGgfsMetricsSelfTest.java | 4 +- .../fs/GridGgfsModeResolverSelfTest.java | 4 +- .../processors/fs/GridGgfsModesSelfTest.java | 4 +- .../GridGgfsPrimaryOffheapTieredSelfTest.java | 2 +- .../GridGgfsPrimaryOffheapValuesSelfTest.java | 2 +- .../processors/fs/GridGgfsPrimarySelfTest.java | 2 +- .../fs/GridGgfsProcessorSelfTest.java | 2 +- .../fs/GridGgfsProcessorValidationSelfTest.java | 8 +- ...IpcEndpointRegistrationAbstractSelfTest.java | 4 +- ...dpointRegistrationOnLinuxAndMacSelfTest.java | 2 +- .../processors/fs/GridGgfsSizeSelfTest.java | 2 +- .../processors/fs/GridGgfsStreamsSelfTest.java | 2 +- .../processors/fs/GridGgfsTaskSelfTest.java | 10 +- .../GridGgfsAbstractRecordResolverSelfTest.java | 6 +- ...GgfsByteDelimiterRecordResolverSelfTest.java | 6 +- ...idGgfsFixedLengthRecordResolverSelfTest.java | 6 +- ...sNewLineDelimiterRecordResolverSelfTest.java | 8 +- ...fsStringDelimiterRecordResolverSelfTest.java | 6 +- .../ipc/shmem/IpcSharedMemoryNodeStartup.java | 1 - .../ignite/testsuites/IgniteFsTestSuite.java | 2 +- .../fs/hadoop/GridGgfsHadoopParameters.java | 94 - .../org/apache/ignite/fs/hadoop/package.html | 23 - .../fs/hadoop/v1/GridGgfsHadoopFileSystem.java | 1253 ---------- .../org/apache/ignite/fs/hadoop/v1/package.html | 23 - .../fs/hadoop/v2/GridGgfsHadoopFileSystem.java | 1007 -------- .../org/apache/ignite/fs/hadoop/v2/package.html | 23 - .../main/java/org/apache/ignite/fs/package.html | 23 - .../hadoop/GridGgfsHadoopParameters.java | 94 + .../apache/ignite/ignitefs/hadoop/package.html | 23 + .../hadoop/v1/GridGgfsHadoopFileSystem.java | 1253 ++++++++++ .../ignite/ignitefs/hadoop/v1/package.html | 23 + .../hadoop/v2/GridGgfsHadoopFileSystem.java | 1007 ++++++++ .../ignite/ignitefs/hadoop/v2/package.html | 23 + .../org/apache/ignite/ignitefs/package.html | 23 + .../internal/fs/hadoop/GridGgfsHadoop.java | 2 +- .../fs/hadoop/GridGgfsHadoopEndpoint.java | 2 +- .../hadoop/GridGgfsHadoopFileSystemWrapper.java | 4 +- .../fs/hadoop/GridGgfsHadoopInProc.java | 2 +- .../fs/hadoop/GridGgfsHadoopOutProc.java | 2 +- .../fs/hadoop/GridGgfsHadoopReader.java | 2 +- .../internal/fs/hadoop/GridGgfsHadoopUtils.java | 2 +- .../fs/hadoop/GridGgfsHadoopWrapper.java | 2 +- .../fs/GridHadoopDistributedFileSystem.java | 2 +- .../hadoop/fs/GridHadoopFileSystemsUtils.java | 2 +- .../GridHadoopDefaultMapReducePlanner.java | 2 +- .../hadoop/v2/GridHadoopV2TaskContext.java | 2 +- .../GridHadoopClientProtocolSelfTest.java | 2 +- .../apache/ignite/fs/GridFileSystemLoad.java | 549 ---- ...dGgfsHadoop20FileSystemAbstractSelfTest.java | 1967 --------------- ...doop20FileSystemLoopbackPrimarySelfTest.java | 74 - ...sHadoop20FileSystemShmemPrimarySelfTest.java | 74 - .../fs/GridGgfsHadoopDualAbstractSelfTest.java | 304 --- .../fs/GridGgfsHadoopDualAsyncSelfTest.java | 32 - .../fs/GridGgfsHadoopDualSyncSelfTest.java | 32 - ...ridGgfsHadoopFileSystemAbstractSelfTest.java | 2365 ------------------ .../GridGgfsHadoopFileSystemClientSelfTest.java | 200 -- ...idGgfsHadoopFileSystemHandshakeSelfTest.java | 311 --- ...ridGgfsHadoopFileSystemIpcCacheSelfTest.java | 207 -- .../GridGgfsHadoopFileSystemLoggerSelfTest.java | 287 --- ...GgfsHadoopFileSystemLoggerStateSelfTest.java | 325 --- ...adoopFileSystemLoopbackAbstractSelfTest.java | 46 - ...SystemLoopbackEmbeddedDualAsyncSelfTest.java | 33 - ...eSystemLoopbackEmbeddedDualSyncSelfTest.java | 33 - ...leSystemLoopbackEmbeddedPrimarySelfTest.java | 33 - ...SystemLoopbackEmbeddedSecondarySelfTest.java | 34 - ...SystemLoopbackExternalDualAsyncSelfTest.java | 33 - ...eSystemLoopbackExternalDualSyncSelfTest.java | 33 - ...leSystemLoopbackExternalPrimarySelfTest.java | 33 - ...SystemLoopbackExternalSecondarySelfTest.java | 34 - ...fsHadoopFileSystemSecondaryModeSelfTest.java | 319 --- ...fsHadoopFileSystemShmemAbstractSelfTest.java | 88 - ...ileSystemShmemEmbeddedDualAsyncSelfTest.java | 33 - ...FileSystemShmemEmbeddedDualSyncSelfTest.java | 33 - ...pFileSystemShmemEmbeddedPrimarySelfTest.java | 33 - ...ileSystemShmemEmbeddedSecondarySelfTest.java | 33 - ...ileSystemShmemExternalDualAsyncSelfTest.java | 33 - ...FileSystemShmemExternalDualSyncSelfTest.java | 33 - ...pFileSystemShmemExternalPrimarySelfTest.java | 33 - ...ileSystemShmemExternalSecondarySelfTest.java | 33 - .../fs/GridGgfsNearOnlyMultiNodeSelfTest.java | 212 -- .../ignite/fs/IgniteFsEventsTestSuite.java | 267 -- .../ignite/ignitefs/GridFileSystemLoad.java | 549 ++++ ...dGgfsHadoop20FileSystemAbstractSelfTest.java | 1967 +++++++++++++++ ...doop20FileSystemLoopbackPrimarySelfTest.java | 74 + ...sHadoop20FileSystemShmemPrimarySelfTest.java | 74 + .../GridGgfsHadoopDualAbstractSelfTest.java | 304 +++ .../GridGgfsHadoopDualAsyncSelfTest.java | 32 + .../GridGgfsHadoopDualSyncSelfTest.java | 32 + ...ridGgfsHadoopFileSystemAbstractSelfTest.java | 2365 ++++++++++++++++++ .../GridGgfsHadoopFileSystemClientSelfTest.java | 200 ++ ...idGgfsHadoopFileSystemHandshakeSelfTest.java | 311 +++ ...ridGgfsHadoopFileSystemIpcCacheSelfTest.java | 207 ++ .../GridGgfsHadoopFileSystemLoggerSelfTest.java | 287 +++ ...GgfsHadoopFileSystemLoggerStateSelfTest.java | 325 +++ ...adoopFileSystemLoopbackAbstractSelfTest.java | 46 + ...SystemLoopbackEmbeddedDualAsyncSelfTest.java | 33 + ...eSystemLoopbackEmbeddedDualSyncSelfTest.java | 33 + ...leSystemLoopbackEmbeddedPrimarySelfTest.java | 33 + ...SystemLoopbackEmbeddedSecondarySelfTest.java | 34 + ...SystemLoopbackExternalDualAsyncSelfTest.java | 33 + ...eSystemLoopbackExternalDualSyncSelfTest.java | 33 + ...leSystemLoopbackExternalPrimarySelfTest.java | 33 + ...SystemLoopbackExternalSecondarySelfTest.java | 34 + ...fsHadoopFileSystemSecondaryModeSelfTest.java | 319 +++ ...fsHadoopFileSystemShmemAbstractSelfTest.java | 88 + ...ileSystemShmemEmbeddedDualAsyncSelfTest.java | 33 + ...FileSystemShmemEmbeddedDualSyncSelfTest.java | 33 + ...pFileSystemShmemEmbeddedPrimarySelfTest.java | 33 + ...ileSystemShmemEmbeddedSecondarySelfTest.java | 33 + ...ileSystemShmemExternalDualAsyncSelfTest.java | 33 + ...FileSystemShmemExternalDualSyncSelfTest.java | 33 + ...pFileSystemShmemExternalPrimarySelfTest.java | 33 + ...ileSystemShmemExternalSecondarySelfTest.java | 33 + .../GridGgfsNearOnlyMultiNodeSelfTest.java | 212 ++ .../ignitefs/IgniteFsEventsTestSuite.java | 267 ++ .../hadoop/GridHadoopAbstractSelfTest.java | 6 +- .../hadoop/GridHadoopAbstractWordCountTest.java | 2 +- .../hadoop/GridHadoopCommandLineTest.java | 2 +- ...idHadoopDefaultMapReducePlannerSelfTest.java | 5 +- .../GridHadoopMapReduceEmbeddedSelfTest.java | 2 +- .../hadoop/GridHadoopMapReduceTest.java | 2 +- .../processors/hadoop/GridHadoopStartup.java | 4 +- .../hadoop/GridHadoopTaskExecutionSelfTest.java | 6 +- .../hadoop/GridHadoopTasksAllVersionsTest.java | 2 +- ...GridHadoopExternalTaskExecutionSelfTest.java | 2 +- .../IgniteFsLinuxAndMacOSTestSuite.java | 2 +- .../testsuites/IgniteHadoopTestSuite.java | 2 +- pom.xml | 4 +- 285 files changed, 16346 insertions(+), 16343 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/afa29526/config/hadoop/default-config.xml ---------------------------------------------------------------------- diff --git a/config/hadoop/default-config.xml b/config/hadoop/default-config.xml index 6eb7f46..fcb31e2 100644 --- a/config/hadoop/default-config.xml +++ b/config/hadoop/default-config.xml @@ -52,7 +52,7 @@ <!-- Abstract GGFS file system configuration to be used as a template. --> - <bean id="ggfsCfgBase" class="org.apache.ignite.fs.IgniteFsConfiguration" abstract="true"> + <bean id="ggfsCfgBase" class="org.apache.ignite.configuration.IgniteFsConfiguration" abstract="true"> <!-- Must correlate with cache affinity mapper. --> <property name="blockSize" value="#{128 * 1024}"/> <property name="perNodeBatchSize" value="512"/> @@ -70,7 +70,7 @@ <property name="writeSynchronizationMode" value="FULL_SYNC"/> <property name="backups" value="0"/> <property name="affinityMapper"> - <bean class="org.apache.ignite.fs.IgniteFsGroupDataBlocksKeyMapper"> + <bean class="org.apache.ignite.ignitefs.IgniteFsGroupDataBlocksKeyMapper"> <!-- How many sequential blocks will be stored on the same node. --> <constructor-arg value="512"/> </bean> @@ -110,7 +110,7 @@ --> <property name="ggfsConfiguration"> <list> - <bean class="org.apache.ignite.fs.IgniteFsConfiguration" parent="ggfsCfgBase"> + <bean class="org.apache.ignite.configuration.IgniteFsConfiguration" parent="ggfsCfgBase"> <property name="name" value="ggfs"/> <!-- Caches with these names must be configured. --> http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/afa29526/docs/core-site.ignite.xml ---------------------------------------------------------------------- diff --git a/docs/core-site.ignite.xml b/docs/core-site.ignite.xml index 93620a6..7b84bba 100644 --- a/docs/core-site.ignite.xml +++ b/docs/core-site.ignite.xml @@ -47,7 +47,7 @@ --> <property> <name>fs.ignitefs.impl</name> - <value>org.apache.ignite.fs.hadoop.v1.GridGgfsHadoopFileSystem</value> + <value>org.apache.ignite.ignitefs.hadoop.v1.GridGgfsHadoopFileSystem</value> </property> <!-- @@ -55,7 +55,7 @@ --> <property> <name>fs.AbstractFileSystem.ggfs.impl</name> - <value>org.apache.ignite.fs.hadoop.v2.GridGgfsHadoopFileSystem</value> + <value>org.apache.ignite.ignitefs.hadoop.v2.GridGgfsHadoopFileSystem</value> </property> <!-- http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/afa29526/examples/config/filesystem/core-site.xml ---------------------------------------------------------------------- diff --git a/examples/config/filesystem/core-site.xml b/examples/config/filesystem/core-site.xml index a422077..71eb0ac 100644 --- a/examples/config/filesystem/core-site.xml +++ b/examples/config/filesystem/core-site.xml @@ -31,12 +31,12 @@ <property> <!-- FS driver class for the 'ignitefs://' URIs. --> <name>fs.ignitefs.impl</name> - <value>org.apache.ignite.fs.hadoop.v1.GridGgfsHadoopFileSystem</value> + <value>org.apache.ignite.ignitefs.hadoop.v1.GridGgfsHadoopFileSystem</value> </property> <property> <!-- FS driver class for the 'ignitefs://' URIs in Hadoop2.x --> <name>fs.AbstractFileSystem.ggfs.impl</name> - <value>org.apache.ignite.fs.hadoop.v2.GridGgfsHadoopFileSystem</value> + <value>org.apache.ignite.ignitefs.hadoop.v2.GridGgfsHadoopFileSystem</value> </property> </configuration> http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/afa29526/examples/config/filesystem/example-ignitefs.xml ---------------------------------------------------------------------- diff --git a/examples/config/filesystem/example-ignitefs.xml b/examples/config/filesystem/example-ignitefs.xml index 0cdb774..ba136f5 100644 --- a/examples/config/filesystem/example-ignitefs.xml +++ b/examples/config/filesystem/example-ignitefs.xml @@ -65,7 +65,7 @@ <property name="ggfsConfiguration"> <list> - <bean class="org.apache.ignite.fs.IgniteFsConfiguration"> + <bean class="org.apache.ignite.configuration.IgniteFsConfiguration"> <property name="name" value="ignitefs"/> <property name="metaCacheName" value="ignitefs-meta"/> <property name="dataCacheName" value="ignitefs-data"/> @@ -122,7 +122,7 @@ <property name="distributionMode" value="PARTITIONED_ONLY"/> <property name="backups" value="0"/> <property name="affinityMapper"> - <bean class="org.apache.ignite.fs.IgniteFsGroupDataBlocksKeyMapper"> + <bean class="org.apache.ignite.ignitefs.IgniteFsGroupDataBlocksKeyMapper"> <!-- Haw many blocks in row will be stored on the same node. --> <constructor-arg value="512"/> </bean> http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/afa29526/examples/src/main/java/org/apache/ignite/examples/fs/IgniteFsExample.java ---------------------------------------------------------------------- diff --git a/examples/src/main/java/org/apache/ignite/examples/fs/IgniteFsExample.java b/examples/src/main/java/org/apache/ignite/examples/fs/IgniteFsExample.java index cb67ae3..9a017dc 100644 --- a/examples/src/main/java/org/apache/ignite/examples/fs/IgniteFsExample.java +++ b/examples/src/main/java/org/apache/ignite/examples/fs/IgniteFsExample.java @@ -18,7 +18,7 @@ package org.apache.ignite.examples.fs; import org.apache.ignite.*; -import org.apache.ignite.fs.*; +import org.apache.ignite.ignitefs.*; import org.jetbrains.annotations.*; import java.io.*; http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/afa29526/examples/src/main/java/org/apache/ignite/examples/fs/IgniteFsMapReduceExample.java ---------------------------------------------------------------------- diff --git a/examples/src/main/java/org/apache/ignite/examples/fs/IgniteFsMapReduceExample.java b/examples/src/main/java/org/apache/ignite/examples/fs/IgniteFsMapReduceExample.java index 1570c6f..07fec09 100644 --- a/examples/src/main/java/org/apache/ignite/examples/fs/IgniteFsMapReduceExample.java +++ b/examples/src/main/java/org/apache/ignite/examples/fs/IgniteFsMapReduceExample.java @@ -19,9 +19,9 @@ package org.apache.ignite.examples.fs; import org.apache.ignite.*; import org.apache.ignite.compute.*; -import org.apache.ignite.fs.*; -import org.apache.ignite.fs.mapreduce.*; -import org.apache.ignite.fs.mapreduce.records.*; +import org.apache.ignite.ignitefs.*; +import org.apache.ignite.ignitefs.mapreduce.*; +import org.apache.ignite.ignitefs.mapreduce.records.*; import java.io.*; import java.util.*; http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/afa29526/modules/core/src/main/java/org/apache/ignite/Ignite.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/Ignite.java b/modules/core/src/main/java/org/apache/ignite/Ignite.java index 8e1a82e..10eced1 100644 --- a/modules/core/src/main/java/org/apache/ignite/Ignite.java +++ b/modules/core/src/main/java/org/apache/ignite/Ignite.java @@ -21,7 +21,7 @@ import org.apache.ignite.cache.*; import org.apache.ignite.cache.affinity.*; import org.apache.ignite.cluster.*; import org.apache.ignite.configuration.*; -import org.apache.ignite.fs.IgniteFsConfiguration; +import org.apache.ignite.configuration.IgniteFsConfiguration; import org.apache.ignite.plugin.*; import org.apache.ignite.internal.product.*; import org.apache.ignite.plugin.security.*; http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/afa29526/modules/core/src/main/java/org/apache/ignite/IgniteFs.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteFs.java b/modules/core/src/main/java/org/apache/ignite/IgniteFs.java index e78bd27..94d33ff 100644 --- a/modules/core/src/main/java/org/apache/ignite/IgniteFs.java +++ b/modules/core/src/main/java/org/apache/ignite/IgniteFs.java @@ -17,8 +17,9 @@ package org.apache.ignite; -import org.apache.ignite.fs.*; -import org.apache.ignite.fs.mapreduce.*; +import org.apache.ignite.configuration.*; +import org.apache.ignite.ignitefs.*; +import org.apache.ignite.ignitefs.mapreduce.*; import org.apache.ignite.lang.*; import org.jetbrains.annotations.*; @@ -283,7 +284,7 @@ public interface IgniteFs extends IgniteFsFileSystem, IgniteAsyncSupport { /** * Executes GGFS task with overridden maximum range length (see - * {@link org.apache.ignite.fs.IgniteFsConfiguration#getMaximumTaskRangeLength()} for more information). + * {@link org.apache.ignite.configuration.IgniteFsConfiguration#getMaximumTaskRangeLength()} for more information). * <p> * Supports asynchronous execution (see {@link IgniteAsyncSupport}). * @@ -321,7 +322,7 @@ public interface IgniteFs extends IgniteFsFileSystem, IgniteAsyncSupport { /** * Executes GGFS task with overridden maximum range length (see - * {@link org.apache.ignite.fs.IgniteFsConfiguration#getMaximumTaskRangeLength()} for more information). + * {@link org.apache.ignite.configuration.IgniteFsConfiguration#getMaximumTaskRangeLength()} for more information). * <p> * Supports asynchronous execution (see {@link IgniteAsyncSupport}). * http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/afa29526/modules/core/src/main/java/org/apache/ignite/cache/eviction/ggfs/CacheGgfsPerBlockLruEvictionPolicy.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/cache/eviction/ggfs/CacheGgfsPerBlockLruEvictionPolicy.java b/modules/core/src/main/java/org/apache/ignite/cache/eviction/ggfs/CacheGgfsPerBlockLruEvictionPolicy.java index f1d5cdf..23e22d6 100644 --- a/modules/core/src/main/java/org/apache/ignite/cache/eviction/ggfs/CacheGgfsPerBlockLruEvictionPolicy.java +++ b/modules/core/src/main/java/org/apache/ignite/cache/eviction/ggfs/CacheGgfsPerBlockLruEvictionPolicy.java @@ -20,7 +20,7 @@ package org.apache.ignite.cache.eviction.ggfs; import org.apache.ignite.*; import org.apache.ignite.cache.*; import org.apache.ignite.cache.eviction.*; -import org.apache.ignite.fs.*; +import org.apache.ignite.ignitefs.*; import org.apache.ignite.internal.processors.fs.*; import org.jdk8.backport.*; import org.jdk8.backport.ConcurrentLinkedDeque8.*; http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/afa29526/modules/core/src/main/java/org/apache/ignite/configuration/IgniteConfiguration.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/configuration/IgniteConfiguration.java b/modules/core/src/main/java/org/apache/ignite/configuration/IgniteConfiguration.java index a671e2b..f086bbc 100644 --- a/modules/core/src/main/java/org/apache/ignite/configuration/IgniteConfiguration.java +++ b/modules/core/src/main/java/org/apache/ignite/configuration/IgniteConfiguration.java @@ -21,7 +21,6 @@ import org.apache.ignite.*; import org.apache.ignite.cache.*; import org.apache.ignite.client.ssl.*; import org.apache.ignite.events.*; -import org.apache.ignite.fs.*; import org.apache.ignite.internal.processors.hadoop.*; import org.apache.ignite.internal.managers.eventstorage.*; import org.apache.ignite.internal.util.typedef.internal.*; http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/afa29526/modules/core/src/main/java/org/apache/ignite/configuration/IgniteFsConfiguration.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/configuration/IgniteFsConfiguration.java b/modules/core/src/main/java/org/apache/ignite/configuration/IgniteFsConfiguration.java new file mode 100644 index 0000000..497037b --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/configuration/IgniteFsConfiguration.java @@ -0,0 +1,807 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.configuration; + +import org.apache.ignite.ignitefs.*; +import org.apache.ignite.internal.util.typedef.internal.*; +import org.jetbrains.annotations.*; + +import java.util.*; +import java.util.concurrent.*; + +/** + * {@code GGFS} configuration. More than one file system can be configured within grid. + * {@code GGFS} configuration is provided via {@link org.apache.ignite.configuration.IgniteConfiguration#getGgfsConfiguration()} + * method. + * <p> + * Refer to {@code config/hadoop/default-config.xml} or {@code config/hadoop/default-config-client.xml} + * configuration files under GridGain installation to see sample {@code GGFS} configuration. + */ +public class IgniteFsConfiguration { + /** Default file system user name. */ + public static final String DFLT_USER_NAME = System.getProperty("user.name", "anonymous"); + + /** Default IPC port. */ + public static final int DFLT_IPC_PORT = 10500; + + /** Default fragmentizer throttling block length. */ + public static final long DFLT_FRAGMENTIZER_THROTTLING_BLOCK_LENGTH = 16 * 1024 * 1024; + + /** Default fragmentizer throttling delay. */ + public static final long DFLT_FRAGMENTIZER_THROTTLING_DELAY = 200; + + /** Default fragmentizer concurrent files. */ + public static final int DFLT_FRAGMENTIZER_CONCURRENT_FILES = 0; + + /** Default fragmentizer local writes ratio. */ + public static final float DFLT_FRAGMENTIZER_LOCAL_WRITES_RATIO = 0.8f; + + /** Fragmentizer enabled property. */ + public static final boolean DFLT_FRAGMENTIZER_ENABLED = true; + + /** Default batch size for logging. */ + public static final int DFLT_GGFS_LOG_BATCH_SIZE = 100; + + /** Default {@code GGFS} log directory. */ + public static final String DFLT_GGFS_LOG_DIR = "work/ggfs/log"; + + /** Default per node buffer size. */ + public static final int DFLT_PER_NODE_BATCH_SIZE = 100; + + /** Default number of per node parallel operations. */ + public static final int DFLT_PER_NODE_PARALLEL_BATCH_CNT = 8; + + /** Default GGFS mode. */ + public static final IgniteFsMode DFLT_MODE = IgniteFsMode.DUAL_ASYNC; + + /** Default file's data block size (bytes). */ + public static final int DFLT_BLOCK_SIZE = 1 << 16; + + /** Default read/write buffers size (bytes). */ + public static final int DFLT_BUF_SIZE = 1 << 16; + + /** Default trash directory purge await timeout in case data cache oversize is detected. */ + public static final long DFLT_TRASH_PURGE_TIMEOUT = 1000; + + /** Default management port. */ + public static final int DFLT_MGMT_PORT = 11400; + + /** Default IPC endpoint enabled flag. */ + public static final boolean DFLT_IPC_ENDPOINT_ENABLED = true; + + /** GGFS instance name. */ + private String name; + + /** Cache name to store GGFS meta information. */ + private String metaCacheName; + + /** Cache name to store file's data blocks. */ + private String dataCacheName; + + /** File's data block size (bytes). */ + private int blockSize = DFLT_BLOCK_SIZE; + + /** The number of pre-fetched blocks if specific file's chunk is requested. */ + private int prefetchBlocks; + + /** Amount of sequential block reads before prefetch is triggered. */ + private int seqReadsBeforePrefetch; + + /** Read/write buffers size for stream operations (bytes). */ + private int bufSize = DFLT_BUF_SIZE; + + /** Per node buffer size. */ + private int perNodeBatchSize = DFLT_PER_NODE_BATCH_SIZE; + + /** Per node parallel operations. */ + private int perNodeParallelBatchCnt = DFLT_PER_NODE_PARALLEL_BATCH_CNT; + + /** IPC endpoint properties to publish GGFS over. */ + private Map<String, String> ipcEndpointCfg; + + /** IPC endpoint enabled flag. */ + private boolean ipcEndpointEnabled = DFLT_IPC_ENDPOINT_ENABLED; + + /** Management port. */ + private int mgmtPort = DFLT_MGMT_PORT; + + /** Secondary file system */ + private IgniteFsFileSystem secondaryFs; + + /** GGFS mode. */ + private IgniteFsMode dfltMode = DFLT_MODE; + + /** Fragmentizer throttling block length. */ + private long fragmentizerThrottlingBlockLen = DFLT_FRAGMENTIZER_THROTTLING_BLOCK_LENGTH; + + /** Fragmentizer throttling delay. */ + private long fragmentizerThrottlingDelay = DFLT_FRAGMENTIZER_THROTTLING_DELAY; + + /** Fragmentizer concurrent files. */ + private int fragmentizerConcurrentFiles = DFLT_FRAGMENTIZER_CONCURRENT_FILES; + + /** Fragmentizer local writes ratio. */ + private float fragmentizerLocWritesRatio = DFLT_FRAGMENTIZER_LOCAL_WRITES_RATIO; + + /** Fragmentizer enabled flag. */ + private boolean fragmentizerEnabled = DFLT_FRAGMENTIZER_ENABLED; + + /** Path modes. */ + private Map<String, IgniteFsMode> pathModes; + + /** Maximum space. */ + private long maxSpace; + + /** Trash purge await timeout. */ + private long trashPurgeTimeout = DFLT_TRASH_PURGE_TIMEOUT; + + /** Dual mode PUT operations executor service. */ + private ExecutorService dualModePutExec; + + /** Dual mode PUT operations executor service shutdown flag. */ + private boolean dualModePutExecShutdown; + + /** Maximum amount of data in pending puts. */ + private long dualModeMaxPendingPutsSize; + + /** Maximum range length. */ + private long maxTaskRangeLen; + + /** + * Constructs default configuration. + */ + public IgniteFsConfiguration() { + // No-op. + } + + /** + * Constructs the copy of the configuration. + * + * @param cfg Configuration to copy. + */ + public IgniteFsConfiguration(IgniteFsConfiguration cfg) { + assert cfg != null; + + /* + * Must preserve alphabetical order! + */ + blockSize = cfg.getBlockSize(); + bufSize = cfg.getStreamBufferSize(); + dataCacheName = cfg.getDataCacheName(); + dfltMode = cfg.getDefaultMode(); + dualModeMaxPendingPutsSize = cfg.getDualModeMaxPendingPutsSize(); + dualModePutExec = cfg.getDualModePutExecutorService(); + dualModePutExecShutdown = cfg.getDualModePutExecutorServiceShutdown(); + fragmentizerConcurrentFiles = cfg.getFragmentizerConcurrentFiles(); + fragmentizerLocWritesRatio = cfg.getFragmentizerLocalWritesRatio(); + fragmentizerEnabled = cfg.isFragmentizerEnabled(); + fragmentizerThrottlingBlockLen = cfg.getFragmentizerThrottlingBlockLength(); + fragmentizerThrottlingDelay = cfg.getFragmentizerThrottlingDelay(); + secondaryFs = cfg.getSecondaryFileSystem(); + ipcEndpointCfg = cfg.getIpcEndpointConfiguration(); + ipcEndpointEnabled = cfg.isIpcEndpointEnabled(); + maxSpace = cfg.getMaxSpaceSize(); + maxTaskRangeLen = cfg.getMaximumTaskRangeLength(); + metaCacheName = cfg.getMetaCacheName(); + mgmtPort = cfg.getManagementPort(); + name = cfg.getName(); + pathModes = cfg.getPathModes(); + perNodeBatchSize = cfg.getPerNodeBatchSize(); + perNodeParallelBatchCnt = cfg.getPerNodeParallelBatchCount(); + prefetchBlocks = cfg.getPrefetchBlocks(); + seqReadsBeforePrefetch = cfg.getSequentialReadsBeforePrefetch(); + trashPurgeTimeout = cfg.getTrashPurgeTimeout(); + } + + /** + * Gets GGFS instance name. If {@code null}, then instance with default + * name will be used. + * + * @return GGFS instance name. + */ + @Nullable public String getName() { + return name; + } + + /** + * Sets GGFS instance name. + * + * @param name GGFS instance name. + */ + public void setName(String name) { + this.name = name; + } + + /** + * Cache name to store GGFS meta information. If {@code null}, then instance + * with default meta-cache name will be used. + * + * @return Cache name to store GGFS meta information. + */ + @Nullable public String getMetaCacheName() { + return metaCacheName; + } + + /** + * Sets cache name to store GGFS meta information. + * + * @param metaCacheName Cache name to store GGFS meta information. + */ + public void setMetaCacheName(String metaCacheName) { + this.metaCacheName = metaCacheName; + } + + /** + * Cache name to store GGFS data. + * + * @return Cache name to store GGFS data. + */ + @Nullable public String getDataCacheName() { + return dataCacheName; + } + + /** + * Sets cache name to store GGFS data. + * + * @param dataCacheName Cache name to store GGFS data. + */ + public void setDataCacheName(String dataCacheName) { + this.dataCacheName = dataCacheName; + } + + /** + * Get file's data block size. + * + * @return File's data block size. + */ + public int getBlockSize() { + return blockSize; + } + + /** + * Sets file's data block size. + * + * @param blockSize File's data block size (bytes) or {@code 0} to reset default value. + */ + public void setBlockSize(int blockSize) { + A.ensure(blockSize >= 0, "blockSize >= 0"); + + this.blockSize = blockSize == 0 ? DFLT_BLOCK_SIZE : blockSize; + } + + /** + * Get number of pre-fetched blocks if specific file's chunk is requested. + * + * @return The number of pre-fetched blocks. + */ + public int getPrefetchBlocks() { + return prefetchBlocks; + } + + /** + * Sets the number of pre-fetched blocks if specific file's chunk is requested. + * + * @param prefetchBlocks New number of pre-fetched blocks. + */ + public void setPrefetchBlocks(int prefetchBlocks) { + A.ensure(prefetchBlocks >= 0, "prefetchBlocks >= 0"); + + this.prefetchBlocks = prefetchBlocks; + } + + /** + * Get amount of sequential block reads before prefetch is triggered. The + * higher this value, the longer GGFS will wait before starting to prefetch + * values ahead of time. Depending on the use case, this can either help + * or hurt performance. + * <p> + * Default is {@code 0} which means that pre-fetching will start right away. + * <h1 class="header">Integration With Hadoop</h1> + * This parameter can be also overridden for individual Hadoop MapReduce tasks by passing + * {@code org.apache.ignite.fs.hadoop.GridGgfsHadoopParameters.PARAM_GGFS_SEQ_READS_BEFORE_PREFETCH} + * configuration property directly to Hadoop MapReduce task. + * <p> + * <b>NOTE:</b> Integration with Hadoop is available only in {@code In-Memory Accelerator For Hadoop} edition. + * + * @return Amount of sequential block reads. + */ + public int getSequentialReadsBeforePrefetch() { + return seqReadsBeforePrefetch; + } + + /** + * Sets amount of sequential block reads before prefetch is triggered. The + * higher this value, the longer GGFS will wait before starting to prefetch + * values ahead of time. Depending on the use case, this can either help + * or hurt performance. + * <p> + * Default is {@code 0} which means that pre-fetching will start right away. + * <h1 class="header">Integration With Hadoop</h1> + * This parameter can be also overridden for individual Hadoop MapReduce tasks by passing + * {@code org.apache.ignite.fs.hadoop.GridGgfsHadoopParameters.PARAM_GGFS_SEQ_READS_BEFORE_PREFETCH} + * configuration property directly to Hadoop MapReduce task. + * <p> + * <b>NOTE:</b> Integration with Hadoop is available only in {@code In-Memory Accelerator For Hadoop} edition. + * + * @param seqReadsBeforePrefetch Amount of sequential block reads before prefetch is triggered. + */ + public void setSequentialReadsBeforePrefetch(int seqReadsBeforePrefetch) { + A.ensure(seqReadsBeforePrefetch >= 0, "seqReadsBeforePrefetch >= 0"); + + this.seqReadsBeforePrefetch = seqReadsBeforePrefetch; + } + + /** + * Get read/write buffer size for {@code GGFS} stream operations in bytes. + * + * @return Read/write buffers size (bytes). + */ + public int getStreamBufferSize() { + return bufSize; + } + + /** + * Sets read/write buffers size for {@code GGFS} stream operations (bytes). + * + * @param bufSize Read/write buffers size for stream operations (bytes) or {@code 0} to reset default value. + */ + public void setStreamBufferSize(int bufSize) { + A.ensure(bufSize >= 0, "bufSize >= 0"); + + this.bufSize = bufSize == 0 ? DFLT_BUF_SIZE : bufSize; + } + + /** + * Gets number of file blocks buffered on local node before sending batch to remote node. + * + * @return Per node buffer size. + */ + public int getPerNodeBatchSize() { + return perNodeBatchSize; + } + + /** + * Sets number of file blocks collected on local node before sending batch to remote node. + * + * @param perNodeBatchSize Per node buffer size. + */ + public void setPerNodeBatchSize(int perNodeBatchSize) { + this.perNodeBatchSize = perNodeBatchSize; + } + + /** + * Gets number of batches that can be concurrently sent to remote node. + * + * @return Number of batches for each node. + */ + public int getPerNodeParallelBatchCount() { + return perNodeParallelBatchCnt; + } + + /** + * Sets number of file block batches that can be concurrently sent to remote node. + * + * @param perNodeParallelBatchCnt Per node parallel load operations. + */ + public void setPerNodeParallelBatchCount(int perNodeParallelBatchCnt) { + this.perNodeParallelBatchCnt = perNodeParallelBatchCnt; + } + + /** + * Gets map of IPC endpoint configuration properties. There are 2 different + * types of endpoint supported: {@code shared-memory}, and {@code TCP}. + * <p> + * The following configuration properties are supported for {@code shared-memory} + * endpoint: + * <ul> + * <li>{@code type} - value is {@code shmem} to specify {@code shared-memory} approach.</li> + * <li>{@code port} - endpoint port.</li> + * <li>{@code size} - memory size allocated for single endpoint communication.</li> + * <li> + * {@code tokenDirectoryPath} - path, either absolute or relative to {@code IGNITE_HOME} to + * store shared memory tokens. + * </li> + * </ul> + * <p> + * The following configuration properties are supported for {@code TCP} approach: + * <ul> + * <li>{@code type} - value is {@code tcp} to specify {@code TCP} approach.</li> + * <li>{@code port} - endpoint bind port.</li> + * <li> + * {@code host} - endpoint bind host. If omitted '127.0.0.1' will be used. + * </li> + * </ul> + * <p> + * Note that {@code shared-memory} approach is not supported on Windows environments. + * In case GGFS is failed to bind to particular port, further attempts will be performed every 3 seconds. + * + * @return Map of IPC endpoint configuration properties. In case the value is not set, defaults will be used. Default + * type for Windows is "tcp", for all other platforms - "shmem". Default port is {@link #DFLT_IPC_PORT}. + */ + @Nullable public Map<String,String> getIpcEndpointConfiguration() { + return ipcEndpointCfg; + } + + /** + * Sets IPC endpoint configuration to publish GGFS over. + * + * @param ipcEndpointCfg Map of IPC endpoint config properties. + */ + public void setIpcEndpointConfiguration(@Nullable Map<String,String> ipcEndpointCfg) { + this.ipcEndpointCfg = ipcEndpointCfg; + } + + /** + * Get IPC endpoint enabled flag. In case it is set to {@code true} endpoint will be created and bound to specific + * port. Otherwise endpoint will not be created. Default value is {@link #DFLT_IPC_ENDPOINT_ENABLED}. + * + * @return {@code True} in case endpoint is enabled. + */ + public boolean isIpcEndpointEnabled() { + return ipcEndpointEnabled; + } + + /** + * Set IPC endpoint enabled flag. See {@link #isIpcEndpointEnabled()}. + * + * @param ipcEndpointEnabled IPC endpoint enabled flag. + */ + public void setIpcEndpointEnabled(boolean ipcEndpointEnabled) { + this.ipcEndpointEnabled = ipcEndpointEnabled; + } + + /** + * Gets port number for management endpoint. All GGFS nodes should have this port open + * for Visor Management Console to work with GGFS. + * <p> + * Default value is {@link #DFLT_MGMT_PORT} + * + * @return Port number or {@code -1} if management endpoint should be disabled. + */ + public int getManagementPort() { + return mgmtPort; + } + + /** + * Sets management endpoint port. + * + * @param mgmtPort port number or {@code -1} to disable management endpoint. + */ + public void setManagementPort(int mgmtPort) { + this.mgmtPort = mgmtPort; + } + + /** + * Gets mode to specify how {@code GGFS} interacts with Hadoop file system, like {@code HDFS}. + * Secondary Hadoop file system is provided for pass-through, write-through, and read-through + * purposes. + * <p> + * Default mode is {@link IgniteFsMode#DUAL_ASYNC}. If secondary Hadoop file system is + * not configured, this mode will work just like {@link IgniteFsMode#PRIMARY} mode. + * + * @return Mode to specify how GGFS interacts with secondary HDFS file system. + */ + public IgniteFsMode getDefaultMode() { + return dfltMode; + } + + /** + * Sets {@code GGFS} mode to specify how it should interact with secondary + * Hadoop file system, like {@code HDFS}. Secondary Hadoop file system is provided + * for pass-through, write-through, and read-through purposes. + * + * @param dfltMode {@code GGFS} mode. + */ + public void setDefaultMode(IgniteFsMode dfltMode) { + this.dfltMode = dfltMode; + } + + /** + * Gets the secondary file system. Secondary file system is provided for pass-through, write-through, + * and read-through purposes. + * + * @return Secondary file system. + */ + public IgniteFsFileSystem getSecondaryFileSystem() { + return secondaryFs; + } + + /** + * Sets the secondary file system. Secondary file system is provided for pass-through, write-through, + * and read-through purposes. + * + * @param fileSystem + */ + public void setSecondaryFileSystem(IgniteFsFileSystem fileSystem) { + secondaryFs = fileSystem; + } + + /** + * Gets map of path prefixes to {@code GGFS} modes used for them. + * <p> + * If path doesn't correspond to any specified prefix or mappings are not provided, then + * {@link #getDefaultMode()} is used. + * <p> + * Several folders under {@code '/gridgain'} folder have predefined mappings which cannot be overridden. + * <li>{@code /gridgain/primary} and all it's sub-folders will always work in {@code PRIMARY} mode.</li> + * <p> + * And in case secondary file system URI is provided: + * <li>{@code /gridgain/proxy} and all it's sub-folders will always work in {@code PROXY} mode.</li> + * <li>{@code /gridgain/sync} and all it's sub-folders will always work in {@code DUAL_SYNC} mode.</li> + * <li>{@code /gridgain/async} and all it's sub-folders will always work in {@code DUAL_ASYNC} mode.</li> + * + * @return Map of paths to {@code GGFS} modes. + */ + @Nullable public Map<String, IgniteFsMode> getPathModes() { + return pathModes; + } + + /** + * Sets map of path prefixes to {@code GGFS} modes used for them. + * <p> + * If path doesn't correspond to any specified prefix or mappings are not provided, then + * {@link #getDefaultMode()} is used. + * + * @param pathModes Map of paths to {@code GGFS} modes. + */ + public void setPathModes(Map<String, IgniteFsMode> pathModes) { + this.pathModes = pathModes; + } + + /** + * Gets the length of file chunk to send before delaying the fragmentizer. + * + * @return File chunk length in bytes. + */ + public long getFragmentizerThrottlingBlockLength() { + return fragmentizerThrottlingBlockLen; + } + + /** + * Sets length of file chunk to transmit before throttling is delayed. + * + * @param fragmentizerThrottlingBlockLen Block length in bytes. + */ + public void setFragmentizerThrottlingBlockLength(long fragmentizerThrottlingBlockLen) { + this.fragmentizerThrottlingBlockLen = fragmentizerThrottlingBlockLen; + } + + /** + * Gets throttle delay for fragmentizer. + * + * @return Throttle delay in milliseconds. + */ + public long getFragmentizerThrottlingDelay() { + return fragmentizerThrottlingDelay; + } + + /** + * Sets delay in milliseconds for which fragmentizer is paused. + * + * @param fragmentizerThrottlingDelay Delay in milliseconds. + */ + public void setFragmentizerThrottlingDelay(long fragmentizerThrottlingDelay) { + this.fragmentizerThrottlingDelay = fragmentizerThrottlingDelay; + } + + /** + * Gets number of files that can be processed by fragmentizer concurrently. + * + * @return Number of files to process concurrently. + */ + public int getFragmentizerConcurrentFiles() { + return fragmentizerConcurrentFiles; + } + + /** + * Sets number of files to process concurrently by fragmentizer. + * + * @param fragmentizerConcurrentFiles Number of files to process concurrently. + */ + public void setFragmentizerConcurrentFiles(int fragmentizerConcurrentFiles) { + this.fragmentizerConcurrentFiles = fragmentizerConcurrentFiles; + } + + /** + * Gets amount of local memory (in % of local GGFS max space size) available for local writes + * during file creation. + * <p> + * If current GGFS space size is less than {@code fragmentizerLocalWritesRatio * maxSpaceSize}, + * then file blocks will be written to the local node first and then asynchronously distributed + * among cluster nodes (fragmentized). + * <p> + * Default value is {@link #DFLT_FRAGMENTIZER_LOCAL_WRITES_RATIO}. + * + * @return Ratio for local writes space. + */ + public float getFragmentizerLocalWritesRatio() { + return fragmentizerLocWritesRatio; + } + + /** + * Sets ratio for space available for local file writes. + * + * @param fragmentizerLocWritesRatio Ratio for local file writes. + * @see #getFragmentizerLocalWritesRatio() + */ + public void setFragmentizerLocalWritesRatio(float fragmentizerLocWritesRatio) { + this.fragmentizerLocWritesRatio = fragmentizerLocWritesRatio; + } + + /** + * Gets flag indicating whether GGFS fragmentizer is enabled. If fragmentizer is disabled, files will be + * written in distributed fashion. + * + * @return Flag indicating whether fragmentizer is enabled. + */ + public boolean isFragmentizerEnabled() { + return fragmentizerEnabled; + } + + /** + * Sets property indicating whether fragmentizer is enabled. + * + * @param fragmentizerEnabled {@code True} if fragmentizer is enabled. + */ + public void setFragmentizerEnabled(boolean fragmentizerEnabled) { + this.fragmentizerEnabled = fragmentizerEnabled; + } + + /** + * Get maximum space available for data cache to store file system entries. + * + * @return Maximum space available for data cache. + */ + public long getMaxSpaceSize() { + return maxSpace; + } + + /** + * Set maximum space in bytes available in data cache. + * + * @param maxSpace Maximum space available in data cache. + */ + public void setMaxSpaceSize(long maxSpace) { + this.maxSpace = maxSpace; + } + + /** + * Gets maximum timeout awaiting for trash purging in case data cache oversize is detected. + * + * @return Maximum timeout awaiting for trash purging in case data cache oversize is detected. + */ + public long getTrashPurgeTimeout() { + return trashPurgeTimeout; + } + + /** + * Sets maximum timeout awaiting for trash purging in case data cache oversize is detected. + * + * @param trashPurgeTimeout Maximum timeout awaiting for trash purging in case data cache oversize is detected. + */ + public void setTrashPurgeTimeout(long trashPurgeTimeout) { + this.trashPurgeTimeout = trashPurgeTimeout; + } + + /** + * Get DUAL mode put operation executor service. This executor service will process cache PUT requests for + * data which came from the secondary file system and about to be written to GGFS data cache. + * In case no executor service is provided, default one will be created with maximum amount of threads equals + * to amount of processor cores. + * + * @return Get DUAL mode put operation executor service + */ + @Nullable public ExecutorService getDualModePutExecutorService() { + return dualModePutExec; + } + + /** + * Set DUAL mode put operations executor service. + * + * @param dualModePutExec Dual mode put operations executor service. + */ + public void setDualModePutExecutorService(ExecutorService dualModePutExec) { + this.dualModePutExec = dualModePutExec; + } + + /** + * Get DUAL mode put operation executor service shutdown flag. + * + * @return DUAL mode put operation executor service shutdown flag. + */ + public boolean getDualModePutExecutorServiceShutdown() { + return dualModePutExecShutdown; + } + + /** + * Set DUAL mode put operations executor service shutdown flag. + * + * @param dualModePutExecShutdown Dual mode put operations executor service shutdown flag. + */ + public void setDualModePutExecutorServiceShutdown(boolean dualModePutExecShutdown) { + this.dualModePutExecShutdown = dualModePutExecShutdown; + } + + /** + * Get maximum amount of pending data read from the secondary file system and waiting to be written to data + * cache. {@code 0} or negative value stands for unlimited size. + * <p> + * By default this value is set to {@code 0}. It is recommended to set positive value in case your + * application performs frequent reads of large amount of data from the secondary file system in order to + * avoid issues with increasing GC pauses or out-of-memory error. + * + * @return Maximum amount of pending data read from the secondary file system + */ + public long getDualModeMaxPendingPutsSize() { + return dualModeMaxPendingPutsSize; + } + + /** + * Set maximum amount of data in pending put operations. + * + * @param dualModeMaxPendingPutsSize Maximum amount of data in pending put operations. + */ + public void setDualModeMaxPendingPutsSize(long dualModeMaxPendingPutsSize) { + this.dualModeMaxPendingPutsSize = dualModeMaxPendingPutsSize; + } + + /** + * Get maximum default range size of a file being split during GGFS task execution. When GGFS task is about to + * be executed, it requests file block locations first. Each location is defined as {@link org.apache.ignite.ignitefs.mapreduce.IgniteFsFileRange} which + * has length. In case this parameter is set to positive value, then GGFS will split single file range into smaller + * ranges with length not greater that this parameter. The only exception to this case is when maximum task range + * length is smaller than file block size. In this case maximum task range size will be overridden and set to file + * block size. + * <p> + * Note that this parameter is applied when task is split into jobs before {@link org.apache.ignite.ignitefs.mapreduce.IgniteFsRecordResolver} is + * applied. Therefore, final file ranges being assigned to particular jobs could be greater than value of this + * parameter depending on file data layout and selected resolver type. + * <p> + * Setting this parameter might be useful when file is highly colocated and have very long consequent data chunks + * so that task execution suffers from insufficient parallelism. E.g., in case you have one GGFS node in topology + * and want to process 1Gb file, then only single range of length 1Gb will be returned. This will result in + * a single job which will be processed in one thread. But in case you provide this configuration parameter and set + * maximum range length to 16Mb, then 64 ranges will be returned resulting in 64 jobs which could be executed in + * parallel. + * <p> + * Note that some {@code GridGgfs.execute()} methods can override value of this parameter. + * <p> + * In case value of this parameter is set to {@code 0} or negative value, it is simply ignored. Default value is + * {@code 0}. + * + * @return Maximum range size of a file being split during GGFS task execution. + */ + public long getMaximumTaskRangeLength() { + return maxTaskRangeLen; + } + + /** + * Set maximum default range size of a file being split during GGFS task execution. + * See {@link #getMaximumTaskRangeLength()} for more details. + * + * @param maxTaskRangeLen Set maximum default range size of a file being split during GGFS task execution. + */ + public void setMaximumTaskRangeLength(long maxTaskRangeLen) { + this.maxTaskRangeLen = maxTaskRangeLen; + } + + /** {@inheritDoc} */ + @Override public String toString() { + return S.toString(IgniteFsConfiguration.class, this); + } +} http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/afa29526/modules/core/src/main/java/org/apache/ignite/events/IgniteFsEvent.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/events/IgniteFsEvent.java b/modules/core/src/main/java/org/apache/ignite/events/IgniteFsEvent.java index face12f..d738d44 100644 --- a/modules/core/src/main/java/org/apache/ignite/events/IgniteFsEvent.java +++ b/modules/core/src/main/java/org/apache/ignite/events/IgniteFsEvent.java @@ -18,7 +18,7 @@ package org.apache.ignite.events; import org.apache.ignite.cluster.*; -import org.apache.ignite.fs.*; +import org.apache.ignite.ignitefs.*; import org.apache.ignite.internal.util.tostring.*; import org.apache.ignite.internal.util.typedef.internal.*; import org.jetbrains.annotations.*; http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/afa29526/modules/core/src/main/java/org/apache/ignite/fs/IgniteFsBlockLocation.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/fs/IgniteFsBlockLocation.java b/modules/core/src/main/java/org/apache/ignite/fs/IgniteFsBlockLocation.java deleted file mode 100644 index 5904e15..0000000 --- a/modules/core/src/main/java/org/apache/ignite/fs/IgniteFsBlockLocation.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ignite.fs; - -import java.util.*; - -/** - * {@code GGFS} file's data block location in the grid. It is used to determine - * node affinity of a certain file block within the Grid by calling - * {@link org.apache.ignite.IgniteFs#affinity(IgniteFsPath, long, long)} method. - */ -public interface IgniteFsBlockLocation { - /** - * Start position in the file this block relates to. - * - * @return Start position in the file this block relates to. - */ - public long start(); - - /** - * Length of the data block in the file. - * - * @return Length of the data block in the file. - */ - public long length(); - - /** - * Nodes this block belongs to. First node id in collection is - * primary node id. - * - * @return Nodes this block belongs to. - */ - public Collection<UUID> nodeIds(); - - /** - * Compliant with Hadoop interface. - * - * @return Collection of host:port addresses. - */ - public Collection<String> names(); - - /** - * Compliant with Hadoop interface. - * - * @return Collection of host names. - */ - public Collection<String> hosts(); -} http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/afa29526/modules/core/src/main/java/org/apache/ignite/fs/IgniteFsConcurrentModificationException.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/fs/IgniteFsConcurrentModificationException.java b/modules/core/src/main/java/org/apache/ignite/fs/IgniteFsConcurrentModificationException.java deleted file mode 100644 index 34d527d..0000000 --- a/modules/core/src/main/java/org/apache/ignite/fs/IgniteFsConcurrentModificationException.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ignite.fs; - -/** - * {@code GGFS} exception indicating that file system structure was modified concurrently. This error - * indicates that an operation performed in DUAL mode cannot proceed due to these changes. - */ -public class IgniteFsConcurrentModificationException extends IgniteFsException { - /** */ - private static final long serialVersionUID = 0L; - - /** - * Creates new exception. - * - * @param path Affected path. - */ - public IgniteFsConcurrentModificationException(IgniteFsPath path) { - super("File system entry has been modified concurrently: " + path, null); - } -}
