ACCUMULO-3871 move ITs into distro jar, stop building test jar
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/01ae5b85 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/01ae5b85 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/01ae5b85 Branch: refs/heads/master Commit: 01ae5b85897c318da639f875097edb5c7c3212aa Parents: ab5a867 Author: Eric Newton <eric.new...@gmail.com> Authored: Thu Jun 4 14:52:27 2015 -0400 Committer: Eric Newton <eric.new...@gmail.com> Committed: Thu Jun 4 14:52:27 2015 -0400 ---------------------------------------------------------------------- assemble/pom.xml | 11 - test/pom.xml | 89 +- .../harness/AccumuloClusterHarness.java | 338 +++ .../apache/accumulo/harness/AccumuloITBase.java | 104 + .../MiniClusterConfigurationCallback.java | 41 + .../accumulo/harness/MiniClusterHarness.java | 242 ++ .../accumulo/harness/SharedMiniClusterBase.java | 185 ++ .../org/apache/accumulo/harness/TestingKdc.java | 210 ++ .../conf/AccumuloClusterConfiguration.java | 35 + .../AccumuloClusterPropertyConfiguration.java | 195 ++ .../conf/AccumuloMiniClusterConfiguration.java | 117 + .../StandaloneAccumuloClusterConfiguration.java | 252 ++ .../accumulo/test/AccumuloOutputFormatIT.java | 125 + .../test/ArbitraryTablePropertiesIT.java | 198 ++ .../accumulo/test/AssignmentThreadsIT.java | 94 + .../apache/accumulo/test/AuditMessageIT.java | 506 ++++ .../test/BadDeleteMarkersCreatedIT.java | 176 ++ .../apache/accumulo/test/BalanceFasterIT.java | 94 + .../org/apache/accumulo/test/BalanceIT.java | 52 + .../test/BalanceWithOfflineTableIT.java | 90 + .../org/apache/accumulo/test/BatchWriterIT.java | 49 + .../accumulo/test/BulkImportVolumeIT.java | 95 + .../org/apache/accumulo/test/CleanWalIT.java | 146 ++ .../accumulo/test/ConditionalWriterIT.java | 1349 +++++++++++ .../test/ConfigurableMajorCompactionIT.java | 119 + .../test/CreateTableWithNewTableConfigIT.java | 193 ++ .../org/apache/accumulo/test/DumpConfigIT.java | 69 + .../org/apache/accumulo/test/ExistingMacIT.java | 169 ++ .../org/apache/accumulo/test/FileArchiveIT.java | 271 +++ .../accumulo/test/GarbageCollectWALIT.java | 81 + .../apache/accumulo/test/ImportExportIT.java | 198 ++ .../accumulo/test/IntegrationTestMapReduce.java | 146 ++ .../accumulo/test/InterruptibleScannersIT.java | 102 + .../accumulo/test/KeyValueEqualityIT.java | 77 + .../apache/accumulo/test/LargeSplitRowIT.java | 286 +++ .../test/MasterRepairsDualAssignmentIT.java | 161 ++ .../accumulo/test/MetaConstraintRetryIT.java | 63 + .../apache/accumulo/test/MetaGetsReadersIT.java | 116 + .../org/apache/accumulo/test/MetaSplitIT.java | 137 ++ .../MissingWalHeaderCompletesRecoveryIT.java | 211 ++ .../accumulo/test/MultiTableBatchWriterIT.java | 518 ++++ .../accumulo/test/MultiTableRecoveryIT.java | 134 ++ .../org/apache/accumulo/test/NamespacesIT.java | 1362 +++++++++++ .../test/RecoveryCompactionsAreFlushesIT.java | 101 + .../test/RewriteTabletDirectoriesIT.java | 168 ++ .../apache/accumulo/test/ScanIteratorIT.java | 170 ++ .../org/apache/accumulo/test/ShellConfigIT.java | 103 + .../org/apache/accumulo/test/ShellServerIT.java | 1609 +++++++++++++ .../accumulo/test/SplitCancelsMajCIT.java | 89 + .../apache/accumulo/test/SplitRecoveryIT.java | 137 ++ .../test/TableConfigurationUpdateIT.java | 139 ++ .../apache/accumulo/test/TableOperationsIT.java | 375 +++ .../accumulo/test/TabletServerGivesUpIT.java | 73 + .../org/apache/accumulo/test/TotalQueuedIT.java | 131 + .../test/TracerRecoversAfterOfflineTableIT.java | 127 + .../accumulo/test/TransportCachingIT.java | 120 + .../org/apache/accumulo/test/UnusedWALIT.java | 153 ++ .../accumulo/test/UserCompactionStrategyIT.java | 296 +++ .../java/org/apache/accumulo/test/UsersIT.java | 60 + .../accumulo/test/VerifySerialRecoveryIT.java | 107 + .../apache/accumulo/test/VolumeChooserIT.java | 392 +++ .../java/org/apache/accumulo/test/VolumeIT.java | 568 +++++ .../apache/accumulo/test/WaitForBalanceIT.java | 118 + .../test/functional/AccumuloInputFormatIT.java | 210 ++ .../accumulo/test/functional/AddSplitIT.java | 142 ++ .../test/functional/BackupMasterIT.java | 68 + .../test/functional/BadIteratorMincIT.java | 107 + .../functional/BalanceAfterCommsFailureIT.java | 138 ++ .../BalanceInPresenceOfOfflineTableIT.java | 201 ++ .../test/functional/BatchScanSplitIT.java | 129 + .../test/functional/BatchWriterFlushIT.java | 178 ++ .../test/functional/BigRootTabletIT.java | 66 + .../accumulo/test/functional/BinaryIT.java | 86 + .../test/functional/BinaryStressIT.java | 107 + .../accumulo/test/functional/BloomFilterIT.java | 256 ++ .../accumulo/test/functional/BulkFileIT.java | 130 + .../apache/accumulo/test/functional/BulkIT.java | 120 + .../functional/BulkSplitOptimizationIT.java | 142 ++ .../test/functional/ChaoticBalancerIT.java | 85 + .../accumulo/test/functional/ClassLoaderIT.java | 104 + .../accumulo/test/functional/CleanTmpIT.java | 112 + .../accumulo/test/functional/CleanUpIT.java | 151 ++ .../accumulo/test/functional/CloneTestIT.java | 295 +++ .../accumulo/test/functional/CombinerIT.java | 76 + .../accumulo/test/functional/CompactionIT.java | 184 ++ .../accumulo/test/functional/ConcurrencyIT.java | 158 ++ .../functional/ConfigurableCompactionIT.java | 164 ++ .../test/functional/ConfigurableMacBase.java | 182 ++ .../accumulo/test/functional/ConstraintIT.java | 335 +++ .../test/functional/CreateAndUseIT.java | 130 + .../test/functional/CreateManyScannersIT.java | 41 + .../accumulo/test/functional/CredentialsIT.java | 124 + .../test/functional/DeleteEverythingIT.java | 117 + .../accumulo/test/functional/DeleteIT.java | 106 + .../accumulo/test/functional/DeleteRowsIT.java | 154 ++ .../test/functional/DeleteRowsSplitIT.java | 147 ++ .../functional/DeleteTableDuringSplitIT.java | 109 + .../functional/DeletedTablesDontFlushIT.java | 62 + .../accumulo/test/functional/DurabilityIT.java | 222 ++ .../test/functional/DynamicThreadPoolsIT.java | 126 + .../accumulo/test/functional/ExamplesIT.java | 660 +++++ .../test/functional/FateStarvationIT.java | 80 + .../test/functional/FunctionalTestUtils.java | 192 ++ .../test/functional/GarbageCollectorIT.java | 301 +++ .../test/functional/HalfDeadTServerIT.java | 218 ++ .../accumulo/test/functional/KerberosIT.java | 573 +++++ .../test/functional/KerberosProxyIT.java | 426 ++++ .../accumulo/test/functional/LargeRowIT.java | 219 ++ .../test/functional/LateLastContactIT.java | 49 + .../accumulo/test/functional/LogicalTimeIT.java | 109 + .../accumulo/test/functional/MapReduceIT.java | 92 + .../test/functional/MasterAssignmentIT.java | 100 + .../test/functional/MasterFailoverIT.java | 80 + .../accumulo/test/functional/MaxOpenIT.java | 176 ++ .../accumulo/test/functional/MergeIT.java | 194 ++ .../accumulo/test/functional/MetadataIT.java | 148 ++ .../test/functional/MetadataMaxFilesIT.java | 116 + .../test/functional/MetadataSplitIT.java | 56 + .../test/functional/MonitorLoggingIT.java | 121 + .../accumulo/test/functional/MonitorSslIT.java | 132 + .../accumulo/test/functional/NativeMapIT.java | 613 +++++ .../accumulo/test/functional/PermissionsIT.java | 707 ++++++ .../accumulo/test/functional/ReadWriteIT.java | 456 ++++ .../functional/RecoveryWithEmptyRFileIT.java | 109 + .../test/functional/RegexGroupBalanceIT.java | 192 ++ .../accumulo/test/functional/RenameIT.java | 74 + .../accumulo/test/functional/RestartIT.java | 367 +++ .../test/functional/RestartStressIT.java | 153 ++ .../accumulo/test/functional/RowDeleteIT.java | 109 + .../accumulo/test/functional/ScanIdIT.java | 385 +++ .../test/functional/ScanIteratorIT.java | 134 ++ .../accumulo/test/functional/ScanRangeIT.java | 244 ++ .../test/functional/ScanSessionTimeOutIT.java | 142 ++ .../accumulo/test/functional/ScannerIT.java | 121 + .../test/functional/ServerSideErrorIT.java | 128 + .../test/functional/SessionDurabilityIT.java | 153 ++ .../accumulo/test/functional/ShutdownIT.java | 121 + .../functional/SimpleBalancerFairnessIT.java | 117 + .../test/functional/SparseColumnFamilyIT.java | 98 + .../accumulo/test/functional/SplitIT.java | 223 ++ .../test/functional/SplitRecoveryIT.java | 279 +++ .../apache/accumulo/test/functional/SslIT.java | 72 + .../test/functional/SslWithClientAuthIT.java | 77 + .../accumulo/test/functional/StartIT.java | 43 + .../accumulo/test/functional/TableIT.java | 108 + .../accumulo/test/functional/TabletIT.java | 101 + .../functional/TabletStateChangeIteratorIT.java | 192 ++ .../accumulo/test/functional/TimeoutIT.java | 120 + .../accumulo/test/functional/VisibilityIT.java | 323 +++ .../accumulo/test/functional/WALSunnyDayIT.java | 234 ++ .../test/functional/WatchTheWatchCountIT.java | 80 + .../test/functional/WriteAheadLogIT.java | 79 + .../accumulo/test/functional/WriteLotsIT.java | 89 + .../accumulo/test/functional/ZooCacheIT.java | 75 + .../test/functional/ZookeeperRestartIT.java | 87 + .../test/performance/RollWALPerformanceIT.java | 120 + .../performance/metadata/FastBulkImportIT.java | 103 + .../accumulo/test/proxy/ProxyDurabilityIT.java | 145 ++ .../accumulo/test/proxy/SimpleProxyBase.java | 2273 ++++++++++++++++++ .../accumulo/test/proxy/TBinaryProxyIT.java | 33 + .../accumulo/test/proxy/TCompactProxyIT.java | 32 + .../test/proxy/TJsonProtocolProxyIT.java | 33 + .../accumulo/test/proxy/TTupleProxyIT.java | 33 + .../accumulo/test/proxy/TestProxyClient.java | 204 ++ .../test/proxy/TestProxyInstanceOperations.java | 84 + .../accumulo/test/proxy/TestProxyReadWrite.java | 468 ++++ .../test/proxy/TestProxySecurityOperations.java | 147 ++ .../test/proxy/TestProxyTableOperations.java | 202 ++ .../test/replication/CyclicReplicationIT.java | 332 +++ ...bageCollectorCommunicatesWithTServersIT.java | 417 ++++ .../test/replication/KerberosReplicationIT.java | 233 ++ .../replication/MultiInstanceReplicationIT.java | 731 ++++++ .../replication/MultiTserverReplicationIT.java | 115 + .../test/replication/ReplicationIT.java | 1436 +++++++++++ .../replication/ReplicationRandomWalkIT.java | 67 + .../test/replication/StatusCombinerMacIT.java | 118 + .../UnorderedWorkAssignerReplicationIT.java | 731 ++++++ ...UnusedWalDoesntCloseReplicationStatusIT.java | 219 ++ .../server/security/SystemCredentialsIT.java | 233 ++ .../accumulo/test/start/KeywordStartIT.java | 197 ++ .../apache/accumulo/test/util/CertUtils.java | 348 +++ test/src/main/resources/FooConstraint.jar | Bin 0 -> 2130 bytes test/src/main/resources/FooFilter.jar | Bin 0 -> 1645 bytes test/src/main/resources/TestCombinerX.jar | Bin 0 -> 4335 bytes test/src/main/resources/TestCombinerY.jar | Bin 0 -> 4129 bytes test/src/main/resources/TestCompactionStrat.jar | Bin 0 -> 2530 bytes test/src/main/resources/conf/accumulo-site.xml | 123 + test/src/main/resources/conf/generic_logger.xml | 83 + test/src/main/resources/conf/monitor_logger.xml | 64 + test/src/main/resources/log4j.properties | 55 + test/src/main/resources/randomwalk/Basic.xml | 37 + test/src/main/resources/randomwalk/Simple.xml | 43 + test/src/main/resources/unit/Basic.xml | 37 + test/src/main/resources/unit/Simple.xml | 43 + .../harness/AccumuloClusterHarness.java | 338 --- .../apache/accumulo/harness/AccumuloITBase.java | 104 - .../MiniClusterConfigurationCallback.java | 41 - .../accumulo/harness/MiniClusterHarness.java | 242 -- .../accumulo/harness/SharedMiniClusterBase.java | 185 -- .../org/apache/accumulo/harness/TestingKdc.java | 210 -- .../conf/AccumuloClusterConfiguration.java | 35 - .../AccumuloClusterPropertyConfiguration.java | 195 -- .../conf/AccumuloMiniClusterConfiguration.java | 117 - .../StandaloneAccumuloClusterConfiguration.java | 252 -- .../accumulo/test/AccumuloOutputFormatIT.java | 125 - .../test/ArbitraryTablePropertiesIT.java | 198 -- .../accumulo/test/AssignmentThreadsIT.java | 94 - .../apache/accumulo/test/AuditMessageIT.java | 506 ---- .../test/BadDeleteMarkersCreatedIT.java | 176 -- .../apache/accumulo/test/BalanceFasterIT.java | 94 - .../org/apache/accumulo/test/BalanceIT.java | 52 - .../test/BalanceWithOfflineTableIT.java | 90 - .../org/apache/accumulo/test/BatchWriterIT.java | 49 - .../accumulo/test/BulkImportVolumeIT.java | 95 - .../org/apache/accumulo/test/CleanWalIT.java | 146 -- .../accumulo/test/ConditionalWriterIT.java | 1349 ----------- .../test/ConfigurableMajorCompactionIT.java | 119 - .../test/CreateTableWithNewTableConfigIT.java | 193 -- .../org/apache/accumulo/test/DumpConfigIT.java | 69 - .../org/apache/accumulo/test/ExistingMacIT.java | 169 -- .../org/apache/accumulo/test/FileArchiveIT.java | 271 --- .../accumulo/test/GarbageCollectWALIT.java | 81 - .../apache/accumulo/test/ImportExportIT.java | 198 -- .../accumulo/test/IntegrationTestMapReduce.java | 146 -- .../accumulo/test/InterruptibleScannersIT.java | 102 - .../accumulo/test/KeyValueEqualityIT.java | 77 - .../apache/accumulo/test/LargeSplitRowIT.java | 286 --- .../test/MasterRepairsDualAssignmentIT.java | 161 -- .../accumulo/test/MetaConstraintRetryIT.java | 63 - .../apache/accumulo/test/MetaGetsReadersIT.java | 116 - .../org/apache/accumulo/test/MetaSplitIT.java | 137 -- .../MissingWalHeaderCompletesRecoveryIT.java | 211 -- .../accumulo/test/MultiTableBatchWriterIT.java | 518 ---- .../accumulo/test/MultiTableRecoveryIT.java | 134 -- .../org/apache/accumulo/test/NamespacesIT.java | 1362 ----------- .../test/RecoveryCompactionsAreFlushesIT.java | 101 - .../test/RewriteTabletDirectoriesIT.java | 168 -- .../apache/accumulo/test/ScanIteratorIT.java | 170 -- .../org/apache/accumulo/test/ShellConfigIT.java | 103 - .../org/apache/accumulo/test/ShellServerIT.java | 1609 ------------- .../accumulo/test/SplitCancelsMajCIT.java | 89 - .../apache/accumulo/test/SplitRecoveryIT.java | 137 -- .../test/TableConfigurationUpdateIT.java | 139 -- .../apache/accumulo/test/TableOperationsIT.java | 375 --- .../accumulo/test/TabletServerGivesUpIT.java | 73 - .../org/apache/accumulo/test/TotalQueuedIT.java | 131 - .../test/TracerRecoversAfterOfflineTableIT.java | 127 - .../accumulo/test/TransportCachingIT.java | 120 - .../org/apache/accumulo/test/UnusedWALIT.java | 153 -- .../accumulo/test/UserCompactionStrategyIT.java | 296 --- .../java/org/apache/accumulo/test/UsersIT.java | 60 - .../accumulo/test/VerifySerialRecoveryIT.java | 107 - .../apache/accumulo/test/VolumeChooserIT.java | 392 --- .../java/org/apache/accumulo/test/VolumeIT.java | 568 ----- .../apache/accumulo/test/WaitForBalanceIT.java | 118 - .../test/functional/AccumuloInputFormatIT.java | 210 -- .../accumulo/test/functional/AddSplitIT.java | 142 -- .../test/functional/BackupMasterIT.java | 68 - .../test/functional/BadIteratorMincIT.java | 107 - .../functional/BalanceAfterCommsFailureIT.java | 138 -- .../BalanceInPresenceOfOfflineTableIT.java | 201 -- .../test/functional/BatchScanSplitIT.java | 129 - .../test/functional/BatchWriterFlushIT.java | 178 -- .../test/functional/BigRootTabletIT.java | 66 - .../accumulo/test/functional/BinaryIT.java | 86 - .../test/functional/BinaryStressIT.java | 107 - .../accumulo/test/functional/BloomFilterIT.java | 256 -- .../accumulo/test/functional/BulkFileIT.java | 130 - .../apache/accumulo/test/functional/BulkIT.java | 120 - .../functional/BulkSplitOptimizationIT.java | 142 -- .../test/functional/ChaoticBalancerIT.java | 85 - .../accumulo/test/functional/ClassLoaderIT.java | 104 - .../accumulo/test/functional/CleanTmpIT.java | 112 - .../accumulo/test/functional/CleanUpIT.java | 151 -- .../accumulo/test/functional/CloneTestIT.java | 295 --- .../accumulo/test/functional/CombinerIT.java | 76 - .../accumulo/test/functional/CompactionIT.java | 184 -- .../accumulo/test/functional/ConcurrencyIT.java | 158 -- .../functional/ConfigurableCompactionIT.java | 164 -- .../test/functional/ConfigurableMacBase.java | 182 -- .../accumulo/test/functional/ConstraintIT.java | 335 --- .../test/functional/CreateAndUseIT.java | 130 - .../test/functional/CreateManyScannersIT.java | 41 - .../accumulo/test/functional/CredentialsIT.java | 124 - .../test/functional/DeleteEverythingIT.java | 117 - .../accumulo/test/functional/DeleteIT.java | 106 - .../accumulo/test/functional/DeleteRowsIT.java | 154 -- .../test/functional/DeleteRowsSplitIT.java | 147 -- .../functional/DeleteTableDuringSplitIT.java | 109 - .../functional/DeletedTablesDontFlushIT.java | 62 - .../accumulo/test/functional/DurabilityIT.java | 222 -- .../test/functional/DynamicThreadPoolsIT.java | 126 - .../accumulo/test/functional/ExamplesIT.java | 660 ----- .../test/functional/FateStarvationIT.java | 80 - .../test/functional/FunctionalTestUtils.java | 192 -- .../test/functional/GarbageCollectorIT.java | 301 --- .../test/functional/HalfDeadTServerIT.java | 218 -- .../accumulo/test/functional/KerberosIT.java | 573 ----- .../test/functional/KerberosProxyIT.java | 426 ---- .../accumulo/test/functional/LargeRowIT.java | 219 -- .../test/functional/LateLastContactIT.java | 49 - .../accumulo/test/functional/LogicalTimeIT.java | 109 - .../accumulo/test/functional/MapReduceIT.java | 92 - .../test/functional/MasterAssignmentIT.java | 100 - .../test/functional/MasterFailoverIT.java | 80 - .../accumulo/test/functional/MaxOpenIT.java | 176 -- .../accumulo/test/functional/MergeIT.java | 194 -- .../accumulo/test/functional/MetadataIT.java | 148 -- .../test/functional/MetadataMaxFilesIT.java | 116 - .../test/functional/MetadataSplitIT.java | 56 - .../test/functional/MonitorLoggingIT.java | 121 - .../accumulo/test/functional/MonitorSslIT.java | 132 - .../accumulo/test/functional/NativeMapIT.java | 613 ----- .../accumulo/test/functional/PermissionsIT.java | 707 ------ .../accumulo/test/functional/ReadWriteIT.java | 456 ---- .../functional/RecoveryWithEmptyRFileIT.java | 109 - .../test/functional/RegexGroupBalanceIT.java | 192 -- .../accumulo/test/functional/RenameIT.java | 74 - .../accumulo/test/functional/RestartIT.java | 367 --- .../test/functional/RestartStressIT.java | 153 -- .../accumulo/test/functional/RowDeleteIT.java | 109 - .../accumulo/test/functional/ScanIdIT.java | 385 --- .../test/functional/ScanIteratorIT.java | 134 -- .../accumulo/test/functional/ScanRangeIT.java | 244 -- .../test/functional/ScanSessionTimeOutIT.java | 142 -- .../accumulo/test/functional/ScannerIT.java | 121 - .../test/functional/ServerSideErrorIT.java | 128 - .../test/functional/SessionDurabilityIT.java | 153 -- .../accumulo/test/functional/ShutdownIT.java | 121 - .../functional/SimpleBalancerFairnessIT.java | 117 - .../test/functional/SparseColumnFamilyIT.java | 98 - .../accumulo/test/functional/SplitIT.java | 223 -- .../test/functional/SplitRecoveryIT.java | 279 --- .../apache/accumulo/test/functional/SslIT.java | 72 - .../test/functional/SslWithClientAuthIT.java | 77 - .../accumulo/test/functional/StartIT.java | 43 - .../accumulo/test/functional/TableIT.java | 108 - .../accumulo/test/functional/TabletIT.java | 101 - .../functional/TabletStateChangeIteratorIT.java | 192 -- .../accumulo/test/functional/TimeoutIT.java | 120 - .../accumulo/test/functional/VisibilityIT.java | 323 --- .../accumulo/test/functional/WALSunnyDayIT.java | 234 -- .../test/functional/WatchTheWatchCountIT.java | 80 - .../test/functional/WriteAheadLogIT.java | 79 - .../accumulo/test/functional/WriteLotsIT.java | 89 - .../accumulo/test/functional/ZooCacheIT.java | 75 - .../test/functional/ZookeeperRestartIT.java | 87 - .../test/performance/RollWALPerformanceIT.java | 120 - .../performance/metadata/FastBulkImportIT.java | 103 - .../accumulo/test/proxy/ProxyDurabilityIT.java | 145 -- .../accumulo/test/proxy/SimpleProxyBase.java | 2273 ------------------ .../accumulo/test/proxy/TBinaryProxyIT.java | 33 - .../accumulo/test/proxy/TCompactProxyIT.java | 32 - .../test/proxy/TJsonProtocolProxyIT.java | 33 - .../accumulo/test/proxy/TTupleProxyIT.java | 33 - .../accumulo/test/proxy/TestProxyClient.java | 204 -- .../test/proxy/TestProxyInstanceOperations.java | 84 - .../accumulo/test/proxy/TestProxyReadWrite.java | 468 ---- .../test/proxy/TestProxySecurityOperations.java | 147 -- .../test/proxy/TestProxyTableOperations.java | 202 -- .../test/replication/CyclicReplicationIT.java | 332 --- ...bageCollectorCommunicatesWithTServersIT.java | 417 ---- .../test/replication/KerberosReplicationIT.java | 233 -- .../replication/MultiInstanceReplicationIT.java | 731 ------ .../replication/MultiTserverReplicationIT.java | 115 - .../test/replication/ReplicationIT.java | 1436 ----------- .../replication/ReplicationRandomWalkIT.java | 67 - .../test/replication/StatusCombinerMacIT.java | 118 - .../UnorderedWorkAssignerReplicationIT.java | 731 ------ ...UnusedWalDoesntCloseReplicationStatusIT.java | 219 -- .../server/security/SystemCredentialsIT.java | 233 -- .../accumulo/test/start/KeywordStartIT.java | 197 -- .../apache/accumulo/test/util/CertUtils.java | 348 --- test/src/test/resources/FooConstraint.jar | Bin 2130 -> 0 bytes test/src/test/resources/FooFilter.jar | Bin 1645 -> 0 bytes test/src/test/resources/TestCombinerX.jar | Bin 4335 -> 0 bytes test/src/test/resources/TestCombinerY.jar | Bin 4129 -> 0 bytes test/src/test/resources/TestCompactionStrat.jar | Bin 2530 -> 0 bytes test/src/test/resources/conf/accumulo-site.xml | 123 - test/src/test/resources/conf/generic_logger.xml | 83 - test/src/test/resources/conf/monitor_logger.xml | 64 - test/src/test/resources/log4j.properties | 55 - test/src/test/resources/randomwalk/Basic.xml | 37 - test/src/test/resources/randomwalk/Simple.xml | 43 - test/src/test/resources/unit/Basic.xml | 37 - test/src/test/resources/unit/Simple.xml | 43 - 386 files changed, 39618 insertions(+), 39666 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo/blob/01ae5b85/assemble/pom.xml ---------------------------------------------------------------------- diff --git a/assemble/pom.xml b/assemble/pom.xml index 525b443..b965fe6 100644 --- a/assemble/pom.xml +++ b/assemble/pom.xml @@ -223,17 +223,6 @@ </build> <profiles> <profile> - <id>test-jar</id> - <dependencies> - <dependency> - <groupId>org.apache.accumulo</groupId> - <artifactId>accumulo-test</artifactId> - <version>${project.version}</version> - <classifier>tests</classifier> - </dependency> - </dependencies> - </profile> - <profile> <id>apache-release</id> <build> <plugins> http://git-wip-us.apache.org/repos/asf/accumulo/blob/01ae5b85/test/pom.xml ---------------------------------------------------------------------- diff --git a/test/pom.xml b/test/pom.xml index c68e158..57cfbbd 100644 --- a/test/pom.xml +++ b/test/pom.xml @@ -43,6 +43,10 @@ <artifactId>guava</artifactId> </dependency> <dependency> + <groupId>commons-cli</groupId> + <artifactId>commons-cli</artifactId> + </dependency> + <dependency> <groupId>commons-codec</groupId> <artifactId>commons-codec</artifactId> </dependency> @@ -51,6 +55,10 @@ <artifactId>commons-configuration</artifactId> </dependency> <dependency> + <groupId>commons-httpclient</groupId> + <artifactId>commons-httpclient</artifactId> + </dependency> + <dependency> <groupId>commons-io</groupId> <artifactId>commons-io</artifactId> </dependency> @@ -63,6 +71,10 @@ <artifactId>jline</artifactId> </dependency> <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + </dependency> + <dependency> <groupId>log4j</groupId> <artifactId>log4j</artifactId> </dependency> @@ -72,6 +84,10 @@ </dependency> <dependency> <groupId>org.apache.accumulo</groupId> + <artifactId>accumulo-examples-simple</artifactId> + </dependency> + <dependency> + <groupId>org.apache.accumulo</groupId> <artifactId>accumulo-fate</artifactId> </dependency> <dependency> @@ -128,42 +144,12 @@ <artifactId>hadoop-client</artifactId> </dependency> <dependency> - <groupId>org.apache.thrift</groupId> - <artifactId>libthrift</artifactId> - </dependency> - <dependency> - <groupId>org.apache.zookeeper</groupId> - <artifactId>zookeeper</artifactId> - </dependency> - <dependency> - <groupId>commons-cli</groupId> - <artifactId>commons-cli</artifactId> - <scope>test</scope> - </dependency> - <dependency> - <groupId>commons-httpclient</groupId> - <artifactId>commons-httpclient</artifactId> - <scope>test</scope> - </dependency> - <dependency> - <groupId>junit</groupId> - <artifactId>junit</artifactId> - <scope>test</scope> - </dependency> - <dependency> - <groupId>org.apache.accumulo</groupId> - <artifactId>accumulo-examples-simple</artifactId> - <scope>test</scope> - </dependency> - <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-minicluster</artifactId> - <scope>test</scope> </dependency> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-minikdc</artifactId> - <scope>test</scope> <exclusions> <!-- Pulls in an older bouncycastle version --> <exclusion> @@ -173,29 +159,32 @@ </exclusions> </dependency> <dependency> + <groupId>org.apache.thrift</groupId> + <artifactId>libthrift</artifactId> + </dependency> + <dependency> + <groupId>org.apache.zookeeper</groupId> + <artifactId>zookeeper</artifactId> + </dependency> + <dependency> <groupId>org.bouncycastle</groupId> <artifactId>bcpkix-jdk15on</artifactId> - <scope>test</scope> </dependency> <dependency> <groupId>org.bouncycastle</groupId> <artifactId>bcprov-jdk15on</artifactId> - <scope>test</scope> </dependency> <dependency> <groupId>org.easymock</groupId> <artifactId>easymock</artifactId> - <scope>test</scope> </dependency> <dependency> <groupId>org.eclipse.jetty</groupId> <artifactId>jetty-server</artifactId> - <scope>test</scope> </dependency> <dependency> <groupId>org.slf4j</groupId> <artifactId>slf4j-log4j12</artifactId> - <scope>test</scope> </dependency> </dependencies> <build> @@ -216,6 +205,8 @@ <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-failsafe-plugin</artifactId> <configuration> + <testSourceDirectory>${project.basedir}/src/main/java/</testSourceDirectory> + <testClassesDirectory>${project.build.directory}/classes/</testClassesDirectory> <systemPropertyVariables> <timeout.factor>${timeout.factor}</timeout.factor> <org.apache.accumulo.test.functional.useCredProviderForIT>${useCredProviderForIT}</org.apache.accumulo.test.functional.useCredProviderForIT> @@ -240,32 +231,6 @@ </build> <profiles> <profile> - <id>test-jar</id> - <build> - <plugins> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-jar-plugin</artifactId> - <configuration> - <archive> - <manifestEntries> - <Sealed>false</Sealed> - </manifestEntries> - </archive> - </configuration> - <executions> - <execution> - <id>make-test-jar</id> - <goals> - <goal>test-jar</goal> - </goals> - </execution> - </executions> - </plugin> - </plugins> - </build> - </profile> - <profile> <id>shared-mini-for-it</id> <!-- <activation> @@ -336,7 +301,6 @@ <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-distcp</artifactId> - <scope>test</scope> </dependency> </dependencies> </profile> @@ -355,7 +319,6 @@ <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-distcp</artifactId> - <scope>test</scope> </dependency> </dependencies> </profile> http://git-wip-us.apache.org/repos/asf/accumulo/blob/01ae5b85/test/src/main/java/org/apache/accumulo/harness/AccumuloClusterHarness.java ---------------------------------------------------------------------- diff --git a/test/src/main/java/org/apache/accumulo/harness/AccumuloClusterHarness.java b/test/src/main/java/org/apache/accumulo/harness/AccumuloClusterHarness.java new file mode 100644 index 0000000..30058db --- /dev/null +++ b/test/src/main/java/org/apache/accumulo/harness/AccumuloClusterHarness.java @@ -0,0 +1,338 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.accumulo.harness; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.junit.Assert.fail; + +import java.io.IOException; + +import org.apache.accumulo.cluster.AccumuloCluster; +import org.apache.accumulo.cluster.ClusterControl; +import org.apache.accumulo.cluster.ClusterUser; +import org.apache.accumulo.cluster.ClusterUsers; +import org.apache.accumulo.cluster.standalone.StandaloneAccumuloCluster; +import org.apache.accumulo.core.client.ClientConfiguration; +import org.apache.accumulo.core.client.ClientConfiguration.ClientProperty; +import org.apache.accumulo.core.client.Connector; +import org.apache.accumulo.core.client.admin.SecurityOperations; +import org.apache.accumulo.core.client.admin.TableOperations; +import org.apache.accumulo.core.client.security.tokens.AuthenticationToken; +import org.apache.accumulo.core.client.security.tokens.KerberosToken; +import org.apache.accumulo.core.client.security.tokens.PasswordToken; +import org.apache.accumulo.core.conf.Property; +import org.apache.accumulo.core.security.TablePermission; +import org.apache.accumulo.harness.conf.AccumuloClusterConfiguration; +import org.apache.accumulo.harness.conf.AccumuloClusterPropertyConfiguration; +import org.apache.accumulo.harness.conf.AccumuloMiniClusterConfiguration; +import org.apache.accumulo.harness.conf.StandaloneAccumuloClusterConfiguration; +import org.apache.accumulo.minicluster.impl.MiniAccumuloClusterImpl; +import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.security.UserGroupInformation; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Assume; +import org.junit.Before; +import org.junit.BeforeClass; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.base.Preconditions; + +/** + * General Integration-Test base class that provides access to an Accumulo instance for testing. This instance could be MAC or a standalone instance. + */ +public abstract class AccumuloClusterHarness extends AccumuloITBase implements MiniClusterConfigurationCallback, ClusterUsers { + private static final Logger log = LoggerFactory.getLogger(AccumuloClusterHarness.class); + private static final String TRUE = Boolean.toString(true); + + public static enum ClusterType { + MINI, STANDALONE; + + public boolean isDynamic() { + return this == MINI; + } + } + + private static boolean initialized = false; + + protected static AccumuloCluster cluster; + protected static ClusterType type; + protected static AccumuloClusterPropertyConfiguration clusterConf; + protected static TestingKdc krb; + + @BeforeClass + public static void setUp() throws Exception { + clusterConf = AccumuloClusterPropertyConfiguration.get(); + type = clusterConf.getClusterType(); + + if (ClusterType.MINI == type && TRUE.equals(System.getProperty(MiniClusterHarness.USE_KERBEROS_FOR_IT_OPTION))) { + krb = new TestingKdc(); + krb.start(); + log.info("MiniKdc started"); + } + + initialized = true; + } + + @AfterClass + public static void tearDownKdc() throws Exception { + if (null != krb) { + krb.stop(); + } + } + + /** + * The {@link TestingKdc} used for this {@link AccumuloCluster}. Might be null. + */ + public static TestingKdc getKdc() { + return krb; + } + + @Before + public void setupCluster() throws Exception { + // Before we try to instantiate the cluster, check to see if the test even wants to run against this type of cluster + Assume.assumeTrue(canRunTest(type)); + + switch (type) { + case MINI: + MiniClusterHarness miniClusterHarness = new MiniClusterHarness(); + // Intrinsically performs the callback to let tests alter MiniAccumuloConfig and core-site.xml + MiniAccumuloClusterImpl impl = miniClusterHarness.create(this, getAdminToken(), krb); + cluster = impl; + // MAC makes a ClientConf for us, just set it + ((AccumuloMiniClusterConfiguration) clusterConf).setClientConf(impl.getClientConfig()); + // Login as the "root" user + if (null != krb) { + ClusterUser rootUser = krb.getRootUser(); + // Log in the 'client' user + UserGroupInformation.loginUserFromKeytab(rootUser.getPrincipal(), rootUser.getKeytab().getAbsolutePath()); + } + break; + case STANDALONE: + StandaloneAccumuloClusterConfiguration conf = (StandaloneAccumuloClusterConfiguration) clusterConf; + ClientConfiguration clientConf = conf.getClientConf(); + StandaloneAccumuloCluster standaloneCluster = new StandaloneAccumuloCluster(conf.getInstance(), clientConf, conf.getTmpDirectory(), conf.getUsers(), + conf.getAccumuloServerUser()); + // If these are provided in the configuration, pass them into the cluster + standaloneCluster.setAccumuloHome(conf.getAccumuloHome()); + standaloneCluster.setClientAccumuloConfDir(conf.getClientAccumuloConfDir()); + standaloneCluster.setServerAccumuloConfDir(conf.getServerAccumuloConfDir()); + standaloneCluster.setHadoopConfDir(conf.getHadoopConfDir()); + + // For SASL, we need to get the Hadoop configuration files as well otherwise UGI will log in as SIMPLE instead of KERBEROS + Configuration hadoopConfiguration = standaloneCluster.getHadoopConfiguration(); + if (clientConf.getBoolean(ClientProperty.INSTANCE_RPC_SASL_ENABLED.getKey(), false)) { + UserGroupInformation.setConfiguration(hadoopConfiguration); + // Login as the admin user to start the tests + UserGroupInformation.loginUserFromKeytab(conf.getAdminPrincipal(), conf.getAdminKeytab().getAbsolutePath()); + } + + // Set the implementation + cluster = standaloneCluster; + break; + default: + throw new RuntimeException("Unhandled type"); + } + + if (type.isDynamic()) { + cluster.start(); + } else { + log.info("Removing tables which appear to be from a previous test run"); + cleanupTables(); + log.info("Removing users which appear to be from a previous test run"); + cleanupUsers(); + } + + switch (type) { + case MINI: + if (null != krb) { + final String traceTable = Property.TRACE_TABLE.getDefaultValue(); + final ClusterUser systemUser = krb.getAccumuloServerUser(), rootUser = krb.getRootUser(); + + // Login as the trace user + UserGroupInformation.loginUserFromKeytab(systemUser.getPrincipal(), systemUser.getKeytab().getAbsolutePath()); + + // Open a connector as the system user (ensures the user will exist for us to assign permissions to) + Connector conn = cluster.getConnector(systemUser.getPrincipal(), new KerberosToken(systemUser.getPrincipal(), systemUser.getKeytab(), true)); + + // Then, log back in as the "root" user and do the grant + UserGroupInformation.loginUserFromKeytab(rootUser.getPrincipal(), rootUser.getKeytab().getAbsolutePath()); + conn = getConnector(); + + // Create the trace table + conn.tableOperations().create(traceTable); + + // Trace user (which is the same kerberos principal as the system user, but using a normal KerberosToken) needs + // to have the ability to read, write and alter the trace table + conn.securityOperations().grantTablePermission(systemUser.getPrincipal(), traceTable, TablePermission.READ); + conn.securityOperations().grantTablePermission(systemUser.getPrincipal(), traceTable, TablePermission.WRITE); + conn.securityOperations().grantTablePermission(systemUser.getPrincipal(), traceTable, TablePermission.ALTER_TABLE); + } + break; + default: + // do nothing + } + } + + public void cleanupTables() throws Exception { + final String tablePrefix = this.getClass().getSimpleName() + "_"; + final TableOperations tops = getConnector().tableOperations(); + for (String table : tops.list()) { + if (table.startsWith(tablePrefix)) { + log.debug("Removing table {}", table); + tops.delete(table); + } + } + } + + public void cleanupUsers() throws Exception { + final String userPrefix = this.getClass().getSimpleName(); + final SecurityOperations secOps = getConnector().securityOperations(); + for (String user : secOps.listLocalUsers()) { + if (user.startsWith(userPrefix)) { + log.info("Dropping local user {}", user); + secOps.dropLocalUser(user); + } + } + } + + @After + public void teardownCluster() throws Exception { + if (null != cluster) { + if (type.isDynamic()) { + cluster.stop(); + } else { + log.info("Removing tables which appear to be from the current test"); + cleanupTables(); + log.info("Removing users which appear to be from the current test"); + cleanupUsers(); + } + } + } + + public static AccumuloCluster getCluster() { + Preconditions.checkState(initialized); + return cluster; + } + + public static ClusterControl getClusterControl() { + Preconditions.checkState(initialized); + return cluster.getClusterControl(); + } + + public static ClusterType getClusterType() { + Preconditions.checkState(initialized); + return type; + } + + public static String getAdminPrincipal() { + Preconditions.checkState(initialized); + return clusterConf.getAdminPrincipal(); + } + + public static AuthenticationToken getAdminToken() { + Preconditions.checkState(initialized); + return clusterConf.getAdminToken(); + } + + @Override + public ClusterUser getAdminUser() { + switch (type) { + case MINI: + if (null == krb) { + PasswordToken passwordToken = (PasswordToken) getAdminToken(); + return new ClusterUser(getAdminPrincipal(), new String(passwordToken.getPassword(), UTF_8)); + } + return krb.getRootUser(); + case STANDALONE: + return new ClusterUser(getAdminPrincipal(), ((StandaloneAccumuloClusterConfiguration) clusterConf).getAdminKeytab()); + default: + throw new RuntimeException("Unknown cluster type"); + } + } + + @Override + public ClusterUser getUser(int offset) { + switch (type) { + case MINI: + if (null != krb) { + // Defer to the TestingKdc when kerberos is on so we can get the keytab instead of a password + return krb.getClientPrincipal(offset); + } else { + // Come up with a mostly unique name + String principal = getClass().getSimpleName() + "_" + testName.getMethodName() + "_" + offset; + // Username and password are the same + return new ClusterUser(principal, principal); + } + case STANDALONE: + return ((StandaloneAccumuloCluster) cluster).getUser(offset); + default: + throw new RuntimeException("Unknown cluster type"); + } + } + + public static FileSystem getFileSystem() throws IOException { + Preconditions.checkState(initialized); + return cluster.getFileSystem(); + } + + public static AccumuloClusterConfiguration getClusterConfiguration() { + Preconditions.checkState(initialized); + return clusterConf; + } + + public Connector getConnector() { + try { + String princ = getAdminPrincipal(); + AuthenticationToken token = getAdminToken(); + log.debug("Creating connector as {} with {}", princ, token); + return cluster.getConnector(princ, token); + } catch (Exception e) { + log.error("Could not connect to Accumulo", e); + fail("Could not connect to Accumulo: " + e.getMessage()); + + throw new RuntimeException("Could not connect to Accumulo", e); + } + } + + // TODO Really don't want this here. Will ultimately need to abstract configuration method away from MAConfig + // and change over to something more generic + @Override + public void configureMiniCluster(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {} + + /** + * A test may not be capable of running against a given AccumuloCluster. Implementations can override this method to advertise that they cannot (or perhaps do + * not) want to run the test. + */ + public boolean canRunTest(ClusterType type) { + return true; + } + + /** + * Tries to give a reasonable directory which can be used to create temporary files for the test. Makes a basic attempt to create the directory if it does not + * already exist. + * + * @return A directory which can be expected to exist on the Cluster's FileSystem + */ + public Path getUsableDir() throws IllegalArgumentException, IOException { + return cluster.getTemporaryPath(); + } +} http://git-wip-us.apache.org/repos/asf/accumulo/blob/01ae5b85/test/src/main/java/org/apache/accumulo/harness/AccumuloITBase.java ---------------------------------------------------------------------- diff --git a/test/src/main/java/org/apache/accumulo/harness/AccumuloITBase.java b/test/src/main/java/org/apache/accumulo/harness/AccumuloITBase.java new file mode 100644 index 0000000..8e2f6e0 --- /dev/null +++ b/test/src/main/java/org/apache/accumulo/harness/AccumuloITBase.java @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.accumulo.harness; + +import static org.junit.Assert.assertTrue; + +import java.io.File; + +import org.apache.commons.io.FileUtils; +import org.junit.Rule; +import org.junit.rules.TestName; +import org.junit.rules.Timeout; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Methods, setup and/or infrastructure which are common to any Accumulo integration test. + */ +public class AccumuloITBase { + private static final Logger log = LoggerFactory.getLogger(AccumuloITBase.class); + + @Rule + public TestName testName = new TestName(); + + public String[] getUniqueNames(int num) { + String[] names = new String[num]; + for (int i = 0; i < num; i++) + names[i] = this.getClass().getSimpleName() + "_" + testName.getMethodName() + i; + return names; + } + + /** + * Determines an appropriate directory name for holding generated ssl files for a test. The directory returned will have the same name as the provided + * directory, but with the suffix "-ssl" appended. This new directory is not created here, but is expected to be created as needed. + * + * @param baseDir + * the original directory, which the new directory will be created next to; it should exist + * @return the new directory (is not created) + */ + public static File getSslDir(File baseDir) { + assertTrue(baseDir.exists() && baseDir.isDirectory()); + return new File(baseDir.getParentFile(), baseDir.getName() + "-ssl"); + } + + public static File createTestDir(String name) { + File baseDir = new File(System.getProperty("user.dir") + "/target/mini-tests"); + assertTrue(baseDir.mkdirs() || baseDir.isDirectory()); + if (name == null) + return baseDir; + File testDir = new File(baseDir, name); + FileUtils.deleteQuietly(testDir); + assertTrue(testDir.mkdir()); + return testDir; + } + + /** + * If a given IT test has a method that takes longer than a class-set default timeout, declare it failed. + * + * Note that this provides a upper bound on test times, even in the presence of Test annotations with a timeout. That is, the Test annotatation can make the + * timing tighter but will not be able to allow a timeout that takes longer. + * + * Defaults to no timeout and can be changed via two mechanisms + * + * 1) A given IT class can override the defaultTimeoutSeconds method if test methods in that class should have a timeout. 2) The system property + * "timeout.factor" is used as a multiplier for the class provided default + * + * Note that if either of these values is '0' tests will run with no timeout. The default class level timeout is set to 0. + * + */ + @Rule + public Timeout testsShouldTimeout() { + int waitLonger = 0; + try { + String timeoutString = System.getProperty("timeout.factor"); + if (timeoutString != null && !timeoutString.isEmpty()) { + waitLonger = Integer.parseInt(timeoutString); + } + } catch (NumberFormatException exception) { + log.warn("Could not parse timeout.factor, defaulting to no timeout."); + } + return new Timeout(waitLonger * defaultTimeoutSeconds() * 1000); + } + + /** + * time to wait per-method before declaring a timeout, in seconds. + */ + protected int defaultTimeoutSeconds() { + return 0; + } +} http://git-wip-us.apache.org/repos/asf/accumulo/blob/01ae5b85/test/src/main/java/org/apache/accumulo/harness/MiniClusterConfigurationCallback.java ---------------------------------------------------------------------- diff --git a/test/src/main/java/org/apache/accumulo/harness/MiniClusterConfigurationCallback.java b/test/src/main/java/org/apache/accumulo/harness/MiniClusterConfigurationCallback.java new file mode 100644 index 0000000..5fa6eb5 --- /dev/null +++ b/test/src/main/java/org/apache/accumulo/harness/MiniClusterConfigurationCallback.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.accumulo.harness; + +import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl; +import org.apache.hadoop.conf.Configuration; + +/** + * Callback interface to inject configuration into the MiniAccumuloCluster or Hadoop core-site.xml file used by the MiniAccumuloCluster + */ +public interface MiniClusterConfigurationCallback { + + public static class NoCallback implements MiniClusterConfigurationCallback { + + private NoCallback() {} + + @Override + public void configureMiniCluster(MiniAccumuloConfigImpl cfg, Configuration coreSite) { + return; + } + } + + public static final MiniClusterConfigurationCallback NO_CALLBACK = new NoCallback(); + + void configureMiniCluster(MiniAccumuloConfigImpl cfg, Configuration coreSite); + +} http://git-wip-us.apache.org/repos/asf/accumulo/blob/01ae5b85/test/src/main/java/org/apache/accumulo/harness/MiniClusterHarness.java ---------------------------------------------------------------------- diff --git a/test/src/main/java/org/apache/accumulo/harness/MiniClusterHarness.java b/test/src/main/java/org/apache/accumulo/harness/MiniClusterHarness.java new file mode 100644 index 0000000..d923593 --- /dev/null +++ b/test/src/main/java/org/apache/accumulo/harness/MiniClusterHarness.java @@ -0,0 +1,242 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.accumulo.harness; + +import static org.junit.Assert.assertTrue; + +import java.io.BufferedOutputStream; +import java.io.File; +import java.io.FileOutputStream; +import java.io.OutputStream; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.atomic.AtomicLong; + +import org.apache.accumulo.cluster.ClusterUser; +import org.apache.accumulo.core.client.security.tokens.AuthenticationToken; +import org.apache.accumulo.core.client.security.tokens.KerberosToken; +import org.apache.accumulo.core.client.security.tokens.PasswordToken; +import org.apache.accumulo.core.conf.Property; +import org.apache.accumulo.minicluster.impl.MiniAccumuloClusterImpl; +import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl; +import org.apache.accumulo.server.security.handler.KerberosAuthenticator; +import org.apache.accumulo.server.security.handler.KerberosAuthorizor; +import org.apache.accumulo.server.security.handler.KerberosPermissionHandler; +import org.apache.accumulo.test.functional.NativeMapIT; +import org.apache.accumulo.test.util.CertUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CommonConfigurationKeysPublic; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.base.Charsets; +import com.google.common.base.Preconditions; + +/** + * Harness that sets up a MiniAccumuloCluster in a manner expected for Accumulo integration tests. + */ +public class MiniClusterHarness { + private static final Logger log = LoggerFactory.getLogger(MiniClusterHarness.class); + + private static final AtomicLong COUNTER = new AtomicLong(0); + + public static final String USE_SSL_FOR_IT_OPTION = "org.apache.accumulo.test.functional.useSslForIT", + USE_CRED_PROVIDER_FOR_IT_OPTION = "org.apache.accumulo.test.functional.useCredProviderForIT", + USE_KERBEROS_FOR_IT_OPTION = "org.apache.accumulo.test.functional.useKrbForIT", TRUE = Boolean.toString(true); + + // TODO These are defined in MiniKdc >= 2.6.0. Can be removed when minimum Hadoop dependency is increased to that. + public static final String JAVA_SECURITY_KRB5_CONF = "java.security.krb5.conf", SUN_SECURITY_KRB5_DEBUG = "sun.security.krb5.debug"; + + /** + * Create a MiniAccumuloCluster using the given Token as the credentials for the root user. + */ + public MiniAccumuloClusterImpl create(AuthenticationToken token) throws Exception { + return create(MiniClusterHarness.class.getName(), Long.toString(COUNTER.incrementAndGet()), token); + } + + public MiniAccumuloClusterImpl create(AuthenticationToken token, TestingKdc kdc) throws Exception { + return create(MiniClusterHarness.class.getName(), Long.toString(COUNTER.incrementAndGet()), token, kdc); + } + + public MiniAccumuloClusterImpl create(AccumuloITBase testBase, AuthenticationToken token) throws Exception { + return create(testBase.getClass().getName(), testBase.testName.getMethodName(), token); + } + + public MiniAccumuloClusterImpl create(AccumuloITBase testBase, AuthenticationToken token, TestingKdc kdc) throws Exception { + return create(testBase, token, kdc, MiniClusterConfigurationCallback.NO_CALLBACK); + } + + public MiniAccumuloClusterImpl create(AccumuloITBase testBase, AuthenticationToken token, TestingKdc kdc, MiniClusterConfigurationCallback configCallback) + throws Exception { + return create(testBase.getClass().getName(), testBase.testName.getMethodName(), token, configCallback, kdc); + } + + public MiniAccumuloClusterImpl create(AccumuloClusterHarness testBase, AuthenticationToken token, TestingKdc kdc) throws Exception { + return create(testBase.getClass().getName(), testBase.testName.getMethodName(), token, testBase, kdc); + } + + public MiniAccumuloClusterImpl create(AccumuloClusterHarness testBase, AuthenticationToken token, MiniClusterConfigurationCallback callback) throws Exception { + return create(testBase.getClass().getName(), testBase.testName.getMethodName(), token, callback); + } + + public MiniAccumuloClusterImpl create(String testClassName, String testMethodName, AuthenticationToken token) throws Exception { + return create(testClassName, testMethodName, token, MiniClusterConfigurationCallback.NO_CALLBACK); + } + + public MiniAccumuloClusterImpl create(String testClassName, String testMethodName, AuthenticationToken token, TestingKdc kdc) throws Exception { + return create(testClassName, testMethodName, token, MiniClusterConfigurationCallback.NO_CALLBACK, kdc); + } + + public MiniAccumuloClusterImpl create(String testClassName, String testMethodName, AuthenticationToken token, MiniClusterConfigurationCallback configCallback) + throws Exception { + return create(testClassName, testMethodName, token, configCallback, null); + } + + public MiniAccumuloClusterImpl create(String testClassName, String testMethodName, AuthenticationToken token, + MiniClusterConfigurationCallback configCallback, TestingKdc kdc) throws Exception { + Preconditions.checkNotNull(token); + Preconditions.checkArgument(token instanceof PasswordToken || token instanceof KerberosToken, "A PasswordToken or KerberosToken is required"); + + String rootPasswd; + if (token instanceof PasswordToken) { + rootPasswd = new String(((PasswordToken) token).getPassword(), Charsets.UTF_8); + } else { + rootPasswd = UUID.randomUUID().toString(); + } + + File baseDir = AccumuloClusterHarness.createTestDir(testClassName + "_" + testMethodName); + MiniAccumuloConfigImpl cfg = new MiniAccumuloConfigImpl(baseDir, rootPasswd); + + // Enable native maps by default + cfg.setNativeLibPaths(NativeMapIT.nativeMapLocation().getAbsolutePath()); + cfg.setProperty(Property.TSERV_NATIVEMAP_ENABLED, Boolean.TRUE.toString()); + + Configuration coreSite = new Configuration(false); + + // Setup SSL and credential providers if the properties request such + configureForEnvironment(cfg, getClass(), AccumuloClusterHarness.getSslDir(baseDir), coreSite, kdc); + + // Invoke the callback for tests to configure MAC before it starts + configCallback.configureMiniCluster(cfg, coreSite); + + MiniAccumuloClusterImpl miniCluster = new MiniAccumuloClusterImpl(cfg); + + // Write out any configuration items to a file so HDFS will pick them up automatically (from the classpath) + if (coreSite.size() > 0) { + File csFile = new File(miniCluster.getConfig().getConfDir(), "core-site.xml"); + if (csFile.exists()) + throw new RuntimeException(csFile + " already exist"); + + OutputStream out = new BufferedOutputStream(new FileOutputStream(new File(miniCluster.getConfig().getConfDir(), "core-site.xml"))); + coreSite.writeXml(out); + out.close(); + } + + return miniCluster; + } + + protected void configureForEnvironment(MiniAccumuloConfigImpl cfg, Class<?> testClass, File folder, Configuration coreSite, TestingKdc kdc) { + if (TRUE.equals(System.getProperty(USE_SSL_FOR_IT_OPTION))) { + configureForSsl(cfg, folder); + } + if (TRUE.equals(System.getProperty(USE_CRED_PROVIDER_FOR_IT_OPTION))) { + cfg.setUseCredentialProvider(true); + } + + if (TRUE.equals(System.getProperty(USE_KERBEROS_FOR_IT_OPTION))) { + if (TRUE.equals(System.getProperty(USE_SSL_FOR_IT_OPTION))) { + throw new RuntimeException("Cannot use both SSL and Kerberos"); + } + + try { + configureForKerberos(cfg, folder, coreSite, kdc); + } catch (Exception e) { + throw new RuntimeException("Failed to initialize KDC", e); + } + } + } + + protected void configureForSsl(MiniAccumuloConfigImpl cfg, File folder) { + Map<String,String> siteConfig = cfg.getSiteConfig(); + if (TRUE.equals(siteConfig.get(Property.INSTANCE_RPC_SSL_ENABLED.getKey()))) { + // already enabled; don't mess with it + return; + } + + File sslDir = new File(folder, "ssl"); + assertTrue(sslDir.mkdirs() || sslDir.isDirectory()); + File rootKeystoreFile = new File(sslDir, "root-" + cfg.getInstanceName() + ".jks"); + File localKeystoreFile = new File(sslDir, "local-" + cfg.getInstanceName() + ".jks"); + File publicTruststoreFile = new File(sslDir, "public-" + cfg.getInstanceName() + ".jks"); + final String rootKeystorePassword = "root_keystore_password", truststorePassword = "truststore_password"; + try { + new CertUtils(Property.RPC_SSL_KEYSTORE_TYPE.getDefaultValue(), "o=Apache Accumulo,cn=MiniAccumuloCluster", "RSA", 2048, "sha1WithRSAEncryption") + .createAll(rootKeystoreFile, localKeystoreFile, publicTruststoreFile, cfg.getInstanceName(), rootKeystorePassword, cfg.getRootPassword(), + truststorePassword); + } catch (Exception e) { + throw new RuntimeException("error creating MAC keystore", e); + } + + siteConfig.put(Property.INSTANCE_RPC_SSL_ENABLED.getKey(), "true"); + siteConfig.put(Property.RPC_SSL_KEYSTORE_PATH.getKey(), localKeystoreFile.getAbsolutePath()); + siteConfig.put(Property.RPC_SSL_KEYSTORE_PASSWORD.getKey(), cfg.getRootPassword()); + siteConfig.put(Property.RPC_SSL_TRUSTSTORE_PATH.getKey(), publicTruststoreFile.getAbsolutePath()); + siteConfig.put(Property.RPC_SSL_TRUSTSTORE_PASSWORD.getKey(), truststorePassword); + cfg.setSiteConfig(siteConfig); + } + + protected void configureForKerberos(MiniAccumuloConfigImpl cfg, File folder, Configuration coreSite, TestingKdc kdc) throws Exception { + Map<String,String> siteConfig = cfg.getSiteConfig(); + if (TRUE.equals(siteConfig.get(Property.INSTANCE_RPC_SSL_ENABLED.getKey()))) { + throw new RuntimeException("Cannot use both SSL and SASL/Kerberos"); + } + + if (TRUE.equals(siteConfig.get(Property.INSTANCE_RPC_SASL_ENABLED.getKey()))) { + // already enabled + return; + } + + if (null == kdc) { + throw new IllegalStateException("MiniClusterKdc was null"); + } + + log.info("Enabling Kerberos/SASL for minicluster"); + + // Turn on SASL and set the keytab/principal information + cfg.setProperty(Property.INSTANCE_RPC_SASL_ENABLED, "true"); + ClusterUser serverUser = kdc.getAccumuloServerUser(); + cfg.setProperty(Property.GENERAL_KERBEROS_KEYTAB, serverUser.getKeytab().getAbsolutePath()); + cfg.setProperty(Property.GENERAL_KERBEROS_PRINCIPAL, serverUser.getPrincipal()); + cfg.setProperty(Property.INSTANCE_SECURITY_AUTHENTICATOR, KerberosAuthenticator.class.getName()); + cfg.setProperty(Property.INSTANCE_SECURITY_AUTHORIZOR, KerberosAuthorizor.class.getName()); + cfg.setProperty(Property.INSTANCE_SECURITY_PERMISSION_HANDLER, KerberosPermissionHandler.class.getName()); + // Piggy-back on the "system user" credential, but use it as a normal KerberosToken, not the SystemToken. + cfg.setProperty(Property.TRACE_USER, serverUser.getPrincipal()); + cfg.setProperty(Property.TRACE_TOKEN_TYPE, KerberosToken.CLASS_NAME); + + // Pass down some KRB5 debug properties + Map<String,String> systemProperties = cfg.getSystemProperties(); + systemProperties.put(JAVA_SECURITY_KRB5_CONF, System.getProperty(JAVA_SECURITY_KRB5_CONF, "")); + systemProperties.put(SUN_SECURITY_KRB5_DEBUG, System.getProperty(SUN_SECURITY_KRB5_DEBUG, "false")); + cfg.setSystemProperties(systemProperties); + + // Make sure UserGroupInformation will do the correct login + coreSite.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos"); + + cfg.setRootUserName(kdc.getRootUser().getPrincipal()); + } +} http://git-wip-us.apache.org/repos/asf/accumulo/blob/01ae5b85/test/src/main/java/org/apache/accumulo/harness/SharedMiniClusterBase.java ---------------------------------------------------------------------- diff --git a/test/src/main/java/org/apache/accumulo/harness/SharedMiniClusterBase.java b/test/src/main/java/org/apache/accumulo/harness/SharedMiniClusterBase.java new file mode 100644 index 0000000..79340f2 --- /dev/null +++ b/test/src/main/java/org/apache/accumulo/harness/SharedMiniClusterBase.java @@ -0,0 +1,185 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.accumulo.harness; + +import static org.junit.Assert.assertTrue; + +import java.io.File; +import java.io.IOException; +import java.util.Random; + +import org.apache.accumulo.cluster.ClusterUser; +import org.apache.accumulo.cluster.ClusterUsers; +import org.apache.accumulo.core.client.Connector; +import org.apache.accumulo.core.client.security.tokens.AuthenticationToken; +import org.apache.accumulo.core.client.security.tokens.KerberosToken; +import org.apache.accumulo.core.client.security.tokens.PasswordToken; +import org.apache.accumulo.core.conf.Property; +import org.apache.accumulo.core.security.TablePermission; +import org.apache.accumulo.minicluster.impl.MiniAccumuloClusterImpl; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CommonConfigurationKeysPublic; +import org.apache.hadoop.security.UserGroupInformation; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Convenience class which starts a single MAC instance for a test to leverage. + * + * There isn't a good way to build this off of the {@link AccumuloClusterHarness} (as would be the logical place) because we need to start the + * MiniAccumuloCluster in a static BeforeClass-annotated method. Because it is static and invoked before any other BeforeClass methods in the implementation, + * the actual test classes can't expose any information to tell the base class that it is to perform the one-MAC-per-class semantics. + */ +public abstract class SharedMiniClusterBase extends AccumuloITBase implements ClusterUsers { + private static final Logger log = LoggerFactory.getLogger(SharedMiniClusterBase.class); + public static final String TRUE = Boolean.toString(true); + + private static String principal = "root"; + private static String rootPassword; + private static AuthenticationToken token; + private static MiniAccumuloClusterImpl cluster; + private static TestingKdc krb; + + @BeforeClass + public static void startMiniCluster() throws Exception { + File baseDir = new File(System.getProperty("user.dir") + "/target/mini-tests"); + assertTrue(baseDir.mkdirs() || baseDir.isDirectory()); + + // Make a shared MAC instance instead of spinning up one per test method + MiniClusterHarness harness = new MiniClusterHarness(); + + if (TRUE.equals(System.getProperty(MiniClusterHarness.USE_KERBEROS_FOR_IT_OPTION))) { + krb = new TestingKdc(); + krb.start(); + // Enabled krb auth + Configuration conf = new Configuration(false); + conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos"); + UserGroupInformation.setConfiguration(conf); + // Login as the client + ClusterUser rootUser = krb.getRootUser(); + // Get the krb token + principal = rootUser.getPrincipal(); + token = new KerberosToken(principal, rootUser.getKeytab(), true); + } else { + rootPassword = "rootPasswordShared1"; + token = new PasswordToken(rootPassword); + } + + cluster = harness.create(SharedMiniClusterBase.class.getName(), System.currentTimeMillis() + "_" + new Random().nextInt(Short.MAX_VALUE), token, krb); + cluster.start(); + + if (null != krb) { + final String traceTable = Property.TRACE_TABLE.getDefaultValue(); + final ClusterUser systemUser = krb.getAccumuloServerUser(), rootUser = krb.getRootUser(); + // Login as the trace user + // Open a connector as the system user (ensures the user will exist for us to assign permissions to) + Connector conn = cluster.getConnector(systemUser.getPrincipal(), new KerberosToken(systemUser.getPrincipal(), systemUser.getKeytab(), true)); + + // Then, log back in as the "root" user and do the grant + UserGroupInformation.loginUserFromKeytab(rootUser.getPrincipal(), rootUser.getKeytab().getAbsolutePath()); + conn = cluster.getConnector(principal, token); + + // Create the trace table + conn.tableOperations().create(traceTable); + + // Trace user (which is the same kerberos principal as the system user, but using a normal KerberosToken) needs + // to have the ability to read, write and alter the trace table + conn.securityOperations().grantTablePermission(systemUser.getPrincipal(), traceTable, TablePermission.READ); + conn.securityOperations().grantTablePermission(systemUser.getPrincipal(), traceTable, TablePermission.WRITE); + conn.securityOperations().grantTablePermission(systemUser.getPrincipal(), traceTable, TablePermission.ALTER_TABLE); + } + } + + @AfterClass + public static void stopMiniCluster() throws Exception { + if (null != cluster) { + try { + cluster.stop(); + } catch (Exception e) { + log.error("Failed to stop minicluster", e); + } + } + if (null != krb) { + try { + krb.stop(); + } catch (Exception e) { + log.error("Failed to stop KDC", e); + } + } + } + + public static String getRootPassword() { + return rootPassword; + } + + public static AuthenticationToken getToken() { + if (token instanceof KerberosToken) { + try { + UserGroupInformation.loginUserFromKeytab(getPrincipal(), krb.getRootUser().getKeytab().getAbsolutePath()); + } catch (IOException e) { + throw new RuntimeException("Failed to login", e); + } + } + return token; + } + + public static String getPrincipal() { + return principal; + } + + public static MiniAccumuloClusterImpl getCluster() { + return cluster; + } + + public static File getMiniClusterDir() { + return cluster.getConfig().getDir(); + } + + public static Connector getConnector() { + try { + return getCluster().getConnector(principal, getToken()); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + public static TestingKdc getKdc() { + return krb; + } + + @Override + public ClusterUser getAdminUser() { + if (null == krb) { + return new ClusterUser(getPrincipal(), getRootPassword()); + } else { + return krb.getRootUser(); + } + } + + @Override + public ClusterUser getUser(int offset) { + if (null == krb) { + String user = SharedMiniClusterBase.class.getName() + "_" + testName.getMethodName() + "_" + offset; + // Password is the username + return new ClusterUser(user, user); + } else { + return krb.getClientPrincipal(offset); + } + } +} http://git-wip-us.apache.org/repos/asf/accumulo/blob/01ae5b85/test/src/main/java/org/apache/accumulo/harness/TestingKdc.java ---------------------------------------------------------------------- diff --git a/test/src/main/java/org/apache/accumulo/harness/TestingKdc.java b/test/src/main/java/org/apache/accumulo/harness/TestingKdc.java new file mode 100644 index 0000000..9471274 --- /dev/null +++ b/test/src/main/java/org/apache/accumulo/harness/TestingKdc.java @@ -0,0 +1,210 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.accumulo.harness; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkNotNull; +import static org.junit.Assert.assertTrue; + +import java.io.File; +import java.net.InetAddress; +import java.util.ArrayList; +import java.util.List; +import java.util.Properties; + +import org.apache.accumulo.cluster.ClusterUser; +import org.apache.hadoop.minikdc.MiniKdc; +import org.junit.Assert; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Creates a {@link MiniKdc} for tests to use to exercise secure Accumulo + */ +public class TestingKdc { + private static final Logger log = LoggerFactory.getLogger(TestingKdc.class); + + public static final int NUM_USERS = 10; + + protected MiniKdc kdc = null; + protected ClusterUser accumuloServerUser = null, accumuloAdmin = null; + protected List<ClusterUser> clientPrincipals = null; + + public final String ORG_NAME = "EXAMPLE", ORG_DOMAIN = "COM"; + + private String hostname; + private File keytabDir; + private boolean started = false; + + public TestingKdc() throws Exception { + this(computeKdcDir(), computeKeytabDir()); + } + + private static File computeKdcDir() { + File targetDir = new File(System.getProperty("user.dir"), "target"); + Assert.assertTrue("Could not find Maven target directory: " + targetDir, targetDir.exists() && targetDir.isDirectory()); + + // Create the directories: target/kerberos/minikdc + File kdcDir = new File(new File(targetDir, "kerberos"), "minikdc"); + + assertTrue(kdcDir.mkdirs() || kdcDir.isDirectory()); + + return kdcDir; + } + + private static File computeKeytabDir() { + File targetDir = new File(System.getProperty("user.dir"), "target"); + Assert.assertTrue("Could not find Maven target directory: " + targetDir, targetDir.exists() && targetDir.isDirectory()); + + // Create the directories: target/kerberos/keytabs + File keytabDir = new File(new File(targetDir, "kerberos"), "keytabs"); + + assertTrue(keytabDir.mkdirs() || keytabDir.isDirectory()); + + return keytabDir; + } + + public TestingKdc(File kdcDir, File keytabDir) throws Exception { + checkNotNull(kdcDir, "KDC directory was null"); + checkNotNull(keytabDir, "Keytab directory was null"); + + this.keytabDir = keytabDir; + this.hostname = InetAddress.getLocalHost().getCanonicalHostName(); + + log.debug("Starting MiniKdc in {} with keytabs in {}", kdcDir, keytabDir); + + Properties kdcConf = MiniKdc.createConf(); + kdcConf.setProperty(MiniKdc.ORG_NAME, ORG_NAME); + kdcConf.setProperty(MiniKdc.ORG_DOMAIN, ORG_DOMAIN); + // kdcConf.setProperty(MiniKdc.DEBUG, "true"); + kdc = new MiniKdc(kdcConf, kdcDir); + } + + /** + * Starts the KDC and creates the principals and their keytabs + */ + public synchronized void start() throws Exception { + checkArgument(!started, "KDC was already started"); + kdc.start(); + Thread.sleep(1000); + + // Create the identity for accumulo servers + File accumuloKeytab = new File(keytabDir, "accumulo.keytab"); + String accumuloPrincipal = String.format("accumulo/%s", hostname); + + log.info("Creating Kerberos principal {} with keytab {}", accumuloPrincipal, accumuloKeytab); + kdc.createPrincipal(accumuloKeytab, accumuloPrincipal); + + accumuloServerUser = new ClusterUser(qualifyUser(accumuloPrincipal), accumuloKeytab); + + // Create the identity for the "root" user + String rootPrincipal = "root"; + File rootKeytab = new File(keytabDir, rootPrincipal + ".keytab"); + + log.info("Creating Kerberos principal {} with keytab {}", rootPrincipal, rootKeytab); + kdc.createPrincipal(rootKeytab, rootPrincipal); + + accumuloAdmin = new ClusterUser(qualifyUser(rootPrincipal), rootKeytab); + + clientPrincipals = new ArrayList<>(NUM_USERS); + // Create a number of unprivileged users for tests to use + for (int i = 1; i <= NUM_USERS; i++) { + String clientPrincipal = "client" + i; + File clientKeytab = new File(keytabDir, clientPrincipal + ".keytab"); + + log.info("Creating Kerberos principal {} with keytab {}", clientPrincipal, clientKeytab); + kdc.createPrincipal(clientKeytab, clientPrincipal); + + clientPrincipals.add(new ClusterUser(qualifyUser(clientPrincipal), clientKeytab)); + } + + started = true; + } + + public synchronized void stop() throws Exception { + checkArgument(started, "KDC is not started"); + kdc.stop(); + started = false; + } + + /** + * A directory where the automatically-created keytab files are written + */ + public File getKeytabDir() { + return keytabDir; + } + + /** + * A {@link ClusterUser} for Accumulo server processes to use + */ + public ClusterUser getAccumuloServerUser() { + checkArgument(started, "The KDC is not started"); + return accumuloServerUser; + } + + /** + * A {@link ClusterUser} which is the Accumulo "root" user + */ + public ClusterUser getRootUser() { + checkArgument(started, "The KDC is not started"); + return accumuloAdmin; + } + + /** + * The {@link ClusterUser} corresponding to the given offset. Represents an unprivileged user. + * + * @param offset + * The offset to fetch credentials for, valid through {@link #NUM_USERS} + */ + public ClusterUser getClientPrincipal(int offset) { + checkArgument(started, "Client principal is not initialized, is the KDC started?"); + checkArgument(offset >= 0 && offset < NUM_USERS, "Offset is invalid, must be non-negative and less than " + NUM_USERS); + return clientPrincipals.get(offset); + } + + /** + * @see MiniKdc#createPrincipal(File, String...) + */ + public void createPrincipal(File keytabFile, String... principals) throws Exception { + checkArgument(started, "KDC is not started"); + kdc.createPrincipal(keytabFile, principals); + } + + /** + * @return the name for the realm + */ + public String getOrgName() { + return ORG_NAME; + } + + /** + * @return the domain for the realm + */ + public String getOrgDomain() { + return ORG_DOMAIN; + } + + /** + * Qualify a username (only the primary from the kerberos principal) with the proper realm + * + * @param primary + * The primary or primary and instance + */ + public String qualifyUser(String primary) { + return String.format("%s@%s.%s", primary, getOrgName(), getOrgDomain()); + } +} http://git-wip-us.apache.org/repos/asf/accumulo/blob/01ae5b85/test/src/main/java/org/apache/accumulo/harness/conf/AccumuloClusterConfiguration.java ---------------------------------------------------------------------- diff --git a/test/src/main/java/org/apache/accumulo/harness/conf/AccumuloClusterConfiguration.java b/test/src/main/java/org/apache/accumulo/harness/conf/AccumuloClusterConfiguration.java new file mode 100644 index 0000000..31ed94a --- /dev/null +++ b/test/src/main/java/org/apache/accumulo/harness/conf/AccumuloClusterConfiguration.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.accumulo.harness.conf; + +import org.apache.accumulo.core.client.ClientConfiguration; +import org.apache.accumulo.core.client.security.tokens.AuthenticationToken; +import org.apache.accumulo.harness.AccumuloClusterHarness.ClusterType; + +/** + * Base functionality that must be provided as configuration to the test + */ +public interface AccumuloClusterConfiguration { + + ClusterType getClusterType(); + + String getAdminPrincipal(); + + AuthenticationToken getAdminToken(); + + ClientConfiguration getClientConf(); +} http://git-wip-us.apache.org/repos/asf/accumulo/blob/01ae5b85/test/src/main/java/org/apache/accumulo/harness/conf/AccumuloClusterPropertyConfiguration.java ---------------------------------------------------------------------- diff --git a/test/src/main/java/org/apache/accumulo/harness/conf/AccumuloClusterPropertyConfiguration.java b/test/src/main/java/org/apache/accumulo/harness/conf/AccumuloClusterPropertyConfiguration.java new file mode 100644 index 0000000..2300da3 --- /dev/null +++ b/test/src/main/java/org/apache/accumulo/harness/conf/AccumuloClusterPropertyConfiguration.java @@ -0,0 +1,195 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.accumulo.harness.conf; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileReader; +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Properties; + +import org.apache.accumulo.harness.AccumuloClusterHarness.ClusterType; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.base.Preconditions; + +/** + * Base class for extracting configuration values from Java Properties + */ +public abstract class AccumuloClusterPropertyConfiguration implements AccumuloClusterConfiguration { + private static final Logger log = LoggerFactory.getLogger(AccumuloClusterPropertyConfiguration.class); + + public static final String ACCUMULO_IT_PROPERTIES_FILE = "accumulo.it.properties"; + + public static final String ACCUMULO_CLUSTER_TYPE_KEY = "accumulo.it.cluster.type"; + + public static final String ACCUMULO_MINI_PREFIX = "accumulo.it.cluster.mini."; + public static final String ACCUMULO_STANDALONE_PREFIX = "accumulo.it.cluster.standalone."; + + public static final String ACCUMULO_CLUSTER_CLIENT_CONF_KEY = "accumulo.it.cluster.clientconf"; + + protected ClusterType clusterType; + + public static AccumuloClusterPropertyConfiguration get() { + Properties systemProperties = System.getProperties(); + + String clusterTypeValue = null, clientConf = null; + String propertyFile = systemProperties.getProperty(ACCUMULO_IT_PROPERTIES_FILE); + + if (null != propertyFile) { + // Check for properties provided in a file + File f = new File(propertyFile); + if (f.exists() && f.isFile() && f.canRead()) { + Properties fileProperties = new Properties(); + FileReader reader = null; + try { + reader = new FileReader(f); + } catch (FileNotFoundException e) { + log.warn("Could not read properties from specified file: {}", propertyFile, e); + } + + if (null != reader) { + try { + fileProperties.load(reader); + } catch (IOException e) { + log.warn("Could not load properties from specified file: {}", propertyFile, e); + } finally { + try { + reader.close(); + } catch (IOException e) { + log.warn("Could not close reader", e); + } + } + + clusterTypeValue = fileProperties.getProperty(ACCUMULO_CLUSTER_TYPE_KEY); + clientConf = fileProperties.getProperty(ACCUMULO_CLUSTER_CLIENT_CONF_KEY); + } + } else { + log.debug("Property file ({}) is not a readable file", propertyFile); + } + } else { + log.debug("No properties file found in {}", ACCUMULO_IT_PROPERTIES_FILE); + } + + if (null == clusterTypeValue) { + clusterTypeValue = systemProperties.getProperty(ACCUMULO_CLUSTER_TYPE_KEY); + } + + if (null == clientConf) { + clientConf = systemProperties.getProperty(ACCUMULO_CLUSTER_CLIENT_CONF_KEY); + } + + ClusterType type; + if (null == clusterTypeValue) { + type = ClusterType.MINI; + } else { + type = ClusterType.valueOf(clusterTypeValue); + } + + log.info("Using {} cluster type from system properties", type); + + switch (type) { + case MINI: + // we'll let no client conf pass through and expect that the caller will set it after MAC is started + return new AccumuloMiniClusterConfiguration(); + case STANDALONE: + if (null == clientConf) { + throw new RuntimeException("Expected client configuration to be provided: " + ACCUMULO_CLUSTER_CLIENT_CONF_KEY); + } + File clientConfFile = new File(clientConf); + if (!clientConfFile.exists() || !clientConfFile.isFile()) { + throw new RuntimeException("Client configuration should be a normal file: " + clientConfFile); + } + return new StandaloneAccumuloClusterConfiguration(clientConfFile); + default: + throw new RuntimeException("Clusters other than MiniAccumuloCluster are not yet implemented"); + } + } + + public Map<String,String> getConfiguration(ClusterType type) { + Preconditions.checkNotNull(type); + + String prefix; + switch (type) { + case MINI: + prefix = ACCUMULO_MINI_PREFIX; + break; + case STANDALONE: + prefix = ACCUMULO_STANDALONE_PREFIX; + break; + default: + throw new IllegalArgumentException("Unknown ClusterType: " + type); + } + + Map<String,String> configuration = new HashMap<String,String>(); + + Properties systemProperties = System.getProperties(); + + String propertyFile = systemProperties.getProperty(ACCUMULO_IT_PROPERTIES_FILE); + + // Check for properties provided in a file + if (null != propertyFile) { + File f = new File(propertyFile); + if (f.exists() && f.isFile() && f.canRead()) { + Properties fileProperties = new Properties(); + FileReader reader = null; + try { + reader = new FileReader(f); + } catch (FileNotFoundException e) { + log.warn("Could not read properties from specified file: {}", propertyFile, e); + } + + if (null != reader) { + try { + fileProperties.load(reader); + loadFromProperties(prefix, fileProperties, configuration); + } catch (IOException e) { + log.warn("Could not load properties from specified file: {}", propertyFile, e); + } finally { + try { + reader.close(); + } catch (IOException e) { + log.warn("Could not close reader", e); + } + } + } + } + } + + // Load any properties specified directly in the system properties + loadFromProperties(prefix, systemProperties, configuration); + + return configuration; + } + + protected void loadFromProperties(String desiredPrefix, Properties properties, Map<String,String> configuration) { + for (Entry<Object,Object> entry : properties.entrySet()) { + if (!(entry.getKey() instanceof String)) { + continue; + } + + String key = (String) entry.getKey(); + if (key.startsWith(desiredPrefix)) { + configuration.put(key, (String) entry.getValue()); + } + } + } +}
