ACCUMULO-2558 Add unit tests for server/gc Code changes accompanying unit tests: - visibility of methods under test changed from private to package - some methods refactored to take arguments, to permit testability - GarbageCollectWriteAheadLogs.isUUID() improved to check for null and to check length of potential UUID, since UUID.fromString() will accept UUIDs with extraneous leading zeroes in components - SimpleGarbageCollector.getZooLock() finds ZK root using instance field of class instead of calling HdfsZooInstance.getInstance() again - SimpleGarbageCollector.init() no longer declares throwing IOException
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/88f24d23 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/88f24d23 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/88f24d23 Branch: refs/heads/master Commit: 88f24d2323efd1aa33de1bf2cd5a0241cbd78306 Parents: 8d6a4cf Author: Bill Havanki <bhava...@cloudera.com> Authored: Thu Mar 27 11:27:15 2014 -0400 Committer: Bill Havanki <bhava...@cloudera.com> Committed: Mon Apr 7 17:05:03 2014 -0400 ---------------------------------------------------------------------- server/gc/pom.xml | 5 + .../gc/GarbageCollectWriteAheadLogs.java | 89 ++++++- .../accumulo/gc/SimpleGarbageCollector.java | 134 +++++++++- .../gc/GarbageCollectWriteAheadLogsTest.java | 264 +++++++++++++++++++ .../gc/SimpleGarbageCollectorOptsTest.java | 37 +++ .../accumulo/gc/SimpleGarbageCollectorTest.java | 146 ++++++++++ 6 files changed, 657 insertions(+), 18 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo/blob/88f24d23/server/gc/pom.xml ---------------------------------------------------------------------- diff --git a/server/gc/pom.xml b/server/gc/pom.xml index 6f95273..33cc596 100644 --- a/server/gc/pom.xml +++ b/server/gc/pom.xml @@ -73,6 +73,11 @@ <scope>test</scope> </dependency> <dependency> + <groupId>org.easymock</groupId> + <artifactId>easymock</artifactId> + <scope>test</scope> + </dependency> + <dependency> <groupId>org.slf4j</groupId> <artifactId>slf4j-api</artifactId> <scope>test</scope> http://git-wip-us.apache.org/repos/asf/accumulo/blob/88f24d23/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java ---------------------------------------------------------------------- diff --git a/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java b/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java index ab2ab42..ae850af 100644 --- a/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java +++ b/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java @@ -64,12 +64,44 @@ public class GarbageCollectWriteAheadLogs { private boolean useTrash; + /** + * Creates a new GC WAL object. + * + * @param instance instance to use + * @param fs volume manager to use + * @param useTrash true to move files to trash rather than delete them + */ GarbageCollectWriteAheadLogs(Instance instance, VolumeManager fs, boolean useTrash) throws IOException { this.instance = instance; this.fs = fs; this.useTrash = useTrash; } + /** + * Gets the instance used by this object. + * + * @return instance + */ + Instance getInstance() { + return instance; + } + /** + * Gets the volume manager used by this object. + * + * @return volume manager + */ + VolumeManager getVolumeManager() { + return fs; + } + /** + * Checks if the volume manager should move files to the trash rather than + * delete them. + * + * @return true if trash is used + */ + boolean isUsingTrash() { + return useTrash; + } public void collect(GCStatus status) { Span span = Trace.start("scanServers"); @@ -205,14 +237,29 @@ public class GarbageCollectWriteAheadLogs { return 0; } - private List<String> paths2strings(ArrayList<Path> paths) { + /** + * Converts a list of paths to their corresponding strings. + * + * @param paths list of paths + * @return string forms of paths + */ + static List<String> paths2strings(List<Path> paths) { List<String> result = new ArrayList<String>(paths.size()); for (Path path : paths) result.add(path.toString()); return result; } - private static Map<String,ArrayList<Path>> mapServersToFiles(Map<Path,String> fileToServerMap, Map<String,Path> nameToFileMap) { + /** + * Reverses the given mapping of file paths to servers. The returned map + * provides a list of file paths for each server. Any path whose name is not + * in the mapping of file names to paths is skipped. + * + * @param fileToServerMap map of file paths to servers + * @param nameToFileMap map of file names to paths + * @return map of servers to lists of file paths + */ + static Map<String,ArrayList<Path>> mapServersToFiles(Map<Path,String> fileToServerMap, Map<String,Path> nameToFileMap) { Map<String,ArrayList<Path>> result = new HashMap<String,ArrayList<Path>>(); for (Entry<Path,String> fileServer : fileToServerMap.entrySet()) { if (!nameToFileMap.containsKey(fileServer.getKey().getName())) @@ -251,11 +298,23 @@ public class GarbageCollectWriteAheadLogs { return count; } + private int scanServers(Map<Path,String> fileToServerMap, Map<String,Path> nameToFileMap) throws Exception { + return scanServers(ServerConstants.getWalDirs(), fileToServerMap, nameToFileMap); + } //TODO Remove deprecation warning suppression when Hadoop1 support is dropped @SuppressWarnings("deprecation") - private int scanServers(Map<Path,String> fileToServerMap, Map<String,Path> nameToFileMap) throws Exception { + /** + * Scans write-ahead log directories for logs. The maps passed in are + * populated with scan information. + * + * @param walDirs write-ahead log directories + * @param fileToServerMap map of file paths to servers + * @param nameToFileMap map of file names to paths + * @return number of servers located (including those with no logs present) + */ + int scanServers(String[] walDirs, Map<Path,String> fileToServerMap, Map<String,Path> nameToFileMap) throws Exception { Set<String> servers = new HashSet<String>(); - for (String walDir : ServerConstants.getWalDirs()) { + for (String walDir : walDirs) { Path walRoot = new Path(walDir); FileStatus[] listing = null; try { @@ -290,9 +349,18 @@ public class GarbageCollectWriteAheadLogs { } private Map<String, Path> getSortedWALogs() throws IOException { + return getSortedWALogs(ServerConstants.getRecoveryDirs()); + } + /** + * Looks for write-ahead logs in recovery directories. + * + * @param recoveryDirs recovery directories + * @return map of log file names to paths + */ + Map<String, Path> getSortedWALogs(String[] recoveryDirs) throws IOException { Map<String, Path> result = new HashMap<String, Path>(); - for (String dir : ServerConstants.getRecoveryDirs()) { + for (String dir : recoveryDirs) { Path recoveryDir = new Path(dir); if (fs.exists(recoveryDir)) { @@ -309,7 +377,16 @@ public class GarbageCollectWriteAheadLogs { return result; } - static private boolean isUUID(String name) { + /** + * Checks if a string is a valid UUID. + * + * @param name string to check + * @return true if string is a UUID + */ + static boolean isUUID(String name) { + if (name == null || name.length() != 36) { + return false; + } try { UUID.fromString(name); return true; http://git-wip-us.apache.org/repos/asf/accumulo/blob/88f24d23/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java ---------------------------------------------------------------------- diff --git a/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java b/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java index 89925b4..39716eb 100644 --- a/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java +++ b/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java @@ -108,6 +108,9 @@ import com.google.common.net.HostAndPort; public class SimpleGarbageCollector implements Iface { private static final Text EMPTY_TEXT = new Text(); + /** + * Options for the garbage collector. + */ static class Opts extends ServerOpts { @Parameter(names = {"-v", "--verbose"}, description = "extra information will get printed to stdout also") boolean verbose = false; @@ -115,8 +118,11 @@ public class SimpleGarbageCollector implements Iface { boolean safeMode = false; } - // how much of the JVM's available memory should it use gathering candidates - private static final float CANDIDATE_MEMORY_PERCENTAGE = 0.75f; + /** + * A fraction representing how much of the JVM's available memory should be + * used for gathering candidates. + */ + static final float CANDIDATE_MEMORY_PERCENTAGE = 0.75f; private static final Logger log = Logger.getLogger(SimpleGarbageCollector.class); @@ -148,18 +154,100 @@ public class SimpleGarbageCollector implements Iface { gc.run(); } + /** + * Creates a new garbage collector. + * + * @param opts options + */ public SimpleGarbageCollector(Opts opts) { this.opts = opts; } - public void init(VolumeManager fs, Instance instance, Credentials credentials, boolean noTrash) throws IOException { + /** + * Gets the credentials used by this GC. + * + * @return credentials + */ + Credentials getCredentials() { + return credentials; + } + /** + * Gets the delay before the first collection. + * + * @return start delay, in milliseconds + */ + long getStartDelay() { + return gcStartDelay; + } + /** + * Gets the volume manager used by this GC. + * + * @return volume manager + */ + VolumeManager getVolumeManager() { + return fs; + } + /** + * Checks if the volume manager should move files to the trash rather than + * delete them. + * + * @return true if trash is used + */ + boolean isUsingTrash() { + return useTrash; + } + /** + * Gets the options for this garbage collector. + */ + Opts getOpts() { + return opts; + } + /** + * Gets the number of threads used for deleting files. + * + * @return number of delete threads + */ + int getNumDeleteThreads() { + return numDeleteThreads; + } + /** + * Gets the instance used by this GC. + * + * @return instance + */ + Instance getInstance() { + return instance; + } + + /** + * Initializes this garbage collector with the current system configuration. + * + * @param fs volume manager + * @param instance instance + * @param credentials credentials + * @param noTrash true to not move files to trash instead of deleting + */ + public void init(VolumeManager fs, Instance instance, Credentials credentials, boolean noTrash) { + init(fs, instance, credentials, noTrash, ServerConfiguration.getSystemConfiguration(instance)); + } + + /** + * Initializes this garbage collector. + * + * @param fs volume manager + * @param instance instance + * @param credentials credentials + * @param noTrash true to not move files to trash instead of deleting + * @param systemConfig system configuration + */ + public void init(VolumeManager fs, Instance instance, Credentials credentials, boolean noTrash, AccumuloConfiguration systemConfig) { this.fs = fs; this.credentials = credentials; this.instance = instance; - gcStartDelay = ServerConfiguration.getSystemConfiguration(instance).getTimeInMillis(Property.GC_CYCLE_START); - long gcDelay = ServerConfiguration.getSystemConfiguration(instance).getTimeInMillis(Property.GC_CYCLE_DELAY); - numDeleteThreads = ServerConfiguration.getSystemConfiguration(instance).getCount(Property.GC_DELETE_THREADS); + gcStartDelay = systemConfig.getTimeInMillis(Property.GC_CYCLE_START); + long gcDelay = systemConfig.getTimeInMillis(Property.GC_CYCLE_DELAY); + numDeleteThreads = systemConfig.getCount(Property.GC_DELETE_THREADS); log.info("start delay: " + gcStartDelay + " milliseconds"); log.info("time delay: " + gcDelay + " milliseconds"); log.info("safemode: " + opts.safeMode); @@ -194,7 +282,7 @@ public class SimpleGarbageCollector implements Iface { for (Entry<Key,Value> entry : scanner) { String cand = entry.getKey().getRow().toString().substring(MetadataSchema.DeletesSection.getRowPrefix().length()); result.add(cand); - if (almostOutOfMemory()) { + if (almostOutOfMemory(Runtime.getRuntime())) { log.info("List of delete candidates has exceeded the memory threshold. Attempting to delete what has been gathered so far."); break; } @@ -506,7 +594,16 @@ public class SimpleGarbageCollector implements Iface { } } - private boolean moveToTrash(Path path) throws IOException { + /** + * Moves a file to trash. If this garbage collector is not using trash, this + * method returns false and leaves the file alone. If the file is missing, + * this method returns false as opposed to throwing an exception. + * + * @param path + * @return true if the file was moved to trash + * @throws IOException if the volume manager encountered a problem + */ + boolean moveToTrash(Path path) throws IOException { if (!useTrash) return false; try { @@ -517,7 +614,7 @@ public class SimpleGarbageCollector implements Iface { } private void getZooLock(HostAndPort addr) throws KeeperException, InterruptedException { - String path = ZooUtil.getRoot(HdfsZooInstance.getInstance()) + Constants.ZGC_LOCK; + String path = ZooUtil.getRoot(instance) + Constants.ZGC_LOCK; LockWatcher lockWatcher = new LockWatcher() { @Override @@ -563,8 +660,14 @@ public class SimpleGarbageCollector implements Iface { } } - static public boolean almostOutOfMemory() { - Runtime runtime = Runtime.getRuntime(); + /** + * Checks if the system is almost out of memory. + * + * @param runtime Java runtime + * @return true if system is almost out of memory + * @see #CANDIDATE_MEMORY_PERCENTAGE + */ + static boolean almostOutOfMemory(Runtime runtime) { return runtime.totalMemory() - runtime.freeMemory() > CANDIDATE_MEMORY_PERCENTAGE * runtime.maxMemory(); } @@ -576,7 +679,14 @@ public class SimpleGarbageCollector implements Iface { writer.addMutation(m); } - private boolean isDir(String delete) { + /** + * Checks if the given string is a directory. + * + * @param delete possible directory + * @return true if string is a directory + */ + static boolean isDir(String delete) { + if (delete == null) { return false; } int slashCount = 0; for (int i = 0; i < delete.length(); i++) if (delete.charAt(i) == '/') http://git-wip-us.apache.org/repos/asf/accumulo/blob/88f24d23/server/gc/src/test/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogsTest.java ---------------------------------------------------------------------- diff --git a/server/gc/src/test/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogsTest.java b/server/gc/src/test/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogsTest.java new file mode 100644 index 0000000..f90b965 --- /dev/null +++ b/server/gc/src/test/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogsTest.java @@ -0,0 +1,264 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.accumulo.gc; + +import java.io.FileNotFoundException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import org.apache.accumulo.core.client.Instance; +import org.apache.accumulo.server.fs.VolumeManager; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.Path; + +import org.junit.Before; +import org.junit.Test; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; +import static org.easymock.EasyMock.createMock; +import static org.easymock.EasyMock.expect; +import static org.easymock.EasyMock.replay; + +public class GarbageCollectWriteAheadLogsTest { + private static final long BLOCK_SIZE = 64000000L; + + private static final Path DIR_1_PATH = new Path("/dir1"); + private static final Path DIR_2_PATH = new Path("/dir2"); + private static final Path DIR_3_PATH = new Path("/dir3"); + private static final String UUID1 = UUID.randomUUID().toString(); + private static final String UUID2 = UUID.randomUUID().toString(); + private static final String UUID3 = UUID.randomUUID().toString(); + + private Instance instance; + private VolumeManager volMgr; + private GarbageCollectWriteAheadLogs gcwal; + private long modTime; + + @Before + public void setUp() throws Exception { + instance = createMock(Instance.class); + volMgr = createMock(VolumeManager.class); + gcwal = new GarbageCollectWriteAheadLogs(instance, volMgr, false); + modTime = System.currentTimeMillis(); + } + + @Test + public void testGetters() { + assertSame(instance, gcwal.getInstance()); + assertSame(volMgr, gcwal.getVolumeManager()); + assertFalse(gcwal.isUsingTrash()); + } + + @Test + public void testPathsToStrings() { + ArrayList<Path> paths = new ArrayList<Path>(); + paths.add(new Path(DIR_1_PATH, "file1")); + paths.add(DIR_2_PATH); + paths.add(new Path(DIR_3_PATH, "file3")); + List<String> strings = GarbageCollectWriteAheadLogs.paths2strings(paths); + int len = 3; + assertEquals(len, strings.size()); + for (int i = 0; i < len; i++) { + assertEquals(paths.get(i).toString(), strings.get(i)); + } + } + + @Test + public void testMapServersToFiles() { + /* + * Test fileToServerMap: + * /dir1/server1/uuid1 -> server1 (new-style) + * /dir1/uuid2 -> "" (old-style) + * /dir3/server3/uuid3 -> server3 (new-style) + */ + Map<Path,String> fileToServerMap = new java.util.HashMap<Path,String>(); + Path path1 = new Path(new Path(DIR_1_PATH, "server1"), UUID1); + fileToServerMap.put(path1, "server1"); // new-style + Path path2 = new Path(DIR_1_PATH, UUID2); + fileToServerMap.put(path2, ""); // old-style + Path path3 = new Path(new Path(DIR_3_PATH, "server3"), UUID3); + fileToServerMap.put(path3, "server3"); // old-style + /* + * Test nameToFileMap: + * uuid1 -> /dir1/server1/uuid1 + * uuid3 -> /dir3/server3/uuid3 + */ + Map<String,Path> nameToFileMap = new java.util.HashMap<String,Path>(); + nameToFileMap.put(UUID1, path1); + nameToFileMap.put(UUID3, path3); + + /** + * Expected map: + * server1 -> [ /dir1/server1/uuid1 ] + * server3 -> [ /dir3/server3/uuid3 ] + */ + Map<String,ArrayList<Path>> result = GarbageCollectWriteAheadLogs.mapServersToFiles(fileToServerMap, nameToFileMap); + assertEquals(2, result.size()); + ArrayList<Path> list1 = result.get("server1"); + assertEquals(1, list1.size()); + assertTrue(list1.contains(path1)); + ArrayList<Path> list3 = result.get("server3"); + assertEquals(1, list3.size()); + assertTrue(list3.contains(path3)); + } + + private FileStatus makeFileStatus(int size, Path path) { + boolean isDir = (size == 0); + return new FileStatus(size, isDir, 3, BLOCK_SIZE, modTime, path); + } + private void mockListStatus(Path dir, FileStatus... fileStatuses) throws Exception { + expect(volMgr.listStatus(dir)).andReturn(fileStatuses); + } + + @Test + public void testScanServers_NewStyle() throws Exception { + String[] walDirs = new String[] {"/dir1", "/dir2", "/dir3"}; + /* + * Test directory layout: + * /dir1/ + * server1/ + * uuid1 + * file2 + * subdir2/ + * /dir2/ missing + * /dir3/ + * server3/ + * uuid3 + */ + Path serverDir1Path = new Path(DIR_1_PATH, "server1"); + FileStatus serverDir1 = makeFileStatus(0, serverDir1Path); + Path subDir2Path = new Path(DIR_1_PATH, "subdir2"); + FileStatus serverDir2 = makeFileStatus(0, subDir2Path); + mockListStatus(DIR_1_PATH, serverDir1, serverDir2); + Path path1 = new Path(serverDir1Path, UUID1); + FileStatus file1 = makeFileStatus(100, path1); + FileStatus file2 = makeFileStatus(200, new Path(serverDir1Path, "file2")); + mockListStatus(serverDir1Path, file1, file2); + mockListStatus(subDir2Path); + expect(volMgr.listStatus(DIR_2_PATH)).andThrow(new FileNotFoundException()); + Path serverDir3Path = new Path(DIR_3_PATH, "server3"); + FileStatus serverDir3 = makeFileStatus(0, serverDir3Path); + mockListStatus(DIR_3_PATH, serverDir3); + Path path3 = new Path(serverDir3Path, UUID3); + FileStatus file3 = makeFileStatus(300, path3); + mockListStatus(serverDir3Path, file3); + replay(volMgr); + + Map<Path,String> fileToServerMap = new java.util.HashMap<Path,String>(); + Map<String,Path> nameToFileMap = new java.util.HashMap<String,Path>(); + int count = gcwal.scanServers(walDirs, fileToServerMap, nameToFileMap); + assertEquals(3, count); + /* + * Expected fileToServerMap: + * /dir1/server1/uuid1 -> server1 + * /dir3/server3/uuid3 -> server3 + */ + assertEquals(2, fileToServerMap.size()); + assertEquals("server1", fileToServerMap.get(path1)); + assertEquals("server3", fileToServerMap.get(path3)); + /* + * Expected nameToFileMap: + * uuid1 -> /dir1/server1/uuid1 + * uuid3 -> /dir3/server3/uuid3 + */ + assertEquals(2, nameToFileMap.size()); + assertEquals(path1, nameToFileMap.get(UUID1)); + assertEquals(path3, nameToFileMap.get(UUID3)); + } + + @Test + public void testScanServers_OldStyle() throws Exception { + /* + * Test directory layout: + * /dir1/ + * uuid1 + * /dir3/ + * uuid3 + */ + String[] walDirs = new String[] {"/dir1", "/dir3"}; + Path serverFile1Path = new Path(DIR_1_PATH, UUID1); + FileStatus serverFile1 = makeFileStatus(100, serverFile1Path); + mockListStatus(DIR_1_PATH, serverFile1); + Path serverFile3Path = new Path(DIR_3_PATH, UUID3); + FileStatus serverFile3 = makeFileStatus(300, serverFile3Path); + mockListStatus(DIR_3_PATH, serverFile3); + replay(volMgr); + + Map<Path,String> fileToServerMap = new java.util.HashMap<Path,String>(); + Map<String,Path> nameToFileMap = new java.util.HashMap<String,Path>(); + int count = gcwal.scanServers(walDirs, fileToServerMap, nameToFileMap); + /* + * Expected fileToServerMap: + * /dir1/uuid1 -> "" + * /dir3/uuid3 -> "" + */ + assertEquals(2, count); + assertEquals(2, fileToServerMap.size()); + assertEquals("", fileToServerMap.get(serverFile1Path)); + assertEquals("", fileToServerMap.get(serverFile3Path)); + /* + * Expected nameToFileMap: empty + */ + assertEquals(0, nameToFileMap.size()); + } + + @Test + public void testGetSortedWALogs() throws Exception { + String[] recoveryDirs = new String[] {"/dir1", "/dir2", "/dir3"}; + /* + * Test directory layout: + * /dir1/ + * uuid1 + * file2 + * /dir2/ missing + * /dir3/ + * uuid3 + */ + expect(volMgr.exists(DIR_1_PATH)).andReturn(true); + expect(volMgr.exists(DIR_2_PATH)).andReturn(false); + expect(volMgr.exists(DIR_3_PATH)).andReturn(true); + Path path1 = new Path(DIR_1_PATH, UUID1); + FileStatus file1 = makeFileStatus(100, path1); + FileStatus file2 = makeFileStatus(200, new Path(DIR_1_PATH, "file2")); + mockListStatus(DIR_1_PATH, file1, file2); + Path path3 = new Path(DIR_3_PATH, UUID3); + FileStatus file3 = makeFileStatus(300, path3); + mockListStatus(DIR_3_PATH, file3); + replay(volMgr); + + Map<String,Path> sortedWalogs = gcwal.getSortedWALogs(recoveryDirs); + /** + * Expected map: + * uuid1 -> /dir1/uuid1 + * uuid3 -> /dir3/uuid3 + */ + assertEquals(2, sortedWalogs.size()); + assertEquals(path1, sortedWalogs.get(UUID1)); + assertEquals(path3, sortedWalogs.get(UUID3)); + } + + @Test + public void testIsUUID() { + assertTrue(GarbageCollectWriteAheadLogs.isUUID(UUID.randomUUID().toString())); + assertFalse(GarbageCollectWriteAheadLogs.isUUID("foo")); + assertFalse(GarbageCollectWriteAheadLogs.isUUID("0" + UUID.randomUUID().toString())); + assertFalse(GarbageCollectWriteAheadLogs.isUUID(null)); + } +} http://git-wip-us.apache.org/repos/asf/accumulo/blob/88f24d23/server/gc/src/test/java/org/apache/accumulo/gc/SimpleGarbageCollectorOptsTest.java ---------------------------------------------------------------------- diff --git a/server/gc/src/test/java/org/apache/accumulo/gc/SimpleGarbageCollectorOptsTest.java b/server/gc/src/test/java/org/apache/accumulo/gc/SimpleGarbageCollectorOptsTest.java new file mode 100644 index 0000000..d484741 --- /dev/null +++ b/server/gc/src/test/java/org/apache/accumulo/gc/SimpleGarbageCollectorOptsTest.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.accumulo.gc; + +import org.apache.accumulo.gc.SimpleGarbageCollector.Opts; +import org.junit.Before; +import org.junit.Test; +import static org.junit.Assert.assertFalse; + +public class SimpleGarbageCollectorOptsTest { + private Opts opts; + + @Before + public void setUp() { + opts = new Opts(); + } + + @Test + public void testIt() { + assertFalse(opts.verbose); + assertFalse(opts.safeMode); + } +} http://git-wip-us.apache.org/repos/asf/accumulo/blob/88f24d23/server/gc/src/test/java/org/apache/accumulo/gc/SimpleGarbageCollectorTest.java ---------------------------------------------------------------------- diff --git a/server/gc/src/test/java/org/apache/accumulo/gc/SimpleGarbageCollectorTest.java b/server/gc/src/test/java/org/apache/accumulo/gc/SimpleGarbageCollectorTest.java new file mode 100644 index 0000000..532eeba --- /dev/null +++ b/server/gc/src/test/java/org/apache/accumulo/gc/SimpleGarbageCollectorTest.java @@ -0,0 +1,146 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.accumulo.gc; + +import java.io.FileNotFoundException; +import org.apache.accumulo.core.client.Instance; +import org.apache.accumulo.core.conf.AccumuloConfiguration; +import org.apache.accumulo.core.conf.Property; +import org.apache.accumulo.core.security.Credentials; +import org.apache.accumulo.core.security.thrift.TCredentials; +import org.apache.accumulo.gc.SimpleGarbageCollector.Opts; +import static org.apache.accumulo.gc.SimpleGarbageCollector.CANDIDATE_MEMORY_PERCENTAGE; +import org.apache.accumulo.server.fs.VolumeManager; +import org.apache.accumulo.trace.thrift.TInfo; +import org.apache.hadoop.fs.Path; +import org.junit.Before; +import org.junit.Test; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; +import static org.easymock.EasyMock.createMock; +import static org.easymock.EasyMock.expect; +import static org.easymock.EasyMock.expectLastCall; +import static org.easymock.EasyMock.replay; +import static org.easymock.EasyMock.verify; + +public class SimpleGarbageCollectorTest { + private VolumeManager volMgr; + private Instance instance; + private Credentials credentials; + private Opts opts; + private SimpleGarbageCollector gc; + private AccumuloConfiguration systemConfig; + + @Before + public void setUp() { + volMgr = createMock(VolumeManager.class); + instance = createMock(Instance.class); + credentials = createMock(Credentials.class); + + opts = new Opts(); + gc = new SimpleGarbageCollector(opts); + systemConfig = mockSystemConfig(); + } + + @Test + public void testConstruction() { + assertSame(opts, gc.getOpts()); + assertNotNull(gc.getStatus(createMock(TInfo.class), createMock(TCredentials.class))); + } + + private AccumuloConfiguration mockSystemConfig() { + AccumuloConfiguration systemConfig = createMock(AccumuloConfiguration.class); + expect(systemConfig.getTimeInMillis(Property.GC_CYCLE_START)).andReturn(1000L); + expect(systemConfig.getTimeInMillis(Property.GC_CYCLE_DELAY)).andReturn(20000L); + expect(systemConfig.getCount(Property.GC_DELETE_THREADS)).andReturn(2); + replay(systemConfig); + return systemConfig; + } + + @Test + public void testInit() throws Exception { + gc.init(volMgr, instance, credentials, false, systemConfig); + assertSame(volMgr, gc.getVolumeManager()); + assertSame(instance, gc.getInstance()); + assertSame(credentials, gc.getCredentials()); + assertTrue(gc.isUsingTrash()); + assertEquals(1000L, gc.getStartDelay()); + assertEquals(2, gc.getNumDeleteThreads()); + } + + @Test + public void testMoveToTrash_UsingTrash() throws Exception { + gc.init(volMgr, instance, credentials, false, systemConfig); + Path path = createMock(Path.class); + expect(volMgr.moveToTrash(path)).andReturn(true); + replay(volMgr); + assertTrue(gc.moveToTrash(path)); + verify(volMgr); + } + + @Test + public void testMoveToTrash_UsingTrash_VolMgrFailure() throws Exception { + gc.init(volMgr, instance, credentials, false, systemConfig); + Path path = createMock(Path.class); + expect(volMgr.moveToTrash(path)).andThrow(new FileNotFoundException()); + replay(volMgr); + assertFalse(gc.moveToTrash(path)); + verify(volMgr); + } + + @Test + public void testMoveToTrash_NotUsingTrash() throws Exception { + gc.init(volMgr, instance, credentials, true, systemConfig); + Path path = createMock(Path.class); + assertFalse(gc.moveToTrash(path)); + } + + @Test + public void testAlmostOutOfMemory_Pass() { + testAlmostOutOfMemory(1.0f - (CANDIDATE_MEMORY_PERCENTAGE - 0.05f), false); + } + + @Test + public void testAlmostOutOfMemory_Fail() { + testAlmostOutOfMemory(1.0f - (CANDIDATE_MEMORY_PERCENTAGE + 0.05f), true); + } + + private void testAlmostOutOfMemory(float freeFactor, boolean expected) { + Runtime runtime = createMock(Runtime.class); + expect(runtime.totalMemory()).andReturn(1000L); + expectLastCall().anyTimes(); + expect(runtime.maxMemory()).andReturn(1000L); + expectLastCall().anyTimes(); + expect(runtime.freeMemory()).andReturn((long) (freeFactor * 1000.0f)); + expectLastCall().anyTimes(); + replay(runtime); + + assertEquals(expected, SimpleGarbageCollector.almostOutOfMemory(runtime)); + } + + @Test + public void testIsDir() { + assertTrue(SimpleGarbageCollector.isDir("/dir1")); + assertFalse(SimpleGarbageCollector.isDir("file1")); + assertFalse(SimpleGarbageCollector.isDir("/dir1/file1")); + assertFalse(SimpleGarbageCollector.isDir("")); + assertFalse(SimpleGarbageCollector.isDir(null)); + } +}
