Merge branch '1.4.5-SNAPSHOT' into 1.5.1-SNAPSHOT
Conflicts:
start/src/main/java/org/apache/accumulo/start/classloader/vfs/providers/HdfsFileAttributes.java
test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/CheckBalance.java
The conflict for HdfsFileAttributes above was Git erroneously merging updates
for
ConcurrentFixture into it. Also, the State class merged cleanly but required the
addition of getInteger(String).
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/9aa4b70e
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/9aa4b70e
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/9aa4b70e
Branch: refs/heads/master
Commit: 9aa4b70e260dd82d5da9bdb5381ff9d32bf0ed89
Parents: fe1348f cd4eac0
Author: Bill Havanki <bhava...@cloudera.com>
Authored: Thu Jan 16 15:08:51 2014 -0500
Committer: Bill Havanki <bhava...@cloudera.com>
Committed: Thu Jan 16 15:08:51 2014 -0500
----------------------------------------------------------------------
.../apache/accumulo/test/randomwalk/State.java | 8 +++++
.../randomwalk/concurrent/CheckBalance.java | 31 ++++++++++++--------
.../concurrent/ConcurrentFixture.java | 5 +++-
3 files changed, 31 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9aa4b70e/test/src/main/java/org/apache/accumulo/test/randomwalk/State.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/randomwalk/State.java
index 65d26c6,0000000..b799d04
mode 100644,000000..100644
--- a/test/src/main/java/org/apache/accumulo/test/randomwalk/State.java
+++ b/test/src/main/java/org/apache/accumulo/test/randomwalk/State.java
@@@ -1,190 -1,0 +1,198 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.randomwalk;
+
+import java.io.File;
+import java.lang.management.ManagementFactory;
+import java.util.HashMap;
+import java.util.Properties;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.MultiTableBatchWriter;
+import org.apache.accumulo.core.client.MutationsRejectedException;
+import org.apache.accumulo.core.client.ZooKeeperInstance;
+import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.accumulo.core.security.CredentialHelper;
+import org.apache.accumulo.core.security.thrift.TCredentials;
+import org.apache.log4j.Logger;
+
+public class State {
+
+ private static final Logger log = Logger.getLogger(State.class);
+ private HashMap<String,Object> stateMap = new HashMap<String,Object>();
+ private Properties props;
+ private int numVisits = 0;
+ private int maxVisits = Integer.MAX_VALUE;
+
+ private MultiTableBatchWriter mtbw = null;
+ private Connector connector = null;
+ private Instance instance = null;
+
+ State(Properties props) {
+ this.props = props;
+ }
+
+ public void setMaxVisits(int num) {
+ maxVisits = num;
+ }
+
+ public void visitedNode() throws Exception {
+ numVisits++;
+ if (numVisits > maxVisits) {
+ log.debug("Visited max number (" + maxVisits + ") of nodes");
+ throw new Exception("Visited max number (" + maxVisits + ") of nodes");
+ }
+ }
+
+ public String getPid() {
+ return ManagementFactory.getRuntimeMXBean().getName().split("@")[0];
+ }
+
+ public void set(String key, Object value) {
+ stateMap.put(key, value);
+ }
++
++ public void remove(String key) {
++ stateMap.remove(key);
++ }
+
+ public Object get(String key) {
+ if (stateMap.containsKey(key) == false) {
+ throw new RuntimeException("State does not contain " + key);
+ }
+ return stateMap.get(key);
+ }
+
+ public HashMap<String,Object> getMap() {
+ return stateMap;
+ }
+
+ /**
+ *
+ * @return a copy of Properties, so accidental changes don't affect the
framework
+ */
+ public Properties getProperties() {
+ return new Properties(props);
+ }
+
+ public String getString(String key) {
+ return (String) stateMap.get(key);
+ }
+
++ public Integer getInteger(String key) {
++ return (Integer) stateMap.get(key);
++ }
++
+ public Long getLong(String key) {
+ return (Long) stateMap.get(key);
+ }
+
+ public String getProperty(String key) {
+ return props.getProperty(key);
+ }
+
+ public Connector getConnector() throws AccumuloException,
AccumuloSecurityException {
+ if (connector == null) {
+ connector = getInstance().getConnector(getUserName(), getToken());
+ }
+ return connector;
+ }
+
+ public TCredentials getCredentials() {
+ String username = getUserName();
+ AuthenticationToken password = getToken();
+ return CredentialHelper.createSquelchError(username, password,
getInstance().getInstanceID());
+ }
+
+ public String getUserName() {
+ return props.getProperty("USERNAME");
+ }
+
+ public AuthenticationToken getToken() {
+ return new PasswordToken(props.getProperty("PASSWORD"));
+ }
+
+ public Instance getInstance() {
+ if (instance == null) {
+ String instance = props.getProperty("INSTANCE");
+ String zookeepers = props.getProperty("ZOOKEEPERS");
+ this.instance = new ZooKeeperInstance(instance, zookeepers);
+ }
+ return instance;
+ }
+
+ public MultiTableBatchWriter getMultiTableBatchWriter() {
+ if (mtbw == null) {
+ long maxMem = Long.parseLong(props.getProperty("MAX_MEM"));
+ long maxLatency = Long.parseLong(props.getProperty("MAX_LATENCY"));
+ int numThreads = Integer.parseInt(props.getProperty("NUM_THREADS"));
+ mtbw = connector.createMultiTableBatchWriter(new
BatchWriterConfig().setMaxMemory(maxMem).setMaxLatency(maxLatency,
TimeUnit.MILLISECONDS)
+ .setMaxWriteThreads(numThreads));
+ }
+ return mtbw;
+ }
+
+ public boolean isMultiTableBatchWriterInitialized() {
+ return mtbw != null;
+ }
+
+ public void resetMultiTableBatchWriter() {
+ if (!mtbw.isClosed()) {
+ log.warn("Setting non-closed MultiTableBatchWriter to null (leaking
resources)");
+ }
+
+ mtbw = null;
+ }
+
+ public String getMapReduceJars() {
+
+ String acuHome = System.getenv("ACCUMULO_HOME");
+ String zkHome = System.getenv("ZOOKEEPER_HOME");
+
+ if (acuHome == null || zkHome == null) {
+ throw new RuntimeException("ACCUMULO or ZOOKEEPER home not set!");
+ }
+
+ String retval = null;
+
+ File zkLib = new File(zkHome);
+ String[] files = zkLib.list();
+ for (int i = 0; i < files.length; i++) {
+ String f = files[i];
+ if (f.matches("^zookeeper-.+jar$")) {
+ if (retval == null) {
+ retval = String.format("%s/%s", zkLib.getAbsolutePath(), f);
+ } else {
+ retval += String.format(",%s/%s", zkLib.getAbsolutePath(), f);
+ }
+ }
+ }
+
+ File libdir = new File(acuHome + "/lib");
+ for (String jar : "accumulo-core accumulo-server accumulo-fate
accumulo-trace libthrift".split(" ")) {
+ retval += String.format(",%s/%s.jar", libdir.getAbsolutePath(), jar);
+ }
+
+ return retval;
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9aa4b70e/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/CheckBalance.java
----------------------------------------------------------------------
diff --cc
test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/CheckBalance.java
index 3017d73,0000000..8278843
mode 100644,000000..100644
---
a/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/CheckBalance.java
+++
b/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/CheckBalance.java
@@@ -1,91 -1,0 +1,98 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.randomwalk.concurrent;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Properties;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.test.randomwalk.State;
+import org.apache.accumulo.test.randomwalk.Test;
+
+/**
+ *
+ */
+public class CheckBalance extends Test {
+
- private static final String LAST_UNBALANCED_TIME = "lastUnbalancedTime";
- private static final String UNBALANCED_COUNT = "unbalancedCount";
++ static final String LAST_UNBALANCED_TIME = "lastUnbalancedTime";
++ static final String UNBALANCED_COUNT = "unbalancedCount";
+
+ /* (non-Javadoc)
+ * @see
org.apache.accumulo.test.randomwalk.Node#visit(org.apache.accumulo.test.randomwalk.State,
java.util.Properties)
+ */
+ @Override
+ public void visit(State state, Properties props) throws Exception {
+ log.debug("checking balance");
+ Map<String,Long> counts = new HashMap<String,Long>();
+ Scanner scanner =
state.getConnector().createScanner(Constants.METADATA_TABLE_NAME,
Constants.NO_AUTHS);
+
scanner.fetchColumnFamily(Constants.METADATA_CURRENT_LOCATION_COLUMN_FAMILY);
+ for (Entry<Key,Value> entry : scanner) {
+ String location = entry.getKey().getColumnQualifier().toString();
+ Long count = counts.get(location);
+ if (count == null)
+ count = new Long(0);
+ counts.put(location, count + 1);
+ }
+ double total = 0.;
+ for (Long count : counts.values()) {
+ total += count.longValue();
+ }
+ final double average = total / counts.size();
+
+ // Check for even # of tablets on each node
++ double maxDifference = Math.max(1, average / 5);
++ String unbalancedLocation = null;
++ long lastCount = 0L;
+ boolean balanced = true;
+ for (Entry<String,Long> entry : counts.entrySet()) {
- if (Math.abs(entry.getValue().longValue() - average) > Math.max(1,
average / 5)) {
++ long thisCount = entry.getValue().longValue();
++ if (Math.abs(thisCount - average) > maxDifference) {
+ balanced = false;
+ log.debug("unbalanced: " + entry.getKey() + " has " +
entry.getValue() + " tablets and the average is " + average);
++ unbalancedLocation = entry.getKey();
++ lastCount = thisCount;
+ }
+ }
+
+ // It is expected that the number of tablets will be uneven for short
+ // periods of time. Don't complain unless we've seen it only unbalanced
+ // over a 15 minute period and it's been at least three checks.
+ if (!balanced) {
- String last = props.getProperty(LAST_UNBALANCED_TIME);
- if (last != null && System.currentTimeMillis() - Long.parseLong(last) >
15 * 60 * 1000) {
- String countString = props.getProperty(UNBALANCED_COUNT, "0");
- int count = Integer.parseInt(countString);
++ Long last = state.getLong(LAST_UNBALANCED_TIME);
++ if (last != null && System.currentTimeMillis() - last > 15 * 60 * 1000)
{
++ Integer count = state.getInteger(UNBALANCED_COUNT);
++ if (count == null)
++ count = Integer.valueOf(0);
+ if (count > 3)
- throw new Exception("servers are unbalanced!");
++ throw new Exception("servers are unbalanced! location " +
unbalancedLocation + " count " + lastCount + " too far from average " +
average);
+ count++;
- props.setProperty(UNBALANCED_COUNT, "" + count);
++ state.set(UNBALANCED_COUNT, count);
+ }
+ if (last == null)
- props.setProperty(LAST_UNBALANCED_TIME,
Long.toString(System.currentTimeMillis()));
++ state.set(LAST_UNBALANCED_TIME, System.currentTimeMillis());
+ } else {
- props.remove(LAST_UNBALANCED_TIME);
- props.remove(UNBALANCED_COUNT);
++ state.remove(LAST_UNBALANCED_TIME);
++ state.remove(UNBALANCED_COUNT);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9aa4b70e/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/ConcurrentFixture.java
----------------------------------------------------------------------
diff --cc
test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/ConcurrentFixture.java
index 7438ce6,0000000..df7ddd7
mode 100644,000000..100644
---
a/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/ConcurrentFixture.java
+++
b/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/ConcurrentFixture.java
@@@ -1,74 -1,0 +1,77 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.randomwalk.concurrent;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+
+import org.apache.accumulo.test.randomwalk.Fixture;
+import org.apache.accumulo.test.randomwalk.State;
+import org.apache.hadoop.io.Text;
+
+/**
+ * When multiple instance of this test suite are run, all instances will
operate on the same set of table names.
+ *
+ *
+ */
+
+public class ConcurrentFixture extends Fixture {
+
+ @Override
+ public void setUp(State state) throws Exception {}
+
+ @Override
- public void tearDown(State state) throws Exception {}
++ public void tearDown(State state) throws Exception {
++ state.remove(CheckBalance.LAST_UNBALANCED_TIME);
++ state.remove(CheckBalance.UNBALANCED_COUNT);
++ }
+
+ /**
+ *
+ * @param rand
+ * A Random to use
+ * @return
+ * A two element list with first being smaller than the second, but either
value (or both) can be null
+ */
+ public static List<Text> generateRange(Random rand) {
+ ArrayList<Text> toRet = new ArrayList<Text>(2);
+
+ long firstLong = rand.nextLong();
+
+
+ long secondLong = rand.nextLong();
+ Text first = null, second = null;
+
+ // Having all negative values = null might be too frequent
+ if (firstLong >= 0)
+ first = new Text(String.format("%016x", firstLong &
0x7fffffffffffffffl));
+ if (secondLong >= 0)
+ second = new Text(String.format("%016x", secondLong &
0x7fffffffffffffffl));
+
+ if (first != null && second != null && first.compareTo(second) > 0) {
+ Text swap = first;
+ first = second;
+ second = swap;
+ }
+
+ toRet.add(first);
+ toRet.add(second);
+
+ return toRet;
+ }
+}
