Merge branch '1.6.1-SNAPSHOT' Conflicts: server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/6104c98b Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/6104c98b Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/6104c98b Branch: refs/heads/master Commit: 6104c98b9c9ab87c81d4ba631d802ea4d6ae3bc8 Parents: 32f2769 560c268 Author: Josh Elser <els...@apache.org> Authored: Wed Sep 10 23:29:59 2014 -0700 Committer: Josh Elser <els...@apache.org> Committed: Wed Sep 10 23:29:59 2014 -0700 ---------------------------------------------------------------------- .../java/org/apache/accumulo/server/GarbageCollectionLogger.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo/blob/6104c98b/server/tserver/src/main/java/org/apache/accumulo/server/GarbageCollectionLogger.java ---------------------------------------------------------------------- diff --cc server/tserver/src/main/java/org/apache/accumulo/server/GarbageCollectionLogger.java index d53f240,0000000..cd7dfc8 mode 100644,000000..100644 --- a/server/tserver/src/main/java/org/apache/accumulo/server/GarbageCollectionLogger.java +++ b/server/tserver/src/main/java/org/apache/accumulo/server/GarbageCollectionLogger.java @@@ -1,116 -1,0 +1,117 @@@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.accumulo.server; + +import java.lang.management.GarbageCollectorMXBean; +import java.lang.management.ManagementFactory; +import java.util.HashMap; +import java.util.List; + +import org.apache.accumulo.core.conf.AccumuloConfiguration; +import org.apache.accumulo.core.conf.Property; +import org.apache.accumulo.server.util.Halt; +import org.apache.log4j.Logger; + +public class GarbageCollectionLogger { + private static final Logger log = Logger.getLogger(GarbageCollectionLogger.class); + + private final HashMap<String,Long> prevGcTime = new HashMap<String,Long>(); + private long lastMemorySize = 0; + private long gcTimeIncreasedCount = 0; + private static long lastMemoryCheckTime = 0; + + public GarbageCollectionLogger() { + } + + public synchronized void logGCInfo(AccumuloConfiguration conf) { + final long now = System.currentTimeMillis(); + + List<GarbageCollectorMXBean> gcmBeans = ManagementFactory.getGarbageCollectorMXBeans(); + Runtime rt = Runtime.getRuntime(); + + StringBuilder sb = new StringBuilder("gc"); + + boolean sawChange = false; + + long maxIncreaseInCollectionTime = 0; + + for (GarbageCollectorMXBean gcBean : gcmBeans) { + Long prevTime = prevGcTime.get(gcBean.getName()); + long pt = 0; + if (prevTime != null) { + pt = prevTime; + } + + long time = gcBean.getCollectionTime(); + + if (time - pt != 0) { + sawChange = true; + } + + long increaseInCollectionTime = time - pt; + sb.append(String.format(" %s=%,.2f(+%,.2f) secs", gcBean.getName(), time / 1000.0, increaseInCollectionTime / 1000.0)); + maxIncreaseInCollectionTime = Math.max(increaseInCollectionTime, maxIncreaseInCollectionTime); + prevGcTime.put(gcBean.getName(), time); + } + + long mem = rt.freeMemory(); + if (maxIncreaseInCollectionTime == 0) { + gcTimeIncreasedCount = 0; + } else { + gcTimeIncreasedCount++; + if (gcTimeIncreasedCount > 3 && mem < rt.maxMemory() * 0.05) { + log.warn("Running low on memory"); + gcTimeIncreasedCount = 0; + } + } + + if (mem > lastMemorySize) { + sawChange = true; + } + + String sign = "+"; + if (mem - lastMemorySize <= 0) { + sign = ""; + } + + sb.append(String.format(" freemem=%,d(%s%,d) totalmem=%,d", mem, sign, (mem - lastMemorySize), rt.totalMemory())); + + if (sawChange) { + log.debug(sb.toString()); + } + + final long keepAliveTimeout = conf.getTimeInMillis(Property.INSTANCE_ZK_TIMEOUT); + if (lastMemoryCheckTime > 0 && lastMemoryCheckTime < now) { + final long diff = now - lastMemoryCheckTime; + if (diff > keepAliveTimeout) { - log.warn(String.format("Check for long GC pauses not called in a timely fashion %.1f", diff / 1000.)); ++ log.warn(String.format("Check for long GC pauses not called in a timely fashion. Expected every %.1f seconds but was %.1f seconds since last check", ++ keepAliveTimeout / 1000., diff / 1000.)); + } + lastMemoryCheckTime = now; + return; + } + + if (maxIncreaseInCollectionTime > keepAliveTimeout) { + Halt.halt("Garbage collection may be interfering with lock keep-alive. Halting.", -1); + } + + lastMemorySize = mem; + lastMemoryCheckTime = now; + } + + +}