ACCUMULO-4657 Update MetadataTableUtil.java Outputting every "loaded" entry in the table is excessive, especially for tables with multiple simultaneous bulk imports and multiple references to the same file. This has been seen to cause performance problems. Even when the log level was reduced, there was blocking within log4j. By doing that check once outside the loop and only logging at trace level, we have seen bulk import performance improvements.
Updated String to byte array for faster comparison Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/6c31cd49 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/6c31cd49 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/6c31cd49 Branch: refs/heads/master Commit: 6c31cd49ce0b8eae13d7a6cca50bc802745a6e72 Parents: d5ddca1 Author: matthpeterson <mpeter...@appliedtech-group.com> Authored: Fri Jun 16 15:51:38 2017 -0400 Committer: Michael Wall <mjw...@apache.org> Committed: Tue Jun 20 16:17:38 2017 -0400 ---------------------------------------------------------------------- .../accumulo/server/util/MetadataTableUtil.java | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo/blob/6c31cd49/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java ---------------------------------------------------------------------- diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java b/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java index c07114b..54ca37c 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java +++ b/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java @@ -24,6 +24,7 @@ import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSec import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.HashMap; import java.util.HashSet; @@ -894,12 +895,19 @@ public class MetadataTableUtil { BatchWriter bw = conn.createBatchWriter(MetadataTable.NAME, new BatchWriterConfig())) { mscanner.setRange(new KeyExtent(tableId, null, null).toMetadataRange()); mscanner.fetchColumnFamily(TabletsSection.BulkFileColumnFamily.NAME); + boolean shouldTrace = log.isTraceEnabled(); + byte[] tidAsBytes = Long.toString(tid).getBytes(UTF_8); for (Entry<Key,Value> entry : mscanner) { - log.debug("Looking at entry " + entry + " with tid " + tid); - if (Long.parseLong(entry.getValue().toString()) == tid) { - log.debug("deleting entry " + entry); - Mutation m = new Mutation(entry.getKey().getRow()); - m.putDelete(entry.getKey().getColumnFamily(), entry.getKey().getColumnQualifier()); + if (shouldTrace) { + log.trace("Looking at entry {} with tid {}", entry, tid); + } + if (Arrays.equals(entry.getValue().get(), tidAsBytes)) { + if (shouldTrace) { + log.trace("deleting entry {}", entry); + } + Key key = entry.getKey(); + Mutation m = new Mutation(key.getRow()); + m.putDelete(key.getColumnFamily(), key.getColumnQualifier()); bw.addMutation(m); } }
