This is an automated email from the ASF dual-hosted git repository.
ddanielr pushed a commit to branch 2.1
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/2.1 by this push:
new fcaabbe427 More Detailed Compaction Errors (#4148)
fcaabbe427 is described below
commit fcaabbe427177eabf8f3dc24a29fb668677eb86e
Author: Kevin Rathbun <43969518+kevinrr...@users.noreply.github.com>
AuthorDate: Mon Jan 22 11:07:52 2024 -0500
More Detailed Compaction Errors (#4148)
* More Detailed Compaction Errors
Closes #4075
Changes:
- CompactionFinalizer
- Added compactionId to log of compaction failure (tableId
is already included from 'ecfs')
- CompactionCoordinator
- Added extent/tableId to log of compaction failure
(compactionId already included)
- CompactableImpl
- Added extent/tableId to two logs of compaction failure
(compactionId already included)
- Compactor
- Added extent/tableId and compactionId to three of the logs
- ExternalDoNothingCompactor
- Added extent/tableId and compactionId to log of
compaction failure
- DeadCompactionDetector
- Added extent/tableId to log (compactionId already
included)
- FileCompactor
- Added extent/tableId, input files, output file,
iterators, and thread start time to log when error occurs
---
.../apache/accumulo/server/compaction/FileCompactor.java | 16 +++++++++++++---
.../accumulo/coordinator/CompactionCoordinator.java | 3 ++-
.../apache/accumulo/coordinator/CompactionFinalizer.java | 3 ++-
.../accumulo/coordinator/DeadCompactionDetector.java | 4 ++--
.../java/org/apache/accumulo/compactor/Compactor.java | 11 ++++++++---
.../org/apache/accumulo/compactor/CompactorTest.java | 5 +++--
.../apache/accumulo/tserver/tablet/CompactableImpl.java | 5 +++--
.../test/compaction/ExternalDoNothingCompactor.java | 5 ++++-
8 files changed, 37 insertions(+), 15 deletions(-)
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/compaction/FileCompactor.java
b/server/base/src/main/java/org/apache/accumulo/server/compaction/FileCompactor.java
index 27eef8def0..7c16351ccf 100644
---
a/server/base/src/main/java/org/apache/accumulo/server/compaction/FileCompactor.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/compaction/FileCompactor.java
@@ -75,6 +75,8 @@ import org.apache.hadoop.fs.FileSystem;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.google.common.collect.Collections2;
+
import io.opentelemetry.api.trace.Span;
import io.opentelemetry.context.Scope;
@@ -207,11 +209,13 @@ public class FileCompactor implements
Callable<CompactionStats> {
boolean remove = runningCompactions.add(this);
+ String threadStartDate = dateFormatter.format(new Date());
+
clearStats();
String oldThreadName = Thread.currentThread().getName();
- String newThreadName = "MajC compacting " + extent + " started "
- + dateFormatter.format(new Date()) + " file: " + outputFile;
+ String newThreadName =
+ "MajC compacting " + extent + " started " + threadStartDate + " file:
" + outputFile;
Thread.currentThread().setName(newThreadName);
thread = Thread.currentThread();
try {
@@ -279,7 +283,13 @@ public class FileCompactor implements
Callable<CompactionStats> {
log.debug("Compaction canceled {}", extent);
throw e;
} catch (IOException | RuntimeException e) {
- log.error("{}", e.getMessage(), e);
+ Collection<String> inputFileNames =
+ Collections2.transform(getFilesToCompact(),
StoredTabletFile::getFileName);
+ String outputFileName = outputFile.getFileName();
+ log.error(
+ "Compaction error. Compaction info: "
+ + "extent: {}, input files: {}, output file: {}, iterators: {},
start date: {}",
+ getExtent(), inputFileNames, outputFileName, getIterators(),
threadStartDate, e);
throw e;
} finally {
Thread.currentThread().setName(oldThreadName);
diff --git
a/server/compaction-coordinator/src/main/java/org/apache/accumulo/coordinator/CompactionCoordinator.java
b/server/compaction-coordinator/src/main/java/org/apache/accumulo/coordinator/CompactionCoordinator.java
index 88df289a56..df94ccb824 100644
---
a/server/compaction-coordinator/src/main/java/org/apache/accumulo/coordinator/CompactionCoordinator.java
+++
b/server/compaction-coordinator/src/main/java/org/apache/accumulo/coordinator/CompactionCoordinator.java
@@ -537,7 +537,8 @@ public class CompactionCoordinator extends AbstractServer
throw new AccumuloSecurityException(credentials.getPrincipal(),
SecurityErrorCode.PERMISSION_DENIED).asThriftException();
}
- LOG.info("Compaction failed, id: {}", externalCompactionId);
+ KeyExtent fromThriftExtent = KeyExtent.fromThrift(extent);
+ LOG.info("Compaction failed: id: {}, extent: {}", externalCompactionId,
fromThriftExtent);
final var ecid = ExternalCompactionId.of(externalCompactionId);
compactionFailed(Map.of(ecid, KeyExtent.fromThrift(extent)));
}
diff --git
a/server/compaction-coordinator/src/main/java/org/apache/accumulo/coordinator/CompactionFinalizer.java
b/server/compaction-coordinator/src/main/java/org/apache/accumulo/coordinator/CompactionFinalizer.java
index 329c9fb4eb..5c259b13f4 100644
---
a/server/compaction-coordinator/src/main/java/org/apache/accumulo/coordinator/CompactionFinalizer.java
+++
b/server/compaction-coordinator/src/main/java/org/apache/accumulo/coordinator/CompactionFinalizer.java
@@ -128,7 +128,8 @@ public class CompactionFinalizer {
ecfs.getExternalCompactionId().canonical(),
ecfs.getExtent().toThrift(),
ecfs.getFileSize(), ecfs.getEntries());
} else if (ecfs.getFinalState() == FinalState.FAILED) {
- LOG.debug("Notifying tserver {} that compaction {} has failed.", loc,
ecfs);
+ LOG.debug("Notifying tserver {} that compaction {} with {} has
failed.", loc,
+ ecfs.getExternalCompactionId(), ecfs);
client.compactionJobFailed(TraceUtil.traceInfo(), context.rpcCreds(),
ecfs.getExternalCompactionId().canonical(),
ecfs.getExtent().toThrift());
} else {
diff --git
a/server/compaction-coordinator/src/main/java/org/apache/accumulo/coordinator/DeadCompactionDetector.java
b/server/compaction-coordinator/src/main/java/org/apache/accumulo/coordinator/DeadCompactionDetector.java
index 3141559d31..ba4a575ddf 100644
---
a/server/compaction-coordinator/src/main/java/org/apache/accumulo/coordinator/DeadCompactionDetector.java
+++
b/server/compaction-coordinator/src/main/java/org/apache/accumulo/coordinator/DeadCompactionDetector.java
@@ -128,8 +128,8 @@ public class DeadCompactionDetector {
this.deadCompactions.entrySet().stream().filter(e -> e.getValue() >
2).map(e -> e.getKey())
.collect(Collectors.toCollection(TreeSet::new));
tabletCompactions.keySet().retainAll(toFail);
- tabletCompactions.forEach((eci, v) -> {
- log.warn("Compaction {} believed to be dead, failing it.", eci);
+ tabletCompactions.forEach((ecid, extent) -> {
+ log.warn("Compaction believed to be dead, failing it: id: {}, extent:
{}", ecid, extent);
});
coordinator.compactionFailed(tabletCompactions);
this.deadCompactions.keySet().removeAll(toFail);
diff --git
a/server/compactor/src/main/java/org/apache/accumulo/compactor/Compactor.java
b/server/compactor/src/main/java/org/apache/accumulo/compactor/Compactor.java
index 9662757d3d..823f9fd58d 100644
---
a/server/compactor/src/main/java/org/apache/accumulo/compactor/Compactor.java
+++
b/server/compactor/src/main/java/org/apache/accumulo/compactor/Compactor.java
@@ -559,7 +559,9 @@ public class Compactor extends AbstractServer implements
MetricsProducer, Compac
} catch (FileCompactor.CompactionCanceledException cce) {
LOG.debug("Compaction canceled {}", job.getExternalCompactionId());
} catch (Exception e) {
- LOG.error("Compaction failed", e);
+ KeyExtent fromThriftExtent = KeyExtent.fromThrift(job.getExtent());
+ LOG.error("Compaction failed: id: {}, extent: {}",
job.getExternalCompactionId(),
+ fromThriftExtent, e);
err.set(e);
} finally {
stopped.countDown();
@@ -728,14 +730,17 @@ public class Compactor extends AbstractServer implements
MetricsProducer, Compac
currentCompactionId.set(null);
}
} else if (err.get() != null) {
+ KeyExtent fromThriftExtent = KeyExtent.fromThrift(job.getExtent());
try {
- LOG.info("Updating coordinator with compaction failure.");
+ LOG.info("Updating coordinator with compaction failure: id: {},
extent: {}",
+ job.getExternalCompactionId(), fromThriftExtent);
TCompactionStatusUpdate update = new
TCompactionStatusUpdate(TCompactionState.FAILED,
"Compaction failed due to: " + err.get().getMessage(), -1,
-1, -1);
updateCompactionState(job, update);
updateCompactionFailed(job);
} catch (RetriesExceededException e) {
- LOG.error("Error updating coordinator with compaction failure.",
e);
+ LOG.error("Error updating coordinator with compaction failure:
id: {}, extent: {}",
+ job.getExternalCompactionId(), fromThriftExtent, e);
} finally {
currentCompactionId.set(null);
}
diff --git
a/server/compactor/src/test/java/org/apache/accumulo/compactor/CompactorTest.java
b/server/compactor/src/test/java/org/apache/accumulo/compactor/CompactorTest.java
index ac2384add6..174afc8593 100644
---
a/server/compactor/src/test/java/org/apache/accumulo/compactor/CompactorTest.java
+++
b/server/compactor/src/test/java/org/apache/accumulo/compactor/CompactorTest.java
@@ -39,6 +39,8 @@ import org.apache.accumulo.core.conf.AccumuloConfiguration;
import org.apache.accumulo.core.conf.ConfigurationCopy;
import org.apache.accumulo.core.conf.DefaultConfiguration;
import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.TableId;
+import org.apache.accumulo.core.dataImpl.KeyExtent;
import org.apache.accumulo.core.dataImpl.thrift.TKeyExtent;
import org.apache.accumulo.core.fate.zookeeper.ZooReaderWriter;
import org.apache.accumulo.core.metadata.schema.ExternalCompactionId;
@@ -367,8 +369,7 @@ public class CompactorTest {
expect(client.getAddress()).andReturn(address);
TExternalCompactionJob job =
PowerMock.createNiceMock(TExternalCompactionJob.class);
- TKeyExtent extent = PowerMock.createNiceMock(TKeyExtent.class);
- expect(extent.getTable()).andReturn("testTable".getBytes()).anyTimes();
+ TKeyExtent extent = new KeyExtent(TableId.of("testTable"), null,
null).toThrift();
expect(job.isSetExternalCompactionId()).andReturn(true).anyTimes();
expect(job.getExternalCompactionId()).andReturn(eci.toString()).anyTimes();
diff --git
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/CompactableImpl.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/CompactableImpl.java
index cb055d080c..73a2efa0e4 100644
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/CompactableImpl.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/CompactableImpl.java
@@ -1408,7 +1408,8 @@ public class CompactableImpl implements Compactable {
successful = true;
} catch (Exception e) {
metaFile = Optional.empty();
- log.error("Error committing external compaction {}",
extCompactionId, e);
+ log.error("Error committing external compaction: id: {}, extent:
{}", extCompactionId,
+ getExtent(), e);
throw new RuntimeException(e);
} finally {
completeCompaction(ecInfo.job, ecInfo.meta.getJobFiles(), metaFile,
successful);
@@ -1450,7 +1451,7 @@ public class CompactableImpl implements Compactable {
.mutate();
completeCompaction(ecInfo.job, ecInfo.meta.getJobFiles(),
Optional.empty(), false);
externalCompactions.remove(ecid);
- log.debug("Processed external compaction failure {}", ecid);
+ log.debug("Processed external compaction failure: id: {}, extent: {}",
ecid, getExtent());
} else {
log.debug("Ignoring request to fail external compaction that is
unknown {}", ecid);
}
diff --git
a/test/src/main/java/org/apache/accumulo/test/compaction/ExternalDoNothingCompactor.java
b/test/src/main/java/org/apache/accumulo/test/compaction/ExternalDoNothingCompactor.java
index b03e181d40..6b4547252b 100644
---
a/test/src/main/java/org/apache/accumulo/test/compaction/ExternalDoNothingCompactor.java
+++
b/test/src/main/java/org/apache/accumulo/test/compaction/ExternalDoNothingCompactor.java
@@ -29,6 +29,7 @@ import org.apache.accumulo.compactor.Compactor;
import org.apache.accumulo.core.compaction.thrift.CompactorService.Iface;
import org.apache.accumulo.core.compaction.thrift.TCompactionState;
import org.apache.accumulo.core.compaction.thrift.TCompactionStatusUpdate;
+import org.apache.accumulo.core.dataImpl.KeyExtent;
import org.apache.accumulo.core.tabletserver.thrift.TExternalCompactionJob;
import org.apache.accumulo.core.util.UtilWaitThread;
import
org.apache.accumulo.server.compaction.FileCompactor.CompactionCanceledException;
@@ -77,7 +78,9 @@ public class ExternalDoNothingCompactor extends Compactor
implements Iface {
throw new CompactionCanceledException();
} catch (Exception e) {
- LOG.error("Compaction failed", e);
+ KeyExtent fromThriftExtent = KeyExtent.fromThrift(job.getExtent());
+ LOG.error("Compaction failed: id: {}, extent: {}",
job.getExternalCompactionId(),
+ fromThriftExtent, e);
err.set(e);
} finally {
stopped.countDown();
