Repository: kylin Updated Branches: refs/heads/fix-sonar-1 a7f0f49e8 -> 7972ac5bb (forced update)
fix FileSystem.close Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7972ac5b Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7972ac5b Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7972ac5b Branch: refs/heads/fix-sonar-1 Commit: 7972ac5bb86aee57127eeafa6cee7ca3b5548cd9 Parents: 6766894 Author: Billy Liu <billy...@apache.org> Authored: Tue Jan 3 20:37:13 2017 +0800 Committer: Billy Liu <billy...@apache.org> Committed: Tue Jan 3 20:37:13 2017 +0800 ---------------------------------------------------------------------- .../apache/kylin/common/util/CliCommandExecutor.java | 15 +++++++++------ .../org/apache/kylin/dict/AppendTrieDictionary.java | 11 ++++++++++- .../java/org/apache/kylin/engine/mr/HadoopUtil.java | 1 + .../kylin/engine/mr/steps/MergeStatisticsStep.java | 1 + .../kylin/engine/mr/steps/SaveStatisticsStep.java | 1 + .../mr/steps/UpdateCubeInfoAfterBuildStep.java | 1 + .../apache/kylin/engine/spark/SparkCountDemo.java | 2 +- .../org/apache/kylin/source/hive/HiveMRInput.java | 1 + .../cardinality/HiveColumnCardinalityUpdateJob.java | 1 + .../org/apache/kylin/source/kafka/KafkaMRInput.java | 1 + .../apache/kylin/storage/hbase/HBaseConnection.java | 5 ++++- .../org/apache/kylin/tool/StorageCleanupJob.java | 2 ++ 12 files changed, 33 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/7972ac5b/core-common/src/main/java/org/apache/kylin/common/util/CliCommandExecutor.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/util/CliCommandExecutor.java b/core-common/src/main/java/org/apache/kylin/common/util/CliCommandExecutor.java index f97b609..488af5f 100644 --- a/core-common/src/main/java/org/apache/kylin/common/util/CliCommandExecutor.java +++ b/core-common/src/main/java/org/apache/kylin/common/util/CliCommandExecutor.java @@ -131,18 +131,21 @@ public class CliCommandExecutor { BufferedReader reader = new BufferedReader(new InputStreamReader(proc.getInputStream())); String line; StringBuilder result = new StringBuilder(); - while ((line = reader.readLine()) != null) { - result.append(line).append('\n'); - if (logAppender != null) { - logAppender.log(line); + try { + while ((line = reader.readLine()) != null) { + result.append(line).append('\n'); + if (logAppender != null) { + logAppender.log(line); + } } - } - try { int exitCode = proc.waitFor(); return Pair.newPair(exitCode, result.toString()); } catch (InterruptedException e) { + Thread.currentThread().interrupt(); throw new IOException(e); + } finally { + reader.close(); } } http://git-wip-us.apache.org/repos/asf/kylin/blob/7972ac5b/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java b/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java index 32bfde6..1d17569 100644 --- a/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java +++ b/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java @@ -161,6 +161,15 @@ public class AppendTrieDictionary<T> extends Dictionary<T> { } @Override + public boolean equals(Object obj) { + if (!(obj instanceof DictSliceKey)){ + return false; + } + DictSliceKey other = (DictSliceKey)obj; + return Bytes.equals(key, other.key); + } + + @Override public int compareTo(Object o) { if (!(o instanceof DictSliceKey)) { return -1; @@ -1219,7 +1228,7 @@ public class AppendTrieDictionary<T> extends Dictionary<T> { dstFs.delete(dstPath, true); } FileUtil.copy(FileSystem.get(srcPath.toUri(), conf), srcPath, FileSystem.get(dstPath.toUri(), conf), dstPath, false, true, conf); - + dstFs.close(); return newDict; } http://git-wip-us.apache.org/repos/asf/kylin/blob/7972ac5b/engine-mr/src/main/java/org/apache/kylin/engine/mr/HadoopUtil.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/HadoopUtil.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/HadoopUtil.java index 88692a0..55a73ad 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/HadoopUtil.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/HadoopUtil.java @@ -106,6 +106,7 @@ public class HadoopUtil { if (fs.exists(path)) { fs.delete(path, true); } + fs.close(); } public static byte[] toBytes(Writable writable) { http://git-wip-us.apache.org/repos/asf/kylin/blob/7972ac5b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeStatisticsStep.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeStatisticsStep.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeStatisticsStep.java index 811fc24..8227b68 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeStatisticsStep.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeStatisticsStep.java @@ -115,6 +115,7 @@ public class MergeStatisticsStep extends AbstractExecutable { e.printStackTrace(); throw e; } finally { + fs.close(); IOUtils.closeStream(reader); if (tempFile != null) tempFile.delete(); http://git-wip-us.apache.org/repos/asf/kylin/blob/7972ac5b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java index 020c62c..1a4e502 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java @@ -70,6 +70,7 @@ public class SaveStatisticsStep extends AbstractExecutable { String statisticsFileName = newSegment.getStatisticsResourcePath(); rs.putResource(statisticsFileName, is, System.currentTimeMillis()); } finally { + fs.close(); IOUtils.closeStream(is); } http://git-wip-us.apache.org/repos/asf/kylin/blob/7972ac5b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java index dcc9190..dbeec53 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java @@ -91,6 +91,7 @@ public class UpdateCubeInfoAfterBuildStep extends AbstractExecutable { minValue = Math.min(min, minValue); maxValue = Math.max(max, maxValue); } finally { + fs.close(); IOUtils.closeQuietly(is); } logger.info("updateTimeRange step. minValue:" + minValue + " maxValue:" + maxValue); http://git-wip-us.apache.org/repos/asf/kylin/blob/7972ac5b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCountDemo.java ---------------------------------------------------------------------- diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCountDemo.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCountDemo.java index 6478c10..8a1a42e 100644 --- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCountDemo.java +++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCountDemo.java @@ -75,6 +75,6 @@ public class SparkCountDemo extends AbstractApplication { return new Tuple2(key, value); } }).saveAsNewAPIHadoopFile("hdfs://10.249.65.231:8020/tmp/hfile", ImmutableBytesWritable.class, KeyValue.class, HFileOutputFormat.class); - + sc.close(); } } http://git-wip-us.apache.org/repos/asf/kylin/blob/7972ac5b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java ---------------------------------------------------------------------- diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java index fc2b982..7c7a5d1 100644 --- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java +++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java @@ -383,6 +383,7 @@ public class HiveMRInput implements IMRInput { if (fs.exists(externalDataPath)) { fs.delete(externalDataPath, true); } + fs.close(); } public void setIntermediateTableIdentity(String tableIdentity) { http://git-wip-us.apache.org/repos/asf/kylin/blob/7972ac5b/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/HiveColumnCardinalityUpdateJob.java ---------------------------------------------------------------------- diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/HiveColumnCardinalityUpdateJob.java b/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/HiveColumnCardinalityUpdateJob.java index 246822c..b52f204 100644 --- a/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/HiveColumnCardinalityUpdateJob.java +++ b/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/HiveColumnCardinalityUpdateJob.java @@ -155,6 +155,7 @@ public class HiveColumnCardinalityUpdateJob extends AbstractHadoopJob { results.add(str); } } + fileSystem.close(); return results; } http://git-wip-us.apache.org/repos/asf/kylin/blob/7972ac5b/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaMRInput.java ---------------------------------------------------------------------- diff --git a/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaMRInput.java b/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaMRInput.java index dde3584..62038db 100644 --- a/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaMRInput.java +++ b/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaMRInput.java @@ -227,6 +227,7 @@ public class KafkaMRInput implements IMRInput { if (fs.exists(externalDataPath)) { fs.delete(externalDataPath, true); } + fs.close(); } public void setDataPath(String externalDataPath) { http://git-wip-us.apache.org/repos/asf/kylin/blob/7972ac5b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java index cbf81b6..972d637 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java @@ -124,6 +124,7 @@ public class HBaseConnection { coprocessorPool.shutdownNow(); } } catch (InterruptedException e) { + Thread.currentThread().interrupt(); coprocessorPool.shutdownNow(); } } @@ -205,7 +206,9 @@ public class HBaseConnection { public static String makeQualifiedPathInHBaseCluster(String path) { try { FileSystem fs = FileSystem.get(getCurrentHBaseConfiguration()); - return fs.makeQualified(new Path(path)).toString(); + String result = fs.makeQualified(new Path(path)).toString(); + fs.close(); + return result; } catch (IOException e) { throw new IllegalArgumentException("Cannot create FileSystem from current hbase cluster conf", e); } http://git-wip-us.apache.org/repos/asf/kylin/blob/7972ac5b/tool/src/main/java/org/apache/kylin/tool/StorageCleanupJob.java ---------------------------------------------------------------------- diff --git a/tool/src/main/java/org/apache/kylin/tool/StorageCleanupJob.java b/tool/src/main/java/org/apache/kylin/tool/StorageCleanupJob.java index 05e0142..5b993fb 100644 --- a/tool/src/main/java/org/apache/kylin/tool/StorageCleanupJob.java +++ b/tool/src/main/java/org/apache/kylin/tool/StorageCleanupJob.java @@ -243,6 +243,7 @@ public class StorageCleanupJob extends AbstractApplication { } System.out.println("-------------------------------------------------------"); } + fs.close(); } private void cleanUnusedIntermediateHiveTable(Configuration conf) throws Exception { @@ -349,6 +350,7 @@ public class StorageCleanupJob extends AbstractApplication { } else { logger.info("Hive table {}'s external path {} not exist. It's normal if kylin.source.hive.keep-flat-table set false (By default)", tableToDelete, path); } + fs.close(); } else { logger.warn("Hive table {}'s job ID not found, segmentId2JobId: {}", tableToDelete, segmentId2JobId.toString()); }
