fix-2 close FileSystem
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c8d95251 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c8d95251 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c8d95251 Branch: refs/heads/fix-sonar-1 Commit: c8d95251b1366b59d42b26f6514a99509277c60f Parents: b32435c Author: Billy Liu <billy...@apache.org> Authored: Fri Dec 30 18:28:27 2016 +0800 Committer: Billy Liu <billy...@apache.org> Committed: Fri Dec 30 18:28:27 2016 +0800 ---------------------------------------------------------------------- .../src/main/java/org/apache/kylin/engine/mr/DFSFileTable.java | 1 + engine-mr/src/main/java/org/apache/kylin/engine/mr/HadoopUtil.java | 1 + .../main/java/org/apache/kylin/engine/mr/SortedColumnDFSFile.java | 1 + .../java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java | 2 ++ .../apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java | 2 ++ .../java/org/apache/kylin/engine/mr/steps/MergeStatisticsStep.java | 1 + .../java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java | 1 + .../apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java | 1 + .../main/java/org/apache/kylin/engine/spark/SparkCountDemo.java | 2 +- .../src/main/java/org/apache/kylin/source/hive/HiveMRInput.java | 1 + .../source/hive/cardinality/HiveColumnCardinalityUpdateJob.java | 1 + .../src/main/java/org/apache/kylin/source/kafka/KafkaMRInput.java | 1 + tool/src/main/java/org/apache/kylin/tool/StorageCleanupJob.java | 2 ++ 13 files changed, 16 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/c8d95251/engine-mr/src/main/java/org/apache/kylin/engine/mr/DFSFileTable.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/DFSFileTable.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/DFSFileTable.java index ee932ac..3358ffc 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/DFSFileTable.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/DFSFileTable.java @@ -105,6 +105,7 @@ public class DFSFileTable implements ReadableTable { lastModified = Math.max(lastModified, file.getModificationTime()); } + fs.close(); return Pair.newPair(size, lastModified); } http://git-wip-us.apache.org/repos/asf/kylin/blob/c8d95251/engine-mr/src/main/java/org/apache/kylin/engine/mr/HadoopUtil.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/HadoopUtil.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/HadoopUtil.java index 88692a0..55a73ad 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/HadoopUtil.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/HadoopUtil.java @@ -106,6 +106,7 @@ public class HadoopUtil { if (fs.exists(path)) { fs.delete(path, true); } + fs.close(); } public static byte[] toBytes(Writable writable) { http://git-wip-us.apache.org/repos/asf/kylin/blob/c8d95251/engine-mr/src/main/java/org/apache/kylin/engine/mr/SortedColumnDFSFile.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/SortedColumnDFSFile.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/SortedColumnDFSFile.java index 62c309a..27370d3 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/SortedColumnDFSFile.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/SortedColumnDFSFile.java @@ -76,6 +76,7 @@ public class SortedColumnDFSFile implements ReadableTable { DFSFileTableReader reader = new DFSFileTableReader(f.getPath().toString(), -1); readers.add(reader); } + fs.close(); return new SortedColumnDFSFileReader(readers, comparator); } http://git-wip-us.apache.org/repos/asf/kylin/blob/c8d95251/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java index e4e5e82..9a2c391 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java @@ -317,6 +317,8 @@ public abstract class AbstractHadoopJob extends Configured implements Tool { appendTmpFiles(fileList.toString(), jobConf); appendTmpJars(jarList.toString(), jobConf); + localfs.close(); + hdfs.close(); } catch (IOException e) { throw new RuntimeException(e); } http://git-wip-us.apache.org/repos/asf/kylin/blob/c8d95251/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java index 5d42797..a5a2d23 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java @@ -238,6 +238,7 @@ public class FactDistinctColumnsReducer extends KylinReducer<SelfDefineSortableK out.write('\n'); } } finally { + fs.close(); IOUtils.closeQuietly(out); } } @@ -358,6 +359,7 @@ public class FactDistinctColumnsReducer extends KylinReducer<SelfDefineSortableK } } finally { + fs.close(); IOUtils.closeQuietly(out); } } http://git-wip-us.apache.org/repos/asf/kylin/blob/c8d95251/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeStatisticsStep.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeStatisticsStep.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeStatisticsStep.java index 811fc24..8227b68 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeStatisticsStep.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeStatisticsStep.java @@ -115,6 +115,7 @@ public class MergeStatisticsStep extends AbstractExecutable { e.printStackTrace(); throw e; } finally { + fs.close(); IOUtils.closeStream(reader); if (tempFile != null) tempFile.delete(); http://git-wip-us.apache.org/repos/asf/kylin/blob/c8d95251/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java index 020c62c..1a4e502 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java @@ -70,6 +70,7 @@ public class SaveStatisticsStep extends AbstractExecutable { String statisticsFileName = newSegment.getStatisticsResourcePath(); rs.putResource(statisticsFileName, is, System.currentTimeMillis()); } finally { + fs.close(); IOUtils.closeStream(is); } http://git-wip-us.apache.org/repos/asf/kylin/blob/c8d95251/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java index dcc9190..dbeec53 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java @@ -91,6 +91,7 @@ public class UpdateCubeInfoAfterBuildStep extends AbstractExecutable { minValue = Math.min(min, minValue); maxValue = Math.max(max, maxValue); } finally { + fs.close(); IOUtils.closeQuietly(is); } logger.info("updateTimeRange step. minValue:" + minValue + " maxValue:" + maxValue); http://git-wip-us.apache.org/repos/asf/kylin/blob/c8d95251/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCountDemo.java ---------------------------------------------------------------------- diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCountDemo.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCountDemo.java index 6478c10..8a1a42e 100644 --- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCountDemo.java +++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCountDemo.java @@ -75,6 +75,6 @@ public class SparkCountDemo extends AbstractApplication { return new Tuple2(key, value); } }).saveAsNewAPIHadoopFile("hdfs://10.249.65.231:8020/tmp/hfile", ImmutableBytesWritable.class, KeyValue.class, HFileOutputFormat.class); - + sc.close(); } } http://git-wip-us.apache.org/repos/asf/kylin/blob/c8d95251/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java ---------------------------------------------------------------------- diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java index fc2b982..7c7a5d1 100644 --- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java +++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java @@ -383,6 +383,7 @@ public class HiveMRInput implements IMRInput { if (fs.exists(externalDataPath)) { fs.delete(externalDataPath, true); } + fs.close(); } public void setIntermediateTableIdentity(String tableIdentity) { http://git-wip-us.apache.org/repos/asf/kylin/blob/c8d95251/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/HiveColumnCardinalityUpdateJob.java ---------------------------------------------------------------------- diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/HiveColumnCardinalityUpdateJob.java b/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/HiveColumnCardinalityUpdateJob.java index 246822c..b52f204 100644 --- a/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/HiveColumnCardinalityUpdateJob.java +++ b/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/HiveColumnCardinalityUpdateJob.java @@ -155,6 +155,7 @@ public class HiveColumnCardinalityUpdateJob extends AbstractHadoopJob { results.add(str); } } + fileSystem.close(); return results; } http://git-wip-us.apache.org/repos/asf/kylin/blob/c8d95251/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaMRInput.java ---------------------------------------------------------------------- diff --git a/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaMRInput.java b/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaMRInput.java index dde3584..62038db 100644 --- a/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaMRInput.java +++ b/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaMRInput.java @@ -227,6 +227,7 @@ public class KafkaMRInput implements IMRInput { if (fs.exists(externalDataPath)) { fs.delete(externalDataPath, true); } + fs.close(); } public void setDataPath(String externalDataPath) { http://git-wip-us.apache.org/repos/asf/kylin/blob/c8d95251/tool/src/main/java/org/apache/kylin/tool/StorageCleanupJob.java ---------------------------------------------------------------------- diff --git a/tool/src/main/java/org/apache/kylin/tool/StorageCleanupJob.java b/tool/src/main/java/org/apache/kylin/tool/StorageCleanupJob.java index 05e0142..5b993fb 100644 --- a/tool/src/main/java/org/apache/kylin/tool/StorageCleanupJob.java +++ b/tool/src/main/java/org/apache/kylin/tool/StorageCleanupJob.java @@ -243,6 +243,7 @@ public class StorageCleanupJob extends AbstractApplication { } System.out.println("-------------------------------------------------------"); } + fs.close(); } private void cleanUnusedIntermediateHiveTable(Configuration conf) throws Exception { @@ -349,6 +350,7 @@ public class StorageCleanupJob extends AbstractApplication { } else { logger.info("Hive table {}'s external path {} not exist. It's normal if kylin.source.hive.keep-flat-table set false (By default)", tableToDelete, path); } + fs.close(); } else { logger.warn("Hive table {}'s job ID not found, segmentId2JobId: {}", tableToDelete, segmentId2JobId.toString()); }
