http://git-wip-us.apache.org/repos/asf/kylin/blob/826f23f1/engine-mr/src/main/java/org/apache/kylin/engine/mr/DFSFileTableReader.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/DFSFileTableReader.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/DFSFileTableReader.java index 173c908..67fedbd 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/DFSFileTableReader.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/DFSFileTableReader.java @@ -1,252 +1,252 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -package org.apache.kylin.engine.mr; - -import java.io.BufferedReader; -import java.io.Closeable; -import java.io.EOFException; -import java.io.IOException; -import java.io.InputStreamReader; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - -import org.apache.commons.io.IOUtils; -import org.apache.commons.lang.StringEscapeUtils; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FSDataInputStream; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.io.SequenceFile; -import org.apache.hadoop.io.SequenceFile.Reader; -import org.apache.hadoop.io.Text; -import org.apache.hadoop.io.Writable; -import org.apache.hadoop.util.ReflectionUtils; -import org.apache.kylin.common.util.Bytes; -import org.apache.kylin.common.util.StringSplitter; -import org.apache.kylin.source.ReadableTable.TableReader; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Tables are typically CSV or SEQ file. - * - * @author yangli9 - */ -public class DFSFileTableReader implements TableReader { - - private static final Logger logger = LoggerFactory.getLogger(DFSFileTableReader.class); - private static final char CSV_QUOTE = '"'; - private static final String[] DETECT_DELIMS = new String[] { "\177", "|", "\t", "," }; - - private String filePath; - private String delim; - private List<RowReader> readerList; - - private String curLine; - private String[] curColumns; - private int expectedColumnNumber = -1; // helps delimiter detection - - public DFSFileTableReader(String filePath, int expectedColumnNumber) throws IOException { - this(filePath, DFSFileTable.DELIM_AUTO, expectedColumnNumber); - } - - public DFSFileTableReader(String filePath, String delim, int expectedColumnNumber) throws IOException { - filePath = HadoopUtil.fixWindowsPath(filePath); - this.filePath = filePath; - this.delim = delim; - this.expectedColumnNumber = expectedColumnNumber; - this.readerList = new ArrayList<RowReader>(); - - FileSystem fs = HadoopUtil.getFileSystem(filePath); - - ArrayList<FileStatus> allFiles = new ArrayList<>(); - FileStatus status = fs.getFileStatus(new Path(filePath)); - if (status.isFile()) { - allFiles.add(status); - } else { - FileStatus[] listStatus = fs.listStatus(new Path(filePath)); - allFiles.addAll(Arrays.asList(listStatus)); - } - - try { - for (FileStatus f : allFiles) { - RowReader rowReader = new SeqRowReader(HadoopUtil.getCurrentConfiguration(), fs, f.getPath().toString()); - this.readerList.add(rowReader); - } - } catch (IOException e) { - if (isExceptionSayingNotSeqFile(e) == false) - throw e; - - this.readerList = new ArrayList<RowReader>(); - for (FileStatus f : allFiles) { - RowReader rowReader = new CsvRowReader(fs, f.getPath().toString()); - this.readerList.add(rowReader); - } - } - } - - private boolean isExceptionSayingNotSeqFile(IOException e) { - if (e.getMessage() != null && e.getMessage().contains("not a SequenceFile")) - return true; - - if (e instanceof EOFException) // in case the file is very very small - return true; - - return false; - } - - @Override - public boolean next() throws IOException { - int curReaderIndex = -1; - RowReader curReader; - - while (++curReaderIndex < readerList.size()) { - curReader = readerList.get(curReaderIndex); - curLine = curReader.nextLine(); - curColumns = null; - - if (curLine != null) { - return true; - } - } - - return false; - } - - public String getLine() { - return curLine; - } - - @Override - public String[] getRow() { - if (curColumns == null) { - if (DFSFileTable.DELIM_AUTO.equals(delim)) - delim = autoDetectDelim(curLine); - - if (delim == null) - curColumns = new String[] { curLine }; - else - curColumns = split(curLine, delim); - } - return curColumns; - } - - private String[] split(String line, String delim) { - // FIXME CVS line should be parsed considering escapes - String[] str = StringSplitter.split(line, delim); - - // un-escape CSV - if (DFSFileTable.DELIM_COMMA.equals(delim)) { - for (int i = 0; i < str.length; i++) { - str[i] = unescapeCsv(str[i]); - } - } - - return str; - } - - private String unescapeCsv(String str) { - if (str == null || str.length() < 2) - return str; - - str = StringEscapeUtils.unescapeCsv(str); - - // unescapeCsv may not remove the outer most quotes - if (str.charAt(0) == CSV_QUOTE && str.charAt(str.length() - 1) == CSV_QUOTE) - str = str.substring(1, str.length() - 1); - - return str; - } - - @Override - public void close() { - for (RowReader reader : readerList) { - IOUtils.closeQuietly(reader); - } - } - - private String autoDetectDelim(String line) { - if (expectedColumnNumber > 0) { - for (String delim : DETECT_DELIMS) { - if (StringSplitter.split(line, delim).length == expectedColumnNumber) { - logger.info("Auto detect delim to be '" + delim + "', split line to " + expectedColumnNumber + " columns -- " + line); - return delim; - } - } - } - - logger.info("Auto detect delim to be null, will take THE-WHOLE-LINE as a single value, for " + filePath); - return null; - } - - // ============================================================================ - - private interface RowReader extends Closeable { - String nextLine() throws IOException; // return null on EOF - } - - private class SeqRowReader implements RowReader { - Reader reader; - Writable key; - Text value; - - SeqRowReader(Configuration hconf, FileSystem fs, String path) throws IOException { - reader = new Reader(hconf, SequenceFile.Reader.file(new Path(path))); - key = (Writable) ReflectionUtils.newInstance(reader.getKeyClass(), hconf); - value = new Text(); - } - - @Override - public String nextLine() throws IOException { - boolean hasNext = reader.next(key, value); - if (hasNext) - return Bytes.toString(value.getBytes(), 0, value.getLength()); - else - return null; - } - - @Override - public void close() throws IOException { - reader.close(); - } - } - - private class CsvRowReader implements RowReader { - BufferedReader reader; - - CsvRowReader(FileSystem fs, String path) throws IOException { - FSDataInputStream in = fs.open(new Path(path)); - reader = new BufferedReader(new InputStreamReader(in, "UTF-8")); - } - - @Override - public String nextLine() throws IOException { - return reader.readLine(); - } - - @Override - public void close() throws IOException { - reader.close(); - } - - } - -} +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +package org.apache.kylin.engine.mr; + +import java.io.BufferedReader; +import java.io.Closeable; +import java.io.EOFException; +import java.io.IOException; +import java.io.InputStreamReader; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang.StringEscapeUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.SequenceFile; +import org.apache.hadoop.io.SequenceFile.Reader; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.io.Writable; +import org.apache.hadoop.util.ReflectionUtils; +import org.apache.kylin.common.util.Bytes; +import org.apache.kylin.common.util.StringSplitter; +import org.apache.kylin.source.ReadableTable.TableReader; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Tables are typically CSV or SEQ file. + * + * @author yangli9 + */ +public class DFSFileTableReader implements TableReader { + + private static final Logger logger = LoggerFactory.getLogger(DFSFileTableReader.class); + private static final char CSV_QUOTE = '"'; + private static final String[] DETECT_DELIMS = new String[] { "\177", "|", "\t", "," }; + + private String filePath; + private String delim; + private List<RowReader> readerList; + + private String curLine; + private String[] curColumns; + private int expectedColumnNumber = -1; // helps delimiter detection + + public DFSFileTableReader(String filePath, int expectedColumnNumber) throws IOException { + this(filePath, DFSFileTable.DELIM_AUTO, expectedColumnNumber); + } + + public DFSFileTableReader(String filePath, String delim, int expectedColumnNumber) throws IOException { + filePath = HadoopUtil.fixWindowsPath(filePath); + this.filePath = filePath; + this.delim = delim; + this.expectedColumnNumber = expectedColumnNumber; + this.readerList = new ArrayList<RowReader>(); + + FileSystem fs = HadoopUtil.getFileSystem(filePath); + + ArrayList<FileStatus> allFiles = new ArrayList<>(); + FileStatus status = fs.getFileStatus(new Path(filePath)); + if (status.isFile()) { + allFiles.add(status); + } else { + FileStatus[] listStatus = fs.listStatus(new Path(filePath)); + allFiles.addAll(Arrays.asList(listStatus)); + } + + try { + for (FileStatus f : allFiles) { + RowReader rowReader = new SeqRowReader(HadoopUtil.getCurrentConfiguration(), fs, f.getPath().toString()); + this.readerList.add(rowReader); + } + } catch (IOException e) { + if (isExceptionSayingNotSeqFile(e) == false) + throw e; + + this.readerList = new ArrayList<RowReader>(); + for (FileStatus f : allFiles) { + RowReader rowReader = new CsvRowReader(fs, f.getPath().toString()); + this.readerList.add(rowReader); + } + } + } + + private boolean isExceptionSayingNotSeqFile(IOException e) { + if (e.getMessage() != null && e.getMessage().contains("not a SequenceFile")) + return true; + + if (e instanceof EOFException) // in case the file is very very small + return true; + + return false; + } + + @Override + public boolean next() throws IOException { + int curReaderIndex = -1; + RowReader curReader; + + while (++curReaderIndex < readerList.size()) { + curReader = readerList.get(curReaderIndex); + curLine = curReader.nextLine(); + curColumns = null; + + if (curLine != null) { + return true; + } + } + + return false; + } + + public String getLine() { + return curLine; + } + + @Override + public String[] getRow() { + if (curColumns == null) { + if (DFSFileTable.DELIM_AUTO.equals(delim)) + delim = autoDetectDelim(curLine); + + if (delim == null) + curColumns = new String[] { curLine }; + else + curColumns = split(curLine, delim); + } + return curColumns; + } + + private String[] split(String line, String delim) { + // FIXME CVS line should be parsed considering escapes + String[] str = StringSplitter.split(line, delim); + + // un-escape CSV + if (DFSFileTable.DELIM_COMMA.equals(delim)) { + for (int i = 0; i < str.length; i++) { + str[i] = unescapeCsv(str[i]); + } + } + + return str; + } + + private String unescapeCsv(String str) { + if (str == null || str.length() < 2) + return str; + + str = StringEscapeUtils.unescapeCsv(str); + + // unescapeCsv may not remove the outer most quotes + if (str.charAt(0) == CSV_QUOTE && str.charAt(str.length() - 1) == CSV_QUOTE) + str = str.substring(1, str.length() - 1); + + return str; + } + + @Override + public void close() { + for (RowReader reader : readerList) { + IOUtils.closeQuietly(reader); + } + } + + private String autoDetectDelim(String line) { + if (expectedColumnNumber > 0) { + for (String delim : DETECT_DELIMS) { + if (StringSplitter.split(line, delim).length == expectedColumnNumber) { + logger.info("Auto detect delim to be '" + delim + "', split line to " + expectedColumnNumber + " columns -- " + line); + return delim; + } + } + } + + logger.info("Auto detect delim to be null, will take THE-WHOLE-LINE as a single value, for " + filePath); + return null; + } + + // ============================================================================ + + private interface RowReader extends Closeable { + String nextLine() throws IOException; // return null on EOF + } + + private class SeqRowReader implements RowReader { + Reader reader; + Writable key; + Text value; + + SeqRowReader(Configuration hconf, FileSystem fs, String path) throws IOException { + reader = new Reader(hconf, SequenceFile.Reader.file(new Path(path))); + key = (Writable) ReflectionUtils.newInstance(reader.getKeyClass(), hconf); + value = new Text(); + } + + @Override + public String nextLine() throws IOException { + boolean hasNext = reader.next(key, value); + if (hasNext) + return Bytes.toString(value.getBytes(), 0, value.getLength()); + else + return null; + } + + @Override + public void close() throws IOException { + reader.close(); + } + } + + private class CsvRowReader implements RowReader { + BufferedReader reader; + + CsvRowReader(FileSystem fs, String path) throws IOException { + FSDataInputStream in = fs.open(new Path(path)); + reader = new BufferedReader(new InputStreamReader(in, "UTF-8")); + } + + @Override + public String nextLine() throws IOException { + return reader.readLine(); + } + + @Override + public void close() throws IOException { + reader.close(); + } + + } + +}
http://git-wip-us.apache.org/repos/asf/kylin/blob/826f23f1/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsJob.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsJob.java index 863ab8f..2eb694e 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsJob.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsJob.java @@ -1,160 +1,160 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -package org.apache.kylin.engine.mr.steps; - -import java.io.IOException; -import java.util.List; - -import org.apache.commons.cli.Options; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.io.NullWritable; -import org.apache.hadoop.io.Text; -import org.apache.hadoop.mapreduce.Job; -import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; -import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat; -import org.apache.hadoop.util.ToolRunner; -import org.apache.kylin.common.KylinConfig; -import org.apache.kylin.cube.CubeInstance; -import org.apache.kylin.cube.CubeManager; -import org.apache.kylin.cube.CubeSegment; -import org.apache.kylin.engine.mr.IMRInput.IMRTableInputFormat; -import org.apache.kylin.engine.mr.MRUtil; -import org.apache.kylin.engine.mr.common.AbstractHadoopJob; -import org.apache.kylin.engine.mr.common.BatchConstants; -import org.apache.kylin.metadata.model.TblColRef; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - */ -public class FactDistinctColumnsJob extends AbstractHadoopJob { - protected static final Logger logger = LoggerFactory.getLogger(FactDistinctColumnsJob.class); - - @Override - public int run(String[] args) throws Exception { - Options options = new Options(); - - try { - options.addOption(OPTION_JOB_NAME); - options.addOption(OPTION_CUBE_NAME); - options.addOption(OPTION_CUBING_JOB_ID); - options.addOption(OPTION_OUTPUT_PATH); - options.addOption(OPTION_SEGMENT_ID); - options.addOption(OPTION_STATISTICS_ENABLED); - options.addOption(OPTION_STATISTICS_OUTPUT); - options.addOption(OPTION_STATISTICS_SAMPLING_PERCENT); - parseOptions(options, args); - - job = Job.getInstance(getConf(), getOptionValue(OPTION_JOB_NAME)); - String job_id = getOptionValue(OPTION_CUBING_JOB_ID); - job.getConfiguration().set(BatchConstants.ARG_CUBING_JOB_ID, job_id); - String cubeName = getOptionValue(OPTION_CUBE_NAME); - Path output = new Path(getOptionValue(OPTION_OUTPUT_PATH)); - - String segmentID = getOptionValue(OPTION_SEGMENT_ID); - String statistics_enabled = getOptionValue(OPTION_STATISTICS_ENABLED); - String statistics_output = getOptionValue(OPTION_STATISTICS_OUTPUT); - String statistics_sampling_percent = getOptionValue(OPTION_STATISTICS_SAMPLING_PERCENT); - - // ---------------------------------------------------------------------------- - // add metadata to distributed cache - CubeManager cubeMgr = CubeManager.getInstance(KylinConfig.getInstanceFromEnv()); - CubeInstance cube = cubeMgr.getCube(cubeName); - List<TblColRef> columnsNeedDict = cubeMgr.getAllDictColumnsOnFact(cube.getDescriptor()); - - int reducerCount = columnsNeedDict.size(); - int uhcReducerCount = cube.getConfig().getUHCReducerCount(); - - int[] uhcIndex = cubeMgr.getUHCIndex(cube.getDescriptor()); - for(int index : uhcIndex) { - if(index == 1) { - reducerCount += uhcReducerCount - 1; - } - } - - if (reducerCount > 255) { - throw new IllegalArgumentException("The max reducer number for FactDistinctColumnsJob is 255, but now it is " + reducerCount + ", decrease 'kylin.job.uhc.reducer.count'"); - } - - - job.getConfiguration().set(BatchConstants.CFG_CUBE_NAME, cubeName); - job.getConfiguration().set(BatchConstants.CFG_CUBE_SEGMENT_ID, segmentID); - job.getConfiguration().set(BatchConstants.CFG_STATISTICS_ENABLED, statistics_enabled); - job.getConfiguration().set(BatchConstants.CFG_STATISTICS_OUTPUT, statistics_output); - job.getConfiguration().set(BatchConstants.CFG_STATISTICS_SAMPLING_PERCENT, statistics_sampling_percent); - - logger.info("Starting: " + job.getJobName()); - - setJobClasspath(job, cube.getConfig()); - - CubeSegment segment = cube.getSegmentById(segmentID); - if (segment == null) { - logger.error("Failed to find {} in cube {}", segmentID, cube); - for (CubeSegment s : cube.getSegments()) { - logger.error(s.getName() + " with status " + s.getStatus()); - } - throw new IllegalStateException(); - } else { - logger.info("Found segment: " + segment); - } - setupMapper(cube.getSegmentById(segmentID)); - setupReducer(output, "true".equalsIgnoreCase(statistics_enabled) ? reducerCount + 2 : reducerCount); - - attachKylinPropsAndMetadata(cube, job.getConfiguration()); - - return waitForCompletion(job); - - } finally { - if (job != null) - cleanupTempConfFile(job.getConfiguration()); - } - - } - - private void setupMapper(CubeSegment cubeSeg) throws IOException { - IMRTableInputFormat flatTableInputFormat = MRUtil.getBatchCubingInputSide(cubeSeg).getFlatTableInputFormat(); - flatTableInputFormat.configureJob(job); - - job.setMapperClass(FactDistinctHiveColumnsMapper.class); - job.setCombinerClass(FactDistinctColumnsCombiner.class); +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +package org.apache.kylin.engine.mr.steps; + +import java.io.IOException; +import java.util.List; + +import org.apache.commons.cli.Options; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.NullWritable; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.mapreduce.Job; +import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; +import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat; +import org.apache.hadoop.util.ToolRunner; +import org.apache.kylin.common.KylinConfig; +import org.apache.kylin.cube.CubeInstance; +import org.apache.kylin.cube.CubeManager; +import org.apache.kylin.cube.CubeSegment; +import org.apache.kylin.engine.mr.IMRInput.IMRTableInputFormat; +import org.apache.kylin.engine.mr.MRUtil; +import org.apache.kylin.engine.mr.common.AbstractHadoopJob; +import org.apache.kylin.engine.mr.common.BatchConstants; +import org.apache.kylin.metadata.model.TblColRef; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + */ +public class FactDistinctColumnsJob extends AbstractHadoopJob { + protected static final Logger logger = LoggerFactory.getLogger(FactDistinctColumnsJob.class); + + @Override + public int run(String[] args) throws Exception { + Options options = new Options(); + + try { + options.addOption(OPTION_JOB_NAME); + options.addOption(OPTION_CUBE_NAME); + options.addOption(OPTION_CUBING_JOB_ID); + options.addOption(OPTION_OUTPUT_PATH); + options.addOption(OPTION_SEGMENT_ID); + options.addOption(OPTION_STATISTICS_ENABLED); + options.addOption(OPTION_STATISTICS_OUTPUT); + options.addOption(OPTION_STATISTICS_SAMPLING_PERCENT); + parseOptions(options, args); + + job = Job.getInstance(getConf(), getOptionValue(OPTION_JOB_NAME)); + String job_id = getOptionValue(OPTION_CUBING_JOB_ID); + job.getConfiguration().set(BatchConstants.ARG_CUBING_JOB_ID, job_id); + String cubeName = getOptionValue(OPTION_CUBE_NAME); + Path output = new Path(getOptionValue(OPTION_OUTPUT_PATH)); + + String segmentID = getOptionValue(OPTION_SEGMENT_ID); + String statistics_enabled = getOptionValue(OPTION_STATISTICS_ENABLED); + String statistics_output = getOptionValue(OPTION_STATISTICS_OUTPUT); + String statistics_sampling_percent = getOptionValue(OPTION_STATISTICS_SAMPLING_PERCENT); + + // ---------------------------------------------------------------------------- + // add metadata to distributed cache + CubeManager cubeMgr = CubeManager.getInstance(KylinConfig.getInstanceFromEnv()); + CubeInstance cube = cubeMgr.getCube(cubeName); + List<TblColRef> columnsNeedDict = cubeMgr.getAllDictColumnsOnFact(cube.getDescriptor()); + + int reducerCount = columnsNeedDict.size(); + int uhcReducerCount = cube.getConfig().getUHCReducerCount(); + + int[] uhcIndex = cubeMgr.getUHCIndex(cube.getDescriptor()); + for(int index : uhcIndex) { + if(index == 1) { + reducerCount += uhcReducerCount - 1; + } + } + + if (reducerCount > 255) { + throw new IllegalArgumentException("The max reducer number for FactDistinctColumnsJob is 255, but now it is " + reducerCount + ", decrease 'kylin.engine.mr.uhc-reducer-count'"); + } + + + job.getConfiguration().set(BatchConstants.CFG_CUBE_NAME, cubeName); + job.getConfiguration().set(BatchConstants.CFG_CUBE_SEGMENT_ID, segmentID); + job.getConfiguration().set(BatchConstants.CFG_STATISTICS_ENABLED, statistics_enabled); + job.getConfiguration().set(BatchConstants.CFG_STATISTICS_OUTPUT, statistics_output); + job.getConfiguration().set(BatchConstants.CFG_STATISTICS_SAMPLING_PERCENT, statistics_sampling_percent); + + logger.info("Starting: " + job.getJobName()); + + setJobClasspath(job, cube.getConfig()); + + CubeSegment segment = cube.getSegmentById(segmentID); + if (segment == null) { + logger.error("Failed to find {} in cube {}", segmentID, cube); + for (CubeSegment s : cube.getSegments()) { + logger.error(s.getName() + " with status " + s.getStatus()); + } + throw new IllegalStateException(); + } else { + logger.info("Found segment: " + segment); + } + setupMapper(cube.getSegmentById(segmentID)); + setupReducer(output, "true".equalsIgnoreCase(statistics_enabled) ? reducerCount + 2 : reducerCount); + + attachKylinPropsAndMetadata(cube, job.getConfiguration()); + + return waitForCompletion(job); + + } finally { + if (job != null) + cleanupTempConfFile(job.getConfiguration()); + } + + } + + private void setupMapper(CubeSegment cubeSeg) throws IOException { + IMRTableInputFormat flatTableInputFormat = MRUtil.getBatchCubingInputSide(cubeSeg).getFlatTableInputFormat(); + flatTableInputFormat.configureJob(job); + + job.setMapperClass(FactDistinctHiveColumnsMapper.class); + job.setCombinerClass(FactDistinctColumnsCombiner.class); job.setMapOutputKeyClass(SelfDefineSortableKey.class); - job.setMapOutputValueClass(Text.class); - } - - private void setupReducer(Path output, int numberOfReducers) throws IOException { - job.setReducerClass(FactDistinctColumnsReducer.class); - job.setOutputFormatClass(SequenceFileOutputFormat.class); - job.setOutputKeyClass(NullWritable.class); - job.setOutputValueClass(Text.class); - job.setPartitionerClass(FactDistinctColumnPartitioner.class); - job.setNumReduceTasks(numberOfReducers); - - FileOutputFormat.setOutputPath(job, output); - job.getConfiguration().set(BatchConstants.CFG_OUTPUT_PATH, output.toString()); - - deletePath(job.getConfiguration(), output); - } - - public static void main(String[] args) throws Exception { - FactDistinctColumnsJob job = new FactDistinctColumnsJob(); - int exitCode = ToolRunner.run(job, args); - System.exit(exitCode); - } - -} + job.setMapOutputValueClass(Text.class); + } + + private void setupReducer(Path output, int numberOfReducers) throws IOException { + job.setReducerClass(FactDistinctColumnsReducer.class); + job.setOutputFormatClass(SequenceFileOutputFormat.class); + job.setOutputKeyClass(NullWritable.class); + job.setOutputValueClass(Text.class); + job.setPartitionerClass(FactDistinctColumnPartitioner.class); + job.setNumReduceTasks(numberOfReducers); + + FileOutputFormat.setOutputPath(job, output); + job.getConfiguration().set(BatchConstants.CFG_OUTPUT_PATH, output.toString()); + + deletePath(job.getConfiguration(), output); + } + + public static void main(String[] args) throws Exception { + FactDistinctColumnsJob job = new FactDistinctColumnsJob(); + int exitCode = ToolRunner.run(job, args); + System.exit(exitCode); + } + +} http://git-wip-us.apache.org/repos/asf/kylin/blob/826f23f1/engine-mr/src/test/java/org/apache/kylin/engine/mr/TableReaderTest.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/test/java/org/apache/kylin/engine/mr/TableReaderTest.java b/engine-mr/src/test/java/org/apache/kylin/engine/mr/TableReaderTest.java index 8790152..4c43dbc 100644 --- a/engine-mr/src/test/java/org/apache/kylin/engine/mr/TableReaderTest.java +++ b/engine-mr/src/test/java/org/apache/kylin/engine/mr/TableReaderTest.java @@ -1,46 +1,46 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -package org.apache.kylin.engine.mr; - -import static org.junit.Assert.assertEquals; - -import java.io.File; -import java.io.IOException; -import java.util.Arrays; - -import org.junit.Test; - -/** - * @author yangli9 - * - */ -public class TableReaderTest { - - @Test - public void testBasicReader() throws IOException { - File f = new File("src/test/resources/dict/DW_SITES"); - DFSFileTableReader reader = new DFSFileTableReader("file://" + f.getAbsolutePath(), DFSFileTable.DELIM_AUTO, 10); - while (reader.next()) { - assertEquals("[-1, Korea Auction.co.kr, S, 48, 0, 111, 2009-02-11, , DW_OFFPLAT, ]", Arrays.toString(reader.getRow())); - break; - } - reader.close(); - - } -} +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +package org.apache.kylin.engine.mr; + +import static org.junit.Assert.assertEquals; + +import java.io.File; +import java.io.IOException; +import java.util.Arrays; + +import org.junit.Test; + +/** + * @author yangli9 + * + */ +public class TableReaderTest { + + @Test + public void testBasicReader() throws IOException { + File f = new File("src/test/resources/dict/DW_SITES"); + DFSFileTableReader reader = new DFSFileTableReader("file://" + f.getAbsolutePath(), DFSFileTable.DELIM_AUTO, 10); + while (reader.next()) { + assertEquals("[-1, Korea Auction.co.kr, S, 48, 0, 111, 2009-02-11, , DW_OFFPLAT, ]", Arrays.toString(reader.getRow())); + break; + } + reader.close(); + + } +} http://git-wip-us.apache.org/repos/asf/kylin/blob/826f23f1/examples/test_case_data/localmeta/kylin.properties ---------------------------------------------------------------------- diff --git a/examples/test_case_data/localmeta/kylin.properties b/examples/test_case_data/localmeta/kylin.properties index c46442e..8854ac1 100644 --- a/examples/test_case_data/localmeta/kylin.properties +++ b/examples/test_case_data/localmeta/kylin.properties @@ -19,10 +19,10 @@ # Optional information for the owner of kylin platform, it can be your team's email # Currently it will be attached to each kylin's htable attribute -kylin.owner=who...@kylin.apache.org +kylin.storage.hbase.owner-tag=who...@kylin.apache.org # List of web servers in use, this enables one web server instance to sync up with other servers. -#kylin.rest.servers=localhost:7070 +#kylin.server.cluster-servers=localhost:7070 ### SOURCE ### @@ -36,10 +36,10 @@ kylin.metadata.url= kylin.storage.url=hbase # Working folder in HDFS, make sure user has the right access to the hdfs directory -kylin.hdfs.working.dir=/kylin +kylin.env.hdfs-working-dir=/kylin # Compression codec for htable, valid value [none, snappy, lzo, gzip, lz4] -kylin.hbase.default.compression.codec=snappy +kylin.storage.hbase.compression-codec=snappy ### JOB ### @@ -47,33 +47,33 @@ kylin.hbase.default.compression.codec=snappy kylin.job.retry=0 # If true, job engine will not assume that hadoop CLI reside on the same server as it self -# you will have to specify kylin.job.remote.cli.hostname, kylin.job.remote.cli.username and kylin.job.remote.cli.password +# you will have to specify kylin.job.remote-cli-hostname, kylin.job.remote-cli-username and kylin.job.remote-cli-password # It should not be set to "true" unless you're NOT running Kylin.sh on a hadoop client machine # (Thus kylin instance has to ssh to another real hadoop client machine to execute hbase,hive,hadoop commands) -kylin.job.run.as.remote.cmd=false +kylin.job.use-remote-cli=false -# Only necessary when kylin.job.run.as.remote.cmd=true -kylin.job.remote.cli.hostname= +# Only necessary when kylin.job.use-remote-cli=true +kylin.job.remote-cli-hostname= -kylin.job.remote.cli.port=22 +kylin.job.remote-cli-port=22 -# Only necessary when kylin.job.run.as.remote.cmd=true -kylin.job.remote.cli.username= +# Only necessary when kylin.job.use-remote-cli=true +kylin.job.remote-cli-username= -# Only necessary when kylin.job.run.as.remote.cmd=true -kylin.job.remote.cli.password= +# Only necessary when kylin.job.use-remote-cli=true +kylin.job.remote-cli-password= # Used by test cases to prepare synthetic data for sample cube -kylin.job.remote.cli.working.dir=/tmp/kylin +kylin.job.remote-cli-working-dir=/tmp/kylin # Max count of concurrent jobs running -kylin.job.concurrent.max.limit=10 +kylin.job.max-concurrent-jobs=10 # Time interval to check hadoop job status -kylin.job.yarn.app.rest.check.interval.seconds=10 +kylin.engine.mr.yarn-check-interval-seconds=10 # for test -kylin.job.uhc.reducer.count=1 +kylin.engine.mr.uhc-reducer-count=1 ### CUBE ### @@ -118,16 +118,16 @@ saml.context.contextPath=/kylin ### MAIL ### # If true, will send email notification; -mail.enabled=false -mail.host=mail.com -mail.username= -mail.password=need_reset -mail.sender= +kylin.job.notification-enabled=false +kylin.job.notification-mail-host=mail.com +kylin.job.notification-mail-username= +kylin.job.notification-mail-password=need_reset +kylin.job.notification-mail-sender= ### OTHER ### # for tests -kylin.test.bcc.old.key=some-value -kylin.job.mr.config.override.test1=test1 -kylin.job.mr.config.override.test2=test2 -kylin.job.controller.lock=org.apache.kylin.job.lock.MockJobLock +kylin.test.bcc.new.key=some-value +kylin.engine.mr.config-override.test1=test1 +kylin.engine.mr.config-override.test2=test2 +kylin.job.lock=org.apache.kylin.job.lock.MockJobLock http://git-wip-us.apache.org/repos/asf/kylin/blob/826f23f1/examples/test_case_data/localmeta/kylin_job_conf.xml ---------------------------------------------------------------------- diff --git a/examples/test_case_data/localmeta/kylin_job_conf.xml b/examples/test_case_data/localmeta/kylin_job_conf.xml index 7755f0c..8a4406a 100644 --- a/examples/test_case_data/localmeta/kylin_job_conf.xml +++ b/examples/test_case_data/localmeta/kylin_job_conf.xml @@ -1,73 +1,73 @@ -<?xml version="1.0"?> -<!-- - Licensed to the Apache Software Foundation (ASF) under one or more - contributor license agreements. See the NOTICE file distributed with - this work for additional information regarding copyright ownership. - The ASF licenses this file to You under the Apache License, Version 2.0 - (the "License"); you may not use this file except in compliance with - the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. ---> -<configuration> - - <property> - <name>mapreduce.job.split.metainfo.maxsize</name> - <value>-1</value> - <description>The maximum permissible size of the split metainfo file. - The JobTracker won't attempt to read split metainfo files bigger than - the configured value. No limits if set to -1. - </description> - </property> - - - <property> +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<configuration> + + <property> + <name>mapreduce.job.split.metainfo.maxsize</name> + <value>-1</value> + <description>The maximum permissible size of the split metainfo file. + The JobTracker won't attempt to read split metainfo files bigger than + the configured value. No limits if set to -1. + </description> + </property> + + + <property> <name>mapreduce.map.output.compress</name> - <value>true</value> - <description>Compress map outputs</description> - </property> - - <property> + <value>true</value> + <description>Compress map outputs</description> + </property> + + <property> <name>mapreduce.map.output.compress.codec</name> - <value>org.apache.hadoop.io.compress.SnappyCodec</value> - <description>The compression codec to use for map outputs - </description> - </property> - - <property> + <value>org.apache.hadoop.io.compress.SnappyCodec</value> + <description>The compression codec to use for map outputs + </description> + </property> + + <property> <name>mapreduce.output.fileoutputformat.compress</name> - <value>true</value> - <description>Compress the output of a MapReduce job</description> - </property> - - <property> + <value>true</value> + <description>Compress the output of a MapReduce job</description> + </property> + + <property> <name>mapreduce.output.fileoutputformat.compress.codec</name> - <value>org.apache.hadoop.io.compress.SnappyCodec</value> - <description>The compression codec to use for job outputs - </description> - </property> - - <property> + <value>org.apache.hadoop.io.compress.SnappyCodec</value> + <description>The compression codec to use for job outputs + </description> + </property> + + <property> <name>mapreduce.output.fileoutputformat.compress.type</name> - <value>BLOCK</value> - <description>The compression type to use for job outputs</description> - </property> - - <property> - <name>mapreduce.job.max.split.locations</name> - <value>2000</value> - <description>No description</description> - </property> - - <property> - <name>dfs.replication</name> - <value>2</value> - <description>Block replication</description> - </property> + <value>BLOCK</value> + <description>The compression type to use for job outputs</description> + </property> + + <property> + <name>mapreduce.job.max.split.locations</name> + <value>2000</value> + <description>No description</description> + </property> + + <property> + <name>dfs.replication</name> + <value>2</value> + <description>Block replication</description> + </property> </configuration> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/826f23f1/examples/test_case_data/sandbox/hive-site.xml ---------------------------------------------------------------------- diff --git a/examples/test_case_data/sandbox/hive-site.xml b/examples/test_case_data/sandbox/hive-site.xml index 1e78107..d4b7745 100644 --- a/examples/test_case_data/sandbox/hive-site.xml +++ b/examples/test_case_data/sandbox/hive-site.xml @@ -118,7 +118,7 @@ <property> <name>hive.conf.restricted.list</name> - <value>hive.security.authenticator.manager,hive.security.authorization.manager,hive.users.in.admin.role</value> + <value>hive.security.authenticator.manager,hive.security.authorization.manager,kylin.source.hive.users.in.admin.role</value> </property> <property> @@ -722,12 +722,12 @@ </property> <property> - <name>hive.user.install.directory</name> + <name>kylin.source.hive.user.install.directory</name> <value>/user/</value> </property> <property> - <name>hive.users.in.admin.role</name> + <name>kylin.source.hive.users.in.admin.role</name> <value>hue,hive</value> </property> http://git-wip-us.apache.org/repos/asf/kylin/blob/826f23f1/examples/test_case_data/sandbox/kylin.properties ---------------------------------------------------------------------- diff --git a/examples/test_case_data/sandbox/kylin.properties b/examples/test_case_data/sandbox/kylin.properties index de1250f..3c066a6 100644 --- a/examples/test_case_data/sandbox/kylin.properties +++ b/examples/test_case_data/sandbox/kylin.properties @@ -22,19 +22,19 @@ kyin.server.mode=all # Optional information for the owner of kylin platform, it can be your team's email # Currently it will be attached to each kylin's htable attribute -kylin.owner=who...@kylin.apache.org +kylin.storage.hbase.owner-tag=who...@kylin.apache.org # List of web servers in use, this enables one web server instance to sync up with other servers. -kylin.rest.servers=localhost:7070 +kylin.server.cluster-servers=localhost:7070 # Display timezone on UI,format like[GMT+N or GMT-N] -kylin.rest.timezone=GMT+8 +kylin.web.timezone=GMT+8 ### SOURCE ### # Hive client, valid value [cli, beeline] -kylin.hive.client=cli -#kylin.hive.beeline.params=-n root --hiveconf hive.security.authorization.sqlstd.confwhitelist.append='mapreduce.job.*|dfs.*' -u 'jdbc:hive2://localhost:10000' +kylin.source.hive.client=cli +#kylin.source.hive.beeline-params=-n root --hiveconf hive.security.authorization.sqlstd.confwhitelist.append='mapreduce.job.*|dfs.*' -u 'jdbc:hive2://localhost:10000' ### STORAGE ### @@ -45,14 +45,14 @@ kylin.metadata.url=kylin_metadata@hbase kylin.storage.url=hbase # Working folder in HDFS, make sure user has the right access to the hdfs directory -kylin.hdfs.working.dir=/kylin +kylin.env.hdfs-working-dir=/kylin # HBase Cluster FileSystem, which serving hbase, format as hdfs://hbase-cluster:8020 # Leave empty if hbase running on same cluster with hive and mapreduce -#kylin.hbase.cluster.fs= +#kylin.storage.hbase.cluster-fs= -kylin.job.mapreduce.default.reduce.input.mb=500 +kylin.engine.mr.reduce-input-mb=500 ### JOB ### @@ -60,66 +60,66 @@ kylin.job.mapreduce.default.reduce.input.mb=500 kylin.job.retry=0 # If true, job engine will not assume that hadoop CLI reside on the same server as it self -# you will have to specify kylin.job.remote.cli.hostname, kylin.job.remote.cli.username and kylin.job.remote.cli.password +# you will have to specify kylin.job.remote-cli-hostname, kylin.job.remote-cli-username and kylin.job.remote-cli-password # It should not be set to "true" unless you're NOT running Kylin.sh on a hadoop client machine # (Thus kylin instance has to ssh to another real hadoop client machine to execute hbase,hive,hadoop commands) -kylin.job.run.as.remote.cmd=false +kylin.job.use-remote-cli=false -# Only necessary when kylin.job.run.as.remote.cmd=true -kylin.job.remote.cli.hostname=sandbox +# Only necessary when kylin.job.use-remote-cli=true +kylin.job.remote-cli-hostname=sandbox -kylin.job.remote.cli.username=root +kylin.job.remote-cli-username=root -# Only necessary when kylin.job.run.as.remote.cmd=true -kylin.job.remote.cli.password=hadoop +# Only necessary when kylin.job.use-remote-cli=true +kylin.job.remote-cli-password=hadoop # Used by test cases to prepare synthetic data for sample cube -kylin.job.remote.cli.working.dir=/tmp/kylin +kylin.job.remote-cli-working-dir=/tmp/kylin # Max count of concurrent jobs running -kylin.job.concurrent.max.limit=10 +kylin.job.max-concurrent-jobs=10 # Time interval to check hadoop job status -kylin.job.yarn.app.rest.check.interval.seconds=10 +kylin.engine.mr.yarn-check-interval-seconds=10 # Hive database name for putting the intermediate flat tables -kylin.job.hive.database.for.intermediatetable=default +kylin.source.hive.database-for-flat-table=default #default compression codec for htable,snappy,lzo,gzip,lz4 -kylin.hbase.default.compression.codec=gzip +kylin.storage.hbase.compression-codec=gzip # Max reducer number -kylin.job.mapreduce.max.reducer.number=5 +kylin.engine.mr.max-reducer-number=5 # The percentage of the sampling, default 100% -kylin.job.cubing.inmem.sampling.percent=100 +kylin.job.sampling-percentage=100 # The cut size for hbase region, in GB. # E.g, for cube whose capacity be marked as "SMALL", split region per 10GB by default -kylin.hbase.region.cut=0.1 -kylin.hbase.region.count.max=5 +kylin.storage.hbase.region-cut-gb=0.1 +kylin.storage.hbase.max-region-count=5 # The hfile size of GB, smaller hfile leading to the converting hfile MR has more reducers and be faster # set 0 to disable this optimization -kylin.hbase.hfile.size.gb=2 +kylin.storage.hbase.hfile-size-gb=2 kylin.query.udf.massin=org.apache.kylin.query.udf.MassInUDF kylin.query.udf.version=org.apache.kylin.query.udf.VersionUDF # for test -kylin.job.controller.lock=org.apache.kylin.job.lock.MockJobLock -kylin.job.uhc.reducer.count=1 +kylin.job.lock=org.apache.kylin.job.lock.MockJobLock +kylin.engine.mr.uhc-reducer-count=1 ### CUBE ### # dictionary forest cut -kylin.dictionary.forest.trie.size.max_mb=500 +kylin.dictionary.forest-trie-max-mb=500 # 'auto', 'inmem', 'layer' or 'random' for testing kylin.cube.algorithm=random # Enable/disable ACL check for cube query -kylin.query.security.enabled=true +kylin.query.security-enabled=true ### SECURITY ### @@ -135,7 +135,7 @@ acl.adminRole=ROLE_ADMIN ### MAIL ### # If true, will send email notification; -mail.enabled=false +kylin.job.notification-enabled=false ### WEB ### @@ -159,7 +159,7 @@ kylin.web.contact_mail= ### OTHER ### # kylin query metrics percentiles intervals default=60, 300, 3600 -kylin.query.metrics.percentiles.intervals=60, 360, 3600 +kylin.server.query-metrics-percentiles-intervals=60, 360, 3600 # Env DEV|QA|PROD -deploy.env=DEV +kylin.env=DEV http://git-wip-us.apache.org/repos/asf/kylin/blob/826f23f1/examples/test_case_data/sandbox/kylin_job_conf.xml ---------------------------------------------------------------------- diff --git a/examples/test_case_data/sandbox/kylin_job_conf.xml b/examples/test_case_data/sandbox/kylin_job_conf.xml index c2046e6..8f5817e 100644 --- a/examples/test_case_data/sandbox/kylin_job_conf.xml +++ b/examples/test_case_data/sandbox/kylin_job_conf.xml @@ -1,79 +1,79 @@ -<?xml version="1.0"?> -<!-- - Licensed to the Apache Software Foundation (ASF) under one or more - contributor license agreements. See the NOTICE file distributed with - this work for additional information regarding copyright ownership. - The ASF licenses this file to You under the Apache License, Version 2.0 - (the "License"); you may not use this file except in compliance with - the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. ---> -<configuration> - - <property> - <name>mapreduce.job.split.metainfo.maxsize</name> - <value>-1</value> - <description>The maximum permissible size of the split metainfo file. - The JobTracker won't attempt to read split metainfo files bigger than - the configured value. No limits if set to -1. - </description> - </property> - - <property> - <name>mapreduce.map.maxattempts</name> - <value>2</value> - </property> - - <property> - <name>mapreduce.map.output.compress</name> - <value>true</value> - <description>Compress map outputs</description> - </property> - - <property> - <name>mapreduce.map.output.compress.codec</name> - <value>org.apache.hadoop.io.compress.SnappyCodec</value> - <description>The compression codec to use for map outputs - </description> - </property> - - <property> - <name>mapreduce.output.fileoutputformat.compress</name> - <value>true</value> - <description>Compress the output of a MapReduce job</description> - </property> - - <property> - <name>mapreduce.output.fileoutputformat.compress.codec</name> - <value>org.apache.hadoop.io.compress.SnappyCodec</value> - <description>The compression codec to use for job outputs - </description> - </property> - - <!-- - <property> - <name>mapreduce.output.fileoutputformat.compress.type</name> - <value>BLOCK</value> - <description>The compression type to use for job outputs</description> - </property> - --> - <property> - <name>mapreduce.job.max.split.locations</name> - <value>2000</value> - <description>No description</description> - </property> - - <property> - <name>dfs.replication</name> - <value>2</value> - <description>Block replication</description> - </property> - +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<configuration> + + <property> + <name>mapreduce.job.split.metainfo.maxsize</name> + <value>-1</value> + <description>The maximum permissible size of the split metainfo file. + The JobTracker won't attempt to read split metainfo files bigger than + the configured value. No limits if set to -1. + </description> + </property> + + <property> + <name>mapreduce.map.maxattempts</name> + <value>2</value> + </property> + + <property> + <name>mapreduce.map.output.compress</name> + <value>true</value> + <description>Compress map outputs</description> + </property> + + <property> + <name>mapreduce.map.output.compress.codec</name> + <value>org.apache.hadoop.io.compress.SnappyCodec</value> + <description>The compression codec to use for map outputs + </description> + </property> + + <property> + <name>mapreduce.output.fileoutputformat.compress</name> + <value>true</value> + <description>Compress the output of a MapReduce job</description> + </property> + + <property> + <name>mapreduce.output.fileoutputformat.compress.codec</name> + <value>org.apache.hadoop.io.compress.SnappyCodec</value> + <description>The compression codec to use for job outputs + </description> + </property> + + <!-- + <property> + <name>mapreduce.output.fileoutputformat.compress.type</name> + <value>BLOCK</value> + <description>The compression type to use for job outputs</description> + </property> + --> + <property> + <name>mapreduce.job.max.split.locations</name> + <value>2000</value> + <description>No description</description> + </property> + + <property> + <name>dfs.replication</name> + <value>2</value> + <description>Block replication</description> + </property> + </configuration> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/826f23f1/examples/test_case_data/sandbox/kylin_job_conf_inmem.xml ---------------------------------------------------------------------- diff --git a/examples/test_case_data/sandbox/kylin_job_conf_inmem.xml b/examples/test_case_data/sandbox/kylin_job_conf_inmem.xml index 42f1cc4..9e8fc84 100644 --- a/examples/test_case_data/sandbox/kylin_job_conf_inmem.xml +++ b/examples/test_case_data/sandbox/kylin_job_conf_inmem.xml @@ -1,97 +1,97 @@ -<?xml version="1.0"?> -<!-- - Licensed to the Apache Software Foundation (ASF) under one or more - contributor license agreements. See the NOTICE file distributed with - this work for additional information regarding copyright ownership. - The ASF licenses this file to You under the Apache License, Version 2.0 - (the "License"); you may not use this file except in compliance with - the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. ---> -<configuration> - - <property> - <name>mapreduce.job.split.metainfo.maxsize</name> - <value>-1</value> - <description>The maximum permissible size of the split metainfo file. - The JobTracker won't attempt to read split metainfo files bigger than - the configured value. No limits if set to -1. - </description> - </property> - - <property> - <name>mapreduce.map.maxattempts</name> - <value>2</value> - </property> - - <property> +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<configuration> + + <property> + <name>mapreduce.job.split.metainfo.maxsize</name> + <value>-1</value> + <description>The maximum permissible size of the split metainfo file. + The JobTracker won't attempt to read split metainfo files bigger than + the configured value. No limits if set to -1. + </description> + </property> + + <property> + <name>mapreduce.map.maxattempts</name> + <value>2</value> + </property> + + <property> <name>mapreduce.map.output.compress</name> - <value>true</value> - <description>Compress map outputs</description> - </property> - - <property> + <value>true</value> + <description>Compress map outputs</description> + </property> + + <property> <name>mapreduce.map.output.compress.codec</name> - <value>org.apache.hadoop.io.compress.SnappyCodec</value> - <description>The compression codec to use for map outputs - </description> - </property> - - <property> + <value>org.apache.hadoop.io.compress.SnappyCodec</value> + <description>The compression codec to use for map outputs + </description> + </property> + + <property> <name>mapreduce.output.fileoutputformat.compress</name> - <value>true</value> - <description>Compress the output of a MapReduce job</description> - </property> - - <property> + <value>true</value> + <description>Compress the output of a MapReduce job</description> + </property> + + <property> <name>mapreduce.output.fileoutputformat.compress.codec</name> - <value>org.apache.hadoop.io.compress.SnappyCodec</value> - <description>The compression codec to use for job outputs - </description> - </property> - - <property> + <value>org.apache.hadoop.io.compress.SnappyCodec</value> + <description>The compression codec to use for job outputs + </description> + </property> + + <property> <name>mapreduce.output.fileoutputformat.compress.type</name> - <value>BLOCK</value> - <description>The compression type to use for job outputs</description> - </property> - - <property> - <name>mapreduce.job.max.split.locations</name> - <value>2000</value> - <description>No description</description> - </property> - - <property> - <name>dfs.replication</name> - <value>2</value> - <description>Block replication</description> - </property> - - <!--Additional config for in-mem cubing, giving mapper more memory --> - <property> - <name>mapreduce.map.memory.mb</name> - <value>512</value> - <description></description> - </property> - - <property> - <name>mapreduce.map.java.opts</name> - <value>-Xmx384m</value> - <description></description> - </property> - - <property> - <name>mapreduce.task.io.sort.mb</name> - <value>100</value> - <description></description> - </property> - + <value>BLOCK</value> + <description>The compression type to use for job outputs</description> + </property> + + <property> + <name>mapreduce.job.max.split.locations</name> + <value>2000</value> + <description>No description</description> + </property> + + <property> + <name>dfs.replication</name> + <value>2</value> + <description>Block replication</description> + </property> + + <!--Additional config for in-mem cubing, giving mapper more memory --> + <property> + <name>mapreduce.map.memory.mb</name> + <value>512</value> + <description></description> + </property> + + <property> + <name>mapreduce.map.java.opts</name> + <value>-Xmx384m</value> + <description></description> + </property> + + <property> + <name>mapreduce.task.io.sort.mb</name> + <value>100</value> + <description></description> + </property> + </configuration> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/826f23f1/jdbc/src/main/resources/org-apache-kylin-jdbc.properties ---------------------------------------------------------------------- diff --git a/jdbc/src/main/resources/org-apache-kylin-jdbc.properties b/jdbc/src/main/resources/org-apache-kylin-jdbc.properties index cab76a1..68918b1 100644 --- a/jdbc/src/main/resources/org-apache-kylin-jdbc.properties +++ b/jdbc/src/main/resources/org-apache-kylin-jdbc.properties @@ -1,28 +1,28 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -driver.name=Kylin JDBC Driver -driver.version=0.1 -product.name=Kylin -product.version=0.1 -jdbc.compliant=true -driver.version.major=0 -driver.version.minor=8 -database.version.major=0 -database.version.minor=8 +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +driver.name=Kylin JDBC Driver +driver.version=0.1 +product.name=Kylin +product.version=0.1 +jdbc.compliant=true +driver.version.major=0 +driver.version.minor=8 +database.version.major=0 +database.version.minor=8 build.timestamp=20140918-2017 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/826f23f1/kylin-it/src/test/java/org/apache/kylin/job/BaseTestDistributedScheduler.java ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/java/org/apache/kylin/job/BaseTestDistributedScheduler.java b/kylin-it/src/test/java/org/apache/kylin/job/BaseTestDistributedScheduler.java index 910db49..1a0a39d 100644 --- a/kylin-it/src/test/java/org/apache/kylin/job/BaseTestDistributedScheduler.java +++ b/kylin-it/src/test/java/org/apache/kylin/job/BaseTestDistributedScheduler.java @@ -72,7 +72,7 @@ public class BaseTestDistributedScheduler extends HBaseMetadataTestCase { @BeforeClass public static void setup() throws Exception { staticCreateTestMetadata(); - System.setProperty("kylin.job.controller.lock", "org.apache.kylin.storage.hbase.util.ZookeeperDistributedJobLock"); + System.setProperty("kylin.job.lock", "org.apache.kylin.storage.hbase.util.ZookeeperDistributedJobLock"); new File(confDstPath1).getParentFile().mkdirs(); new File(confDstPath2).getParentFile().mkdirs(); @@ -131,7 +131,7 @@ public class BaseTestDistributedScheduler extends HBaseMetadataTestCase { zkClient = null; } - System.clearProperty("kylin.job.controller.lock"); + System.clearProperty("kylin.job.lock"); staticCleanupTestMetadata(); } http://git-wip-us.apache.org/repos/asf/kylin/blob/826f23f1/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java index 3b88dab..67b62d5 100644 --- a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java +++ b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java @@ -123,7 +123,7 @@ public class BuildCubeWithEngine { throw new IOException("mkdir fails"); } } catch (IOException e) { - throw new RuntimeException("failed to create kylin.hdfs.working.dir, Please make sure the user has right to access " + KylinConfig.getInstanceFromEnv().getHdfsWorkingDirectory(), e); + throw new RuntimeException("failed to create kylin.env.hdfs-working-dir, Please make sure the user has right to access " + KylinConfig.getInstanceFromEnv().getHdfsWorkingDirectory(), e); } } http://git-wip-us.apache.org/repos/asf/kylin/blob/826f23f1/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java index 9c1b640..90324b5 100644 --- a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java +++ b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java @@ -105,7 +105,7 @@ public class ITKylinQueryTest extends KylinTestBase { toggles.put(BackdoorToggles.DEBUG_TOGGLE_COPROCESSOR_BEHAVIOR, StorageSideBehavior.SCAN_FILTER_AGGR_CHECKMEM_WITHDELAY.toString());//delay 10ms for every scan BackdoorToggles.setToggles(toggles); - KylinConfig.getInstanceFromEnv().setProperty("kylin.query.coprocessor.timeout.seconds", "3"); + KylinConfig.getInstanceFromEnv().setProperty("kylin.storage.hbase.coprocessor-timeout-seconds", "3"); //these two cubes has RAW measure, will disturb limit push down RemoveBlackoutRealizationsRule.blackList.add("CUBE[name=test_kylin_cube_without_slr_left_join_empty]"); @@ -118,7 +118,7 @@ public class ITKylinQueryTest extends KylinTestBase { RemoveBlackoutRealizationsRule.blackList.remove("CUBE[name=test_kylin_cube_without_slr_left_join_empty]"); RemoveBlackoutRealizationsRule.blackList.remove("CUBE[name=test_kylin_cube_without_slr_inner_join_empty]"); - KylinConfig.getInstanceFromEnv().setProperty("kylin.query.coprocessor.timeout.seconds", "0"); // set timeout to default + KylinConfig.getInstanceFromEnv().setProperty("kylin.storage.hbase.coprocessor-timeout-seconds", "0"); // set timeout to default BackdoorToggles.cleanToggles(); } } http://git-wip-us.apache.org/repos/asf/kylin/blob/826f23f1/kylin-it/src/test/resources/logging.properties ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/resources/logging.properties b/kylin-it/src/test/resources/logging.properties index 8f2713d..0a2b1ca 100644 --- a/kylin-it/src/test/resources/logging.properties +++ b/kylin-it/src/test/resources/logging.properties @@ -1,23 +1,23 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -handlers=java.util.logging.ConsoleHandler -.level=INFO -#org.apache.calcite.plan.RelOptPlanner.level=FINE -java.util.logging.ConsoleHandler.level=ALL +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +handlers=java.util.logging.ConsoleHandler +.level=INFO +#org.apache.calcite.plan.RelOptPlanner.level=FINE +java.util.logging.ConsoleHandler.level=ALL java.util.logging.ConsoleHandler.formatter=org.apache.kylin.common.util.MyLogFormatter \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/826f23f1/query/src/test/java/org/apache/kylin/query/aggregate/DimCountDistinctAggFuncTest.java ---------------------------------------------------------------------- diff --git a/query/src/test/java/org/apache/kylin/query/aggregate/DimCountDistinctAggFuncTest.java b/query/src/test/java/org/apache/kylin/query/aggregate/DimCountDistinctAggFuncTest.java index 9e6134a..794f1e5 100644 --- a/query/src/test/java/org/apache/kylin/query/aggregate/DimCountDistinctAggFuncTest.java +++ b/query/src/test/java/org/apache/kylin/query/aggregate/DimCountDistinctAggFuncTest.java @@ -66,7 +66,7 @@ public class DimCountDistinctAggFuncTest extends LocalFileMetadataTestCase { @Test public void testThreshold() { - System.setProperty("kylin.query.dim.distinct.max", "100"); + System.setProperty("kylin.query.max-dimension-count-distinct", "100"); DimCountDistinctAggFunc.DimDistinctCounter counter = DimCountDistinctAggFunc.init(); @@ -77,6 +77,6 @@ public class DimCountDistinctAggFuncTest extends LocalFileMetadataTestCase { counter = DimCountDistinctAggFunc.add(counter, i); } - System.clearProperty("kylin.query.dim.distinct.max"); + System.clearProperty("kylin.query.max-dimension-count-distinct"); } } http://git-wip-us.apache.org/repos/asf/kylin/blob/826f23f1/server/src/main/webapp/WEB-INF/kylin-servlet.xml ---------------------------------------------------------------------- diff --git a/server/src/main/webapp/WEB-INF/kylin-servlet.xml b/server/src/main/webapp/WEB-INF/kylin-servlet.xml index b86505a..c1e76ab 100644 --- a/server/src/main/webapp/WEB-INF/kylin-servlet.xml +++ b/server/src/main/webapp/WEB-INF/kylin-servlet.xml @@ -12,17 +12,17 @@ limitations under the License. See accompanying LICENSE file. --> -<beans xmlns="http://www.springframework.org/schema/beans"; - xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; - xmlns:context="http://www.springframework.org/schema/context"; - xmlns:mvc="http://www.springframework.org/schema/mvc"; - xmlns:task="http://www.springframework.org/schema/task"; - xsi:schemaLocation="http://www.springframework.org/schema/beans - http://www.springframework.org/schema/beans/spring-beans-3.1.xsd - http://www.springframework.org/schema/context - http://www.springframework.org/schema/context/spring-context-3.1.xsd - http://www.springframework.org/schema/task - http://www.springframework.org/schema/task/spring-task-3.1.xsd - http://www.springframework.org/schema/mvc - http://www.springframework.org/schema/mvc/spring-mvc-3.1.xsd";> +<beans xmlns="http://www.springframework.org/schema/beans"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xmlns:context="http://www.springframework.org/schema/context"; + xmlns:mvc="http://www.springframework.org/schema/mvc"; + xmlns:task="http://www.springframework.org/schema/task"; + xsi:schemaLocation="http://www.springframework.org/schema/beans + http://www.springframework.org/schema/beans/spring-beans-3.1.xsd + http://www.springframework.org/schema/context + http://www.springframework.org/schema/context/spring-context-3.1.xsd + http://www.springframework.org/schema/task + http://www.springframework.org/schema/task/spring-task-3.1.xsd + http://www.springframework.org/schema/mvc + http://www.springframework.org/schema/mvc/spring-mvc-3.1.xsd";> </beans> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/826f23f1/server/src/main/webapp/WEB-INF/web.xml ---------------------------------------------------------------------- diff --git a/server/src/main/webapp/WEB-INF/web.xml b/server/src/main/webapp/WEB-INF/web.xml index 7a4420c..8a43b33 100644 --- a/server/src/main/webapp/WEB-INF/web.xml +++ b/server/src/main/webapp/WEB-INF/web.xml @@ -1,4 +1,4 @@ -<?xml version="1.0" encoding="UTF-8"?> +<?xml version="1.0" encoding="UTF-8"?> <!-- Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -13,92 +13,92 @@ limitations under the License. See accompanying LICENSE file. --> -<web-app xmlns="http://java.sun.com/xml/ns/javaee"; - xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; - xsi:schemaLocation="http://java.sun.com/xml/ns/javaee - http://java.sun.com/xml/ns/javaee/web-app_2_5.xsd"; - version="2.5"> - - <display-name>Kylin REST Service</display-name> - - <welcome-file-list> - <welcome-file>/index.html</welcome-file> - </welcome-file-list> - - <error-page> - <error-code>404</error-code> - <location>/index.html</location> - </error-page> - - <context-param> - <param-name>log4jConfigLocation</param-name> - <param-value>classpath:kylin-server-log4j.properties</param-value> - </context-param> - <context-param> - <param-name>contextConfigLocation</param-name> - <param-value> - classpath:applicationContext.xml - classpath:kylinSecurity.xml - classpath*:kylin-*-plugin.xml - </param-value> - </context-param> - - <listener> - <listener-class>org.apache.kylin.rest.util.Log4jConfigListener</listener-class> - </listener> - - <listener> - <listener-class>org.springframework.web.context.ContextLoaderListener</listener-class> - </listener> - <listener> - <listener-class> - org.springframework.security.web.session.HttpSessionEventPublisher - </listener-class> - </listener> - -<filter> - <filter-name>CORS</filter-name> - <filter-class>com.thetransactioncompany.cors.CORSFilter</filter-class> - <init-param> - <param-name>cors.supportedHeaders</param-name> - <param-value>Authorization,Origin, No-Cache, X-Requested-With, If-Modified-Since, Pragma, Last-Modified, Cache-Control, Expires, Content-Type, X-E4M-With, Accept</param-value> - </init-param> - <init-param> - <param-name>cors.supportedMethods</param-name> - <param-value>GET, POST, PUT, DELETE, OPTIONS</param-value> - </init-param> - <init-param> - <param-name>cors.supportsCredentials </param-name> - <param-value>true</param-value> - </init-param> -</filter> - -<filter-mapping> - <filter-name>CORS</filter-name> - <url-pattern>/*</url-pattern> -</filter-mapping> - - <!-- - Apply Spring Security Filter to all Requests - --> - <filter> - <filter-name>springSecurityFilterChain</filter-name> - <filter-class>org.springframework.web.filter.DelegatingFilterProxy</filter-class> - </filter> - <filter-mapping> - <filter-name>springSecurityFilterChain</filter-name> - <url-pattern>/*</url-pattern> - </filter-mapping> - - <servlet> - <servlet-name>kylin</servlet-name> - <servlet-class>org.springframework.web.servlet.DispatcherServlet</servlet-class> - <load-on-startup>1</load-on-startup> - </servlet> - <servlet-mapping> - <servlet-name>kylin</servlet-name> - <url-pattern>/api/*</url-pattern> - </servlet-mapping> - - <distributable /> -</web-app> +<web-app xmlns="http://java.sun.com/xml/ns/javaee"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://java.sun.com/xml/ns/javaee + http://java.sun.com/xml/ns/javaee/web-app_2_5.xsd"; + version="2.5"> + + <display-name>Kylin REST Service</display-name> + + <welcome-file-list> + <welcome-file>/index.html</welcome-file> + </welcome-file-list> + + <error-page> + <error-code>404</error-code> + <location>/index.html</location> + </error-page> + + <context-param> + <param-name>log4jConfigLocation</param-name> + <param-value>classpath:kylin-server-log4j.properties</param-value> + </context-param> + <context-param> + <param-name>contextConfigLocation</param-name> + <param-value> + classpath:applicationContext.xml + classpath:kylinSecurity.xml + classpath*:kylin-*-plugin.xml + </param-value> + </context-param> + + <listener> + <listener-class>org.apache.kylin.rest.util.Log4jConfigListener</listener-class> + </listener> + + <listener> + <listener-class>org.springframework.web.context.ContextLoaderListener</listener-class> + </listener> + <listener> + <listener-class> + org.springframework.security.web.session.HttpSessionEventPublisher + </listener-class> + </listener> + +<filter> + <filter-name>CORS</filter-name> + <filter-class>com.thetransactioncompany.cors.CORSFilter</filter-class> + <init-param> + <param-name>cors.supportedHeaders</param-name> + <param-value>Authorization,Origin, No-Cache, X-Requested-With, If-Modified-Since, Pragma, Last-Modified, Cache-Control, Expires, Content-Type, X-E4M-With, Accept</param-value> + </init-param> + <init-param> + <param-name>cors.supportedMethods</param-name> + <param-value>GET, POST, PUT, DELETE, OPTIONS</param-value> + </init-param> + <init-param> + <param-name>cors.supportsCredentials </param-name> + <param-value>true</param-value> + </init-param> +</filter> + +<filter-mapping> + <filter-name>CORS</filter-name> + <url-pattern>/*</url-pattern> +</filter-mapping> + + <!-- + Apply Spring Security Filter to all Requests + --> + <filter> + <filter-name>springSecurityFilterChain</filter-name> + <filter-class>org.springframework.web.filter.DelegatingFilterProxy</filter-class> + </filter> + <filter-mapping> + <filter-name>springSecurityFilterChain</filter-name> + <url-pattern>/*</url-pattern> + </filter-mapping> + + <servlet> + <servlet-name>kylin</servlet-name> + <servlet-class>org.springframework.web.servlet.DispatcherServlet</servlet-class> + <load-on-startup>1</load-on-startup> + </servlet> + <servlet-mapping> + <servlet-name>kylin</servlet-name> + <url-pattern>/api/*</url-pattern> + </servlet-mapping> + + <distributable /> +</web-app> http://git-wip-us.apache.org/repos/asf/kylin/blob/826f23f1/server/src/test/java/org/apache/kylin/rest/metrics/QueryMetricsTest.java ---------------------------------------------------------------------- diff --git a/server/src/test/java/org/apache/kylin/rest/metrics/QueryMetricsTest.java b/server/src/test/java/org/apache/kylin/rest/metrics/QueryMetricsTest.java index 47d3f7b..bd1da59 100644 --- a/server/src/test/java/org/apache/kylin/rest/metrics/QueryMetricsTest.java +++ b/server/src/test/java/org/apache/kylin/rest/metrics/QueryMetricsTest.java @@ -47,7 +47,7 @@ public class QueryMetricsTest extends ServiceTestBase { @Test public void testQueryMetrics() throws Exception { - System.setProperty("kylin.query.metrics.enabled", "true"); + System.setProperty("kylin.server.query-metrics-enabled", "true"); QueryMetricsFacade.init(); SQLRequest sqlRequest = new SQLRequest(); @@ -108,7 +108,7 @@ public class QueryMetricsTest extends ServiceTestBase { Assert.assertEquals(1L, mBeanServer.getAttribute(objectName, "QuerySuccessCount")); Assert.assertEquals(1L, mBeanServer.getAttribute(objectName, "QueryFailCount")); - System.clearProperty("kylin.query.metrics.enabled"); + System.clearProperty("kylin.server.query-metrics-enabled"); } } http://git-wip-us.apache.org/repos/asf/kylin/blob/826f23f1/server/src/test/java/org/apache/kylin/rest/service/CacheServiceTest.java ---------------------------------------------------------------------- diff --git a/server/src/test/java/org/apache/kylin/rest/service/CacheServiceTest.java b/server/src/test/java/org/apache/kylin/rest/service/CacheServiceTest.java index 19c9c6e..8ea9cd2 100644 --- a/server/src/test/java/org/apache/kylin/rest/service/CacheServiceTest.java +++ b/server/src/test/java/org/apache/kylin/rest/service/CacheServiceTest.java @@ -73,9 +73,9 @@ public class CacheServiceTest extends LocalFileMetadataTestCase { public static void beforeClass() throws Exception { staticCreateTestMetadata(); configA = KylinConfig.getInstanceFromEnv(); - configA.setProperty("kylin.rest.servers", "localhost:7777"); + configA.setProperty("kylin.server.cluster-servers", "localhost:7777"); configB = KylinConfig.createKylinConfig(configA); - configB.setProperty("kylin.rest.servers", "localhost:7777"); + configB.setProperty("kylin.server.cluster-servers", "localhost:7777"); configB.setMetadataUrl("../examples/test_metadata"); server = new Server(7777); http://git-wip-us.apache.org/repos/asf/kylin/blob/826f23f1/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java ---------------------------------------------------------------------- diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java index 3b66287..6eb1a28 100644 --- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java +++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java @@ -284,7 +284,7 @@ public class HiveMRInput implements IMRInput { logger.debug("Row count of table '" + intermediateTable + "' is " + rowCount); if (rowCount == 0) { if (!config.isEmptySegmentAllowed()) { - stepLogger.log("Detect upstream hive table is empty, " + "fail the job because \"kylin.job.allow.empty.segment\" = \"false\""); + stepLogger.log("Detect upstream hive table is empty, " + "fail the job because \"kylin.job.allow-empty-segment\" = \"false\""); return new ExecuteResult(ExecuteResult.State.ERROR, stepLogger.getBufferedLog()); } else { return new ExecuteResult(ExecuteResult.State.SUCCEED, "Row count is 0, no need to redistribute");
