Repository: kylin Updated Branches: refs/heads/master 0d46e4a27 -> 4d1676a66
KYLIN-2312 display server config/environment by order in system tab Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4d1676a6 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4d1676a6 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4d1676a6 Branch: refs/heads/master Commit: 4d1676a66889283647b7c63022505234fa9d328c Parents: 0d46e4a Author: Billy Liu <billy...@apache.org> Authored: Thu Dec 22 18:55:51 2016 +0800 Committer: Billy Liu <billy...@apache.org> Committed: Thu Dec 22 18:55:51 2016 +0800 ---------------------------------------------------------------------- .../common/BackwardCompatibilityConfig.java | 9 + .../org/apache/kylin/common/KylinConfig.java | 21 +- .../kylin/common/util/OrderedProperties.java | 388 +++++++++++++++++++ .../kylin/job/constant/ExecutableConstants.java | 4 - .../apache/kylin/rest/service/AdminService.java | 16 +- 5 files changed, 417 insertions(+), 21 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/4d1676a6/core-common/src/main/java/org/apache/kylin/common/BackwardCompatibilityConfig.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/BackwardCompatibilityConfig.java b/core-common/src/main/java/org/apache/kylin/common/BackwardCompatibilityConfig.java index 21df932..426ebb9 100644 --- a/core-common/src/main/java/org/apache/kylin/common/BackwardCompatibilityConfig.java +++ b/core-common/src/main/java/org/apache/kylin/common/BackwardCompatibilityConfig.java @@ -29,6 +29,7 @@ import java.util.Properties; import java.util.Stack; import org.apache.commons.io.IOUtils; +import org.apache.kylin.common.util.OrderedProperties; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -121,6 +122,14 @@ public class BackwardCompatibilityConfig { return result; } + public OrderedProperties check(OrderedProperties props){ + OrderedProperties result = new OrderedProperties(); + for (Entry<String, String> kv : props.entrySet()) { + result.setProperty(check(kv.getKey()), kv.getValue()); + } + return result; + } + // ============================================================================ public static void main(String[] args) throws IOException { http://git-wip-us.apache.org/repos/asf/kylin/blob/4d1676a6/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java index acd4398..f169142 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java @@ -25,7 +25,6 @@ import java.io.IOException; import java.io.InputStream; import java.io.PrintWriter; import java.io.StringReader; -import java.io.StringWriter; import java.nio.charset.Charset; import java.util.Enumeration; import java.util.Map; @@ -35,6 +34,7 @@ import org.apache.commons.io.IOUtils; import org.apache.commons.lang.StringUtils; import org.apache.kylin.common.restclient.RestClient; import org.apache.kylin.common.util.ClassUtil; +import org.apache.kylin.common.util.OrderedProperties; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -282,18 +282,15 @@ public class KylinConfig extends KylinConfigBase { } public String getConfigAsString() throws IOException { - final StringWriter stringWriter = new StringWriter(); - list(new PrintWriter(stringWriter)); - return stringWriter.toString(); - } - - private void list(PrintWriter out) { - Properties props = getAllProperties(); - for (Enumeration<?> e = props.keys(); e.hasMoreElements();) { - String key = (String) e.nextElement(); - String val = (String) props.get(key); - out.println(key + "=" + val); + File propertiesFile = getKylinPropertiesFile(); + OrderedProperties orderedProperties = new OrderedProperties(); + orderedProperties.load(new FileInputStream(propertiesFile)); + orderedProperties = BCC.check(orderedProperties); + final StringBuilder sb = new StringBuilder(); + for (Map.Entry<String, String> entry : orderedProperties.entrySet()) { + sb.append(entry.getKey() + "=" + entry.getValue()).append('\n'); } + return sb.toString(); } public KylinConfig base() { http://git-wip-us.apache.org/repos/asf/kylin/blob/4d1676a6/core-common/src/main/java/org/apache/kylin/common/util/OrderedProperties.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/util/OrderedProperties.java b/core-common/src/main/java/org/apache/kylin/common/util/OrderedProperties.java new file mode 100644 index 0000000..0e5a3f9 --- /dev/null +++ b/core-common/src/main/java/org/apache/kylin/common/util/OrderedProperties.java @@ -0,0 +1,388 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +package org.apache.kylin.common.util; + +import java.io.IOException; +import java.io.InputStream; +import java.io.InvalidObjectException; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.io.OutputStream; +import java.io.PrintStream; +import java.io.PrintWriter; +import java.io.Reader; +import java.io.Serializable; +import java.io.Writer; +import java.util.Arrays; +import java.util.Comparator; +import java.util.Enumeration; +import java.util.InvalidPropertiesFormatException; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.Map; +import java.util.Properties; +import java.util.Set; +import java.util.TreeMap; +import java.util.Vector; + +/** + * Modified from the etiennestuder/java-ordered-properties in https://github.com/etiennestuder/java-ordered-properties + * + * This class provides an alternative to the JDK's {@link Properties} class. It fixes the design flaw of using + * inheritance over composition, while keeping up the same APIs as the original class. Keys and values are + * guaranteed to be of type {@link String}. + * <p/> + * This class is not synchronized, contrary to the original implementation. + * <p/> + * As additional functionality, this class keeps its properties in a well-defined order. By default, the order + * is the one in which the individual properties have been added, either through explicit API calls or through + * reading them top-to-bottom from a properties file. + * <p/> + * Currently, this class does not support the concept of default properties, contrary to the original implementation. + * <p/> + * <strong>Note that this implementation is not synchronized.</strong> If multiple threads access ordered + * properties concurrently, and at least one of the threads modifies the ordered properties structurally, it + * <em>must</em> be synchronized externally. This is typically accomplished by synchronizing on some object + * that naturally encapsulates the properties. + * <p/> + * Note that the actual (and quite complex) logic of parsing and storing properties from and to a stream + * is delegated to the {@link Properties} class from the JDK. + * + * @see Properties + */ +public final class OrderedProperties implements Serializable { + + private static final long serialVersionUID = 1L; + + private transient Map<String, String> properties; + + public OrderedProperties() { + this(new LinkedHashMap<String, String>()); + } + + public OrderedProperties(Map<String, String> properties) { + this.properties = properties; + } + + /** + * Creates a new instance that will have both the same property entries and + * the same behavior as the given source. + * <p/> + * Note that the source instance and the copy instance will share the same + * comparator instance if a custom ordering had been configured on the source. + * + * @param source the source to copy from + * @return the copy + */ + public static OrderedProperties copyOf(OrderedProperties source) { + // create a copy that has the same behaviour + OrderedPropertiesBuilder builder = new OrderedPropertiesBuilder(); + if (source.properties instanceof TreeMap) { + builder.withOrdering(((TreeMap<String, String>) source.properties).comparator()); + } + OrderedProperties result = builder.build(); + + // copy the properties from the source to the target + for (Map.Entry<String, String> entry : source.entrySet()) { + result.setProperty(entry.getKey(), entry.getValue()); + } + return result; + } + + /** + * See {@link Properties#getProperty(String)}. + */ + public String getProperty(String key) { + return properties.get(key); + } + + /** + * See {@link Properties#getProperty(String, String)}. + */ + public String getProperty(String key, String defaultValue) { + String value = properties.get(key); + return (value == null) ? defaultValue : value; + } + + /** + * See {@link Properties#setProperty(String, String)}. + */ + public String setProperty(String key, String value) { + return properties.put(key, value); + } + + /** + * Removes the property with the specified key, if it is present. Returns + * the value of the property, or <tt>null</tt> if there was no property with + * the specified key. + * + * @param key the key of the property to remove + * @return the previous value of the property, or <tt>null</tt> if there was no property with the specified key + */ + public String removeProperty(String key) { + return properties.remove(key); + } + + /** + * Returns <tt>true</tt> if there is a property with the specified key. + * + * @param key the key whose presence is to be tested + */ + public boolean containsProperty(String key) { + return properties.containsKey(key); + } + + /** + * See {@link Properties#size()}. + */ + public int size() { + return properties.size(); + } + + /** + * See {@link Properties#isEmpty()}. + */ + public boolean isEmpty() { + return properties.isEmpty(); + } + + /** + * See {@link Properties#propertyNames()}. + */ + public Enumeration<String> propertyNames() { + return new Vector<String>(properties.keySet()).elements(); + } + + /** + * See {@link Properties#stringPropertyNames()}. + */ + public Set<String> stringPropertyNames() { + return new LinkedHashSet<String>(properties.keySet()); + } + + /** + * See {@link Properties#entrySet()}. + */ + public Set<Map.Entry<String, String>> entrySet() { + return new LinkedHashSet<Map.Entry<String, String>>(properties.entrySet()); + } + + /** + * See {@link Properties#load(InputStream)}. + */ + public void load(InputStream stream) throws IOException { + CustomProperties customProperties = new CustomProperties(this.properties); + customProperties.load(stream); + } + + /** + * See {@link Properties#load(Reader)}. + */ + public void load(Reader reader) throws IOException { + CustomProperties customProperties = new CustomProperties(this.properties); + customProperties.load(reader); + } + + /** + * See {@link Properties#loadFromXML(InputStream)}. + */ + @SuppressWarnings("DuplicateThrows") + public void loadFromXML(InputStream stream) throws IOException, InvalidPropertiesFormatException { + CustomProperties customProperties = new CustomProperties(this.properties); + customProperties.loadFromXML(stream); + } + + /** + * See {@link Properties#store(OutputStream, String)}. + */ + public void store(OutputStream stream, String comments) throws IOException { + CustomProperties customProperties = new CustomProperties(this.properties); + customProperties.store(stream, comments); + } + + /** + * See {@link Properties#store(Writer, String)}. + */ + public void store(Writer writer, String comments) throws IOException { + CustomProperties customProperties = new CustomProperties(this.properties); + customProperties.store(writer, comments); + } + + /** + * See {@link Properties#storeToXML(OutputStream, String)}. + */ + public void storeToXML(OutputStream stream, String comment) throws IOException { + CustomProperties customProperties = new CustomProperties(this.properties); + customProperties.storeToXML(stream, comment); + } + + /** + * See {@link Properties#storeToXML(OutputStream, String, String)}. + */ + public void storeToXML(OutputStream stream, String comment, String encoding) throws IOException { + CustomProperties customProperties = new CustomProperties(this.properties); + customProperties.storeToXML(stream, comment, encoding); + } + + /** + * See {@link Properties#list(PrintStream)}. + */ + public void list(PrintStream stream) { + CustomProperties customProperties = new CustomProperties(this.properties); + customProperties.list(stream); + } + + /** + * See {@link Properties#list(PrintWriter)}. + */ + public void list(PrintWriter writer) { + CustomProperties customProperties = new CustomProperties(this.properties); + customProperties.list(writer); + } + + /** + * Convert this instance to a {@link Properties} instance. + * + * @return the {@link Properties} instance + */ + public Properties toJdkProperties() { + Properties jdkProperties = new Properties(); + for (Map.Entry<String, String> entry : this.entrySet()) { + jdkProperties.put(entry.getKey(), entry.getValue()); + } + return jdkProperties; + } + + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + + if (other == null || getClass() != other.getClass()) { + return false; + } + + OrderedProperties that = (OrderedProperties) other; + return Arrays.equals(properties.entrySet().toArray(), that.properties.entrySet().toArray()); + } + + @Override + public int hashCode() { + return Arrays.hashCode(properties.entrySet().toArray()); + } + + private void writeObject(ObjectOutputStream stream) throws IOException { + stream.defaultWriteObject(); + stream.writeObject(properties); + } + + @SuppressWarnings("unchecked") + private void readObject(ObjectInputStream stream) throws IOException, ClassNotFoundException { + stream.defaultReadObject(); + properties = (Map<String, String>) stream.readObject(); + } + + private void readObjectNoData() throws InvalidObjectException { + throw new InvalidObjectException("Stream data required"); + } + + /** + * See {@link Properties#toString()}. + */ + @Override + public String toString() { + return properties.toString(); + } + + public Map<String, String> getProperties() { + return properties; + } + + /** + * Builder for {@link OrderedProperties} instances. + */ + public static final class OrderedPropertiesBuilder { + + private Comparator<? super String> comparator; + + /** + * Use a custom ordering of the keys. + * + * @param comparator the ordering to apply on the keys + * @return the builder + */ + public OrderedPropertiesBuilder withOrdering(Comparator<? super String> comparator) { + this.comparator = comparator; + return this; + } + + /** + * Builds a new {@link OrderedProperties} instance. + * + * @return the new instance + */ + public OrderedProperties build() { + Map<String, String> properties = (this.comparator != null) ? new TreeMap<String, String>(comparator) : new LinkedHashMap<String, String>(); + return new OrderedProperties(properties); + } + + } + + /** + * Custom {@link Properties} that delegates reading, writing, and enumerating properties to the + * backing {@link OrderedProperties} instance's properties. + */ + private static final class CustomProperties extends Properties { + + private final Map<String, String> targetProperties; + + private CustomProperties(Map<String, String> targetProperties) { + this.targetProperties = targetProperties; + } + + @Override + public Object get(Object key) { + return targetProperties.get(key); + } + + @Override + public Object put(Object key, Object value) { + return targetProperties.put((String) key, (String) value); + } + + @Override + public String getProperty(String key) { + return targetProperties.get(key); + } + + @Override + public Enumeration<Object> keys() { + return new Vector<Object>(targetProperties.keySet()).elements(); + } + + @SuppressWarnings("NullableProblems") + @Override + public Set<Object> keySet() { + return new LinkedHashSet<Object>(targetProperties.keySet()); + } + + } + +} http://git-wip-us.apache.org/repos/asf/kylin/blob/4d1676a6/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java ---------------------------------------------------------------------- diff --git a/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java b/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java index cec2e5d..4ae3add 100644 --- a/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java +++ b/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java @@ -37,7 +37,6 @@ public final class ExecutableConstants { public static final String STEP_NAME_BUILD_DICTIONARY = "Build Dimension Dictionary"; public static final String STEP_NAME_CREATE_FLAT_HIVE_TABLE = "Create Intermediate Flat Hive Table"; public static final String STEP_NAME_MATERIALIZE_HIVE_VIEW_IN_LOOKUP = "Materialize Hive View in Lookup Tables"; - public static final String STEP_NAME_COUNT_HIVE_TABLE = "Count Source Table"; public static final String STEP_NAME_FACT_DISTINCT_COLUMNS = "Extract Fact Table Distinct Columns"; public static final String STEP_NAME_BUILD_BASE_CUBOID = "Build Base Cuboid Data"; public static final String STEP_NAME_BUILD_IN_MEM_CUBE = "Build Cube"; @@ -55,9 +54,6 @@ public final class ExecutableConstants { public static final String STEP_NAME_KAFKA_CLEANUP = "Kafka Intermediate File Cleanup"; public static final String STEP_NAME_GARBAGE_COLLECTION = "Garbage Collection"; public static final String STEP_NAME_GARBAGE_COLLECTION_HDFS = "Garbage Collection on HDFS"; - public static final String STEP_NAME_BUILD_II = "Build Inverted Index"; - public static final String STEP_NAME_CONVERT_II_TO_HFILE = "Convert Inverted Index Data to HFile"; - public static final String STEP_NAME_UPDATE_II_INFO = "Update Inverted Index Info"; public static final String STEP_NAME_REDISTRIBUTE_FLAT_HIVE_TABLE = "Redistribute Flat Hive Table"; public static final String NOTIFY_EMAIL_TEMPLATE = "<div><b>Build Result of Job ${job_name}</b><pre><ul>" + "<li>Build Result: <b>${result}</b></li>" + "<li>Job Engine: ${job_engine}</li>" + "<li>Env: ${env_name}</li>" + "<li>Project: ${project_name}</li>" + "<li>Cube Name: ${cube_name}</li>" + "<li>Source Records Count: ${source_records_count}</li>" + "<li>Start Time: ${start_time}</li>" + "<li>Duration: ${duration}</li>" + "<li>MR Waiting: ${mr_waiting}</li>" + "<li>Last Update Time: ${last_update_time}</li>" + "<li>Submitter: ${submitter}</li>" + "<li>Error Log: ${error_log}</li>" + "</ul></pre><div/>"; } http://git-wip-us.apache.org/repos/asf/kylin/blob/4d1676a6/server-base/src/main/java/org/apache/kylin/rest/service/AdminService.java ---------------------------------------------------------------------- diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/AdminService.java b/server-base/src/main/java/org/apache/kylin/rest/service/AdminService.java index ace0388..a02804e 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/service/AdminService.java +++ b/server-base/src/main/java/org/apache/kylin/rest/service/AdminService.java @@ -22,10 +22,12 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.util.Map; import java.util.Properties; +import java.util.TreeMap; import org.apache.commons.configuration.ConfigurationException; import org.apache.commons.configuration.PropertiesConfiguration; import org.apache.kylin.common.KylinConfig; +import org.apache.kylin.common.util.OrderedProperties; import org.apache.kylin.rest.constant.Constant; import org.apache.kylin.rest.exception.InternalErrorException; import org.apache.kylin.tool.StorageCleanupJob; @@ -49,7 +51,7 @@ public class AdminService extends BasicService { @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN) public String getEnv() { PropertiesConfiguration tempConfig = new PropertiesConfiguration(); - + OrderedProperties orderedProperties = new OrderedProperties(new TreeMap<String, String>()); // Add Java Env try { @@ -59,14 +61,18 @@ public class AdminService extends BasicService { Map<String, String> env = System.getenv(); for (Map.Entry<String, String> entry : env.entrySet()) { - tempConfig.addProperty(entry.getKey(), entry.getValue()); + orderedProperties.setProperty(entry.getKey(), entry.getValue()); } // properties - Properties proterties = System.getProperties(); + Properties properties = System.getProperties(); - for (Map.Entry<Object, Object> entry : proterties.entrySet()) { - tempConfig.setProperty((String) entry.getKey(), entry.getValue()); + for (Map.Entry<Object, Object> entry : properties.entrySet()) { + orderedProperties.setProperty((String) entry.getKey(), (String) entry.getValue()); + } + + for (Map.Entry<String, String> entry : orderedProperties.entrySet()) { + tempConfig.addProperty(entry.getKey(), entry.getValue()); } // do save
