Repository: kylin Updated Branches: refs/heads/1.5.4.1-rc1 18a14547c -> d33ba6aaa
Revert "KYLIN-2012 more robust approach to hive schema changes" This reverts commit 17569f6c32a373f599ef7689f9506b5af5ed68bd. Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d33ba6aa Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d33ba6aa Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d33ba6aa Branch: refs/heads/1.5.4.1-rc1 Commit: d33ba6aaaddf526b78225eab2a9280a39dd2058d Parents: 18a1454 Author: shaofengshi <shaofeng...@apache.org> Authored: Fri Sep 23 18:11:29 2016 +0800 Committer: shaofengshi <shaofeng...@apache.org> Committed: Fri Sep 23 18:11:29 2016 +0800 ---------------------------------------------------------------------- .../org/apache/kylin/cube/CubeDescManager.java | 62 +++--- .../org/apache/kylin/cube/CubeInstance.java | 11 +- .../java/org/apache/kylin/cube/CubeManager.java | 47 ++-- .../org/apache/kylin/cube/model/CubeDesc.java | 47 ++-- .../model/validation/CubeMetadataValidator.java | 32 ++- .../realization/RealizationStatusEnum.java | 2 +- .../kylin/rest/controller/CubeController.java | 44 ++-- .../apache/kylin/rest/service/CacheService.java | 11 +- .../apache/kylin/rest/service/CubeService.java | 15 ++ .../apache/kylin/rest/service/JobService.java | 6 - .../kylin/rest/service/CubeServiceTest.java | 1 + .../source/hive/HiveSourceTableLoader.java | 33 +-- .../apache/kylin/source/hive/SchemaChecker.java | 216 ------------------- webapp/app/css/AdminLTE.css | 4 +- webapp/app/partials/cubes/cubes.html | 22 +- 15 files changed, 200 insertions(+), 353 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/d33ba6aa/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java index 1b1cf70..33a6830 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java @@ -23,6 +23,7 @@ import java.util.ArrayList; import java.util.List; import java.util.concurrent.ConcurrentHashMap; +import org.apache.commons.lang3.StringUtils; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.persistence.JsonSerializer; import org.apache.kylin.common.persistence.ResourceStore; @@ -35,7 +36,6 @@ import org.apache.kylin.cube.model.validation.CubeMetadataValidator; import org.apache.kylin.cube.model.validation.ValidateContext; import org.apache.kylin.metadata.MetadataConstants; import org.apache.kylin.metadata.MetadataManager; -import org.apache.kylin.metadata.realization.RealizationStatusEnum; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -110,34 +110,30 @@ public class CubeDescManager { * @throws IOException */ public CubeDesc reloadCubeDescLocal(String name) throws IOException { - // Broken CubeDesc is not allowed to be saved and broadcast. - CubeDesc ndesc = loadCubeDesc(CubeDesc.concatResourcePath(name), false); - cubeDescMap.putLocal(ndesc.getName(), ndesc); - Cuboid.reloadCache(name); + // Save Source + String path = CubeDesc.concatResourcePath(name); - // if related cube is in DESCBROKEN state before, change it back to DISABLED - CubeManager cubeManager = CubeManager.getInstance(config); - for (CubeInstance cube : cubeManager.getCubesByDesc(name)) { - if (cube.getStatus() == RealizationStatusEnum.DESCBROKEN) { - cubeManager.reloadCubeLocal(cube.getName()); - } - } + // Reload the CubeDesc + CubeDesc ndesc = loadCubeDesc(path); + // Here replace the old one + cubeDescMap.putLocal(ndesc.getName(), ndesc); + Cuboid.reloadCache(name); return ndesc; } - private CubeDesc loadCubeDesc(String path, boolean allowBroken) throws IOException { + private CubeDesc loadCubeDesc(String path) throws IOException { ResourceStore store = getStore(); CubeDesc ndesc = store.getResource(path, CubeDesc.class, CUBE_DESC_SERIALIZER); - try { - ndesc.init(config, getMetadataManager().getAllTablesMap()); - } catch (Exception e) { - ndesc.addError(e.getMessage()); + if (StringUtils.isBlank(ndesc.getName())) { + throw new IllegalStateException("CubeDesc name must not be blank"); } - if (!allowBroken && !ndesc.getError().isEmpty()) { + ndesc.init(config, getMetadataManager().getAllTablesMap()); + + if (ndesc.getError().isEmpty() == false) { throw new IllegalStateException("Cube desc at " + path + " has issues: " + ndesc.getError()); } @@ -159,8 +155,8 @@ public class CubeDescManager { try { cubeDesc.init(config, getMetadataManager().getAllTablesMap()); - } catch (Exception e) { - cubeDesc.addError(e.getMessage()); + } catch (IllegalStateException e) { + cubeDesc.addError(e.getMessage(), true); } // Check base validation if (!cubeDesc.getError().isEmpty()) { @@ -168,7 +164,7 @@ public class CubeDescManager { } // Semantic validation CubeMetadataValidator validator = new CubeMetadataValidator(); - ValidateContext context = validator.validate(cubeDesc); + ValidateContext context = validator.validate(cubeDesc, true); if (!context.ifPass()) { return cubeDesc; } @@ -204,9 +200,14 @@ public class CubeDescManager { List<String> paths = store.collectResourceRecursively(ResourceStore.CUBE_DESC_RESOURCE_ROOT, MetadataConstants.FILE_SURFIX); for (String path : paths) { - CubeDesc desc = loadCubeDesc(path, true); - - if (!path.equals(desc.getResourcePath())) { + CubeDesc desc; + try { + desc = loadCubeDesc(path); + } catch (Exception e) { + logger.error("Error loading cube desc " + path, e); + continue; + } + if (path.equals(desc.getResourcePath()) == false) { logger.error("Skip suspicious desc at " + path + ", " + desc + " should be at " + desc.getResourcePath()); continue; } @@ -218,7 +219,7 @@ public class CubeDescManager { cubeDescMap.putLocal(desc.getName(), desc); } - logger.info("Loaded " + cubeDescMap.size() + " Cube(s)"); + logger.debug("Loaded " + cubeDescMap.size() + " Cube(s)"); } /** @@ -240,14 +241,17 @@ public class CubeDescManager { try { desc.init(config, getMetadataManager().getAllTablesMap()); - } catch (Exception e) { - desc.addError(e.getMessage()); + } catch (IllegalStateException e) { + desc.addError(e.getMessage(), true); + return desc; + } catch (IllegalArgumentException e) { + desc.addError(e.getMessage(), true); return desc; } // Semantic validation CubeMetadataValidator validator = new CubeMetadataValidator(); - ValidateContext context = validator.validate(desc); + ValidateContext context = validator.validate(desc, true); if (!context.ifPass()) { return desc; } @@ -259,7 +263,7 @@ public class CubeDescManager { getStore().putResource(path, desc, CUBE_DESC_SERIALIZER); // Reload the CubeDesc - CubeDesc ndesc = loadCubeDesc(path, false); + CubeDesc ndesc = loadCubeDesc(path); // Here replace the old one cubeDescMap.put(ndesc.getName(), desc); http://git-wip-us.apache.org/repos/asf/kylin/blob/d33ba6aa/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java index a2ed051..851b016 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java @@ -35,17 +35,17 @@ import org.apache.kylin.metadata.model.SegmentStatusEnum; import org.apache.kylin.metadata.model.TableDesc; import org.apache.kylin.metadata.model.TblColRef; import org.apache.kylin.metadata.realization.CapabilityResult; -import org.apache.kylin.metadata.realization.CapabilityResult.CapabilityInfluence; import org.apache.kylin.metadata.realization.IRealization; import org.apache.kylin.metadata.realization.RealizationStatusEnum; import org.apache.kylin.metadata.realization.RealizationType; import org.apache.kylin.metadata.realization.SQLDigest; +import org.apache.kylin.metadata.realization.CapabilityResult.CapabilityInfluence; import com.fasterxml.jackson.annotation.JsonAutoDetect; -import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonManagedReference; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility; import com.google.common.base.Objects; import com.google.common.collect.Lists; @@ -149,13 +149,6 @@ public class CubeInstance extends RootPersistentEntity implements IRealization, return getStatus() == RealizationStatusEnum.READY; } - // if cube is not online and has no data or any building job, we allow its descriptor to be - // in a temporary broken state, so that user can edit and fix it. Broken state is often due to - // schema changes at source. - public boolean allowBrokenDescriptor() { - return (getStatus() == RealizationStatusEnum.DISABLED || getStatus() == RealizationStatusEnum.DESCBROKEN) && segments.isEmpty(); - } - public String getResourcePath() { return concatResourcePath(name); } http://git-wip-us.apache.org/repos/asf/kylin/blob/d33ba6aa/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java index fd46b54..de801af 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java @@ -18,9 +18,6 @@ package org.apache.kylin.cube; -import static com.google.common.base.Preconditions.checkNotNull; -import static com.google.common.base.Preconditions.checkState; - import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; @@ -33,6 +30,8 @@ import java.util.Random; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; +import javax.annotation.Nullable; + import org.apache.commons.lang3.StringUtils; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.KylinConfigExt; @@ -66,6 +65,8 @@ import org.apache.kylin.source.SourceFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.google.common.base.Function; +import com.google.common.collect.Collections2; import com.google.common.collect.HashMultimap; import com.google.common.collect.Lists; import com.google.common.collect.Multimap; @@ -842,37 +843,39 @@ public class CubeManager implements IRealizationProvider { private synchronized CubeInstance reloadCubeLocalAt(String path) { ResourceStore store = getStore(); - CubeInstance cube; + CubeInstance cubeInstance; try { - cube = store.getResource(path, CubeInstance.class, CUBE_SERIALIZER); - checkNotNull(cube, "cube (at %s) not found", path); + cubeInstance = store.getResource(path, CubeInstance.class, CUBE_SERIALIZER); - String cubeName = cube.getName(); - checkState(StringUtils.isNotBlank(cubeName), "cube (at %s) name must not be blank", path); + CubeDesc cubeDesc = CubeDescManager.getInstance(config).getCubeDesc(cubeInstance.getDescName()); + if (cubeDesc == null) + throw new IllegalStateException("CubeInstance desc not found '" + cubeInstance.getDescName() + "', at " + path); - CubeDesc cubeDesc = CubeDescManager.getInstance(config).getCubeDesc(cube.getDescName()); - checkNotNull(cubeDesc, "cube descriptor '%s' (for cube '%s') not found", cube.getDescName(), cubeName); + cubeInstance.setConfig((KylinConfigExt) cubeDesc.getConfig()); - if (!cubeDesc.getError().isEmpty()) { - cube.setStatus(RealizationStatusEnum.DESCBROKEN); - logger.warn("cube descriptor {} (for cube '{}') is broken", cubeDesc.getResourcePath(), cubeName); + if (StringUtils.isBlank(cubeInstance.getName())) + throw new IllegalStateException("CubeInstance name must not be blank, at " + path); - } else if (cube.getStatus() == RealizationStatusEnum.DESCBROKEN) { - cube.setStatus(RealizationStatusEnum.DISABLED); - logger.info("cube {} changed from DESCBROKEN to DISABLED", cubeName); - } + if (cubeInstance.getDescriptor() == null) + throw new IllegalStateException("CubeInstance desc not found '" + cubeInstance.getDescName() + "', at " + path); - cube.setConfig((KylinConfigExt) cubeDesc.getConfig()); - cubeMap.putLocal(cubeName, cube); + final String cubeName = cubeInstance.getName(); + cubeMap.putLocal(cubeName, cubeInstance); - for (CubeSegment segment : cube.getSegments()) { + for (CubeSegment segment : cubeInstance.getSegments()) { usedStorageLocation.put(cubeName.toUpperCase(), segment.getStorageLocationIdentifier()); } - logger.info("Reloaded cube {} being {} having {} segments", cubeName, cube, cube.getSegments().size()); - return cube; + logger.debug("Reloaded new cube: " + cubeName + " with reference being" + cubeInstance + " having " + cubeInstance.getSegments().size() + " segments:" + StringUtils.join(Collections2.transform(cubeInstance.getSegments(), new Function<CubeSegment, String>() { + @Nullable + @Override + public String apply(CubeSegment input) { + return input.getStorageLocationIdentifier(); + } + }), ",")); + return cubeInstance; } catch (Exception e) { logger.error("Error during load cube instance, skipping : " + path, e); return null; http://git-wip-us.apache.org/repos/asf/kylin/blob/d33ba6aa/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java index 4195451..e6b3d3f 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java @@ -18,10 +18,6 @@ package org.apache.kylin.cube.model; -import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.base.Preconditions.checkNotNull; -import static com.google.common.base.Preconditions.checkState; - import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.util.ArrayList; @@ -33,9 +29,9 @@ import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; -import java.util.Map.Entry; import java.util.Set; import java.util.TreeSet; +import java.util.Map.Entry; import javax.annotation.Nullable; @@ -68,9 +64,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.fasterxml.jackson.annotation.JsonAutoDetect; -import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility; import com.fasterxml.jackson.core.JsonProcessingException; import com.google.common.base.Function; import com.google.common.collect.Collections2; @@ -530,15 +526,19 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware { this.errors.clear(); this.config = KylinConfigExt.createInstance(config, overrideKylinProps); - checkArgument(StringUtils.isNotBlank(name), "CubeDesc name is blank"); - checkArgument(StringUtils.isNotBlank(modelName), "CubeDesc(%s) has blank modelName", name); - - this.model = MetadataManager.getInstance(config).getDataModelDesc(modelName); - checkNotNull(this.model, "DateModelDesc(%s) not found", modelName); + if (this.modelName == null || this.modelName.length() == 0) { + this.addError("The cubeDesc '" + this.getName() + "' doesn't have data model specified."); + } // check if aggregation group is valid validate(); + this.model = MetadataManager.getInstance(config).getDataModelDesc(this.modelName); + + if (this.model == null) { + this.addError("No data model found with name '" + modelName + "'."); + } + for (DimensionDesc dim : dimensions) { dim.init(this, tables); } @@ -559,9 +559,9 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware { // check all dimension columns are presented on rowkey List<TblColRef> dimCols = listDimensionColumnsExcludingDerived(true); - checkState(rowkey.getRowKeyColumns().length == dimCols.size(), - "RowKey columns count (%d) doesn't match dimensions columns count (%d)", - rowkey.getRowKeyColumns().length, dimCols.size()); + if (rowkey.getRowKeyColumns().length != dimCols.size()) { + addError("RowKey columns count (" + rowkey.getRowKeyColumns().length + ") does not match dimension columns count (" + dimCols.size() + "). "); + } initDictionaryDesc(); } @@ -954,8 +954,25 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware { this.autoMergeTimeRanges = autoMergeTimeRanges; } + /** + * Add error info and thrown exception out + * + * @param message + */ public void addError(String message) { - this.errors.add(message); + addError(message, false); + } + + /** + * @param message error message + * @param silent if throw exception + */ + public void addError(String message, boolean silent) { + if (!silent) { + throw new IllegalStateException(message); + } else { + this.errors.add(message); + } } public List<String> getError() { http://git-wip-us.apache.org/repos/asf/kylin/blob/d33ba6aa/core-cube/src/main/java/org/apache/kylin/cube/model/validation/CubeMetadataValidator.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/CubeMetadataValidator.java b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/CubeMetadataValidator.java index c22930a..c631f8d 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/CubeMetadataValidator.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/CubeMetadataValidator.java @@ -35,15 +35,39 @@ public class CubeMetadataValidator { private IValidatorRule<CubeDesc>[] rules = new IValidatorRule[] { new FunctionRule(), new AggregationGroupRule(), new RowKeyAttrRule(), new DictionaryRule() }; public ValidateContext validate(CubeDesc cube) { + return validate(cube, false); + } + + /** + * @param inject inject error into cube desc + * @return + */ + public ValidateContext validate(CubeDesc cube, boolean inject) { ValidateContext context = new ValidateContext(); - for (IValidatorRule<CubeDesc> rule : rules) { + for (int i = 0; i < rules.length; i++) { + IValidatorRule<CubeDesc> rule = rules[i]; rule.validate(cube, context); } - - for (ValidateContext.Result result : context.getResults()) { - cube.addError(result.getLevel() + " : " + result.getMessage()); + if (inject) { + injectResult(cube, context); } return context; } + /** + * + * Inject errors info into cubeDesc + * + * @param cubeDesc + * @param context + */ + public void injectResult(CubeDesc cubeDesc, ValidateContext context) { + ValidateContext.Result[] results = context.getResults(); + for (int i = 0; i < results.length; i++) { + ValidateContext.Result result = results[i]; + cubeDesc.addError(result.getLevel() + " : " + result.getMessage(), true); + } + + } + } http://git-wip-us.apache.org/repos/asf/kylin/blob/d33ba6aa/core-metadata/src/main/java/org/apache/kylin/metadata/realization/RealizationStatusEnum.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/RealizationStatusEnum.java b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/RealizationStatusEnum.java index 27e2d57..e4583f2 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/RealizationStatusEnum.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/RealizationStatusEnum.java @@ -20,6 +20,6 @@ package org.apache.kylin.metadata.realization; public enum RealizationStatusEnum { - DISABLED, READY, DESCBROKEN + DISABLED, BUILDING, READY, DESCBROKEN } http://git-wip-us.apache.org/repos/asf/kylin/blob/d33ba6aa/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java ---------------------------------------------------------------------- diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java index 42b117c..5397df7 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java +++ b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java @@ -22,6 +22,7 @@ import java.io.IOException; import java.net.UnknownHostException; import java.util.ArrayList; import java.util.Collections; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; @@ -42,7 +43,6 @@ import org.apache.kylin.job.JoinedFlatTable; import org.apache.kylin.metadata.model.IJoinedFlatTableDesc; import org.apache.kylin.metadata.model.SegmentStatusEnum; import org.apache.kylin.metadata.project.ProjectInstance; -import org.apache.kylin.metadata.realization.RealizationStatusEnum; import org.apache.kylin.rest.exception.BadRequestException; import org.apache.kylin.rest.exception.ForbiddenException; import org.apache.kylin.rest.exception.InternalErrorException; @@ -74,7 +74,6 @@ import org.springframework.web.bind.annotation.ResponseBody; import com.fasterxml.jackson.core.JsonParseException; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonMappingException; -import com.google.common.base.Joiner; import com.google.common.collect.Sets; /** @@ -346,12 +345,8 @@ public class CubeController extends BasicController { CubeInstance cube = cubeService.getCubeManager().getCube(cubeName); if (cube == null) { - throw new BadRequestException("Cannot find cube " + cubeName); - } - if (cube.getStatus() == RealizationStatusEnum.DESCBROKEN) { - throw new BadRequestException("Broken cube can't be cloned"); + throw new InternalErrorException("Cannot find cube " + cubeName); } - CubeDesc cubeDesc = cube.getDescriptor(); CubeDesc newCubeDesc = CubeDesc.getCopyOf(cubeDesc); newCubeDesc.setName(newCubeName); @@ -451,11 +446,18 @@ public class CubeController extends BasicController { @ResponseBody public CubeRequest updateCubeDesc(@RequestBody CubeRequest cubeRequest) throws JsonProcessingException { + //update cube CubeDesc desc = deserializeCubeDesc(cubeRequest); + CubeDesc oldCubeDesc; + boolean isCubeDescFreeEditable; + if (desc == null) { return cubeRequest; } + // Check if the cube is editable + isCubeDescFreeEditable = cubeService.isCubeDescFreeEditable(desc); + String projectName = (null == cubeRequest.getProject()) ? ProjectInstance.DEFAULT_PROJECT_NAME : cubeRequest.getProject(); try { CubeInstance cube = cubeService.getCubeManager().getCube(cubeRequest.getCubeName()); @@ -473,14 +475,14 @@ public class CubeController extends BasicController { return cubeRequest; } - if (cube.getSegments().size() != 0 && !cube.getDescriptor().consistentWith(desc)) { - String error = "CubeDesc " + desc.getName() + " is inconsistent with existing. Try purge that cube first or avoid updating key cube desc fields."; - updateRequest(cubeRequest, false, error); + oldCubeDesc = cube.getDescriptor(); + if (isCubeDescFreeEditable || oldCubeDesc.consistentWith(desc)) { + desc = cubeService.updateCubeAndDesc(cube, desc, projectName, true); + } else { + logger.warn("Won't update the cube desc due to inconsistency"); + updateRequest(cubeRequest, false, "CubeDesc " + desc.getName() + " is inconsistent with existing. Try purge that cube first or avoid updating key cube desc fields."); return cubeRequest; } - - desc = cubeService.updateCubeAndDesc(cube, desc, projectName, true); - } catch (AccessDeniedException accessDeniedException) { throw new ForbiddenException("You don't have right to update this cube."); } catch (Exception e) { @@ -489,7 +491,8 @@ public class CubeController extends BasicController { } if (!desc.getError().isEmpty()) { - updateRequest(cubeRequest, false, Joiner.on("\n").join(desc.getError())); + logger.warn("Cube " + desc.getName() + " fail to update because " + desc.getError()); + updateRequest(cubeRequest, false, omitMessage(desc.getError())); return cubeRequest; } @@ -596,6 +599,19 @@ public class CubeController extends BasicController { return desc; } + /** + * @return + */ + private String omitMessage(List<String> errors) { + StringBuffer buffer = new StringBuffer(); + for (Iterator<String> iterator = errors.iterator(); iterator.hasNext();) { + String string = (String) iterator.next(); + buffer.append(string); + buffer.append("\n"); + } + return buffer.toString(); + } + private void updateRequest(CubeRequest request, boolean success, String message) { request.setCubeDescData(""); request.setSuccessful(success); http://git-wip-us.apache.org/repos/asf/kylin/blob/d33ba6aa/server-base/src/main/java/org/apache/kylin/rest/service/CacheService.java ---------------------------------------------------------------------- diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/CacheService.java b/server-base/src/main/java/org/apache/kylin/rest/service/CacheService.java index 9d134d6..2160e3d 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/service/CacheService.java +++ b/server-base/src/main/java/org/apache/kylin/rest/service/CacheService.java @@ -189,7 +189,6 @@ public class CacheService extends BasicService { case TABLE: getMetadataManager().reloadTableCache(cacheKey); CubeDescManager.clearCache(); - clearRealizationCache(); break; case EXTERNAL_FILTER: getMetadataManager().reloadExtFilter(cacheKey); @@ -203,7 +202,9 @@ public class CacheService extends BasicService { DictionaryManager.clearCache(); MetadataManager.clearCache(); CubeDescManager.clearCache(); - clearRealizationCache(); + CubeManager.clearCache(); + HybridManager.clearCache(); + RealizationRegistry.clearCache(); Cuboid.clearCache(); ProjectManager.clearCache(); KafkaConfigManager.clearCache(); @@ -221,12 +222,6 @@ public class CacheService extends BasicService { } } - private void clearRealizationCache() { - CubeManager.clearCache(); - HybridManager.clearCache(); - RealizationRegistry.clearCache(); - } - private void rebuildCubeCache(String cubeName) { CubeInstance cube = getCubeManager().reloadCubeLocal(cubeName); getHybridManager().reloadHybridInstanceByChild(RealizationType.CUBE, cubeName); http://git-wip-us.apache.org/repos/asf/kylin/blob/d33ba6aa/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java ---------------------------------------------------------------------- diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java index e446045..4cd527c 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java +++ b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java @@ -273,6 +273,21 @@ public class CubeService extends BasicService { accessService.clean(cube, true); } + public boolean isCubeDescFreeEditable(CubeDesc cd) { + List<CubeInstance> cubes = getCubeManager().getCubesByDesc(cd.getName()); + for (CubeInstance cube : cubes) { + if (cube.getSegments().size() != 0) { + logger.debug("cube '" + cube.getName() + " has " + cube.getSegments().size() + " segments, couldn't edit cube desc."); + return false; + } + } + return true; + } + + public static String getCubeDescNameFromCube(String cubeName) { + return cubeName + DESC_SUFFIX; + } + public static String getCubeNameFromDesc(String descName) { if (descName.toLowerCase().endsWith(DESC_SUFFIX)) { return descName.substring(0, descName.toLowerCase().indexOf(DESC_SUFFIX)); http://git-wip-us.apache.org/repos/asf/kylin/blob/d33ba6aa/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java ---------------------------------------------------------------------- diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java b/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java index 5c704ba..e4fbc98 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java +++ b/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java @@ -48,9 +48,7 @@ import org.apache.kylin.job.execution.DefaultChainedExecutable; import org.apache.kylin.job.execution.ExecutableState; import org.apache.kylin.job.execution.Output; import org.apache.kylin.metadata.model.SegmentStatusEnum; -import org.apache.kylin.metadata.realization.RealizationStatusEnum; import org.apache.kylin.rest.constant.Constant; -import org.apache.kylin.rest.exception.BadRequestException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -201,10 +199,6 @@ public class JobService extends BasicService { public JobInstance submitJob(CubeInstance cube, long startDate, long endDate, long startOffset, long endOffset, // CubeBuildTypeEnum buildType, boolean force, String submitter) throws IOException, JobException { - if (cube.getStatus() == RealizationStatusEnum.DESCBROKEN) { - throw new BadRequestException("Broken cube " + cube.getName() + " can't be built"); - } - checkCubeDescSignature(cube); checkNoRunningJob(cube); http://git-wip-us.apache.org/repos/asf/kylin/blob/d33ba6aa/server/src/test/java/org/apache/kylin/rest/service/CubeServiceTest.java ---------------------------------------------------------------------- diff --git a/server/src/test/java/org/apache/kylin/rest/service/CubeServiceTest.java b/server/src/test/java/org/apache/kylin/rest/service/CubeServiceTest.java index 59e96d6..f98d6b9 100644 --- a/server/src/test/java/org/apache/kylin/rest/service/CubeServiceTest.java +++ b/server/src/test/java/org/apache/kylin/rest/service/CubeServiceTest.java @@ -51,5 +51,6 @@ public class CubeServiceTest extends ServiceTestBase { List<CubeInstance> cubes = cubeService.listAllCubes(null, null, null); Assert.assertNotNull(cubes); CubeInstance cube = cubes.get(0); + cubeService.isCubeDescFreeEditable(cube.getDescriptor()); } } http://git-wip-us.apache.org/repos/asf/kylin/blob/d33ba6aa/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java ---------------------------------------------------------------------- diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java index 8b98e7b..70b097c 100644 --- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java +++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java @@ -28,7 +28,6 @@ import java.util.UUID; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.Table; import org.apache.kylin.common.KylinConfig; -import org.apache.kylin.cube.CubeManager; import org.apache.kylin.engine.mr.HadoopUtil; import org.apache.kylin.metadata.MetadataConstants; import org.apache.kylin.metadata.MetadataManager; @@ -37,10 +36,8 @@ import org.apache.kylin.metadata.model.TableDesc; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.collect.LinkedHashMultimap; import com.google.common.collect.Lists; import com.google.common.collect.Maps; -import com.google.common.collect.SetMultimap; import com.google.common.collect.Sets; /** @@ -54,25 +51,27 @@ public class HiveSourceTableLoader { @SuppressWarnings("unused") private static final Logger logger = LoggerFactory.getLogger(HiveSourceTableLoader.class); - public static Set<String> reloadHiveTables(String[] hiveTables, KylinConfig config) throws IOException { + public static final String OUTPUT_SURFIX = "json"; + public static final String TABLE_FOLDER_NAME = "table"; + public static final String TABLE_EXD_FOLDER_NAME = "table_exd"; - SetMultimap<String, String> db2tables = LinkedHashMultimap.create(); - for (String fullTableName : hiveTables) { - String[] parts = HadoopUtil.parseHiveTableName(fullTableName); - db2tables.put(parts[0], parts[1]); - } + public static Set<String> reloadHiveTables(String[] hiveTables, KylinConfig config) throws IOException { - HiveClient hiveClient = new HiveClient(); - SchemaChecker checker = new SchemaChecker(hiveClient, MetadataManager.getInstance(config), CubeManager.getInstance(config)); - for (Map.Entry<String, String> entry : db2tables.entries()) { - SchemaChecker.CheckResult result = checker.allowReload(entry.getKey(), entry.getValue()); - result.raiseExceptionWhenInvalid(); + Map<String, Set<String>> db2tables = Maps.newHashMap(); + for (String table : hiveTables) { + String[] parts = HadoopUtil.parseHiveTableName(table); + Set<String> set = db2tables.get(parts[0]); + if (set == null) { + set = Sets.newHashSet(); + db2tables.put(parts[0], set); + } + set.add(parts[1]); } // extract from hive Set<String> loadedTables = Sets.newHashSet(); for (String database : db2tables.keySet()) { - List<String> loaded = extractHiveTables(database, db2tables.get(database), hiveClient); + List<String> loaded = extractHiveTables(database, db2tables.get(database), config); loadedTables.addAll(loaded); } @@ -85,12 +84,13 @@ public class HiveSourceTableLoader { metaMgr.removeTableExd(hiveTable); } - private static List<String> extractHiveTables(String database, Set<String> tables, HiveClient hiveClient) throws IOException { + private static List<String> extractHiveTables(String database, Set<String> tables, KylinConfig config) throws IOException { List<String> loadedTables = Lists.newArrayList(); MetadataManager metaMgr = MetadataManager.getInstance(KylinConfig.getInstanceFromEnv()); for (String tableName : tables) { Table table = null; + HiveClient hiveClient = new HiveClient(); List<FieldSchema> partitionFields = null; List<FieldSchema> fields = null; try { @@ -167,4 +167,5 @@ public class HiveSourceTableLoader { return loadedTables; } + } http://git-wip-us.apache.org/repos/asf/kylin/blob/d33ba6aa/source-hive/src/main/java/org/apache/kylin/source/hive/SchemaChecker.java ---------------------------------------------------------------------- diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/SchemaChecker.java b/source-hive/src/main/java/org/apache/kylin/source/hive/SchemaChecker.java deleted file mode 100644 index 3b03551..0000000 --- a/source-hive/src/main/java/org/apache/kylin/source/hive/SchemaChecker.java +++ /dev/null @@ -1,216 +0,0 @@ -package org.apache.kylin.source.hive; - -import static com.google.common.base.Preconditions.checkNotNull; -import static java.lang.String.format; - -import java.util.List; -import java.util.Map; -import java.util.Set; - -import javax.annotation.Nullable; - -import org.apache.hadoop.hive.metastore.api.FieldSchema; -import org.apache.hadoop.hive.metastore.api.Table; -import org.apache.kylin.cube.CubeInstance; -import org.apache.kylin.cube.CubeManager; -import org.apache.kylin.cube.model.CubeDesc; -import org.apache.kylin.metadata.MetadataManager; -import org.apache.kylin.metadata.datatype.DataType; -import org.apache.kylin.metadata.model.ColumnDesc; -import org.apache.kylin.metadata.model.TableDesc; -import org.apache.kylin.metadata.model.TblColRef; - -import com.google.common.base.Predicate; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.Iterables; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; -import com.google.common.collect.Sets; - -public class SchemaChecker { - private final HiveClient hiveClient; - private final MetadataManager metadataManager; - private final CubeManager cubeManager; - - static class CheckResult { - private final boolean valid; - private final String reason; - - private CheckResult(boolean valid, String reason) { - this.valid = valid; - this.reason = reason; - } - - void raiseExceptionWhenInvalid() { - if (!valid) { - throw new RuntimeException(reason); - } - } - - static CheckResult validOnFirstLoad(String tableName) { - return new CheckResult(true, format("Table '%s' hasn't been loaded before", tableName)); - } - - static CheckResult validOnCompatibleSchema(String tableName) { - return new CheckResult(true, format("Table '%s' is compatible with all existing cubes", tableName)); - } - - static CheckResult invalidOnFetchSchema(String tableName, Exception e) { - return new CheckResult(false, format("Failed to fetch metadata of '%s': %s", tableName, e.getMessage())); - } - - static CheckResult invalidOnIncompatibleSchema(String tableName, List<String> reasons) { - StringBuilder buf = new StringBuilder(); - for (String reason : reasons) { - buf.append("- ").append(reason).append("\n"); - } - - return new CheckResult(false, format("Found %d issue(s) with '%s':\n%s Please disable and purge related cube(s) first", reasons.size(), tableName, buf.toString())); - } - } - - SchemaChecker(HiveClient hiveClient, MetadataManager metadataManager, CubeManager cubeManager) { - this.hiveClient = checkNotNull(hiveClient, "hiveClient is null"); - this.metadataManager = checkNotNull(metadataManager, "metadataManager is null"); - this.cubeManager = checkNotNull(cubeManager, "cubeManager is null"); - } - - private List<FieldSchema> fetchSchema(String dbName, String tblName) throws Exception { - List<FieldSchema> fields = Lists.newArrayList(); - fields.addAll(hiveClient.getHiveTableFields(dbName, tblName)); - - Table table = hiveClient.getHiveTable(dbName, tblName); - List<FieldSchema> partitionFields = table.getPartitionKeys(); - if (partitionFields != null) { - fields.addAll(partitionFields); - } - - return fields; - } - - private List<CubeInstance> findCubeByTable(final String fullTableName) { - Iterable<CubeInstance> relatedCubes = Iterables.filter(cubeManager.listAllCubes(), new Predicate<CubeInstance>() { - @Override - public boolean apply(@Nullable CubeInstance cube) { - if (cube == null || cube.allowBrokenDescriptor()) { - return false; - } - CubeDesc desc = cube.getDescriptor(); - - Set<String> usedTables = Sets.newHashSet(); - usedTables.add(desc.getFactTableDesc().getIdentity()); - for (TableDesc lookup : desc.getLookupTableDescs()) { - usedTables.add(lookup.getIdentity()); - } - - return usedTables.contains(fullTableName); - } - }); - - return ImmutableList.copyOf(relatedCubes); - } - - private boolean isColumnCompatible(ColumnDesc column, FieldSchema field) { - if (!column.getName().equalsIgnoreCase(field.getName())) { - return false; - } - - String typeStr = field.getType(); - // kylin uses double internally for float, see HiveSourceTableLoader.java - // TODO should this normalization to be in DataType class ? - if ("float".equalsIgnoreCase(typeStr)) { - typeStr = "double"; - } - DataType fieldType = DataType.getType(typeStr); - - if (column.getType().isIntegerFamily()) { - // OLAPTable.listSourceColumns converts some integer columns to bigint, - // therefore strict type comparison won't work. - // changing from one integer type to another should be fine. - return fieldType.isIntegerFamily(); - } else { - // only compare base type name, changing precision or scale should be fine - return column.getTypeName().equals(fieldType.getName()); - } - } - - private List<String> checkAllUsedColumns(CubeInstance cube, TableDesc table, Map<String, FieldSchema> fieldsMap) { - Set<ColumnDesc> usedColumns = Sets.newHashSet(); - for (TblColRef col : cube.getAllColumns()) { - usedColumns.add(col.getColumnDesc()); - } - - List<String> violateColumns = Lists.newArrayList(); - for (ColumnDesc column : table.getColumns()) { - if (usedColumns.contains(column)) { - FieldSchema field = fieldsMap.get(column.getName()); - if (field == null || !isColumnCompatible(column, field)) { - violateColumns.add(column.getName()); - } - } - } - return violateColumns; - } - - private boolean checkAllColumns(TableDesc table, List<FieldSchema> fields) { - if (table.getColumnCount() != fields.size()) { - return false; - } - - ColumnDesc[] columns = table.getColumns(); - for (int i = 0; i < columns.length; i++) { - if (!isColumnCompatible(columns[i], fields.get(i))) { - return false; - } - } - return true; - } - - public CheckResult allowReload(String dbName, String tblName) { - final String fullTableName = (dbName + "." + tblName).toUpperCase(); - - TableDesc existing = metadataManager.getTableDesc(fullTableName); - if (existing == null) { - return CheckResult.validOnFirstLoad(fullTableName); - } - - List<FieldSchema> currentFields; - Map<String, FieldSchema> currentFieldsMap = Maps.newHashMap(); - try { - currentFields = fetchSchema(dbName, tblName); - } catch (Exception e) { - return CheckResult.invalidOnFetchSchema(fullTableName, e); - } - for (FieldSchema field : currentFields) { - currentFieldsMap.put(field.getName().toUpperCase(), field); - } - - List<String> issues = Lists.newArrayList(); - for (CubeInstance cube : findCubeByTable(fullTableName)) { - TableDesc factTable = cube.getFactTableDesc(); - List<TableDesc> lookupTables = cube.getDescriptor().getLookupTableDescs(); - String modelName = cube.getDataModelDesc().getName(); - - // if user reloads a fact table used by cube, then all used columns - // must match current schema - if (factTable.getIdentity().equals(fullTableName)) { - List<String> violateColumns = checkAllUsedColumns(cube, factTable, currentFieldsMap); - if (!violateColumns.isEmpty()) { - issues.add(format("Column %s used in cube[%s] and model[%s], but changed in hive", violateColumns, cube.getName(), modelName)); - } - } - - // if user reloads a lookup table used by cube, then nearly all changes in schema are disallowed) - for (TableDesc lookupTable : lookupTables) { - if (lookupTable.getIdentity().equals(fullTableName) && !checkAllColumns(lookupTable, currentFields)) { - issues.add(format("Table '%s' is used as Lookup Table in cube[%s] and model[%s], but changed in hive", lookupTable.getIdentity(), cube.getName(), modelName)); - } - } - } - - if (issues.isEmpty()) { - return CheckResult.validOnCompatibleSchema(fullTableName); - } - return CheckResult.invalidOnIncompatibleSchema(fullTableName, issues); - } -} http://git-wip-us.apache.org/repos/asf/kylin/blob/d33ba6aa/webapp/app/css/AdminLTE.css ---------------------------------------------------------------------- diff --git a/webapp/app/css/AdminLTE.css b/webapp/app/css/AdminLTE.css index 6688457..e772ae5 100644 --- a/webapp/app/css/AdminLTE.css +++ b/webapp/app/css/AdminLTE.css @@ -4307,7 +4307,7 @@ fieldset[disabled] .btn-vk.active { .alert-info, .label-danger, .label-info, -.label-warning, +.label-waring, .label-primary, .label-success, .modal-primary .modal-body, @@ -4349,7 +4349,7 @@ fieldset[disabled] .btn-vk.active { .bg-yellow, .callout.callout-warning, .alert-warning, -.label-warning, +.label-waring, .modal-warning .modal-body { background-color: #f39c12 !important; } http://git-wip-us.apache.org/repos/asf/kylin/blob/d33ba6aa/webapp/app/partials/cubes/cubes.html ---------------------------------------------------------------------- diff --git a/webapp/app/partials/cubes/cubes.html b/webapp/app/partials/cubes/cubes.html index de9b99b..30981dc 100644 --- a/webapp/app/partials/cubes/cubes.html +++ b/webapp/app/partials/cubes/cubes.html @@ -67,7 +67,7 @@ </td> <td> <span class="label" - ng-class="{'label-success': cube.status=='READY', 'label-default': cube.status=='DISABLED', 'label-warning': cube.status=='DESCBROKEN'}"> + ng-class="{'label-success': cube.status=='READY', 'label-default': cube.status=='DISABLED'}"> {{ cube.status}} </span> </td> @@ -89,18 +89,18 @@ Action <span class="ace-icon fa fa-caret-down icon-on-right"></span> </button> <ul class="dropdown-menu" role="menu"> - <li ng-if="cube.status!='READY' && userService.hasRole('ROLE_ADMIN') "> + <li ng-if="cube.status=='DISABLED' && userService.hasRole('ROLE_ADMIN') "> <a ng-click="dropCube(cube)" tooltip="Drop the cube, related jobs and data permanently.">Drop</a></li> - <li ng-if="cube.status!='READY' && (userService.hasRole('ROLE_ADMIN') || hasPermission(cube, permissions.ADMINISTRATION.mask, permissions.MANAGEMENT.mask))"> + <li ng-if="cube.status=='DISABLED' && (userService.hasRole('ROLE_ADMIN') || hasPermission(cube, permissions.ADMINISTRATION.mask, permissions.MANAGEMENT.mask))"> <a ng-click="cubeEdit(cube);">Edit</a></li> - <li ng-if="cube.streaming && cube.status=='DISABLED' && (userService.hasRole('ROLE_ADMIN') || hasPermission(cube, permissions.ADMINISTRATION.mask, permissions.MANAGEMENT.mask))"></li> - <li ng-if="cube.status!='DESCBROKEN'"><a ng-click="startJobSubmit(cube);">Build</a></li> - <li ng-if="cube.status!='DESCBROKEN'"><a ng-click="startRefresh(cube)">Refresh</a></li> - <li ng-if="cube.status!='DESCBROKEN'"><a ng-click="startMerge(cube)">Merge</a></li> - <li ng-if="cube.status=='READY'"><a ng-click="disable(cube)">Disable</a></li> + <li ng-if="cube.streaming && cube.status=='DISABLED' && (userService.hasRole('ROLE_ADMIN') || hasPermission(cube, permissions.ADMINISTRATION.mask, permissions.MANAGEMENT.mask))"></li> + <li><a ng-click="startJobSubmit(cube);">Build</a></li> + <li><a ng-click="startRefresh(cube)">Refresh</a></li> + <li><a ng-click="startMerge(cube)">Merge</a></li> + <li ng-if="cube.status!='DISABLED'"><a ng-click="disable(cube)">Disable</a></li> <li ng-if="cube.status=='DISABLED'"><a ng-click="enable(cube)">Enable</a></li> <li ng-if="cube.status=='DISABLED'"><a ng-click="purge(cube)">Purge</a></li> - <li ng-if="cube.status!='DESCBROKEN'"><a ng-click="cloneCube(cube)">Clone</a></li> + <li><a ng-click="cloneCube(cube)">Clone</a></li> </ul> </div> @@ -114,8 +114,8 @@ Action <span class="ace-icon fa fa-caret-down icon-on-right"></span> </button> <ul class="dropdown-menu" role="menu"> - <li ng-if="cube.status!='READY'"><a href="cubes/edit/{{cube.name}}/descriptionjson">Edit CubeDesc</a></li> - <li ng-if="cube.status!='READY'"><a href="cubes/view/{{cube.name}}/instancejson">View Cube</a></li> + <li ng-if="cube.status=='DISABLED'"><a href="cubes/edit/{{cube.name}}/descriptionjson">Edit CubeDesc</a></li> + <li ng-if="cube.status=='DISABLED'"><a href="cubes/view/{{cube.name}}/instancejson">View Cube</a></li> </ul> </div> </td>
