KYLIN-2012 more robust approach to hive schema changes
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/17569f6c Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/17569f6c Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/17569f6c Branch: refs/heads/KYLIN-1726 Commit: 17569f6c32a373f599ef7689f9506b5af5ed68bd Parents: c48baba Author: gaodayue <gaoda...@meituan.com> Authored: Mon Sep 12 12:05:32 2016 +0800 Committer: gaodayue <gaoda...@meituan.com> Committed: Wed Sep 14 15:02:33 2016 +0800 ---------------------------------------------------------------------- .../org/apache/kylin/cube/CubeDescManager.java | 62 +++--- .../org/apache/kylin/cube/CubeInstance.java | 11 +- .../java/org/apache/kylin/cube/CubeManager.java | 47 ++-- .../org/apache/kylin/cube/model/CubeDesc.java | 47 ++-- .../model/validation/CubeMetadataValidator.java | 32 +-- .../realization/RealizationStatusEnum.java | 2 +- .../kylin/rest/controller/CubeController.java | 44 ++-- .../apache/kylin/rest/service/CacheService.java | 11 +- .../apache/kylin/rest/service/CubeService.java | 15 -- .../apache/kylin/rest/service/JobService.java | 6 + .../kylin/rest/service/CubeServiceTest.java | 1 - .../source/hive/HiveSourceTableLoader.java | 33 ++- .../apache/kylin/source/hive/SchemaChecker.java | 216 +++++++++++++++++++ webapp/app/css/AdminLTE.css | 4 +- webapp/app/partials/cubes/cubes.html | 22 +- 15 files changed, 353 insertions(+), 200 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/17569f6c/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java index 33a6830..1b1cf70 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java @@ -23,7 +23,6 @@ import java.util.ArrayList; import java.util.List; import java.util.concurrent.ConcurrentHashMap; -import org.apache.commons.lang3.StringUtils; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.persistence.JsonSerializer; import org.apache.kylin.common.persistence.ResourceStore; @@ -36,6 +35,7 @@ import org.apache.kylin.cube.model.validation.CubeMetadataValidator; import org.apache.kylin.cube.model.validation.ValidateContext; import org.apache.kylin.metadata.MetadataConstants; import org.apache.kylin.metadata.MetadataManager; +import org.apache.kylin.metadata.realization.RealizationStatusEnum; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -110,30 +110,34 @@ public class CubeDescManager { * @throws IOException */ public CubeDesc reloadCubeDescLocal(String name) throws IOException { + // Broken CubeDesc is not allowed to be saved and broadcast. + CubeDesc ndesc = loadCubeDesc(CubeDesc.concatResourcePath(name), false); - // Save Source - String path = CubeDesc.concatResourcePath(name); - - // Reload the CubeDesc - CubeDesc ndesc = loadCubeDesc(path); - - // Here replace the old one cubeDescMap.putLocal(ndesc.getName(), ndesc); Cuboid.reloadCache(name); + + // if related cube is in DESCBROKEN state before, change it back to DISABLED + CubeManager cubeManager = CubeManager.getInstance(config); + for (CubeInstance cube : cubeManager.getCubesByDesc(name)) { + if (cube.getStatus() == RealizationStatusEnum.DESCBROKEN) { + cubeManager.reloadCubeLocal(cube.getName()); + } + } + return ndesc; } - private CubeDesc loadCubeDesc(String path) throws IOException { + private CubeDesc loadCubeDesc(String path, boolean allowBroken) throws IOException { ResourceStore store = getStore(); CubeDesc ndesc = store.getResource(path, CubeDesc.class, CUBE_DESC_SERIALIZER); - if (StringUtils.isBlank(ndesc.getName())) { - throw new IllegalStateException("CubeDesc name must not be blank"); + try { + ndesc.init(config, getMetadataManager().getAllTablesMap()); + } catch (Exception e) { + ndesc.addError(e.getMessage()); } - ndesc.init(config, getMetadataManager().getAllTablesMap()); - - if (ndesc.getError().isEmpty() == false) { + if (!allowBroken && !ndesc.getError().isEmpty()) { throw new IllegalStateException("Cube desc at " + path + " has issues: " + ndesc.getError()); } @@ -155,8 +159,8 @@ public class CubeDescManager { try { cubeDesc.init(config, getMetadataManager().getAllTablesMap()); - } catch (IllegalStateException e) { - cubeDesc.addError(e.getMessage(), true); + } catch (Exception e) { + cubeDesc.addError(e.getMessage()); } // Check base validation if (!cubeDesc.getError().isEmpty()) { @@ -164,7 +168,7 @@ public class CubeDescManager { } // Semantic validation CubeMetadataValidator validator = new CubeMetadataValidator(); - ValidateContext context = validator.validate(cubeDesc, true); + ValidateContext context = validator.validate(cubeDesc); if (!context.ifPass()) { return cubeDesc; } @@ -200,14 +204,9 @@ public class CubeDescManager { List<String> paths = store.collectResourceRecursively(ResourceStore.CUBE_DESC_RESOURCE_ROOT, MetadataConstants.FILE_SURFIX); for (String path : paths) { - CubeDesc desc; - try { - desc = loadCubeDesc(path); - } catch (Exception e) { - logger.error("Error loading cube desc " + path, e); - continue; - } - if (path.equals(desc.getResourcePath()) == false) { + CubeDesc desc = loadCubeDesc(path, true); + + if (!path.equals(desc.getResourcePath())) { logger.error("Skip suspicious desc at " + path + ", " + desc + " should be at " + desc.getResourcePath()); continue; } @@ -219,7 +218,7 @@ public class CubeDescManager { cubeDescMap.putLocal(desc.getName(), desc); } - logger.debug("Loaded " + cubeDescMap.size() + " Cube(s)"); + logger.info("Loaded " + cubeDescMap.size() + " Cube(s)"); } /** @@ -241,17 +240,14 @@ public class CubeDescManager { try { desc.init(config, getMetadataManager().getAllTablesMap()); - } catch (IllegalStateException e) { - desc.addError(e.getMessage(), true); - return desc; - } catch (IllegalArgumentException e) { - desc.addError(e.getMessage(), true); + } catch (Exception e) { + desc.addError(e.getMessage()); return desc; } // Semantic validation CubeMetadataValidator validator = new CubeMetadataValidator(); - ValidateContext context = validator.validate(desc, true); + ValidateContext context = validator.validate(desc); if (!context.ifPass()) { return desc; } @@ -263,7 +259,7 @@ public class CubeDescManager { getStore().putResource(path, desc, CUBE_DESC_SERIALIZER); // Reload the CubeDesc - CubeDesc ndesc = loadCubeDesc(path); + CubeDesc ndesc = loadCubeDesc(path, false); // Here replace the old one cubeDescMap.put(ndesc.getName(), desc); http://git-wip-us.apache.org/repos/asf/kylin/blob/17569f6c/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java index 851b016..a2ed051 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java @@ -35,17 +35,17 @@ import org.apache.kylin.metadata.model.SegmentStatusEnum; import org.apache.kylin.metadata.model.TableDesc; import org.apache.kylin.metadata.model.TblColRef; import org.apache.kylin.metadata.realization.CapabilityResult; +import org.apache.kylin.metadata.realization.CapabilityResult.CapabilityInfluence; import org.apache.kylin.metadata.realization.IRealization; import org.apache.kylin.metadata.realization.RealizationStatusEnum; import org.apache.kylin.metadata.realization.RealizationType; import org.apache.kylin.metadata.realization.SQLDigest; -import org.apache.kylin.metadata.realization.CapabilityResult.CapabilityInfluence; import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonManagedReference; import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility; import com.google.common.base.Objects; import com.google.common.collect.Lists; @@ -149,6 +149,13 @@ public class CubeInstance extends RootPersistentEntity implements IRealization, return getStatus() == RealizationStatusEnum.READY; } + // if cube is not online and has no data or any building job, we allow its descriptor to be + // in a temporary broken state, so that user can edit and fix it. Broken state is often due to + // schema changes at source. + public boolean allowBrokenDescriptor() { + return (getStatus() == RealizationStatusEnum.DISABLED || getStatus() == RealizationStatusEnum.DESCBROKEN) && segments.isEmpty(); + } + public String getResourcePath() { return concatResourcePath(name); } http://git-wip-us.apache.org/repos/asf/kylin/blob/17569f6c/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java index 2ebf5d3..daeca0d 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java @@ -18,6 +18,9 @@ package org.apache.kylin.cube; +import static com.google.common.base.Preconditions.checkNotNull; +import static com.google.common.base.Preconditions.checkState; + import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; @@ -30,8 +33,6 @@ import java.util.Random; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; -import javax.annotation.Nullable; - import org.apache.commons.lang3.StringUtils; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.KylinConfigExt; @@ -65,8 +66,6 @@ import org.apache.kylin.source.SourceFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.base.Function; -import com.google.common.collect.Collections2; import com.google.common.collect.HashMultimap; import com.google.common.collect.Lists; import com.google.common.collect.Multimap; @@ -843,39 +842,37 @@ public class CubeManager implements IRealizationProvider { private synchronized CubeInstance reloadCubeLocalAt(String path) { ResourceStore store = getStore(); + CubeInstance cube; - CubeInstance cubeInstance; try { - cubeInstance = store.getResource(path, CubeInstance.class, CUBE_SERIALIZER); + cube = store.getResource(path, CubeInstance.class, CUBE_SERIALIZER); + checkNotNull(cube, "cube (at %s) not found", path); - CubeDesc cubeDesc = CubeDescManager.getInstance(config).getCubeDesc(cubeInstance.getDescName()); - if (cubeDesc == null) - throw new IllegalStateException("CubeInstance desc not found '" + cubeInstance.getDescName() + "', at " + path); + String cubeName = cube.getName(); + checkState(StringUtils.isNotBlank(cubeName), "cube (at %s) name must not be blank", path); - cubeInstance.setConfig((KylinConfigExt) cubeDesc.getConfig()); + CubeDesc cubeDesc = CubeDescManager.getInstance(config).getCubeDesc(cube.getDescName()); + checkNotNull(cubeDesc, "cube descriptor '%s' (for cube '%s') not found", cube.getDescName(), cubeName); - if (StringUtils.isBlank(cubeInstance.getName())) - throw new IllegalStateException("CubeInstance name must not be blank, at " + path); + if (!cubeDesc.getError().isEmpty()) { + cube.setStatus(RealizationStatusEnum.DESCBROKEN); + logger.warn("cube descriptor {} (for cube '{}') is broken", cubeDesc.getResourcePath(), cubeName); - if (cubeInstance.getDescriptor() == null) - throw new IllegalStateException("CubeInstance desc not found '" + cubeInstance.getDescName() + "', at " + path); + } else if (cube.getStatus() == RealizationStatusEnum.DESCBROKEN) { + cube.setStatus(RealizationStatusEnum.DISABLED); + logger.info("cube {} changed from DESCBROKEN to DISABLED", cubeName); + } - final String cubeName = cubeInstance.getName(); - cubeMap.putLocal(cubeName, cubeInstance); + cube.setConfig((KylinConfigExt) cubeDesc.getConfig()); + cubeMap.putLocal(cubeName, cube); - for (CubeSegment segment : cubeInstance.getSegments()) { + for (CubeSegment segment : cube.getSegments()) { usedStorageLocation.put(cubeName.toUpperCase(), segment.getStorageLocationIdentifier()); } - logger.debug("Reloaded new cube: " + cubeName + " with reference being" + cubeInstance + " having " + cubeInstance.getSegments().size() + " segments:" + StringUtils.join(Collections2.transform(cubeInstance.getSegments(), new Function<CubeSegment, String>() { - @Nullable - @Override - public String apply(CubeSegment input) { - return input.getStorageLocationIdentifier(); - } - }), ",")); + logger.info("Reloaded cube {} being {} having {} segments", cubeName, cube, cube.getSegments().size()); + return cube; - return cubeInstance; } catch (Exception e) { logger.error("Error during load cube instance, skipping : " + path, e); return null; http://git-wip-us.apache.org/repos/asf/kylin/blob/17569f6c/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java index e6b3d3f..4195451 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java @@ -18,6 +18,10 @@ package org.apache.kylin.cube.model; +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkNotNull; +import static com.google.common.base.Preconditions.checkState; + import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.util.ArrayList; @@ -29,9 +33,9 @@ import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import java.util.Set; import java.util.TreeSet; -import java.util.Map.Entry; import javax.annotation.Nullable; @@ -64,9 +68,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility; import com.fasterxml.jackson.core.JsonProcessingException; import com.google.common.base.Function; import com.google.common.collect.Collections2; @@ -526,19 +530,15 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware { this.errors.clear(); this.config = KylinConfigExt.createInstance(config, overrideKylinProps); - if (this.modelName == null || this.modelName.length() == 0) { - this.addError("The cubeDesc '" + this.getName() + "' doesn't have data model specified."); - } + checkArgument(StringUtils.isNotBlank(name), "CubeDesc name is blank"); + checkArgument(StringUtils.isNotBlank(modelName), "CubeDesc(%s) has blank modelName", name); + + this.model = MetadataManager.getInstance(config).getDataModelDesc(modelName); + checkNotNull(this.model, "DateModelDesc(%s) not found", modelName); // check if aggregation group is valid validate(); - this.model = MetadataManager.getInstance(config).getDataModelDesc(this.modelName); - - if (this.model == null) { - this.addError("No data model found with name '" + modelName + "'."); - } - for (DimensionDesc dim : dimensions) { dim.init(this, tables); } @@ -559,9 +559,9 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware { // check all dimension columns are presented on rowkey List<TblColRef> dimCols = listDimensionColumnsExcludingDerived(true); - if (rowkey.getRowKeyColumns().length != dimCols.size()) { - addError("RowKey columns count (" + rowkey.getRowKeyColumns().length + ") does not match dimension columns count (" + dimCols.size() + "). "); - } + checkState(rowkey.getRowKeyColumns().length == dimCols.size(), + "RowKey columns count (%d) doesn't match dimensions columns count (%d)", + rowkey.getRowKeyColumns().length, dimCols.size()); initDictionaryDesc(); } @@ -954,25 +954,8 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware { this.autoMergeTimeRanges = autoMergeTimeRanges; } - /** - * Add error info and thrown exception out - * - * @param message - */ public void addError(String message) { - addError(message, false); - } - - /** - * @param message error message - * @param silent if throw exception - */ - public void addError(String message, boolean silent) { - if (!silent) { - throw new IllegalStateException(message); - } else { - this.errors.add(message); - } + this.errors.add(message); } public List<String> getError() { http://git-wip-us.apache.org/repos/asf/kylin/blob/17569f6c/core-cube/src/main/java/org/apache/kylin/cube/model/validation/CubeMetadataValidator.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/CubeMetadataValidator.java b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/CubeMetadataValidator.java index c631f8d..c22930a 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/CubeMetadataValidator.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/CubeMetadataValidator.java @@ -35,39 +35,15 @@ public class CubeMetadataValidator { private IValidatorRule<CubeDesc>[] rules = new IValidatorRule[] { new FunctionRule(), new AggregationGroupRule(), new RowKeyAttrRule(), new DictionaryRule() }; public ValidateContext validate(CubeDesc cube) { - return validate(cube, false); - } - - /** - * @param inject inject error into cube desc - * @return - */ - public ValidateContext validate(CubeDesc cube, boolean inject) { ValidateContext context = new ValidateContext(); - for (int i = 0; i < rules.length; i++) { - IValidatorRule<CubeDesc> rule = rules[i]; + for (IValidatorRule<CubeDesc> rule : rules) { rule.validate(cube, context); } - if (inject) { - injectResult(cube, context); - } - return context; - } - /** - * - * Inject errors info into cubeDesc - * - * @param cubeDesc - * @param context - */ - public void injectResult(CubeDesc cubeDesc, ValidateContext context) { - ValidateContext.Result[] results = context.getResults(); - for (int i = 0; i < results.length; i++) { - ValidateContext.Result result = results[i]; - cubeDesc.addError(result.getLevel() + " : " + result.getMessage(), true); + for (ValidateContext.Result result : context.getResults()) { + cube.addError(result.getLevel() + " : " + result.getMessage()); } - + return context; } } http://git-wip-us.apache.org/repos/asf/kylin/blob/17569f6c/core-metadata/src/main/java/org/apache/kylin/metadata/realization/RealizationStatusEnum.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/RealizationStatusEnum.java b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/RealizationStatusEnum.java index e4583f2..27e2d57 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/RealizationStatusEnum.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/RealizationStatusEnum.java @@ -20,6 +20,6 @@ package org.apache.kylin.metadata.realization; public enum RealizationStatusEnum { - DISABLED, BUILDING, READY, DESCBROKEN + DISABLED, READY, DESCBROKEN } http://git-wip-us.apache.org/repos/asf/kylin/blob/17569f6c/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java ---------------------------------------------------------------------- diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java index 5397df7..42b117c 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java +++ b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java @@ -22,7 +22,6 @@ import java.io.IOException; import java.net.UnknownHostException; import java.util.ArrayList; import java.util.Collections; -import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; @@ -43,6 +42,7 @@ import org.apache.kylin.job.JoinedFlatTable; import org.apache.kylin.metadata.model.IJoinedFlatTableDesc; import org.apache.kylin.metadata.model.SegmentStatusEnum; import org.apache.kylin.metadata.project.ProjectInstance; +import org.apache.kylin.metadata.realization.RealizationStatusEnum; import org.apache.kylin.rest.exception.BadRequestException; import org.apache.kylin.rest.exception.ForbiddenException; import org.apache.kylin.rest.exception.InternalErrorException; @@ -74,6 +74,7 @@ import org.springframework.web.bind.annotation.ResponseBody; import com.fasterxml.jackson.core.JsonParseException; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonMappingException; +import com.google.common.base.Joiner; import com.google.common.collect.Sets; /** @@ -345,8 +346,12 @@ public class CubeController extends BasicController { CubeInstance cube = cubeService.getCubeManager().getCube(cubeName); if (cube == null) { - throw new InternalErrorException("Cannot find cube " + cubeName); + throw new BadRequestException("Cannot find cube " + cubeName); + } + if (cube.getStatus() == RealizationStatusEnum.DESCBROKEN) { + throw new BadRequestException("Broken cube can't be cloned"); } + CubeDesc cubeDesc = cube.getDescriptor(); CubeDesc newCubeDesc = CubeDesc.getCopyOf(cubeDesc); newCubeDesc.setName(newCubeName); @@ -446,18 +451,11 @@ public class CubeController extends BasicController { @ResponseBody public CubeRequest updateCubeDesc(@RequestBody CubeRequest cubeRequest) throws JsonProcessingException { - //update cube CubeDesc desc = deserializeCubeDesc(cubeRequest); - CubeDesc oldCubeDesc; - boolean isCubeDescFreeEditable; - if (desc == null) { return cubeRequest; } - // Check if the cube is editable - isCubeDescFreeEditable = cubeService.isCubeDescFreeEditable(desc); - String projectName = (null == cubeRequest.getProject()) ? ProjectInstance.DEFAULT_PROJECT_NAME : cubeRequest.getProject(); try { CubeInstance cube = cubeService.getCubeManager().getCube(cubeRequest.getCubeName()); @@ -475,14 +473,14 @@ public class CubeController extends BasicController { return cubeRequest; } - oldCubeDesc = cube.getDescriptor(); - if (isCubeDescFreeEditable || oldCubeDesc.consistentWith(desc)) { - desc = cubeService.updateCubeAndDesc(cube, desc, projectName, true); - } else { - logger.warn("Won't update the cube desc due to inconsistency"); - updateRequest(cubeRequest, false, "CubeDesc " + desc.getName() + " is inconsistent with existing. Try purge that cube first or avoid updating key cube desc fields."); + if (cube.getSegments().size() != 0 && !cube.getDescriptor().consistentWith(desc)) { + String error = "CubeDesc " + desc.getName() + " is inconsistent with existing. Try purge that cube first or avoid updating key cube desc fields."; + updateRequest(cubeRequest, false, error); return cubeRequest; } + + desc = cubeService.updateCubeAndDesc(cube, desc, projectName, true); + } catch (AccessDeniedException accessDeniedException) { throw new ForbiddenException("You don't have right to update this cube."); } catch (Exception e) { @@ -491,8 +489,7 @@ public class CubeController extends BasicController { } if (!desc.getError().isEmpty()) { - logger.warn("Cube " + desc.getName() + " fail to update because " + desc.getError()); - updateRequest(cubeRequest, false, omitMessage(desc.getError())); + updateRequest(cubeRequest, false, Joiner.on("\n").join(desc.getError())); return cubeRequest; } @@ -599,19 +596,6 @@ public class CubeController extends BasicController { return desc; } - /** - * @return - */ - private String omitMessage(List<String> errors) { - StringBuffer buffer = new StringBuffer(); - for (Iterator<String> iterator = errors.iterator(); iterator.hasNext();) { - String string = (String) iterator.next(); - buffer.append(string); - buffer.append("\n"); - } - return buffer.toString(); - } - private void updateRequest(CubeRequest request, boolean success, String message) { request.setCubeDescData(""); request.setSuccessful(success); http://git-wip-us.apache.org/repos/asf/kylin/blob/17569f6c/server-base/src/main/java/org/apache/kylin/rest/service/CacheService.java ---------------------------------------------------------------------- diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/CacheService.java b/server-base/src/main/java/org/apache/kylin/rest/service/CacheService.java index 2160e3d..9d134d6 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/service/CacheService.java +++ b/server-base/src/main/java/org/apache/kylin/rest/service/CacheService.java @@ -189,6 +189,7 @@ public class CacheService extends BasicService { case TABLE: getMetadataManager().reloadTableCache(cacheKey); CubeDescManager.clearCache(); + clearRealizationCache(); break; case EXTERNAL_FILTER: getMetadataManager().reloadExtFilter(cacheKey); @@ -202,9 +203,7 @@ public class CacheService extends BasicService { DictionaryManager.clearCache(); MetadataManager.clearCache(); CubeDescManager.clearCache(); - CubeManager.clearCache(); - HybridManager.clearCache(); - RealizationRegistry.clearCache(); + clearRealizationCache(); Cuboid.clearCache(); ProjectManager.clearCache(); KafkaConfigManager.clearCache(); @@ -222,6 +221,12 @@ public class CacheService extends BasicService { } } + private void clearRealizationCache() { + CubeManager.clearCache(); + HybridManager.clearCache(); + RealizationRegistry.clearCache(); + } + private void rebuildCubeCache(String cubeName) { CubeInstance cube = getCubeManager().reloadCubeLocal(cubeName); getHybridManager().reloadHybridInstanceByChild(RealizationType.CUBE, cubeName); http://git-wip-us.apache.org/repos/asf/kylin/blob/17569f6c/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java ---------------------------------------------------------------------- diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java index 4cd527c..e446045 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java +++ b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java @@ -273,21 +273,6 @@ public class CubeService extends BasicService { accessService.clean(cube, true); } - public boolean isCubeDescFreeEditable(CubeDesc cd) { - List<CubeInstance> cubes = getCubeManager().getCubesByDesc(cd.getName()); - for (CubeInstance cube : cubes) { - if (cube.getSegments().size() != 0) { - logger.debug("cube '" + cube.getName() + " has " + cube.getSegments().size() + " segments, couldn't edit cube desc."); - return false; - } - } - return true; - } - - public static String getCubeDescNameFromCube(String cubeName) { - return cubeName + DESC_SUFFIX; - } - public static String getCubeNameFromDesc(String descName) { if (descName.toLowerCase().endsWith(DESC_SUFFIX)) { return descName.substring(0, descName.toLowerCase().indexOf(DESC_SUFFIX)); http://git-wip-us.apache.org/repos/asf/kylin/blob/17569f6c/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java ---------------------------------------------------------------------- diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java b/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java index e4fbc98..5c704ba 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java +++ b/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java @@ -48,7 +48,9 @@ import org.apache.kylin.job.execution.DefaultChainedExecutable; import org.apache.kylin.job.execution.ExecutableState; import org.apache.kylin.job.execution.Output; import org.apache.kylin.metadata.model.SegmentStatusEnum; +import org.apache.kylin.metadata.realization.RealizationStatusEnum; import org.apache.kylin.rest.constant.Constant; +import org.apache.kylin.rest.exception.BadRequestException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -199,6 +201,10 @@ public class JobService extends BasicService { public JobInstance submitJob(CubeInstance cube, long startDate, long endDate, long startOffset, long endOffset, // CubeBuildTypeEnum buildType, boolean force, String submitter) throws IOException, JobException { + if (cube.getStatus() == RealizationStatusEnum.DESCBROKEN) { + throw new BadRequestException("Broken cube " + cube.getName() + " can't be built"); + } + checkCubeDescSignature(cube); checkNoRunningJob(cube); http://git-wip-us.apache.org/repos/asf/kylin/blob/17569f6c/server/src/test/java/org/apache/kylin/rest/service/CubeServiceTest.java ---------------------------------------------------------------------- diff --git a/server/src/test/java/org/apache/kylin/rest/service/CubeServiceTest.java b/server/src/test/java/org/apache/kylin/rest/service/CubeServiceTest.java index f98d6b9..59e96d6 100644 --- a/server/src/test/java/org/apache/kylin/rest/service/CubeServiceTest.java +++ b/server/src/test/java/org/apache/kylin/rest/service/CubeServiceTest.java @@ -51,6 +51,5 @@ public class CubeServiceTest extends ServiceTestBase { List<CubeInstance> cubes = cubeService.listAllCubes(null, null, null); Assert.assertNotNull(cubes); CubeInstance cube = cubes.get(0); - cubeService.isCubeDescFreeEditable(cube.getDescriptor()); } } http://git-wip-us.apache.org/repos/asf/kylin/blob/17569f6c/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java ---------------------------------------------------------------------- diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java index 70b097c..8b98e7b 100644 --- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java +++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java @@ -28,6 +28,7 @@ import java.util.UUID; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.Table; import org.apache.kylin.common.KylinConfig; +import org.apache.kylin.cube.CubeManager; import org.apache.kylin.engine.mr.HadoopUtil; import org.apache.kylin.metadata.MetadataConstants; import org.apache.kylin.metadata.MetadataManager; @@ -36,8 +37,10 @@ import org.apache.kylin.metadata.model.TableDesc; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.google.common.collect.LinkedHashMultimap; import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import com.google.common.collect.SetMultimap; import com.google.common.collect.Sets; /** @@ -51,27 +54,25 @@ public class HiveSourceTableLoader { @SuppressWarnings("unused") private static final Logger logger = LoggerFactory.getLogger(HiveSourceTableLoader.class); - public static final String OUTPUT_SURFIX = "json"; - public static final String TABLE_FOLDER_NAME = "table"; - public static final String TABLE_EXD_FOLDER_NAME = "table_exd"; - public static Set<String> reloadHiveTables(String[] hiveTables, KylinConfig config) throws IOException { - Map<String, Set<String>> db2tables = Maps.newHashMap(); - for (String table : hiveTables) { - String[] parts = HadoopUtil.parseHiveTableName(table); - Set<String> set = db2tables.get(parts[0]); - if (set == null) { - set = Sets.newHashSet(); - db2tables.put(parts[0], set); - } - set.add(parts[1]); + SetMultimap<String, String> db2tables = LinkedHashMultimap.create(); + for (String fullTableName : hiveTables) { + String[] parts = HadoopUtil.parseHiveTableName(fullTableName); + db2tables.put(parts[0], parts[1]); + } + + HiveClient hiveClient = new HiveClient(); + SchemaChecker checker = new SchemaChecker(hiveClient, MetadataManager.getInstance(config), CubeManager.getInstance(config)); + for (Map.Entry<String, String> entry : db2tables.entries()) { + SchemaChecker.CheckResult result = checker.allowReload(entry.getKey(), entry.getValue()); + result.raiseExceptionWhenInvalid(); } // extract from hive Set<String> loadedTables = Sets.newHashSet(); for (String database : db2tables.keySet()) { - List<String> loaded = extractHiveTables(database, db2tables.get(database), config); + List<String> loaded = extractHiveTables(database, db2tables.get(database), hiveClient); loadedTables.addAll(loaded); } @@ -84,13 +85,12 @@ public class HiveSourceTableLoader { metaMgr.removeTableExd(hiveTable); } - private static List<String> extractHiveTables(String database, Set<String> tables, KylinConfig config) throws IOException { + private static List<String> extractHiveTables(String database, Set<String> tables, HiveClient hiveClient) throws IOException { List<String> loadedTables = Lists.newArrayList(); MetadataManager metaMgr = MetadataManager.getInstance(KylinConfig.getInstanceFromEnv()); for (String tableName : tables) { Table table = null; - HiveClient hiveClient = new HiveClient(); List<FieldSchema> partitionFields = null; List<FieldSchema> fields = null; try { @@ -167,5 +167,4 @@ public class HiveSourceTableLoader { return loadedTables; } - } http://git-wip-us.apache.org/repos/asf/kylin/blob/17569f6c/source-hive/src/main/java/org/apache/kylin/source/hive/SchemaChecker.java ---------------------------------------------------------------------- diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/SchemaChecker.java b/source-hive/src/main/java/org/apache/kylin/source/hive/SchemaChecker.java new file mode 100644 index 0000000..3b03551 --- /dev/null +++ b/source-hive/src/main/java/org/apache/kylin/source/hive/SchemaChecker.java @@ -0,0 +1,216 @@ +package org.apache.kylin.source.hive; + +import static com.google.common.base.Preconditions.checkNotNull; +import static java.lang.String.format; + +import java.util.List; +import java.util.Map; +import java.util.Set; + +import javax.annotation.Nullable; + +import org.apache.hadoop.hive.metastore.api.FieldSchema; +import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.kylin.cube.CubeInstance; +import org.apache.kylin.cube.CubeManager; +import org.apache.kylin.cube.model.CubeDesc; +import org.apache.kylin.metadata.MetadataManager; +import org.apache.kylin.metadata.datatype.DataType; +import org.apache.kylin.metadata.model.ColumnDesc; +import org.apache.kylin.metadata.model.TableDesc; +import org.apache.kylin.metadata.model.TblColRef; + +import com.google.common.base.Predicate; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; + +public class SchemaChecker { + private final HiveClient hiveClient; + private final MetadataManager metadataManager; + private final CubeManager cubeManager; + + static class CheckResult { + private final boolean valid; + private final String reason; + + private CheckResult(boolean valid, String reason) { + this.valid = valid; + this.reason = reason; + } + + void raiseExceptionWhenInvalid() { + if (!valid) { + throw new RuntimeException(reason); + } + } + + static CheckResult validOnFirstLoad(String tableName) { + return new CheckResult(true, format("Table '%s' hasn't been loaded before", tableName)); + } + + static CheckResult validOnCompatibleSchema(String tableName) { + return new CheckResult(true, format("Table '%s' is compatible with all existing cubes", tableName)); + } + + static CheckResult invalidOnFetchSchema(String tableName, Exception e) { + return new CheckResult(false, format("Failed to fetch metadata of '%s': %s", tableName, e.getMessage())); + } + + static CheckResult invalidOnIncompatibleSchema(String tableName, List<String> reasons) { + StringBuilder buf = new StringBuilder(); + for (String reason : reasons) { + buf.append("- ").append(reason).append("\n"); + } + + return new CheckResult(false, format("Found %d issue(s) with '%s':\n%s Please disable and purge related cube(s) first", reasons.size(), tableName, buf.toString())); + } + } + + SchemaChecker(HiveClient hiveClient, MetadataManager metadataManager, CubeManager cubeManager) { + this.hiveClient = checkNotNull(hiveClient, "hiveClient is null"); + this.metadataManager = checkNotNull(metadataManager, "metadataManager is null"); + this.cubeManager = checkNotNull(cubeManager, "cubeManager is null"); + } + + private List<FieldSchema> fetchSchema(String dbName, String tblName) throws Exception { + List<FieldSchema> fields = Lists.newArrayList(); + fields.addAll(hiveClient.getHiveTableFields(dbName, tblName)); + + Table table = hiveClient.getHiveTable(dbName, tblName); + List<FieldSchema> partitionFields = table.getPartitionKeys(); + if (partitionFields != null) { + fields.addAll(partitionFields); + } + + return fields; + } + + private List<CubeInstance> findCubeByTable(final String fullTableName) { + Iterable<CubeInstance> relatedCubes = Iterables.filter(cubeManager.listAllCubes(), new Predicate<CubeInstance>() { + @Override + public boolean apply(@Nullable CubeInstance cube) { + if (cube == null || cube.allowBrokenDescriptor()) { + return false; + } + CubeDesc desc = cube.getDescriptor(); + + Set<String> usedTables = Sets.newHashSet(); + usedTables.add(desc.getFactTableDesc().getIdentity()); + for (TableDesc lookup : desc.getLookupTableDescs()) { + usedTables.add(lookup.getIdentity()); + } + + return usedTables.contains(fullTableName); + } + }); + + return ImmutableList.copyOf(relatedCubes); + } + + private boolean isColumnCompatible(ColumnDesc column, FieldSchema field) { + if (!column.getName().equalsIgnoreCase(field.getName())) { + return false; + } + + String typeStr = field.getType(); + // kylin uses double internally for float, see HiveSourceTableLoader.java + // TODO should this normalization to be in DataType class ? + if ("float".equalsIgnoreCase(typeStr)) { + typeStr = "double"; + } + DataType fieldType = DataType.getType(typeStr); + + if (column.getType().isIntegerFamily()) { + // OLAPTable.listSourceColumns converts some integer columns to bigint, + // therefore strict type comparison won't work. + // changing from one integer type to another should be fine. + return fieldType.isIntegerFamily(); + } else { + // only compare base type name, changing precision or scale should be fine + return column.getTypeName().equals(fieldType.getName()); + } + } + + private List<String> checkAllUsedColumns(CubeInstance cube, TableDesc table, Map<String, FieldSchema> fieldsMap) { + Set<ColumnDesc> usedColumns = Sets.newHashSet(); + for (TblColRef col : cube.getAllColumns()) { + usedColumns.add(col.getColumnDesc()); + } + + List<String> violateColumns = Lists.newArrayList(); + for (ColumnDesc column : table.getColumns()) { + if (usedColumns.contains(column)) { + FieldSchema field = fieldsMap.get(column.getName()); + if (field == null || !isColumnCompatible(column, field)) { + violateColumns.add(column.getName()); + } + } + } + return violateColumns; + } + + private boolean checkAllColumns(TableDesc table, List<FieldSchema> fields) { + if (table.getColumnCount() != fields.size()) { + return false; + } + + ColumnDesc[] columns = table.getColumns(); + for (int i = 0; i < columns.length; i++) { + if (!isColumnCompatible(columns[i], fields.get(i))) { + return false; + } + } + return true; + } + + public CheckResult allowReload(String dbName, String tblName) { + final String fullTableName = (dbName + "." + tblName).toUpperCase(); + + TableDesc existing = metadataManager.getTableDesc(fullTableName); + if (existing == null) { + return CheckResult.validOnFirstLoad(fullTableName); + } + + List<FieldSchema> currentFields; + Map<String, FieldSchema> currentFieldsMap = Maps.newHashMap(); + try { + currentFields = fetchSchema(dbName, tblName); + } catch (Exception e) { + return CheckResult.invalidOnFetchSchema(fullTableName, e); + } + for (FieldSchema field : currentFields) { + currentFieldsMap.put(field.getName().toUpperCase(), field); + } + + List<String> issues = Lists.newArrayList(); + for (CubeInstance cube : findCubeByTable(fullTableName)) { + TableDesc factTable = cube.getFactTableDesc(); + List<TableDesc> lookupTables = cube.getDescriptor().getLookupTableDescs(); + String modelName = cube.getDataModelDesc().getName(); + + // if user reloads a fact table used by cube, then all used columns + // must match current schema + if (factTable.getIdentity().equals(fullTableName)) { + List<String> violateColumns = checkAllUsedColumns(cube, factTable, currentFieldsMap); + if (!violateColumns.isEmpty()) { + issues.add(format("Column %s used in cube[%s] and model[%s], but changed in hive", violateColumns, cube.getName(), modelName)); + } + } + + // if user reloads a lookup table used by cube, then nearly all changes in schema are disallowed) + for (TableDesc lookupTable : lookupTables) { + if (lookupTable.getIdentity().equals(fullTableName) && !checkAllColumns(lookupTable, currentFields)) { + issues.add(format("Table '%s' is used as Lookup Table in cube[%s] and model[%s], but changed in hive", lookupTable.getIdentity(), cube.getName(), modelName)); + } + } + } + + if (issues.isEmpty()) { + return CheckResult.validOnCompatibleSchema(fullTableName); + } + return CheckResult.invalidOnIncompatibleSchema(fullTableName, issues); + } +} http://git-wip-us.apache.org/repos/asf/kylin/blob/17569f6c/webapp/app/css/AdminLTE.css ---------------------------------------------------------------------- diff --git a/webapp/app/css/AdminLTE.css b/webapp/app/css/AdminLTE.css index e772ae5..6688457 100644 --- a/webapp/app/css/AdminLTE.css +++ b/webapp/app/css/AdminLTE.css @@ -4307,7 +4307,7 @@ fieldset[disabled] .btn-vk.active { .alert-info, .label-danger, .label-info, -.label-waring, +.label-warning, .label-primary, .label-success, .modal-primary .modal-body, @@ -4349,7 +4349,7 @@ fieldset[disabled] .btn-vk.active { .bg-yellow, .callout.callout-warning, .alert-warning, -.label-waring, +.label-warning, .modal-warning .modal-body { background-color: #f39c12 !important; } http://git-wip-us.apache.org/repos/asf/kylin/blob/17569f6c/webapp/app/partials/cubes/cubes.html ---------------------------------------------------------------------- diff --git a/webapp/app/partials/cubes/cubes.html b/webapp/app/partials/cubes/cubes.html index 30981dc..de9b99b 100644 --- a/webapp/app/partials/cubes/cubes.html +++ b/webapp/app/partials/cubes/cubes.html @@ -67,7 +67,7 @@ </td> <td> <span class="label" - ng-class="{'label-success': cube.status=='READY', 'label-default': cube.status=='DISABLED'}"> + ng-class="{'label-success': cube.status=='READY', 'label-default': cube.status=='DISABLED', 'label-warning': cube.status=='DESCBROKEN'}"> {{ cube.status}} </span> </td> @@ -89,18 +89,18 @@ Action <span class="ace-icon fa fa-caret-down icon-on-right"></span> </button> <ul class="dropdown-menu" role="menu"> - <li ng-if="cube.status=='DISABLED' && userService.hasRole('ROLE_ADMIN') "> + <li ng-if="cube.status!='READY' && userService.hasRole('ROLE_ADMIN') "> <a ng-click="dropCube(cube)" tooltip="Drop the cube, related jobs and data permanently.">Drop</a></li> - <li ng-if="cube.status=='DISABLED' && (userService.hasRole('ROLE_ADMIN') || hasPermission(cube, permissions.ADMINISTRATION.mask, permissions.MANAGEMENT.mask))"> + <li ng-if="cube.status!='READY' && (userService.hasRole('ROLE_ADMIN') || hasPermission(cube, permissions.ADMINISTRATION.mask, permissions.MANAGEMENT.mask))"> <a ng-click="cubeEdit(cube);">Edit</a></li> - <li ng-if="cube.streaming && cube.status=='DISABLED' && (userService.hasRole('ROLE_ADMIN') || hasPermission(cube, permissions.ADMINISTRATION.mask, permissions.MANAGEMENT.mask))"></li> - <li><a ng-click="startJobSubmit(cube);">Build</a></li> - <li><a ng-click="startRefresh(cube)">Refresh</a></li> - <li><a ng-click="startMerge(cube)">Merge</a></li> - <li ng-if="cube.status!='DISABLED'"><a ng-click="disable(cube)">Disable</a></li> + <li ng-if="cube.streaming && cube.status=='DISABLED' && (userService.hasRole('ROLE_ADMIN') || hasPermission(cube, permissions.ADMINISTRATION.mask, permissions.MANAGEMENT.mask))"></li> + <li ng-if="cube.status!='DESCBROKEN'"><a ng-click="startJobSubmit(cube);">Build</a></li> + <li ng-if="cube.status!='DESCBROKEN'"><a ng-click="startRefresh(cube)">Refresh</a></li> + <li ng-if="cube.status!='DESCBROKEN'"><a ng-click="startMerge(cube)">Merge</a></li> + <li ng-if="cube.status=='READY'"><a ng-click="disable(cube)">Disable</a></li> <li ng-if="cube.status=='DISABLED'"><a ng-click="enable(cube)">Enable</a></li> <li ng-if="cube.status=='DISABLED'"><a ng-click="purge(cube)">Purge</a></li> - <li><a ng-click="cloneCube(cube)">Clone</a></li> + <li ng-if="cube.status!='DESCBROKEN'"><a ng-click="cloneCube(cube)">Clone</a></li> </ul> </div> @@ -114,8 +114,8 @@ Action <span class="ace-icon fa fa-caret-down icon-on-right"></span> </button> <ul class="dropdown-menu" role="menu"> - <li ng-if="cube.status=='DISABLED'"><a href="cubes/edit/{{cube.name}}/descriptionjson">Edit CubeDesc</a></li> - <li ng-if="cube.status=='DISABLED'"><a href="cubes/view/{{cube.name}}/instancejson">View Cube</a></li> + <li ng-if="cube.status!='READY'"><a href="cubes/edit/{{cube.name}}/descriptionjson">Edit CubeDesc</a></li> + <li ng-if="cube.status!='READY'"><a href="cubes/view/{{cube.name}}/instancejson">View Cube</a></li> </ul> </div> </td>
