zgxme commented on code in PR #48023: URL: https://github.com/apache/doris/pull/48023#discussion_r2032605682
########## regression-test/suites/ranger_p2/test_ranger_masking.groovy: ########## @@ -0,0 +1,194 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.ranger.RangerClient +import org.apache.ranger.plugin.model.RangerPolicy + + +suite("test_ranger_masking", "p2,ranger,external") { + String enabled = context.config.otherConfigs.get("enableRangerTest") + String enableHiveTest = context.config.otherConfigs.get("enableHiveTest") + String rangerEndpoint = context.config.otherConfigs.get("rangerEndpoint") + String rangerUser = context.config.otherConfigs.get("rangerUser") + String rangerPassword = context.config.otherConfigs.get("rangerPassword") + String rangerServiceName = context.config.otherConfigs.get("rangerServiceName") + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String HmsPort = context.config.otherConfigs.get("hive3HmsPort") + + if ((enabled != null && enabled.equalsIgnoreCase("true")) && (enableHiveTest != null && enableHiveTest.equalsIgnoreCase("true"))) { + String catalog1 = 'ranger_catalog_6' + // prepare catalog + sql """DROP CATALOG IF EXISTS ${catalog1}""" + sql """CREATE CATALOG `${catalog1}` PROPERTIES ( + "type"="hms", + 'hive.metastore.uris' = 'thrift://${externalEnvIp}:${HmsPort}' + )""" + // prepare database + String catalogDb = 'ranger_catalog_6_db_1' + sql """CREATE DATABASE IF NOT EXISTS ${catalog1}.${catalogDb}""" + + String internalDb = 'ranger_internal_db_5' + sql """CREATE DATABASE IF NOT EXISTS ${internalDb}""" + + // prepare table + String catalogTable = 'ranger_catalog_6_db1_tbl1' + sql """DROP TABLE IF EXISTS ${catalog1}.${catalogDb}.${catalogTable}""" + sql """CREATE TABLE IF NOT EXISTS ${catalog1}.${catalogDb}.${catalogTable} ( + id BIGINT, + c1 VARCHAR(20), + c2 VARCHAR(20), + c3 VARCHAR(20), + c4 VARCHAR(20), + c5 VARCHAR(20), + c6 VARCHAR(20), + c7 VARCHAR(20), + c8 DATE, + c9 DATETIME + ) ENGINE=hive + PROPERTIES ( + 'file_format'='parquet' + );""" + String internalTable = 'ranger_internal_db_5_tbl1' + sql """DROP TABLE IF EXISTS internal.${internalDb}.`${internalTable}`""" + sql """CREATE TABLE IF NOT EXISTS internal.${internalDb}.`${internalTable}` ( + id BIGINT, + c1 VARCHAR(20), + c2 VARCHAR(20), + c3 VARCHAR(20), + c4 VARCHAR(20), + c5 VARCHAR(20), + c6 VARCHAR(20), + c7 VARCHAR(20), + c8 DATE, + c9 DATETIME + ) + DISTRIBUTED BY HASH(id) BUCKETS 2 + PROPERTIES ( + "replication_num" = "1" + );""" + // prepare data + sql """INSERT INTO ${catalog1}.${catalogDb}.${catalogTable} (id, c1, c2, c3, c4, c5, c6, c7, c8, c9) VALUES + (1, 'DataOne01', 'SampleA1', 'Value1X', 'InfoX123', 'DescA1', 'Extra1A', 'AddVal1', '2023-01-01', '2010-12-02 19:28:30'), + (2, 'DataTwo02', 'SampleB2', 'Value2Y', 'InfoY234', 'DescB2', 'Extra2B', 'AddVal2', '2023-02-01', '2010-12-02 19:28:30'), + (3, 'DataThr03', 'SampleC3', 'Value3Z', 'InfoZ345', 'DescC3', 'Extra3C', 'AddVal3', '2023-03-01', '2010-12-02 19:28:30'), + (4, 'DataFou04', 'SampleD4', 'Value4W', 'InfoW456', 'DescD4', 'Extra4D', 'AddVal4', '2023-04-01', '2011-12-02 19:28:30'), + (5, 'DataFiv05', 'SampleE5', 'Value5V', 'InfoV567', 'DescE5', 'Extra5E', 'AddVal5', '2023-05-01', '2012-12-02 19:28:30'), + (6, 'DataSix06', 'SampleF6', 'Value6U', 'InfoU678', 'DescF6', 'Extra6F', 'AddVal6', '2023-06-01', '2013-12-02 19:28:30'), + (7, 'DataSev07', 'SampleG7', 'Value7T', 'InfoT789', 'DescG7', 'Extra7G', 'AddVal7', '2023-07-01', '2014-12-02 19:28:30'), + (8, 'DataEig08', 'SampleH8', 'Value8S', 'InfoS890', 'DescH8', 'Extra8H', 'AddVal8', '2023-08-01', '2015-12-02 19:28:30'), + (9, 'DataNin09', 'SampleI9', 'Value9R', 'InfoR901', 'DescI9', 'Extra9I', 'AddVal9', '2024-09-01', '2016-12-02 19:28:30'), + (10, 'DataTen10', 'SampleJ0', 'Value0Q', 'InfoQ012', 'DescJ0', 'Extra0J', 'AddVal0', '2025-10-01', '2017-12-02 19:28:30');""" + + sql """INSERT INTO internal.${internalDb}.${internalTable} (id, c1, c2, c3, c4, c5, c6, c7, c8, c9) VALUES + (1, 'DataOne01', 'SampleA1', 'Value1X', 'InfoX123', 'DescA1', 'Extra1A', 'AddVal1', '2023-01-01', '2010-12-02 19:28:30'), + (2, 'DataTwo02', 'SampleB2', 'Value2Y', 'InfoY234', 'DescB2', 'Extra2B', 'AddVal2', '2023-02-01', '2010-12-02 19:28:30'), + (3, 'DataThr03', 'SampleC3', 'Value3Z', 'InfoZ345', 'DescC3', 'Extra3C', 'AddVal3', '2023-03-01', '2010-12-02 19:28:30'), + (4, 'DataFou04', 'SampleD4', 'Value4W', 'InfoW456', 'DescD4', 'Extra4D', 'AddVal4', '2023-04-01', '2011-12-02 19:28:30'), + (5, 'DataFiv05', 'SampleE5', 'Value5V', 'InfoV567', 'DescE5', 'Extra5E', 'AddVal5', '2023-05-01', '2012-12-02 19:28:30'), + (6, 'DataSix06', 'SampleF6', 'Value6U', 'InfoU678', 'DescF6', 'Extra6F', 'AddVal6', '2023-06-01', '2013-12-02 19:28:30'), + (7, 'DataSev07', 'SampleG7', 'Value7T', 'InfoT789', 'DescG7', 'Extra7G', 'AddVal7', '2023-07-01', '2014-12-02 19:28:30'), + (8, 'DataEig08', 'SampleH8', 'Value8S', 'InfoS890', 'DescH8', 'Extra8H', 'AddVal8', '2023-08-01', '2015-12-02 19:28:30'), + (9, 'DataNin09', 'SampleI9', 'Value9R', 'InfoR901', 'DescI9', 'Extra9I', 'AddVal9', '2024-09-01', '2016-12-02 19:28:30'), + (10, 'DataTen10', 'SampleJ0', 'Value0Q', 'InfoQ012', 'DescJ0', 'Extra0J', 'AddVal0', '2025-10-01', '2017-12-02 19:28:30');""" + + // prepare user + String user = 'ranger_test_masking_user_1' + String pwd = 'C123_567p' + sql """DROP USER IF EXISTS ${user}""" + sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}'""" + // create policy + RangerClient rangerClient = new RangerClient("http://${rangerEndpoint}";, "simple", rangerUser, rangerPassword, null) + String policy1 = 'ranger_test_masking_policy_access' + List<String> tablePolicy = ["SELECT"] + + Map<String, RangerPolicy.RangerPolicyResource> resource = new HashMap<>() + resource.put("catalog", new RangerPolicy.RangerPolicyResource("*")) + resource.put("database", new RangerPolicy.RangerPolicyResource("*")) + resource.put("table", new RangerPolicy.RangerPolicyResource([internalTable, catalogTable], false, false)) + resource.put("column", new RangerPolicy.RangerPolicyResource("*")) + + RangerPolicy policy = new RangerPolicy() + policy.setService(rangerServiceName) + policy.setName(policy1) + policy.setResources(resource) + + RangerPolicy.RangerPolicyItem policyItem = new RangerPolicy.RangerPolicyItem() + policyItem.setUsers([user]) + List<RangerPolicy.RangerPolicyItemAccess> policyItemAccesses = new ArrayList<RangerPolicy.RangerPolicyItemAccess>() + tablePolicy.forEach { + policyItemAccesses.add(new RangerPolicy.RangerPolicyItemAccess(it)) + } + policyItem.setAccesses(policyItemAccesses) + policy.setPolicyItems([policyItem]) + try { + rangerClient.deletePolicy(rangerServiceName, policy1) + } catch (Exception e) { + log.info("Policy not found: ${e.getMessage()}") + } + RangerPolicy createdPolicy = rangerClient.createPolicy(policy) + println("New Policy created with id: " + createdPolicy.getId()) + + + Map<String, String> columnMaskMap = new HashMap<>() + columnMaskMap.put("c1", "MASK") + columnMaskMap.put("c2", "MASK_SHOW_LAST_4") + columnMaskMap.put("c3", "MASK_SHOW_FIRST_4") + columnMaskMap.put("c4", "MASK_HASH") + columnMaskMap.put("c5", "MASK_NULL") + columnMaskMap.put("c7", "CUSTOM") + columnMaskMap.put("c6", "MASK_NONE") + columnMaskMap.put("c8", "MASK_DATE_SHOW_YEAR") + columnMaskMap.put("c9", "MASK_DATE_SHOW_YEAR") + + policy.setPolicyItems([]) + policy.setPolicyType(RangerPolicy.POLICY_TYPE_DATAMASK) + for (Map.Entry<String, String> entry : columnMaskMap.entrySet()) { + String column = entry.getKey() + String maskType = entry.getValue() + String maskPolicy = "ranger_test_masking_policy_${column}" + policy.setName(maskPolicy) + + try { + rangerClient.deletePolicy(rangerServiceName, maskPolicy) + } catch (Exception e) { + log.info("Policy not found: ${e.getMessage()}") + } + resource.put("column", new RangerPolicy.RangerPolicyResource(column)) + policy.setResources(resource) + RangerPolicy.RangerDataMaskPolicyItem dataMaskPolicyItem = new RangerPolicy.RangerDataMaskPolicyItem() + dataMaskPolicyItem.setUsers([user]) + dataMaskPolicyItem.setAccesses([new RangerPolicy.RangerPolicyItemAccess("SELECT")]) + dataMaskPolicyItem.setDataMaskInfo(new RangerPolicy.RangerPolicyItemDataMaskInfo(maskType, "", "")) + if (maskType == "CUSTOM") { + dataMaskPolicyItem.setDataMaskInfo(new RangerPolicy.RangerPolicyItemDataMaskInfo(maskType, "", "\"ranger test\"")) + } + policy.setDataMaskPolicyItems([dataMaskPolicyItem]) + createdPolicy = rangerClient.createPolicy(policy) + println("New Policy created with id: " + createdPolicy.getId()) + } + sleep(6000) + // check + def tokens = context.config.jdbcUrl.split('/') + def defaultJdbcUrl = tokens[0] + "//" + tokens[2] + "/?" + connect("$user", "$pwd", "$defaultJdbcUrl") { + sql("""SELECT * FROM internal.${internalDb}.${internalTable}""") Review Comment: done ########## regression-test/data/ranger_p2/test_ranger_row_level_filter.out: ########## @@ -0,0 +1,17 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !1 -- +10 DataTen10 SampleJ0 Value0Q InfoQ012 DescJ0 Extra0J AddVal0 2025-10-01 2017-12-02T19:28:30 +5 DataFiv05 SampleE5 Value5V InfoV567 DescE5 Extra5E AddVal5 2023-05-01 2012-12-02T19:28:30 +6 DataSix06 SampleF6 Value6U InfoU678 DescF6 Extra6F AddVal6 2023-06-01 2013-12-02T19:28:30 +7 DataSev07 SampleG7 Value7T InfoT789 DescG7 Extra7G AddVal7 2023-07-01 2014-12-02T19:28:30 +8 DataEig08 SampleH8 Value8S InfoS890 DescH8 Extra8H AddVal8 2023-08-01 2015-12-02T19:28:30 +9 DataNin09 SampleI9 Value9R InfoR901 DescI9 Extra9I AddVal9 2024-09-01 2016-12-02T19:28:30 + +-- !2 -- +10 DataTen10 SampleJ0 Value0Q InfoQ012 DescJ0 Extra0J AddVal0 2025-10-01 2017-12-02T19:28:30 Review Comment: done -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
