szilard-nemeth commented on a change in pull request #3358: URL: https://github.com/apache/hadoop/pull/3358#discussion_r731581051
########## File path: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/QueueCapacityConfigParser.java ########## @@ -0,0 +1,210 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf; + +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacityVector; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacityVector.QueueCapacityType; +import org.apache.hadoop.yarn.util.UnitsConversionUtil; + +import java.util.ArrayList; +import java.util.List; +import java.util.function.Function; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * A class that parses {@code QueueCapacityVector} from the capacity + * configuration property set for a queue. + * + * A new syntax for capacity property could be implemented, by creating a parser + * with a regex to match the pattern and a method that creates a + * {@code QueueCapacityVector} from the matched pattern + * eg. root.capacity 20-50 + * + * A new capacity type for the existing parsers could be added by extending + * the {@code QueueCapacityVector.QueueCapacityType} with a new type and its + * associated postfix symbol. + * eg. root.capacity 20g + */ +public class QueueCapacityConfigParser { + private static final String UNIFORM_REGEX = "^([0-9.]+)(.*)"; + private static final String RESOURCE_REGEX = "^\\[([\\w\\.,\\-_%\\ /]+=[\\w\\.,\\-_%\\ /]+)+\\]$"; + + private static final Pattern RESOURCE_PATTERN = Pattern.compile(RESOURCE_REGEX); + private static final Pattern UNIFORM_PATTERN = Pattern.compile(UNIFORM_REGEX); + + private final List<Parser> parsers = new ArrayList<>(); + + public QueueCapacityConfigParser() { + parsers.add(new Parser(RESOURCE_PATTERN, this::heterogeneousParser)); + parsers.add(new Parser(UNIFORM_PATTERN, this::uniformParser)); + } + + /** + * Creates a {@code QueueCapacityVector} parsed from the capacity configuration + * property set for a queue. + * @param conf configuration object + * @param queuePath queue for which the capacity property is parsed + * @param label node label + * @return a parsed capacity vector + */ + public QueueCapacityVector parse(CapacitySchedulerConfiguration conf, + String queuePath, String label) { + + if (queuePath.equals(CapacitySchedulerConfiguration.ROOT)) { + return QueueCapacityVector.of(100f, QueueCapacityType.PERCENTAGE); + } + + String propertyName = CapacitySchedulerConfiguration.getNodeLabelPrefix( + queuePath, label) + CapacitySchedulerConfiguration.CAPACITY; + String capacityString = conf.get(propertyName); + + if (capacityString == null) { + return new QueueCapacityVector(); + } + + for (Parser parser : parsers) { + Matcher matcher = parser.regex.matcher(capacityString); + if (matcher.find()) { + return parser.parser.apply(matcher); + } + } + + return new QueueCapacityVector(); + } + + /** + * A parser method that is usable on uniform capacity values eg. percentage or + * weight. + * @param matcher a regex matcher that contains parsed value and its possible + * suffix + * @return a parsed resource vector + */ + private QueueCapacityVector uniformParser(Matcher matcher) { + QueueCapacityType capacityType = QueueCapacityType.PERCENTAGE; + String value = matcher.group(1); + if (matcher.groupCount() == 2) { + String matchedSuffix = matcher.group(2); + if (!matchedSuffix.isEmpty()) { + for (QueueCapacityType suffix : QueueCapacityType.values()) { + // when capacity is given in percentage, we do not need % symbol + String uniformSuffix = suffix.getPostfix().replaceAll("%", ""); + if (uniformSuffix.equals(matchedSuffix)) { + capacityType = suffix; + } + } + } + } Review comment: Should we handle errors here? I mean if the suffix is not one of the strings defined in QueueCapacityType, what will happen? ########## File path: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/QueueCapacityConfigParser.java ########## @@ -0,0 +1,210 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf; + +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacityVector; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacityVector.QueueCapacityType; +import org.apache.hadoop.yarn.util.UnitsConversionUtil; + +import java.util.ArrayList; +import java.util.List; +import java.util.function.Function; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * A class that parses {@code QueueCapacityVector} from the capacity + * configuration property set for a queue. + * + * A new syntax for capacity property could be implemented, by creating a parser + * with a regex to match the pattern and a method that creates a + * {@code QueueCapacityVector} from the matched pattern + * eg. root.capacity 20-50 + * + * A new capacity type for the existing parsers could be added by extending + * the {@code QueueCapacityVector.QueueCapacityType} with a new type and its + * associated postfix symbol. + * eg. root.capacity 20g + */ +public class QueueCapacityConfigParser { + private static final String UNIFORM_REGEX = "^([0-9.]+)(.*)"; + private static final String RESOURCE_REGEX = "^\\[([\\w\\.,\\-_%\\ /]+=[\\w\\.,\\-_%\\ /]+)+\\]$"; + + private static final Pattern RESOURCE_PATTERN = Pattern.compile(RESOURCE_REGEX); + private static final Pattern UNIFORM_PATTERN = Pattern.compile(UNIFORM_REGEX); + + private final List<Parser> parsers = new ArrayList<>(); + + public QueueCapacityConfigParser() { + parsers.add(new Parser(RESOURCE_PATTERN, this::heterogeneousParser)); + parsers.add(new Parser(UNIFORM_PATTERN, this::uniformParser)); + } + + /** + * Creates a {@code QueueCapacityVector} parsed from the capacity configuration + * property set for a queue. + * @param conf configuration object + * @param queuePath queue for which the capacity property is parsed + * @param label node label + * @return a parsed capacity vector + */ + public QueueCapacityVector parse(CapacitySchedulerConfiguration conf, + String queuePath, String label) { + + if (queuePath.equals(CapacitySchedulerConfiguration.ROOT)) { + return QueueCapacityVector.of(100f, QueueCapacityType.PERCENTAGE); + } + + String propertyName = CapacitySchedulerConfiguration.getNodeLabelPrefix( + queuePath, label) + CapacitySchedulerConfiguration.CAPACITY; + String capacityString = conf.get(propertyName); + + if (capacityString == null) { + return new QueueCapacityVector(); + } + + for (Parser parser : parsers) { + Matcher matcher = parser.regex.matcher(capacityString); + if (matcher.find()) { + return parser.parser.apply(matcher); + } + } + + return new QueueCapacityVector(); + } + + /** + * A parser method that is usable on uniform capacity values eg. percentage or Review comment: Nit: e.g. ########## File path: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/QueueCapacityConfigParser.java ########## @@ -0,0 +1,210 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf; + +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacityVector; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacityVector.QueueCapacityType; +import org.apache.hadoop.yarn.util.UnitsConversionUtil; + +import java.util.ArrayList; +import java.util.List; +import java.util.function.Function; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * A class that parses {@code QueueCapacityVector} from the capacity + * configuration property set for a queue. + * + * A new syntax for capacity property could be implemented, by creating a parser + * with a regex to match the pattern and a method that creates a + * {@code QueueCapacityVector} from the matched pattern + * eg. root.capacity 20-50 + * + * A new capacity type for the existing parsers could be added by extending + * the {@code QueueCapacityVector.QueueCapacityType} with a new type and its + * associated postfix symbol. + * eg. root.capacity 20g + */ +public class QueueCapacityConfigParser { + private static final String UNIFORM_REGEX = "^([0-9.]+)(.*)"; + private static final String RESOURCE_REGEX = "^\\[([\\w\\.,\\-_%\\ /]+=[\\w\\.,\\-_%\\ /]+)+\\]$"; + + private static final Pattern RESOURCE_PATTERN = Pattern.compile(RESOURCE_REGEX); + private static final Pattern UNIFORM_PATTERN = Pattern.compile(UNIFORM_REGEX); + + private final List<Parser> parsers = new ArrayList<>(); + + public QueueCapacityConfigParser() { + parsers.add(new Parser(RESOURCE_PATTERN, this::heterogeneousParser)); + parsers.add(new Parser(UNIFORM_PATTERN, this::uniformParser)); + } + + /** + * Creates a {@code QueueCapacityVector} parsed from the capacity configuration + * property set for a queue. + * @param conf configuration object + * @param queuePath queue for which the capacity property is parsed + * @param label node label + * @return a parsed capacity vector + */ + public QueueCapacityVector parse(CapacitySchedulerConfiguration conf, + String queuePath, String label) { + + if (queuePath.equals(CapacitySchedulerConfiguration.ROOT)) { + return QueueCapacityVector.of(100f, QueueCapacityType.PERCENTAGE); + } + + String propertyName = CapacitySchedulerConfiguration.getNodeLabelPrefix( + queuePath, label) + CapacitySchedulerConfiguration.CAPACITY; + String capacityString = conf.get(propertyName); + + if (capacityString == null) { + return new QueueCapacityVector(); + } + + for (Parser parser : parsers) { + Matcher matcher = parser.regex.matcher(capacityString); + if (matcher.find()) { + return parser.parser.apply(matcher); + } + } + + return new QueueCapacityVector(); + } + + /** + * A parser method that is usable on uniform capacity values eg. percentage or + * weight. + * @param matcher a regex matcher that contains parsed value and its possible + * suffix + * @return a parsed resource vector + */ + private QueueCapacityVector uniformParser(Matcher matcher) { + QueueCapacityType capacityType = QueueCapacityType.PERCENTAGE; + String value = matcher.group(1); + if (matcher.groupCount() == 2) { + String matchedSuffix = matcher.group(2); + if (!matchedSuffix.isEmpty()) { + for (QueueCapacityType suffix : QueueCapacityType.values()) { + // when capacity is given in percentage, we do not need % symbol + String uniformSuffix = suffix.getPostfix().replaceAll("%", ""); + if (uniformSuffix.equals(matchedSuffix)) { + capacityType = suffix; + } + } + } + } + + return QueueCapacityVector.of(Float.parseFloat(value), capacityType); + } + + /** + * A parser method that is usable on resource capacity values eg. mixed or + * absolute resource. + * @param matcher a regex matcher that contains the matched resource string + * @return a parsed capacity vector + */ + private QueueCapacityVector heterogeneousParser(Matcher matcher) { + // Define resource here. + QueueCapacityVector capacityVector = QueueCapacityVector.newInstance(); + + /* + * Absolute resource configuration for a queue will be grouped by "[]". + * Syntax of absolute resource config could be like below + * "memory=4Gi vcores=2". Ideally this means "4GB of memory and 2 vcores". + */ + // Get the sub-group. + String bracketedGroup = matcher.group(0); + if (bracketedGroup.trim().isEmpty()) { + return capacityVector; + } + bracketedGroup = bracketedGroup.substring(1, bracketedGroup.length() - 1); + // Split by comma and equals delimiter eg. memory=1024, vcores=6 to + // [[memory, 1024], [vcores, 6]] Review comment: Nit: This seems like an example config but as per our discussion you represent the array with it. Please refine the comment. ########## File path: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/QueueCapacityConfigParser.java ########## @@ -0,0 +1,210 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf; + +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacityVector; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacityVector.QueueCapacityType; +import org.apache.hadoop.yarn.util.UnitsConversionUtil; + +import java.util.ArrayList; +import java.util.List; +import java.util.function.Function; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * A class that parses {@code QueueCapacityVector} from the capacity + * configuration property set for a queue. + * + * A new syntax for capacity property could be implemented, by creating a parser + * with a regex to match the pattern and a method that creates a + * {@code QueueCapacityVector} from the matched pattern + * eg. root.capacity 20-50 + * + * A new capacity type for the existing parsers could be added by extending + * the {@code QueueCapacityVector.QueueCapacityType} with a new type and its + * associated postfix symbol. + * eg. root.capacity 20g + */ +public class QueueCapacityConfigParser { + private static final String UNIFORM_REGEX = "^([0-9.]+)(.*)"; + private static final String RESOURCE_REGEX = "^\\[([\\w\\.,\\-_%\\ /]+=[\\w\\.,\\-_%\\ /]+)+\\]$"; + + private static final Pattern RESOURCE_PATTERN = Pattern.compile(RESOURCE_REGEX); + private static final Pattern UNIFORM_PATTERN = Pattern.compile(UNIFORM_REGEX); + + private final List<Parser> parsers = new ArrayList<>(); + + public QueueCapacityConfigParser() { + parsers.add(new Parser(RESOURCE_PATTERN, this::heterogeneousParser)); + parsers.add(new Parser(UNIFORM_PATTERN, this::uniformParser)); + } + + /** + * Creates a {@code QueueCapacityVector} parsed from the capacity configuration + * property set for a queue. + * @param conf configuration object + * @param queuePath queue for which the capacity property is parsed + * @param label node label + * @return a parsed capacity vector + */ + public QueueCapacityVector parse(CapacitySchedulerConfiguration conf, + String queuePath, String label) { + + if (queuePath.equals(CapacitySchedulerConfiguration.ROOT)) { + return QueueCapacityVector.of(100f, QueueCapacityType.PERCENTAGE); + } + + String propertyName = CapacitySchedulerConfiguration.getNodeLabelPrefix( + queuePath, label) + CapacitySchedulerConfiguration.CAPACITY; + String capacityString = conf.get(propertyName); + + if (capacityString == null) { + return new QueueCapacityVector(); + } + + for (Parser parser : parsers) { + Matcher matcher = parser.regex.matcher(capacityString); + if (matcher.find()) { + return parser.parser.apply(matcher); + } + } + + return new QueueCapacityVector(); + } + + /** + * A parser method that is usable on uniform capacity values eg. percentage or + * weight. + * @param matcher a regex matcher that contains parsed value and its possible + * suffix + * @return a parsed resource vector + */ + private QueueCapacityVector uniformParser(Matcher matcher) { + QueueCapacityType capacityType = QueueCapacityType.PERCENTAGE; + String value = matcher.group(1); + if (matcher.groupCount() == 2) { + String matchedSuffix = matcher.group(2); + if (!matchedSuffix.isEmpty()) { + for (QueueCapacityType suffix : QueueCapacityType.values()) { + // when capacity is given in percentage, we do not need % symbol + String uniformSuffix = suffix.getPostfix().replaceAll("%", ""); + if (uniformSuffix.equals(matchedSuffix)) { + capacityType = suffix; + } + } + } + } + + return QueueCapacityVector.of(Float.parseFloat(value), capacityType); + } + + /** + * A parser method that is usable on resource capacity values eg. mixed or + * absolute resource. + * @param matcher a regex matcher that contains the matched resource string + * @return a parsed capacity vector + */ + private QueueCapacityVector heterogeneousParser(Matcher matcher) { + // Define resource here. + QueueCapacityVector capacityVector = QueueCapacityVector.newInstance(); + + /* + * Absolute resource configuration for a queue will be grouped by "[]". + * Syntax of absolute resource config could be like below + * "memory=4Gi vcores=2". Ideally this means "4GB of memory and 2 vcores". + */ + // Get the sub-group. + String bracketedGroup = matcher.group(0); + if (bracketedGroup.trim().isEmpty()) { + return capacityVector; Review comment: Should this be an invalid scenario? ########## File path: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestResourceVector.java ########## @@ -0,0 +1,97 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity; + +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.util.resource.ResourceUtils; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.util.HashMap; +import java.util.Map; + +public class TestResourceVector { + private final YarnConfiguration conf = new YarnConfiguration(); + + @Before + public void setUp() { + conf.set(YarnConfiguration.RESOURCE_TYPES, "custom"); + ResourceUtils.resetResourceTypes(conf); + } + + @Test + public void testCreation() { + ResourceVector zeroResourceVector = ResourceVector.newInstance(); + Assert.assertEquals(0, zeroResourceVector.getValue("memory-mb"), 1e-6); + Assert.assertEquals(0, zeroResourceVector.getValue("vcores"), 1e-6); + Assert.assertEquals(0, zeroResourceVector.getValue("custom"), 1e-6); + + ResourceVector uniformResourceVector = ResourceVector.of(10); + Assert.assertEquals(10, uniformResourceVector.getValue("memory-mb"), 1e-6); + Assert.assertEquals(10, uniformResourceVector.getValue("vcores"), 1e-6); + Assert.assertEquals(10, uniformResourceVector.getValue("custom"), 1e-6); + + Map<String, Long> customResources = new HashMap<>(); + customResources.put("custom", 2L); + Resource resource = Resource.newInstance(10, 5, customResources); + ResourceVector resourceVectorFromResource = ResourceVector.of(resource); + Assert.assertEquals(10, resourceVectorFromResource.getValue("memory-mb"), 1e-6); + Assert.assertEquals(5, resourceVectorFromResource.getValue("vcores"), 1e-6); + Assert.assertEquals(2, resourceVectorFromResource.getValue("custom"), 1e-6); + } + + @Test + public void subtract() { + ResourceVector lhsResourceVector = ResourceVector.of(13); + ResourceVector rhsResourceVector = ResourceVector.of(5); + lhsResourceVector.subtract(rhsResourceVector); + + Assert.assertEquals(8, lhsResourceVector.getValue("memory-mb"), 1e-6); + Assert.assertEquals(8, lhsResourceVector.getValue("vcores"), 1e-6); + Assert.assertEquals(8, lhsResourceVector.getValue("custom"), 1e-6); + } + + @Test + public void increment() { Review comment: Could you add a test that tests the float overflow scenario? ########## File path: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/QueueCapacityConfigParser.java ########## @@ -0,0 +1,210 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf; + +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacityVector; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacityVector.QueueCapacityType; +import org.apache.hadoop.yarn.util.UnitsConversionUtil; + +import java.util.ArrayList; +import java.util.List; +import java.util.function.Function; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * A class that parses {@code QueueCapacityVector} from the capacity + * configuration property set for a queue. + * + * A new syntax for capacity property could be implemented, by creating a parser + * with a regex to match the pattern and a method that creates a + * {@code QueueCapacityVector} from the matched pattern + * eg. root.capacity 20-50 + * + * A new capacity type for the existing parsers could be added by extending + * the {@code QueueCapacityVector.QueueCapacityType} with a new type and its + * associated postfix symbol. + * eg. root.capacity 20g + */ +public class QueueCapacityConfigParser { + private static final String UNIFORM_REGEX = "^([0-9.]+)(.*)"; + private static final String RESOURCE_REGEX = "^\\[([\\w\\.,\\-_%\\ /]+=[\\w\\.,\\-_%\\ /]+)+\\]$"; + + private static final Pattern RESOURCE_PATTERN = Pattern.compile(RESOURCE_REGEX); + private static final Pattern UNIFORM_PATTERN = Pattern.compile(UNIFORM_REGEX); + + private final List<Parser> parsers = new ArrayList<>(); + + public QueueCapacityConfigParser() { + parsers.add(new Parser(RESOURCE_PATTERN, this::heterogeneousParser)); + parsers.add(new Parser(UNIFORM_PATTERN, this::uniformParser)); + } + + /** + * Creates a {@code QueueCapacityVector} parsed from the capacity configuration + * property set for a queue. + * @param conf configuration object + * @param queuePath queue for which the capacity property is parsed + * @param label node label + * @return a parsed capacity vector + */ + public QueueCapacityVector parse(CapacitySchedulerConfiguration conf, + String queuePath, String label) { + + if (queuePath.equals(CapacitySchedulerConfiguration.ROOT)) { + return QueueCapacityVector.of(100f, QueueCapacityType.PERCENTAGE); + } + + String propertyName = CapacitySchedulerConfiguration.getNodeLabelPrefix( + queuePath, label) + CapacitySchedulerConfiguration.CAPACITY; + String capacityString = conf.get(propertyName); + + if (capacityString == null) { + return new QueueCapacityVector(); + } + + for (Parser parser : parsers) { + Matcher matcher = parser.regex.matcher(capacityString); + if (matcher.find()) { + return parser.parser.apply(matcher); + } + } + + return new QueueCapacityVector(); + } + + /** + * A parser method that is usable on uniform capacity values eg. percentage or + * weight. + * @param matcher a regex matcher that contains parsed value and its possible + * suffix + * @return a parsed resource vector + */ + private QueueCapacityVector uniformParser(Matcher matcher) { + QueueCapacityType capacityType = QueueCapacityType.PERCENTAGE; + String value = matcher.group(1); + if (matcher.groupCount() == 2) { + String matchedSuffix = matcher.group(2); + if (!matchedSuffix.isEmpty()) { + for (QueueCapacityType suffix : QueueCapacityType.values()) { + // when capacity is given in percentage, we do not need % symbol + String uniformSuffix = suffix.getPostfix().replaceAll("%", ""); + if (uniformSuffix.equals(matchedSuffix)) { + capacityType = suffix; + } + } + } + } + + return QueueCapacityVector.of(Float.parseFloat(value), capacityType); + } + + /** + * A parser method that is usable on resource capacity values eg. mixed or + * absolute resource. + * @param matcher a regex matcher that contains the matched resource string + * @return a parsed capacity vector + */ + private QueueCapacityVector heterogeneousParser(Matcher matcher) { + // Define resource here. Review comment: Nit: Comment is a bit misleading, resource -> capacityVector. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
