This is an automated email from the ASF dual-hosted git repository. wusheng pushed a commit to branch feature/genai-model-prefix-match-aliases in repository https://gitbox.apache.org/repos/asf/skywalking.git
commit 31b8f2b9bd327545e4af444d9743ebafbbded4ac Author: Wu Sheng <[email protected]> AuthorDate: Thu Mar 26 09:47:27 2026 +0800 Support model prefix matching and aliases for GenAI pricing config Move GenAI model matcher and config loader to library-util for shared access by both agent-based analysis and future MAL-based OTLP processing. - Add GenAIModelMatcher with Trie-based longest-prefix match for both provider identification and model cost lookup (was exact match only) - Add model aliases support so one pricing entry matches multiple naming conventions (e.g., claude-4-sonnet and claude-sonnet-4) - Add GenAIPricingConfig and GenAIPricingConfigLoader in library-util - Add Anthropic API response name aliases to gen-ai-config.yml - gen-ai-analyzer delegates to shared library-util classes --- .../oap/analyzer/genai/config/GenAIConfig.java | 1 + .../analyzer/genai/config/GenAIConfigLoader.java | 87 +++------- .../genai/matcher/GenAIProviderPrefixMatcher.java | 123 +++++++-------- .../src/test/resources/gen-ai-config.yml | 26 +-- .../library/util/genai/GenAIModelMatcher.java | 165 +++++++++++++++++++ .../library/util/genai/GenAIPricingConfig.java} | 11 +- .../util/genai/GenAIPricingConfigLoader.java} | 56 +++---- .../library/util/genai/GenAIModelMatcherTest.java | 175 +++++++++++++++++++++ .../util/genai/GenAIPricingConfigLoaderTest.java | 71 +++++++++ .../src/test}/resources/gen-ai-config.yml | 26 +-- .../src/main/resources/gen-ai-config.yml | 26 +-- 11 files changed, 566 insertions(+), 201 deletions(-) diff --git a/oap-server/analyzer/gen-ai-analyzer/src/main/java/org/apache/skywalking/oap/analyzer/genai/config/GenAIConfig.java b/oap-server/analyzer/gen-ai-analyzer/src/main/java/org/apache/skywalking/oap/analyzer/genai/config/GenAIConfig.java index a4a667a80f..6fb0977f0c 100644 --- a/oap-server/analyzer/gen-ai-analyzer/src/main/java/org/apache/skywalking/oap/analyzer/genai/config/GenAIConfig.java +++ b/oap-server/analyzer/gen-ai-analyzer/src/main/java/org/apache/skywalking/oap/analyzer/genai/config/GenAIConfig.java @@ -44,6 +44,7 @@ public class GenAIConfig extends ModuleConfig { @Setter public static class Model { private String name; + private List<String> aliases = new ArrayList<>(); private double inputEstimatedCostPerM; private double outputEstimatedCostPerM; } diff --git a/oap-server/analyzer/gen-ai-analyzer/src/main/java/org/apache/skywalking/oap/analyzer/genai/config/GenAIConfigLoader.java b/oap-server/analyzer/gen-ai-analyzer/src/main/java/org/apache/skywalking/oap/analyzer/genai/config/GenAIConfigLoader.java index c39d126b8c..6af3c9668d 100644 --- a/oap-server/analyzer/gen-ai-analyzer/src/main/java/org/apache/skywalking/oap/analyzer/genai/config/GenAIConfigLoader.java +++ b/oap-server/analyzer/gen-ai-analyzer/src/main/java/org/apache/skywalking/oap/analyzer/genai/config/GenAIConfigLoader.java @@ -19,16 +19,15 @@ package org.apache.skywalking.oap.analyzer.genai.config; import org.apache.skywalking.oap.server.library.module.ModuleStartException; -import org.apache.skywalking.oap.server.library.util.ResourceUtils; -import org.apache.skywalking.oap.server.library.util.StringUtil; -import org.yaml.snakeyaml.Yaml; +import org.apache.skywalking.oap.server.library.util.genai.GenAIPricingConfig; +import org.apache.skywalking.oap.server.library.util.genai.GenAIPricingConfigLoader; -import java.io.FileNotFoundException; import java.io.IOException; -import java.io.Reader; -import java.util.List; -import java.util.Map; +/** + * Loads {@link GenAIConfig} by delegating to {@link GenAIPricingConfigLoader} + * and converting to the module-specific config (adds baseUrl support). + */ public class GenAIConfigLoader { private final GenAIConfig config; @@ -38,57 +37,28 @@ public class GenAIConfigLoader { } public GenAIConfig loadConfig() throws ModuleStartException { - Map<String, List<Map<String, Object>>> configMap; - try (Reader applicationReader = ResourceUtils.read("gen-ai-config.yml")) { - Yaml yaml = new Yaml(); - configMap = yaml.loadAs(applicationReader, Map.class); - } catch (FileNotFoundException e) { - throw new ModuleStartException( - "Cannot find the GenAI configuration file [gen-ai-config.yml].", e); + GenAIPricingConfig pricingConfig; + try { + pricingConfig = GenAIPricingConfigLoader.load(); } catch (IOException e) { throw new ModuleStartException( - "Failed to read the GenAI configuration file [gen-ai-config.yml].", e); - } - - if (configMap == null || !configMap.containsKey("providers")) { - return config; + "Failed to load GenAI configuration file.", e); + } catch (IllegalArgumentException e) { + throw new ModuleStartException(e.getMessage(), e); } - List<Map<String, Object>> providersConfig = configMap.get("providers"); - for (Map<String, Object> providerMap : providersConfig) { + for (GenAIPricingConfig.Provider pp : pricingConfig.getProviders()) { GenAIConfig.Provider provider = new GenAIConfig.Provider(); - - Object name = providerMap.get("provider"); - if (name == null) { - throw new ModuleStartException("Provider name is missing in [gen-ai-config.yml]."); - } - provider.setProvider(name.toString()); - - Object baseUrl = providerMap.get("base-url"); - if (baseUrl != null && StringUtil.isNotBlank(baseUrl.toString())) { - provider.setBaseUrl(baseUrl.toString()); - } - - Object prefixMatch = providerMap.get("prefix-match"); - if (prefixMatch instanceof List) { - provider.getPrefixMatch().addAll((List<String>) prefixMatch); - } else if (prefixMatch != null) { - throw new ModuleStartException("prefix-match must be a list in [gen-ai-config.yml] for provider: " + name); - } - - // Parse specific model overrides - Object modelsConfig = providerMap.get("models"); - if (modelsConfig instanceof List) { - for (Object modelObj : (List<?>) modelsConfig) { - if (modelObj instanceof Map) { - Map<String, Object> modelMap = (Map<String, Object>) modelObj; - GenAIConfig.Model model = new GenAIConfig.Model(); - model.setName(String.valueOf(modelMap.get("name"))); - model.setInputEstimatedCostPerM(parseCost(modelMap.get("input-estimated-cost-per-m"))); - model.setOutputEstimatedCostPerM(parseCost(modelMap.get("output-estimated-cost-per-m"))); - provider.getModels().add(model); - } - } + provider.setProvider(pp.getProvider()); + provider.setPrefixMatch(pp.getPrefixMatch()); + + for (GenAIPricingConfig.Model pm : pp.getModels()) { + GenAIConfig.Model model = new GenAIConfig.Model(); + model.setName(pm.getName()); + model.setAliases(pm.getAliases()); + model.setInputEstimatedCostPerM(pm.getInputEstimatedCostPerM()); + model.setOutputEstimatedCostPerM(pm.getOutputEstimatedCostPerM()); + provider.getModels().add(model); } config.getProviders().add(provider); @@ -96,15 +66,4 @@ public class GenAIConfigLoader { return config; } - - private double parseCost(Object value) { - if (value == null) { - return 0.0; - } - try { - return Double.parseDouble(value.toString()); - } catch (NumberFormatException e) { - return 0.0; - } - } } diff --git a/oap-server/analyzer/gen-ai-analyzer/src/main/java/org/apache/skywalking/oap/analyzer/genai/matcher/GenAIProviderPrefixMatcher.java b/oap-server/analyzer/gen-ai-analyzer/src/main/java/org/apache/skywalking/oap/analyzer/genai/matcher/GenAIProviderPrefixMatcher.java index 93caa6fef3..9fb36a2e13 100644 --- a/oap-server/analyzer/gen-ai-analyzer/src/main/java/org/apache/skywalking/oap/analyzer/genai/matcher/GenAIProviderPrefixMatcher.java +++ b/oap-server/analyzer/gen-ai-analyzer/src/main/java/org/apache/skywalking/oap/analyzer/genai/matcher/GenAIProviderPrefixMatcher.java @@ -18,32 +18,71 @@ package org.apache.skywalking.oap.analyzer.genai.matcher; -import lombok.Data; import org.apache.skywalking.oap.analyzer.genai.config.GenAIConfig; +import org.apache.skywalking.oap.server.library.util.genai.GenAIModelMatcher; +import org.apache.skywalking.oap.server.library.util.genai.GenAIPricingConfig; -import java.util.HashMap; import java.util.List; -import java.util.Map; +import java.util.stream.Collectors; +/** + * Delegates to {@link GenAIModelMatcher} in library-util. + * Converts module-specific {@link GenAIConfig} to the shared {@link GenAIPricingConfig}. + */ public class GenAIProviderPrefixMatcher { - private static final String UNKNOWN = "unknown"; - private final TrieNode root; - private final Map<String, GenAIConfig.Model> modelMap; - private static final MatchResult UNKNOWN_RESULT = new MatchResult(UNKNOWN, null); + private final GenAIModelMatcher delegate; + + private GenAIProviderPrefixMatcher(GenAIModelMatcher delegate) { + this.delegate = delegate; + } + + public static GenAIProviderPrefixMatcher build(GenAIConfig config) { + GenAIPricingConfig pricingConfig = toPricingConfig(config); + return new GenAIProviderPrefixMatcher(GenAIModelMatcher.build(pricingConfig)); + } + + public MatchResult match(String modelName) { + GenAIModelMatcher.MatchResult result = delegate.match(modelName); + GenAIConfig.Model modelConfig = toModuleModel(result.getModelConfig()); + return new MatchResult(result.getProvider(), modelConfig); + } - private GenAIProviderPrefixMatcher(TrieNode root, Map<String, GenAIConfig.Model> modelMap) { - this.root = root; - this.modelMap = modelMap; + private static GenAIPricingConfig toPricingConfig(GenAIConfig config) { + GenAIPricingConfig pricingConfig = new GenAIPricingConfig(); + pricingConfig.setProviders( + config.getProviders().stream().map(p -> { + GenAIPricingConfig.Provider pp = new GenAIPricingConfig.Provider(); + pp.setProvider(p.getProvider()); + pp.setPrefixMatch(p.getPrefixMatch()); + pp.setModels( + p.getModels().stream().map(m -> { + GenAIPricingConfig.Model pm = new GenAIPricingConfig.Model(); + pm.setName(m.getName()); + pm.setAliases(m.getAliases()); + pm.setInputEstimatedCostPerM(m.getInputEstimatedCostPerM()); + pm.setOutputEstimatedCostPerM(m.getOutputEstimatedCostPerM()); + return pm; + }).collect(Collectors.toList()) + ); + return pp; + }).collect(Collectors.toList()) + ); + return pricingConfig; } - @Data - private static class TrieNode { - private final Map<Character, TrieNode> children = new HashMap<>(); - private String providerName; + private static GenAIConfig.Model toModuleModel(GenAIPricingConfig.Model pm) { + if (pm == null) { + return null; + } + GenAIConfig.Model m = new GenAIConfig.Model(); + m.setName(pm.getName()); + m.setAliases(pm.getAliases()); + m.setInputEstimatedCostPerM(pm.getInputEstimatedCostPerM()); + m.setOutputEstimatedCostPerM(pm.getOutputEstimatedCostPerM()); + return m; } - @Data public static class MatchResult { private final String provider; private final GenAIConfig.Model modelConfig; @@ -61,58 +100,4 @@ public class GenAIProviderPrefixMatcher { return modelConfig; } } - - public static GenAIProviderPrefixMatcher build(GenAIConfig config) { - TrieNode root = new TrieNode(); - Map<String, GenAIConfig.Model> modelMap = new HashMap<>(); - - for (GenAIConfig.Provider p : config.getProviders()) { - List<String> prefixes = p.getPrefixMatch(); - if (prefixes != null) { - for (String prefix : prefixes) { - if (prefix == null || prefix.isEmpty()) continue; - - TrieNode current = root; - for (int i = 0; i < prefix.length(); i++) { - char c = prefix.charAt(i); - current = current.children.computeIfAbsent(c, k -> new TrieNode()); - } - current.providerName = p.getProvider(); - } - } - - List<GenAIConfig.Model> models = p.getModels(); - if (models != null) { - for (GenAIConfig.Model model : models) { - if (model.getName() != null) { - modelMap.put(model.getName(), model); - } - } - } - } - - return new GenAIProviderPrefixMatcher(root, modelMap); - } - - public MatchResult match(String modelName) { - if (modelName == null || modelName.isEmpty()) { - return UNKNOWN_RESULT; - } - - TrieNode current = root; - String matchedProvider = null; - - for (int i = 0; i < modelName.length(); i++) { - current = current.children.get(modelName.charAt(i)); - if (current == null) break; - if (current.providerName != null) { - matchedProvider = current.providerName; - } - } - - String provider = matchedProvider != null ? matchedProvider : UNKNOWN; - GenAIConfig.Model modelConfig = modelMap.get(modelName); - - return new MatchResult(provider, modelConfig); - } } diff --git a/oap-server/analyzer/gen-ai-analyzer/src/test/resources/gen-ai-config.yml b/oap-server/analyzer/gen-ai-analyzer/src/test/resources/gen-ai-config.yml index e04d5a11cd..e9e8e2a1ab 100644 --- a/oap-server/analyzer/gen-ai-analyzer/src/test/resources/gen-ai-config.yml +++ b/oap-server/analyzer/gen-ai-analyzer/src/test/resources/gen-ai-config.yml @@ -113,51 +113,53 @@ providers: - claude models: - name: claude-4.6-opus - # Base Input pricing + aliases: [claude-opus-4-6] input-estimated-cost-per-m: 5.0 output-estimated-cost-per-m: 25.0 - name: claude-4.5-opus - # Base Input pricing + aliases: [claude-opus-4-5] input-estimated-cost-per-m: 5.0 output-estimated-cost-per-m: 25.0 - name: claude-4.1-opus - # Base Input pricing + aliases: [claude-opus-4-1] input-estimated-cost-per-m: 15.0 output-estimated-cost-per-m: 75.0 - name: claude-4-opus - # Base Input pricing + aliases: [claude-opus-4] input-estimated-cost-per-m: 15.0 output-estimated-cost-per-m: 75.0 - name: claude-4.6-sonnet - # Base Input pricing + aliases: [claude-sonnet-4-6] input-estimated-cost-per-m: 3.0 output-estimated-cost-per-m: 15.0 - name: claude-4.5-sonnet - # Base Input pricing + aliases: [claude-sonnet-4-5] input-estimated-cost-per-m: 3.0 output-estimated-cost-per-m: 15.0 - name: claude-4-sonnet - # Base Input pricing + aliases: [claude-sonnet-4] input-estimated-cost-per-m: 3.0 output-estimated-cost-per-m: 15.0 - name: claude-3.7-sonnet - # Deprecated, Base Input pricing + aliases: [claude-sonnet-3-7] + # Deprecated input-estimated-cost-per-m: 3.0 output-estimated-cost-per-m: 15.0 - name: claude-4.5-haiku - # Base Input pricing + aliases: [claude-haiku-4-5] input-estimated-cost-per-m: 1.0 output-estimated-cost-per-m: 5.0 - name: claude-3.5-haiku - # Base Input pricing + aliases: [claude-haiku-3-5] input-estimated-cost-per-m: 0.8 output-estimated-cost-per-m: 4.0 - name: claude-3-opus - # Deprecated, Base Input pricing + aliases: [claude-opus-3] + # Deprecated input-estimated-cost-per-m: 15.0 output-estimated-cost-per-m: 75.0 - name: claude-3-haiku - # Base Input pricing + aliases: [claude-haiku-3] input-estimated-cost-per-m: 0.25 output-estimated-cost-per-m: 1.25 diff --git a/oap-server/server-library/library-util/src/main/java/org/apache/skywalking/oap/server/library/util/genai/GenAIModelMatcher.java b/oap-server/server-library/library-util/src/main/java/org/apache/skywalking/oap/server/library/util/genai/GenAIModelMatcher.java new file mode 100644 index 0000000000..2a0d75a355 --- /dev/null +++ b/oap-server/server-library/library-util/src/main/java/org/apache/skywalking/oap/server/library/util/genai/GenAIModelMatcher.java @@ -0,0 +1,165 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.library.util.genai; + +import lombok.Data; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Trie-based matcher for GenAI provider and model name resolution. + * Uses longest-prefix matching for both provider identification and model cost lookup. + * Supports model aliases so a single pricing entry can match multiple naming conventions + * (e.g., "claude-4-sonnet" from client agents and "claude-sonnet-4" from Anthropic API). + */ +public class GenAIModelMatcher { + private static final String UNKNOWN = "unknown"; + private final TrieNode providerTrie; + private final TrieNode modelTrie; + + private static final MatchResult UNKNOWN_RESULT = new MatchResult(UNKNOWN, null); + + private GenAIModelMatcher(TrieNode providerTrie, TrieNode modelTrie) { + this.providerTrie = providerTrie; + this.modelTrie = modelTrie; + } + + @Data + private static class TrieNode { + private final Map<Character, TrieNode> children = new HashMap<>(); + private String providerName; + private GenAIPricingConfig.Model modelConfig; + } + + @Data + public static class MatchResult { + private final String provider; + private final GenAIPricingConfig.Model modelConfig; + + public MatchResult(String provider, GenAIPricingConfig.Model modelConfig) { + this.provider = provider; + this.modelConfig = modelConfig; + } + + public String getProvider() { + return provider; + } + + public GenAIPricingConfig.Model getModelConfig() { + return modelConfig; + } + } + + public static GenAIModelMatcher build(GenAIPricingConfig config) { + final TrieNode providerTrie = new TrieNode(); + final TrieNode modelTrie = new TrieNode(); + + for (GenAIPricingConfig.Provider p : config.getProviders()) { + List<String> prefixes = p.getPrefixMatch(); + if (prefixes != null) { + for (String prefix : prefixes) { + if (prefix == null || prefix.isEmpty()) continue; + insertProvider(providerTrie, prefix, p.getProvider()); + } + } + + List<GenAIPricingConfig.Model> models = p.getModels(); + if (models != null) { + for (GenAIPricingConfig.Model model : models) { + if (model.getName() != null) { + insertModel(modelTrie, model.getName(), model); + } + List<String> aliases = model.getAliases(); + if (aliases != null) { + for (String alias : aliases) { + if (alias != null && !alias.isEmpty()) { + insertModel(modelTrie, alias, model); + } + } + } + } + } + } + + return new GenAIModelMatcher(providerTrie, modelTrie); + } + + private static void insertProvider(TrieNode root, String key, String providerName) { + TrieNode current = root; + for (int i = 0; i < key.length(); i++) { + current = current.children.computeIfAbsent(key.charAt(i), k -> new TrieNode()); + } + current.providerName = providerName; + } + + private static void insertModel(TrieNode root, String key, GenAIPricingConfig.Model model) { + TrieNode current = root; + for (int i = 0; i < key.length(); i++) { + current = current.children.computeIfAbsent(key.charAt(i), k -> new TrieNode()); + } + current.modelConfig = model; + } + + /** + * Match a model name against provider prefixes and model name/alias prefixes. + * Uses longest-prefix match for both provider and model resolution. + * + * @param modelName the model name to match (e.g., "gpt-4o-2024-08-06", "claude-sonnet-4-20250514") + * @return match result containing the provider name and model pricing config (if found) + */ + public MatchResult match(String modelName) { + if (modelName == null || modelName.isEmpty()) { + return UNKNOWN_RESULT; + } + + String matchedProvider = longestPrefixProvider(modelName); + GenAIPricingConfig.Model matchedModel = longestPrefixModel(modelName); + + String provider = matchedProvider != null ? matchedProvider : UNKNOWN; + return new MatchResult(provider, matchedModel); + } + + private String longestPrefixProvider(String input) { + TrieNode current = providerTrie; + String matched = null; + for (int i = 0; i < input.length(); i++) { + current = current.children.get(input.charAt(i)); + if (current == null) break; + if (current.providerName != null) { + matched = current.providerName; + } + } + return matched; + } + + private GenAIPricingConfig.Model longestPrefixModel(String input) { + TrieNode current = modelTrie; + GenAIPricingConfig.Model matched = null; + for (int i = 0; i < input.length(); i++) { + current = current.children.get(input.charAt(i)); + if (current == null) break; + if (current.modelConfig != null) { + matched = current.modelConfig; + } + } + return matched; + } +} diff --git a/oap-server/analyzer/gen-ai-analyzer/src/main/java/org/apache/skywalking/oap/analyzer/genai/config/GenAIConfig.java b/oap-server/server-library/library-util/src/main/java/org/apache/skywalking/oap/server/library/util/genai/GenAIPricingConfig.java similarity index 82% copy from oap-server/analyzer/gen-ai-analyzer/src/main/java/org/apache/skywalking/oap/analyzer/genai/config/GenAIConfig.java copy to oap-server/server-library/library-util/src/main/java/org/apache/skywalking/oap/server/library/util/genai/GenAIPricingConfig.java index a4a667a80f..2f1b292135 100644 --- a/oap-server/analyzer/gen-ai-analyzer/src/main/java/org/apache/skywalking/oap/analyzer/genai/config/GenAIConfig.java +++ b/oap-server/server-library/library-util/src/main/java/org/apache/skywalking/oap/server/library/util/genai/GenAIPricingConfig.java @@ -16,16 +16,19 @@ * */ -package org.apache.skywalking.oap.analyzer.genai.config; +package org.apache.skywalking.oap.server.library.util.genai; import lombok.Getter; import lombok.Setter; -import org.apache.skywalking.oap.server.library.module.ModuleConfig; import java.util.ArrayList; import java.util.List; -public class GenAIConfig extends ModuleConfig { +/** + * GenAI provider and model pricing configuration. + * Shared by agent-based GenAI analysis and OTLP-based AI Gateway monitoring. + */ +public class GenAIPricingConfig { @Getter @Setter @@ -35,7 +38,6 @@ public class GenAIConfig extends ModuleConfig { @Setter public static class Provider { private String provider; - private String baseUrl; private List<String> prefixMatch = new ArrayList<>(); private List<Model> models = new ArrayList<>(); } @@ -44,6 +46,7 @@ public class GenAIConfig extends ModuleConfig { @Setter public static class Model { private String name; + private List<String> aliases = new ArrayList<>(); private double inputEstimatedCostPerM; private double outputEstimatedCostPerM; } diff --git a/oap-server/analyzer/gen-ai-analyzer/src/main/java/org/apache/skywalking/oap/analyzer/genai/config/GenAIConfigLoader.java b/oap-server/server-library/library-util/src/main/java/org/apache/skywalking/oap/server/library/util/genai/GenAIPricingConfigLoader.java similarity index 64% copy from oap-server/analyzer/gen-ai-analyzer/src/main/java/org/apache/skywalking/oap/analyzer/genai/config/GenAIConfigLoader.java copy to oap-server/server-library/library-util/src/main/java/org/apache/skywalking/oap/server/library/util/genai/GenAIPricingConfigLoader.java index c39d126b8c..26cb018263 100644 --- a/oap-server/analyzer/gen-ai-analyzer/src/main/java/org/apache/skywalking/oap/analyzer/genai/config/GenAIConfigLoader.java +++ b/oap-server/server-library/library-util/src/main/java/org/apache/skywalking/oap/server/library/util/genai/GenAIPricingConfigLoader.java @@ -16,38 +16,38 @@ * */ -package org.apache.skywalking.oap.analyzer.genai.config; +package org.apache.skywalking.oap.server.library.util.genai; -import org.apache.skywalking.oap.server.library.module.ModuleStartException; import org.apache.skywalking.oap.server.library.util.ResourceUtils; import org.apache.skywalking.oap.server.library.util.StringUtil; import org.yaml.snakeyaml.Yaml; -import java.io.FileNotFoundException; import java.io.IOException; import java.io.Reader; import java.util.List; import java.util.Map; -public class GenAIConfigLoader { +/** + * Loads {@link GenAIPricingConfig} from gen-ai-config.yml on the classpath. + */ +public class GenAIPricingConfigLoader { - private final GenAIConfig config; + private static final String CONFIG_FILE = "gen-ai-config.yml"; - public GenAIConfigLoader(GenAIConfig config) { - this.config = config; - } + /** + * Load the GenAI pricing configuration from the classpath. + * + * @return the loaded config, never null + * @throws IOException if the config file cannot be found or read + * @throws IllegalArgumentException if the config file has invalid structure + */ + public static GenAIPricingConfig load() throws IOException { + GenAIPricingConfig config = new GenAIPricingConfig(); - public GenAIConfig loadConfig() throws ModuleStartException { Map<String, List<Map<String, Object>>> configMap; - try (Reader applicationReader = ResourceUtils.read("gen-ai-config.yml")) { + try (Reader reader = ResourceUtils.read(CONFIG_FILE)) { Yaml yaml = new Yaml(); - configMap = yaml.loadAs(applicationReader, Map.class); - } catch (FileNotFoundException e) { - throw new ModuleStartException( - "Cannot find the GenAI configuration file [gen-ai-config.yml].", e); - } catch (IOException e) { - throw new ModuleStartException( - "Failed to read the GenAI configuration file [gen-ai-config.yml].", e); + configMap = yaml.loadAs(reader, Map.class); } if (configMap == null || !configMap.containsKey("providers")) { @@ -56,36 +56,36 @@ public class GenAIConfigLoader { List<Map<String, Object>> providersConfig = configMap.get("providers"); for (Map<String, Object> providerMap : providersConfig) { - GenAIConfig.Provider provider = new GenAIConfig.Provider(); + GenAIPricingConfig.Provider provider = new GenAIPricingConfig.Provider(); Object name = providerMap.get("provider"); if (name == null) { - throw new ModuleStartException("Provider name is missing in [gen-ai-config.yml]."); + throw new IllegalArgumentException( + "Provider name is missing in [" + CONFIG_FILE + "]."); } provider.setProvider(name.toString()); - Object baseUrl = providerMap.get("base-url"); - if (baseUrl != null && StringUtil.isNotBlank(baseUrl.toString())) { - provider.setBaseUrl(baseUrl.toString()); - } - Object prefixMatch = providerMap.get("prefix-match"); if (prefixMatch instanceof List) { provider.getPrefixMatch().addAll((List<String>) prefixMatch); } else if (prefixMatch != null) { - throw new ModuleStartException("prefix-match must be a list in [gen-ai-config.yml] for provider: " + name); + throw new IllegalArgumentException( + "prefix-match must be a list in [" + CONFIG_FILE + "] for provider: " + name); } - // Parse specific model overrides Object modelsConfig = providerMap.get("models"); if (modelsConfig instanceof List) { for (Object modelObj : (List<?>) modelsConfig) { if (modelObj instanceof Map) { Map<String, Object> modelMap = (Map<String, Object>) modelObj; - GenAIConfig.Model model = new GenAIConfig.Model(); + GenAIPricingConfig.Model model = new GenAIPricingConfig.Model(); model.setName(String.valueOf(modelMap.get("name"))); model.setInputEstimatedCostPerM(parseCost(modelMap.get("input-estimated-cost-per-m"))); model.setOutputEstimatedCostPerM(parseCost(modelMap.get("output-estimated-cost-per-m"))); + Object aliases = modelMap.get("aliases"); + if (aliases instanceof List) { + model.getAliases().addAll((List<String>) aliases); + } provider.getModels().add(model); } } @@ -97,7 +97,7 @@ public class GenAIConfigLoader { return config; } - private double parseCost(Object value) { + private static double parseCost(Object value) { if (value == null) { return 0.0; } diff --git a/oap-server/server-library/library-util/src/test/java/org/apache/skywalking/oap/server/library/util/genai/GenAIModelMatcherTest.java b/oap-server/server-library/library-util/src/test/java/org/apache/skywalking/oap/server/library/util/genai/GenAIModelMatcherTest.java new file mode 100644 index 0000000000..6e198fe2c9 --- /dev/null +++ b/oap-server/server-library/library-util/src/test/java/org/apache/skywalking/oap/server/library/util/genai/GenAIModelMatcherTest.java @@ -0,0 +1,175 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.library.util.genai; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.util.Arrays; +import java.util.Collections; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; + +public class GenAIModelMatcherTest { + + private GenAIModelMatcher matcher; + + @BeforeEach + void setUp() { + GenAIPricingConfig config = new GenAIPricingConfig(); + + // OpenAI provider + GenAIPricingConfig.Provider openai = new GenAIPricingConfig.Provider(); + openai.setProvider("openai"); + openai.setPrefixMatch(Collections.singletonList("gpt")); + + GenAIPricingConfig.Model gpt4o = new GenAIPricingConfig.Model(); + gpt4o.setName("gpt-4o"); + gpt4o.setInputEstimatedCostPerM(2.5); + gpt4o.setOutputEstimatedCostPerM(10.0); + + GenAIPricingConfig.Model gpt4oMini = new GenAIPricingConfig.Model(); + gpt4oMini.setName("gpt-4o-mini"); + gpt4oMini.setInputEstimatedCostPerM(0.15); + gpt4oMini.setOutputEstimatedCostPerM(0.6); + + openai.setModels(Arrays.asList(gpt4o, gpt4oMini)); + + // Anthropic provider with aliases + GenAIPricingConfig.Provider anthropic = new GenAIPricingConfig.Provider(); + anthropic.setProvider("anthropic"); + anthropic.setPrefixMatch(Collections.singletonList("claude")); + + GenAIPricingConfig.Model sonnet4 = new GenAIPricingConfig.Model(); + sonnet4.setName("claude-4-sonnet"); + sonnet4.setAliases(Collections.singletonList("claude-sonnet-4")); + sonnet4.setInputEstimatedCostPerM(3.0); + sonnet4.setOutputEstimatedCostPerM(15.0); + + GenAIPricingConfig.Model sonnet45 = new GenAIPricingConfig.Model(); + sonnet45.setName("claude-4.5-sonnet"); + sonnet45.setAliases(Collections.singletonList("claude-sonnet-4-5")); + sonnet45.setInputEstimatedCostPerM(3.0); + sonnet45.setOutputEstimatedCostPerM(15.0); + + GenAIPricingConfig.Model haiku35 = new GenAIPricingConfig.Model(); + haiku35.setName("claude-3.5-haiku"); + haiku35.setAliases(Collections.singletonList("claude-haiku-3-5")); + haiku35.setInputEstimatedCostPerM(0.8); + haiku35.setOutputEstimatedCostPerM(4.0); + + anthropic.setModels(Arrays.asList(sonnet4, sonnet45, haiku35)); + + // DeepSeek provider (exact match, no aliases) + GenAIPricingConfig.Provider deepseek = new GenAIPricingConfig.Provider(); + deepseek.setProvider("deepseek"); + deepseek.setPrefixMatch(Collections.singletonList("deepseek")); + + GenAIPricingConfig.Model dsChat = new GenAIPricingConfig.Model(); + dsChat.setName("deepseek-chat"); + dsChat.setInputEstimatedCostPerM(0.28); + dsChat.setOutputEstimatedCostPerM(0.42); + + deepseek.setModels(Collections.singletonList(dsChat)); + + config.setProviders(Arrays.asList(openai, anthropic, deepseek)); + matcher = GenAIModelMatcher.build(config); + } + + @Test + void testProviderPrefixMatch() { + assertEquals("openai", matcher.match("gpt-4o").getProvider()); + assertEquals("openai", matcher.match("gpt-4o-2024-08-06").getProvider()); + assertEquals("anthropic", matcher.match("claude-sonnet-4-20250514").getProvider()); + assertEquals("anthropic", matcher.match("claude-4-sonnet").getProvider()); + assertEquals("deepseek", matcher.match("deepseek-chat").getProvider()); + assertEquals("unknown", matcher.match("totally-unknown").getProvider()); + } + + @Test + void testModelPrefixMatch() { + // OpenAI: date-stamped response model matches config entry via prefix + GenAIModelMatcher.MatchResult result = matcher.match("gpt-4o-2024-08-06"); + assertNotNull(result.getModelConfig()); + assertEquals("gpt-4o", result.getModelConfig().getName()); + assertEquals(2.5, result.getModelConfig().getInputEstimatedCostPerM(), 0.001); + + // Longer prefix wins: gpt-4o-mini matches gpt-4o-mini, not gpt-4o + result = matcher.match("gpt-4o-mini-2024-07-18"); + assertNotNull(result.getModelConfig()); + assertEquals("gpt-4o-mini", result.getModelConfig().getName()); + assertEquals(0.15, result.getModelConfig().getInputEstimatedCostPerM(), 0.001); + } + + @Test + void testAliasMatch() { + // Anthropic API returns claude-sonnet-4-20250514, alias claude-sonnet-4 matches + GenAIModelMatcher.MatchResult result = matcher.match("claude-sonnet-4-20250514"); + assertNotNull(result.getModelConfig()); + assertEquals("claude-4-sonnet", result.getModelConfig().getName()); + assertEquals(3.0, result.getModelConfig().getInputEstimatedCostPerM(), 0.001); + + // claude-sonnet-4-5 alias matches claude-4.5-sonnet (longer prefix wins over claude-sonnet-4) + result = matcher.match("claude-sonnet-4-5-20250620"); + assertNotNull(result.getModelConfig()); + assertEquals("claude-4.5-sonnet", result.getModelConfig().getName()); + + // claude-haiku-3-5 alias matches claude-3.5-haiku + result = matcher.match("claude-haiku-3-5-20241022"); + assertNotNull(result.getModelConfig()); + assertEquals("claude-3.5-haiku", result.getModelConfig().getName()); + } + + @Test + void testOriginalNameStillWorks() { + // Client agent path uses original config names + GenAIModelMatcher.MatchResult result = matcher.match("claude-4-sonnet"); + assertNotNull(result.getModelConfig()); + assertEquals("claude-4-sonnet", result.getModelConfig().getName()); + + result = matcher.match("gpt-4o"); + assertNotNull(result.getModelConfig()); + assertEquals("gpt-4o", result.getModelConfig().getName()); + } + + @Test + void testExactMatchWithNoSuffix() { + GenAIModelMatcher.MatchResult result = matcher.match("deepseek-chat"); + assertNotNull(result.getModelConfig()); + assertEquals("deepseek-chat", result.getModelConfig().getName()); + assertEquals(0.28, result.getModelConfig().getInputEstimatedCostPerM(), 0.001); + } + + @Test + void testUnknownModel() { + GenAIModelMatcher.MatchResult result = matcher.match("totally-unknown-model"); + assertNull(result.getModelConfig()); + assertEquals("unknown", result.getProvider()); + } + + @Test + void testNullAndEmpty() { + assertEquals("unknown", matcher.match(null).getProvider()); + assertNull(matcher.match(null).getModelConfig()); + assertEquals("unknown", matcher.match("").getProvider()); + assertNull(matcher.match("").getModelConfig()); + } +} diff --git a/oap-server/server-library/library-util/src/test/java/org/apache/skywalking/oap/server/library/util/genai/GenAIPricingConfigLoaderTest.java b/oap-server/server-library/library-util/src/test/java/org/apache/skywalking/oap/server/library/util/genai/GenAIPricingConfigLoaderTest.java new file mode 100644 index 0000000000..425a4e6c72 --- /dev/null +++ b/oap-server/server-library/library-util/src/test/java/org/apache/skywalking/oap/server/library/util/genai/GenAIPricingConfigLoaderTest.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.library.util.genai; + +import org.junit.jupiter.api.Test; + +import java.io.IOException; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +public class GenAIPricingConfigLoaderTest { + + @Test + void testLoadConfig() throws IOException { + GenAIPricingConfig config = GenAIPricingConfigLoader.load(); + assertNotNull(config); + assertFalse(config.getProviders().isEmpty()); + } + + @Test + void testLoadedProviders() throws IOException { + GenAIPricingConfig config = GenAIPricingConfigLoader.load(); + + // Verify key providers exist + boolean hasOpenai = false; + boolean hasAnthropic = false; + for (GenAIPricingConfig.Provider p : config.getProviders()) { + if ("openai".equals(p.getProvider())) hasOpenai = true; + if ("anthropic".equals(p.getProvider())) hasAnthropic = true; + } + assertFalse(!hasOpenai, "OpenAI provider should be loaded"); + assertFalse(!hasAnthropic, "Anthropic provider should be loaded"); + } + + @Test + void testAliasesLoaded() throws IOException { + GenAIPricingConfig config = GenAIPricingConfigLoader.load(); + + // Build matcher from loaded config and verify aliases work + GenAIModelMatcher matcher = GenAIModelMatcher.build(config); + + // Anthropic alias: claude-sonnet-4 → claude-4-sonnet config entry + GenAIModelMatcher.MatchResult result = matcher.match("claude-sonnet-4-20250514"); + assertNotNull(result.getModelConfig()); + assertEquals("claude-4-sonnet", result.getModelConfig().getName()); + assertEquals(3.0, result.getModelConfig().getInputEstimatedCostPerM(), 0.001); + + // OpenAI prefix match: gpt-4o-2024-08-06 → gpt-4o config entry + result = matcher.match("gpt-4o-2024-08-06"); + assertNotNull(result.getModelConfig()); + assertEquals("gpt-4o", result.getModelConfig().getName()); + } +} diff --git a/oap-server/server-starter/src/main/resources/gen-ai-config.yml b/oap-server/server-library/library-util/src/test/resources/gen-ai-config.yml similarity index 96% copy from oap-server/server-starter/src/main/resources/gen-ai-config.yml copy to oap-server/server-library/library-util/src/test/resources/gen-ai-config.yml index e04d5a11cd..e9e8e2a1ab 100644 --- a/oap-server/server-starter/src/main/resources/gen-ai-config.yml +++ b/oap-server/server-library/library-util/src/test/resources/gen-ai-config.yml @@ -113,51 +113,53 @@ providers: - claude models: - name: claude-4.6-opus - # Base Input pricing + aliases: [claude-opus-4-6] input-estimated-cost-per-m: 5.0 output-estimated-cost-per-m: 25.0 - name: claude-4.5-opus - # Base Input pricing + aliases: [claude-opus-4-5] input-estimated-cost-per-m: 5.0 output-estimated-cost-per-m: 25.0 - name: claude-4.1-opus - # Base Input pricing + aliases: [claude-opus-4-1] input-estimated-cost-per-m: 15.0 output-estimated-cost-per-m: 75.0 - name: claude-4-opus - # Base Input pricing + aliases: [claude-opus-4] input-estimated-cost-per-m: 15.0 output-estimated-cost-per-m: 75.0 - name: claude-4.6-sonnet - # Base Input pricing + aliases: [claude-sonnet-4-6] input-estimated-cost-per-m: 3.0 output-estimated-cost-per-m: 15.0 - name: claude-4.5-sonnet - # Base Input pricing + aliases: [claude-sonnet-4-5] input-estimated-cost-per-m: 3.0 output-estimated-cost-per-m: 15.0 - name: claude-4-sonnet - # Base Input pricing + aliases: [claude-sonnet-4] input-estimated-cost-per-m: 3.0 output-estimated-cost-per-m: 15.0 - name: claude-3.7-sonnet - # Deprecated, Base Input pricing + aliases: [claude-sonnet-3-7] + # Deprecated input-estimated-cost-per-m: 3.0 output-estimated-cost-per-m: 15.0 - name: claude-4.5-haiku - # Base Input pricing + aliases: [claude-haiku-4-5] input-estimated-cost-per-m: 1.0 output-estimated-cost-per-m: 5.0 - name: claude-3.5-haiku - # Base Input pricing + aliases: [claude-haiku-3-5] input-estimated-cost-per-m: 0.8 output-estimated-cost-per-m: 4.0 - name: claude-3-opus - # Deprecated, Base Input pricing + aliases: [claude-opus-3] + # Deprecated input-estimated-cost-per-m: 15.0 output-estimated-cost-per-m: 75.0 - name: claude-3-haiku - # Base Input pricing + aliases: [claude-haiku-3] input-estimated-cost-per-m: 0.25 output-estimated-cost-per-m: 1.25 diff --git a/oap-server/server-starter/src/main/resources/gen-ai-config.yml b/oap-server/server-starter/src/main/resources/gen-ai-config.yml index e04d5a11cd..e9e8e2a1ab 100644 --- a/oap-server/server-starter/src/main/resources/gen-ai-config.yml +++ b/oap-server/server-starter/src/main/resources/gen-ai-config.yml @@ -113,51 +113,53 @@ providers: - claude models: - name: claude-4.6-opus - # Base Input pricing + aliases: [claude-opus-4-6] input-estimated-cost-per-m: 5.0 output-estimated-cost-per-m: 25.0 - name: claude-4.5-opus - # Base Input pricing + aliases: [claude-opus-4-5] input-estimated-cost-per-m: 5.0 output-estimated-cost-per-m: 25.0 - name: claude-4.1-opus - # Base Input pricing + aliases: [claude-opus-4-1] input-estimated-cost-per-m: 15.0 output-estimated-cost-per-m: 75.0 - name: claude-4-opus - # Base Input pricing + aliases: [claude-opus-4] input-estimated-cost-per-m: 15.0 output-estimated-cost-per-m: 75.0 - name: claude-4.6-sonnet - # Base Input pricing + aliases: [claude-sonnet-4-6] input-estimated-cost-per-m: 3.0 output-estimated-cost-per-m: 15.0 - name: claude-4.5-sonnet - # Base Input pricing + aliases: [claude-sonnet-4-5] input-estimated-cost-per-m: 3.0 output-estimated-cost-per-m: 15.0 - name: claude-4-sonnet - # Base Input pricing + aliases: [claude-sonnet-4] input-estimated-cost-per-m: 3.0 output-estimated-cost-per-m: 15.0 - name: claude-3.7-sonnet - # Deprecated, Base Input pricing + aliases: [claude-sonnet-3-7] + # Deprecated input-estimated-cost-per-m: 3.0 output-estimated-cost-per-m: 15.0 - name: claude-4.5-haiku - # Base Input pricing + aliases: [claude-haiku-4-5] input-estimated-cost-per-m: 1.0 output-estimated-cost-per-m: 5.0 - name: claude-3.5-haiku - # Base Input pricing + aliases: [claude-haiku-3-5] input-estimated-cost-per-m: 0.8 output-estimated-cost-per-m: 4.0 - name: claude-3-opus - # Deprecated, Base Input pricing + aliases: [claude-opus-3] + # Deprecated input-estimated-cost-per-m: 15.0 output-estimated-cost-per-m: 75.0 - name: claude-3-haiku - # Base Input pricing + aliases: [claude-haiku-3] input-estimated-cost-per-m: 0.25 output-estimated-cost-per-m: 1.25
