sihyeonn commented on code in PR #12895:
URL: https://github.com/apache/apisix/pull/12895#discussion_r2957544704
##########
t/plugin/ai-proxy.openai-compatible.t:
##########
@@ -338,3 +348,80 @@ passed
}
--- response_body_eval
qr/6data: \[DONE\]\n\n/
+
+
+
+=== TEST 6: set route with defaults field
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/routes/1',
+ ngx.HTTP_PUT,
+ [[{
+ "uri": "/anything",
+ "plugins": {
+ "ai-proxy": {
+ "provider": "openai-compatible",
+ "auth": {
+ "header": {
+ "Authorization": "Bearer token"
+ }
+ },
+ "options": {
+ "model": "server-model"
+ },
+ "defaults": {
+ "max_tokens": 512,
+ "temperature": 0.7
+ },
+ "override": {
+ "endpoint":
"http://localhost:6724/v1/chat/completions";
+ },
+ "ssl_verify": false
+ }
+ }
+ }]]
+ )
+
+ if code >= 300 then
+ ngx.status = code
+ end
+ ngx.say(body)
+ }
+ }
+--- response_body
+passed
+
+
+
+=== TEST 7: defaults applied when not set in request
+--- request
+POST /anything
+{ "messages": [ { "role": "user", "content": "hello" } ] }
+--- more_headers
+test-type: defaults
+--- response_body
+{"max_tokens":512,"model":"server-model","temperature":0.7}
+
Review Comment:
Already switched to response_body_like with regex in the previous commit to
handle non-deterministic key ordering.
##########
apisix/plugins/ai-drivers/openai-base.lua:
##########
@@ -315,6 +315,16 @@ function _M.request(self, ctx, conf, request_table,
extra_opts)
ssl_server_name = parsed_url and parsed_url.host or self.host,
}
+ -- defaults: apply only when not set in request
+ if extra_opts.model_defaults then
+ for opt, val in pairs(extra_opts.model_defaults) do
+ if request_table[opt] == nil then
+ request_table[opt] = val
+ end
+ end
+ end
Review Comment:
Moved defaults application to before the stream detection block so ctx vars
are set correctly. This was addressed in the previous commit.
##########
apisix/plugins/ai-proxy/base.lua:
##########
@@ -62,6 +62,7 @@ function _M.before_proxy(conf, ctx, on_error)
name = ai_instance.name,
endpoint = core.table.try_read_attr(ai_instance, "override",
"endpoint"),
model_options = ai_instance.options,
+ model_defaults = ai_instance.defaults,
conf = ai_instance.provider_conf or {},
Review Comment:
Added defaults.model as a fallback in the llm_model assignment. Also removed
the now-unnecessary model_defaults passthrough in extra_opts since defaults are
applied directly to request_body in before_proxy.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
