oscerd commented on code in PR #24844: URL: https://github.com/apache/camel/pull/24844#discussion_r4204324266
########## components/camel-apicurio-registry/src/main/java/org/apache/camel/component/apicurioregistry/ApicurioRegistryEndpoint.java: ########## @@ -0,0 +1,208 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.camel.component.apicurioregistry; + +import java.nio.charset.StandardCharsets; +import java.util.Base64; + +import io.apicurio.registry.client.RegistryClientFactory; +import io.apicurio.registry.client.common.RegistryClientOptions; +import io.apicurio.registry.rest.client.RegistryClient; +import io.vertx.core.Vertx; +import io.vertx.core.net.ProxyOptions; +import io.vertx.ext.auth.oauth2.OAuth2Auth; +import io.vertx.ext.auth.oauth2.OAuth2FlowType; +import io.vertx.ext.auth.oauth2.OAuth2Options; +import io.vertx.ext.auth.oauth2.Oauth2Credentials; +import io.vertx.ext.web.client.OAuth2WebClient; +import io.vertx.ext.web.client.WebClient; +import io.vertx.ext.web.client.WebClientOptions; +import io.vertx.ext.web.client.WebClientSession; +import org.apache.camel.Category; +import org.apache.camel.Consumer; +import org.apache.camel.Processor; +import org.apache.camel.Producer; +import org.apache.camel.component.vertx.common.VertxHelper; +import org.apache.camel.spi.EndpointServiceLocation; +import org.apache.camel.spi.UriEndpoint; +import org.apache.camel.spi.UriParam; +import org.apache.camel.spi.UriPath; +import org.apache.camel.support.ScheduledPollEndpoint; +import org.apache.camel.util.ObjectHelper; + +/** + * Manage artifacts, versions, and groups in Apicurio Registry v3. + */ +@UriEndpoint(firstVersion = "4.23.0", scheme = "apicurio-registry", title = "Apicurio Registry", + syntax = "apicurio-registry:groupId/artifactId", + category = { Category.CLOUD, Category.API }, headersClass = ApicurioRegistryConstants.class) +public class ApicurioRegistryEndpoint extends ScheduledPollEndpoint implements EndpointServiceLocation { + + @UriPath(description = "The artifact group ID") + private String groupId; + + @UriPath(description = "The artifact ID") + private String artifactId; + + @UriParam + private ApicurioRegistryConfiguration configuration; + + @UriParam(label = "advanced", description = "To use a pre-configured RegistryClient instance") + private RegistryClient registryClient; + + private WebClient ownedWebClient; + private OAuth2Auth oauth2; + + ApicurioRegistryEndpoint(String uri, ApicurioRegistryComponent component, + ApicurioRegistryConfiguration configuration, + String groupId, String artifactId) { + super(uri, component); + this.configuration = configuration; + this.groupId = groupId; + this.artifactId = artifactId; + } + + @Override + public Producer createProducer() throws Exception { + return new ApicurioRegistryProducer(this, configuration); + } + + @Override + public Consumer createConsumer(Processor processor) throws Exception { + if (ObjectHelper.isEmpty(groupId) || ObjectHelper.isEmpty(artifactId)) { + throw new IllegalArgumentException( + "Both groupId and artifactId are required for the consumer, for example apicurio-registry:myGroup/myArtifact"); + } + ApicurioRegistryConsumer consumer = new ApicurioRegistryConsumer(this, processor, configuration); + configureConsumer(consumer); + return consumer; + } + + @Override + protected void doStart() throws Exception { + super.doStart(); + if (registryClient == null) { + Vertx vertx = ((ApicurioRegistryComponent) getComponent()).getOrCreateVertx(); + try { + ownedWebClient = createWebClient(vertx); + registryClient = RegistryClientFactory.create( + RegistryClientOptions.create(configuration.getRegistryUrl(), vertx).customWebClient(ownedWebClient)); + } catch (Exception e) { + closeOwnedResources(); + throw e; + } + } + } + + @Override + protected void doStop() throws Exception { + super.doStop(); + if (ownedWebClient != null) { + registryClient = null; + closeOwnedResources(); + } + } + + private void closeOwnedResources() { + // the SDK client has no close method, so the HTTP connection pool lives in the WebClient we created + if (ownedWebClient != null) { + ownedWebClient.close(); + ownedWebClient = null; + } + if (oauth2 != null) { + oauth2.close(); + oauth2 = null; + } + } + + WebClientOptions createWebClientOptions() throws Exception { + WebClientOptions options = new WebClientOptions(); + if (configuration.getSslContextParameters() != null) { + VertxHelper.setupSSLOptions(getCamelContext(), configuration.getSslContextParameters(), options); + } + if (ObjectHelper.isNotEmpty(configuration.getProxyHost())) { + ProxyOptions proxy = new ProxyOptions().setHost(configuration.getProxyHost()); + if (configuration.getProxyPort() != null) { + proxy.setPort(configuration.getProxyPort()); + } + if (ObjectHelper.isNotEmpty(configuration.getProxyUsername())) { + proxy.setUsername(configuration.getProxyUsername()); + proxy.setPassword(configuration.getProxyPassword()); + } + options.setProxyOptions(proxy); + } + return options; + } + + private WebClient createWebClient(Vertx vertx) throws Exception { + WebClientOptions options = createWebClientOptions(); + WebClient webClient = WebClient.create(vertx, options); + String authType = configuration.getAuthType(); + if ("basic".equalsIgnoreCase(authType)) { + String credentials = configuration.getUsername() + ":" + configuration.getPassword(); Review Comment: **customWebClient bypasses the SDK's credential validation** With `customWebClient(...)` the SDK takes its CUSTOM_WEBCLIENT path, where `RegistryClientRequestAdapterFactory.validateAuth` checks nothing. At the previous head, `basicAuth(...)`/`oauth2(...)` made the endpoint fail to start with "Username cannot be null or empty", "Password cannot be null or empty", "Token endpoint cannot be null or empty" or "Client secret cannot be null or empty". Now `authType=basic` without `username`/`password` starts fine and sends `Authorization: Basic bnVsbDpudWxs` (`null:null`) on every request, so every call gets a 401. With `authType=oidc` a missing `tokenEndpoint`/`clientSecret` only fails when a token is requested (Vert.x's `OAuth2Options.validate()` only requires `clientId`). Validating these fields in `createWebClient` before `WebClient.create` would restore the fail-fast. Also, if `OAuth2Auth.create` throws, the plain `webClient` created just before is not yet in `ownedWebClient`, so it stays open on the shared Vert.x until the component stops. A unit test per auth mode through `doStart` would help, since this code used to be the SDK's. ########## components/camel-apicurio-registry/src/main/java/org/apache/camel/component/apicurioregistry/ApicurioRegistryConsumer.java: ########## @@ -0,0 +1,188 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.camel.component.apicurioregistry; + +import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; + +import io.apicurio.registry.rest.client.RegistryClient; +import io.apicurio.registry.rest.client.models.ProblemDetails; +import io.apicurio.registry.rest.client.models.SearchedVersion; +import io.apicurio.registry.rest.client.models.SortOrder; +import io.apicurio.registry.rest.client.models.VersionSearchResults; +import io.apicurio.registry.rest.client.models.VersionSortBy; +import io.apicurio.registry.rest.client.models.VersionState; +import org.apache.camel.Exchange; +import org.apache.camel.Message; +import org.apache.camel.Processor; +import org.apache.camel.support.ScheduledPollConsumer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class ApicurioRegistryConsumer extends ScheduledPollConsumer { + + static final int PAGE_SIZE = 100; + + private static final Logger LOG = LoggerFactory.getLogger(ApicurioRegistryConsumer.class); + + private final ApicurioRegistryEndpoint endpoint; + private final ApicurioRegistryConfiguration configuration; + private volatile Long lastSeenGlobalId; + + public ApicurioRegistryConsumer(ApicurioRegistryEndpoint endpoint, Processor processor, + ApicurioRegistryConfiguration configuration) { + super(endpoint, processor); + this.endpoint = endpoint; + this.configuration = configuration; + } + + @Override + protected int poll() throws Exception { + String groupId = endpoint.getGroupId(); + String artifactId = endpoint.getArtifactId(); + RegistryClient client = endpoint.getRegistryClient(); + + int count = 0; + for (SearchedVersion version : fetchNewVersions(client, groupId, artifactId)) { + Long globalId = version.getGlobalId(); + if (configuration.isFetchContent() && version.getState() == VersionState.DISABLED) { + // the registry answers 404 for the content of a disabled version, so it can never be delivered + LOG.debug("Skipping disabled artifact version {} (globalId {})", version.getVersion(), globalId); + lastSeenGlobalId = globalId; + continue; + } + Exchange exchange = createExchange(false); + try { + Message message = exchange.getIn(); + message.setHeader(ApicurioRegistryConstants.HEADER_GROUP_ID, groupId); + message.setHeader(ApicurioRegistryConstants.HEADER_ARTIFACT_ID, artifactId); + message.setHeader(ApicurioRegistryConstants.HEADER_VERSION, version.getVersion()); + message.setHeader(ApicurioRegistryConstants.HEADER_GLOBAL_ID, globalId); + message.setHeader(ApicurioRegistryConstants.HEADER_CONTENT_ID, version.getContentId()); + message.setHeader(ApicurioRegistryConstants.HEADER_ARTIFACT_TYPE, version.getArtifactType()); + if (version.getState() != null) { + message.setHeader(ApicurioRegistryConstants.HEADER_VERSION_STATE, version.getState().getValue()); + } + + if (configuration.isFetchContent()) { + byte[] content = fetchContent(client, groupId, artifactId, version); + if (content == null) { + lastSeenGlobalId = globalId; + continue; + } + message.setBody(content); + } else { + message.setBody(version); + } + + getProcessor().process(exchange); + if (exchange.getException() != null) { + // do not advance the watermark so the version is retried on the next poll + if (exchange.getExchangeExtension().isErrorHandlerHandledSet()) { + // the error handler has already logged the failure + LOG.debug("Artifact version with globalId {} failed and will be retried on the next poll", globalId); + } else { + getExceptionHandler().handleException( + "Error processing artifact version with globalId " + globalId, exchange, + exchange.getException()); + } + break; + } + lastSeenGlobalId = globalId; + count++; + } finally { + releaseExchange(exchange, false); + } + } + return count; + } + + /** + * Fetches the content of a version, or returns null if the version no longer has retrievable content (it was + * disabled or deleted after the version list was read). + */ + private static byte[] fetchContent( + RegistryClient client, String groupId, String artifactId, SearchedVersion version) + throws Exception { + try (InputStream content = client.groups().byGroupId(groupId).artifacts().byArtifactId(artifactId) + .versions().byVersionExpression(version.getVersion()).content().get()) { + return content.readAllBytes(); + } catch (ProblemDetails e) { + if (e.getStatus() != null && e.getStatus() == 404) { + LOG.debug("Skipping artifact version {} (globalId {}) whose content is no longer available: {}", Review Comment: **Skipped versions are dropped for good with only a DEBUG log** Skipping DISABLED versions and 404 content is the right call, but the skip is permanent (the watermark moves past the version) and only visible at DEBUG. So a DISABLED version that is enabled again later is never delivered (the docs don't say so), and a 404 is not always final: with KafkaSQL storage each replica applies the journal asynchronously, so behind a load balancer a version that one replica listed can briefly 404 on another, and the consumer then loses it without a trace. Logging skips at WARN (at least the 404 path) and documenting that skipped versions are not redelivered would make it visible; optionally treat a 404 as final only when it repeats on the next poll. ########## components/camel-apicurio-registry/pom.xml: ########## @@ -0,0 +1,91 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +--> +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd";> + + <modelVersion>4.0.0</modelVersion> + + <parent> + <artifactId>components</artifactId> + <groupId>org.apache.camel</groupId> + <version>4.23.0-SNAPSHOT</version> + </parent> + + <artifactId>camel-apicurio-registry</artifactId> + <packaging>jar</packaging> + + <name>Camel :: Apicurio Registry</name> + <description>Camel Apicurio Registry component</description> + + <dependencies> + <!-- camel --> + <dependency> + <groupId>org.apache.camel</groupId> + <artifactId>camel-support</artifactId> + </dependency> + + <dependency> + <groupId>org.apache.camel</groupId> + <artifactId>camel-vertx-common</artifactId> Review Comment: **Declare the Vert.x web-client/OAuth2 deps used directly; Vert.x versions are now mixed** Nit: `ApicurioRegistryEndpoint` now imports `io.vertx.ext.web.client.*` and `io.vertx.ext.auth.oauth2.*`, which the module only gets transitively through the SDK's Vert.x adapter (`kiota-http-vertx`, Vert.x 4.5.26 via its Quarkus BOM), while `camel-vertx-common` brings `vertx-core` `${vertx-version}` (4.5.34). The module now resolves vertx-core 4.5.34 next to web-client/auth 4.5.26. Declaring `vertx-web-client` and `vertx-auth-oauth2` with `${vertx-version}`, as `camel-vertx-http` does, would align them and make the dependency on the Vert.x adapter explicit (the SDK pom suggests users may exclude it). -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
