carlesarnal commented on code in PR #24844: URL: https://github.com/apache/camel/pull/24844#discussion_r4204140593
########## components/camel-apicurio-registry/src/main/java/org/apache/camel/component/apicurioregistry/ApicurioRegistryConsumer.java: ########## @@ -0,0 +1,147 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.camel.component.apicurioregistry; + +import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; + +import io.apicurio.registry.rest.client.RegistryClient; +import io.apicurio.registry.rest.client.models.SearchedVersion; +import io.apicurio.registry.rest.client.models.SortOrder; +import io.apicurio.registry.rest.client.models.VersionSearchResults; +import io.apicurio.registry.rest.client.models.VersionSortBy; +import org.apache.camel.Exchange; +import org.apache.camel.Message; +import org.apache.camel.Processor; +import org.apache.camel.support.ScheduledPollConsumer; + +public class ApicurioRegistryConsumer extends ScheduledPollConsumer { + + static final int PAGE_SIZE = 100; + + private final ApicurioRegistryEndpoint endpoint; + private final ApicurioRegistryConfiguration configuration; + private volatile Long lastSeenGlobalId; + + public ApicurioRegistryConsumer(ApicurioRegistryEndpoint endpoint, Processor processor, + ApicurioRegistryConfiguration configuration) { + super(endpoint, processor); + this.endpoint = endpoint; + this.configuration = configuration; + } + + @Override + protected int poll() throws Exception { + String groupId = endpoint.getGroupId(); + String artifactId = endpoint.getArtifactId(); + + if (groupId == null || artifactId == null) { Review Comment: Both fixed in 9ef714c5d5b. createConsumer() now fails at startup when the group or artifact ID is missing. On the producer side, each operation checks the IDs it needs and throws an IllegalArgumentException that names the operation and the header to set, instead of the SDK's NullPointerException. _AI-assisted response prepared with OpenCode on behalf of @carlesarnal._ ########## components/camel-apicurio-registry/src/main/java/org/apache/camel/component/apicurioregistry/ApicurioRegistryConsumer.java: ########## @@ -0,0 +1,147 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.camel.component.apicurioregistry; + +import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; + +import io.apicurio.registry.rest.client.RegistryClient; +import io.apicurio.registry.rest.client.models.SearchedVersion; +import io.apicurio.registry.rest.client.models.SortOrder; +import io.apicurio.registry.rest.client.models.VersionSearchResults; +import io.apicurio.registry.rest.client.models.VersionSortBy; +import org.apache.camel.Exchange; +import org.apache.camel.Message; +import org.apache.camel.Processor; +import org.apache.camel.support.ScheduledPollConsumer; + +public class ApicurioRegistryConsumer extends ScheduledPollConsumer { + + static final int PAGE_SIZE = 100; + + private final ApicurioRegistryEndpoint endpoint; + private final ApicurioRegistryConfiguration configuration; + private volatile Long lastSeenGlobalId; + + public ApicurioRegistryConsumer(ApicurioRegistryEndpoint endpoint, Processor processor, + ApicurioRegistryConfiguration configuration) { + super(endpoint, processor); + this.endpoint = endpoint; + this.configuration = configuration; + } + + @Override + protected int poll() throws Exception { + String groupId = endpoint.getGroupId(); + String artifactId = endpoint.getArtifactId(); + + if (groupId == null || artifactId == null) { + throw new IllegalArgumentException( + "Both groupId and artifactId are required for the consumer"); + } + + RegistryClient client = endpoint.getRegistryClient(); + List<SearchedVersion> versions = fetchNewVersions(client, groupId, artifactId); + + int count = 0; + for (SearchedVersion version : versions) { + Long globalId = version.getGlobalId(); + if (lastSeenGlobalId == null || globalId > lastSeenGlobalId) { + Exchange exchange = createExchange(false); + try { + Message message = exchange.getIn(); + + message.setHeader(ApicurioRegistryConstants.HEADER_GROUP_ID, groupId); + message.setHeader(ApicurioRegistryConstants.HEADER_ARTIFACT_ID, artifactId); + message.setHeader(ApicurioRegistryConstants.HEADER_VERSION, version.getVersion()); + message.setHeader(ApicurioRegistryConstants.HEADER_GLOBAL_ID, globalId); + message.setHeader(ApicurioRegistryConstants.HEADER_CONTENT_ID, version.getContentId()); + message.setHeader(ApicurioRegistryConstants.HEADER_ARTIFACT_TYPE, version.getArtifactType()); + if (version.getState() != null) { + message.setHeader(ApicurioRegistryConstants.HEADER_VERSION_STATE, + version.getState().getValue()); + } + + if (configuration.isFetchContent()) { + try (InputStream content = client.groups().byGroupId(groupId).artifacts() + .byArtifactId(artifactId).versions() + .byVersionExpression(version.getVersion()).content().get()) { + message.setBody(content.readAllBytes()); + } + } else { + message.setBody(version); + } + + getProcessor().process(exchange); + if (exchange.getException() != null) { + // do not advance the watermark so the version is retried on the next poll + getExceptionHandler().handleException( + "Error processing artifact version with globalId " + globalId, exchange, + exchange.getException()); + break; + } + lastSeenGlobalId = globalId; + count++; + } finally { + releaseExchange(exchange, false); + } + } + } + return count; + } + + /** + * Fetches the versions newer than the watermark, in ascending globalId order. Pages are requested newest first so + * that polling stops as soon as an already seen version is reached. + */ + private List<SearchedVersion> fetchNewVersions(RegistryClient client, String groupId, String artifactId) { + // keyed by globalId: a version created between page requests shifts the pages and may repeat one entry + Map<Long, SearchedVersion> answer = new TreeMap<>(); + int offset = 0; + while (true) { + final int pageOffset = offset; + VersionSearchResults page = client.groups().byGroupId(groupId).artifacts().byArtifactId(artifactId) + .versions().get(config -> { + config.queryParameters.orderby = VersionSortBy.GlobalId; + config.queryParameters.order = SortOrder.Desc; + config.queryParameters.offset = pageOffset; + config.queryParameters.limit = PAGE_SIZE; Review Comment: Fixed in 9ef714c5d5b. Once the consumer has a watermark, each poll first requests one version (newest first). An idle poll stops there and only fetches pages when something new appears. I kept the default delay, since an idle poll is now a single small request, but I'm happy to raise it if you'd prefer. _AI-assisted response prepared with OpenCode on behalf of @carlesarnal._ ########## components/camel-apicurio-registry/src/main/java/org/apache/camel/component/apicurioregistry/ApicurioRegistryEndpoint.java: ########## @@ -0,0 +1,139 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.camel.component.apicurioregistry; + +import io.apicurio.registry.client.RegistryClientFactory; +import io.apicurio.registry.client.common.RegistryClientOptions; +import io.apicurio.registry.rest.client.RegistryClient; +import org.apache.camel.Category; +import org.apache.camel.Consumer; +import org.apache.camel.Processor; +import org.apache.camel.Producer; +import org.apache.camel.spi.EndpointServiceLocation; +import org.apache.camel.spi.UriEndpoint; +import org.apache.camel.spi.UriParam; +import org.apache.camel.spi.UriPath; +import org.apache.camel.support.ScheduledPollEndpoint; + +/** + * Manage artifacts, versions, and groups in Apicurio Registry v3. + */ +@UriEndpoint(firstVersion = "4.23.0", scheme = "apicurio-registry", title = "Apicurio Registry", + syntax = "apicurio-registry:groupId/artifactId", + category = { Category.CLOUD, Category.API }, headersClass = ApicurioRegistryConstants.class) +public class ApicurioRegistryEndpoint extends ScheduledPollEndpoint implements EndpointServiceLocation { + + @UriPath(description = "The artifact group ID") + private String groupId; + + @UriPath(description = "The artifact ID") + private String artifactId; + + @UriParam + private ApicurioRegistryConfiguration configuration; + + @UriParam(label = "advanced", description = "To use a pre-configured RegistryClient instance") + private RegistryClient registryClient; + + private boolean managedRegistryClient; + + ApicurioRegistryEndpoint(String uri, ApicurioRegistryComponent component, + ApicurioRegistryConfiguration configuration, + String groupId, String artifactId) { + super(uri, component); + this.configuration = configuration; + this.groupId = groupId; + this.artifactId = artifactId; + } + + @Override + public Producer createProducer() throws Exception { + return new ApicurioRegistryProducer(this, configuration); + } + + @Override + public Consumer createConsumer(Processor processor) throws Exception { + ApicurioRegistryConsumer consumer = new ApicurioRegistryConsumer(this, processor, configuration); + configureConsumer(consumer); + return consumer; + } + + @Override + protected void doStart() throws Exception { + super.doStart(); + if (registryClient == null) { + registryClient = createRegistryClient(); + managedRegistryClient = true; + } + } + + @Override + protected void doStop() throws Exception { + super.doStop(); + if (managedRegistryClient) { + registryClient = null; + managedRegistryClient = false; + } + } + + private RegistryClient createRegistryClient() { + RegistryClientOptions options = RegistryClientOptions.create(configuration.getRegistryUrl(), Review Comment: Added in 9ef714c5d5b, since the endpoint now builds its own WebClient: sslContextParameters (Camel JSSE, for private CAs and mTLS) and proxyHost/proxyPort/proxyUsername/proxyPassword. Both also apply to the OAuth2 token endpoint. I didn't add trustAll or verifyHost options. _AI-assisted response prepared with OpenCode on behalf of @carlesarnal._ ########## components/camel-apicurio-registry/src/main/docs/apicurio-registry-component.adoc: ########## @@ -0,0 +1,241 @@ += Apicurio Registry Component +:doctitle: Apicurio Registry +:shortname: apicurio-registry +:artifactid: camel-apicurio-registry +:description: Manage artifacts, versions, and groups in Apicurio Registry v3. +:since: 4.23 +:supportlevel: Preview +:tabs-sync-option: +:component-header: Both producer and consumer are supported + +*Since Camel {since}* + +*{component-header}* + +The Apicurio Registry component provides integration with https://www.apicur.io/registry/[Apicurio Registry] v3 +using the https://github.com/Apicurio/apicurio-registry[Apicurio Registry Java SDK]. + +It supports CRUD operations on artifacts, groups, and versions, as well as schema validation +and polling for new artifact versions. + +Maven users will need to add the following dependency to their `pom.xml` +for this component: + +[source,xml] +---- +<dependency> + <groupId>org.apache.camel</groupId> + <artifactId>camel-apicurio-registry</artifactId> + <version>x.x.x</version> + <!-- use the same version as your Camel core version --> +</dependency> +---- + +== URI format + +---- +apicurio-registry:groupId/artifactId[?options] +---- + +Where `groupId` and `artifactId` identify the registry artifact. Both are optional for +operations that do not require them (e.g., `searchArtifacts`, `createGroup`). + +The first `/` separates the group ID from the artifact ID. Group IDs containing `/` +cannot be represented in the URI path. Producers can instead supply the group ID using +the `CamelApicurioRegistryGroupId` header, which overrides the URI value. + +// component options: START +include::partial$component-configure-options.adoc[] +include::partial$component-endpoint-options.adoc[] +include::partial$component-endpoint-headers.adoc[] +// component options: END + +== Producer Operations + +The producer supports the following operations, set via the `operation` endpoint option or +the `CamelApicurioRegistryOperation` header: + +[cols="1,3"] +|=== +| Operation | Description + +| `createArtifact` | Create a new artifact. Body contains the artifact content. +| `updateArtifact` | Create a new version of an existing artifact. Body contains the new content. +| `deleteArtifact` | Delete an artifact. Review Comment: Documented in 9ef714c5d5b, under deleteArtifact in the operations table, with the property and environment variable to set. The catalog doc mirror is regenerated. _AI-assisted response prepared with OpenCode on behalf of @carlesarnal._ -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
