This is an automated email from the ASF dual-hosted git repository. acosentino pushed a commit to branch milvus-component in repository https://gitbox.apache.org/repos/asf/camel.git
commit 1f03298b4f1f295ad99e46d9a206d5e675fe04a3 Author: Andrea Cosentino <anco...@gmail.com> AuthorDate: Thu Mar 14 14:10:52 2024 +0100 CAMEL-20485 - Create a Camel-Milvus component Signed-off-by: Andrea Cosentino <anco...@gmail.com> --- catalog/camel-allcomponents/pom.xml | 5 + components/camel-milvus/pom.xml | 122 ++++++++++ .../milvus/MilvusComponentConfigurer.java | 77 ++++++ .../milvus/MilvusConfigurationConfigurer.java | 73 ++++++ .../component/milvus/MilvusEndpointConfigurer.java | 61 +++++ .../component/milvus/MilvusEndpointUriFactory.java | 75 ++++++ .../org/apache/camel/component/milvus/milvus.json | 53 ++++ .../services/org/apache/camel/component.properties | 7 + .../services/org/apache/camel/component/milvus | 2 + .../org/apache/camel/configurer/milvus-component | 2 + .../org/apache/camel/configurer/milvus-endpoint | 2 + ...ache.camel.component.milvus.MilvusConfiguration | 2 + .../org/apache/camel/urifactory/milvus-endpoint | 2 + .../src/main/docs/qdrant-component.adoc | 134 ++++++++++ .../org/apache/camel/component/milvus/Milvus.java | 57 +++++ .../camel/component/milvus/MilvusAction.java | 26 ++ .../camel/component/milvus/MilvusComponent.java | 71 ++++++ .../component/milvus/MilvusConfiguration.java | 116 +++++++++ .../camel/component/milvus/MilvusEndpoint.java | 140 +++++++++++ .../camel/component/milvus/MilvusProducer.java | 192 +++++++++++++++ .../milvus/MilvusCreateCollectionTest.java | 74 ++++++ .../camel/component/milvus/MilvusTestSupport.java | 45 ++++ .../camel/component/milvus/MilvusUpsertTest.java | 42 ++++ .../component/milvus/it/MilvusComponentIT.java | 269 +++++++++++++++++++++ .../src/test/resources/log4j2.properties | 35 +++ parent/pom.xml | 5 + test-infra/camel-test-infra-milvus/pom.xml | 55 +++++ .../src/main/resources/META-INF/MANIFEST.MF | 0 .../test/infra/milvus/common/MilvusProperties.java | 27 +++ .../services/MilvusLocalContainerService.java | 81 +++++++ .../infra/milvus/services/MilvusRemoteService.java | 42 ++++ .../test/infra/milvus/services/MilvusService.java | 27 +++ .../milvus/services/MilvusServiceFactory.java | 55 +++++ .../infra/milvus/services/container.properties | 17 ++ test-infra/pom.xml | 1 + 35 files changed, 1994 insertions(+) diff --git a/catalog/camel-allcomponents/pom.xml b/catalog/camel-allcomponents/pom.xml index a4f3074ba0a..eea65801cb7 100644 --- a/catalog/camel-allcomponents/pom.xml +++ b/catalog/camel-allcomponents/pom.xml @@ -1197,6 +1197,11 @@ <artifactId>camel-microprofile-health</artifactId> <version>${project.version}</version> </dependency> + <dependency> + <groupId>org.apache.camel</groupId> + <artifactId>camel-milvus</artifactId> + <version>${project.version}</version> + </dependency> <dependency> <groupId>org.apache.camel</groupId> <artifactId>camel-mina</artifactId> diff --git a/components/camel-milvus/pom.xml b/components/camel-milvus/pom.xml new file mode 100644 index 00000000000..7e707de0f28 --- /dev/null +++ b/components/camel-milvus/pom.xml @@ -0,0 +1,122 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +--> +<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> + <modelVersion>4.0.0</modelVersion> + + <parent> + <groupId>org.apache.camel</groupId> + <artifactId>components</artifactId> + <version>4.5.0-SNAPSHOT</version> + </parent> + + <artifactId>camel-milvus</artifactId> + <packaging>jar</packaging> + <name>Camel :: Milvus</name> + <description>Camel Milvus support</description> + + <properties> + <camel.surefire.parallel>true</camel.surefire.parallel> + <camel.surefire.parallel.factor>4</camel.surefire.parallel.factor> + + <!-- Qdrant is not available on these platforms --> + <skipITs.ppc64le>true</skipITs.ppc64le> + <skipITs.s390x>true</skipITs.s390x> + <skipTests.ppc64le>true</skipTests.ppc64le> + <skipTests.s390x>true</skipTests.s390x> + </properties> + + <dependencies> + <dependency> + <groupId>org.apache.camel</groupId> + <artifactId>camel-support</artifactId> + </dependency> + + <dependency> + <groupId>io.milvus</groupId> + <artifactId>milvus-sdk-java</artifactId> + <version>2.3.4</version> + <exclusions> + <exclusion> + <groupId>org.apache.logging.log4j</groupId> + <artifactId>log4j-api</artifactId> + </exclusion> + <exclusion> + <groupId>org.apache.logging.log4j</groupId> + <artifactId>log4j-core</artifactId> + </exclusion> + <exclusion> + <groupId>org.apache.logging.log4j</groupId> + <artifactId>log4j-slf4j-impl</artifactId> + </exclusion> + </exclusions> + </dependency> + + <!-- test dependencies --> + <dependency> + <groupId>org.apache.camel</groupId> + <artifactId>camel-direct</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.apache.camel</groupId> + <artifactId>camel-mock</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.apache.camel</groupId> + <artifactId>camel-jackson</artifactId> + <scope>test</scope> + </dependency> + + <dependency> + <groupId>org.apache.camel</groupId> + <artifactId>camel-test-junit5</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.junit.jupiter</groupId> + <artifactId>junit-jupiter</artifactId> + <scope>test</scope> + </dependency> + + <dependency> + <groupId>io.rest-assured</groupId> + <artifactId>rest-assured</artifactId> + <version>${rest-assured-version}</version> + </dependency> + + <dependency> + <groupId>org.assertj</groupId> + <artifactId>assertj-core</artifactId> + <version>${assertj-version}</version> + <scope>test</scope> + </dependency> + + <!-- test infra --> + <dependency> + <groupId>org.apache.camel</groupId> + <artifactId>camel-test-infra-milvus</artifactId> + <version>${project.version}</version> + <type>test-jar</type> + <scope>test</scope> + </dependency> + </dependencies> + +</project> diff --git a/components/camel-milvus/src/generated/java/org/apache/camel/component/milvus/MilvusComponentConfigurer.java b/components/camel-milvus/src/generated/java/org/apache/camel/component/milvus/MilvusComponentConfigurer.java new file mode 100644 index 00000000000..44c50b6fbd3 --- /dev/null +++ b/components/camel-milvus/src/generated/java/org/apache/camel/component/milvus/MilvusComponentConfigurer.java @@ -0,0 +1,77 @@ +/* Generated by camel build tools - do NOT edit this file! */ +package org.apache.camel.component.milvus; + +import java.util.Map; + +import org.apache.camel.CamelContext; +import org.apache.camel.spi.ExtendedPropertyConfigurerGetter; +import org.apache.camel.spi.PropertyConfigurerGetter; +import org.apache.camel.spi.ConfigurerStrategy; +import org.apache.camel.spi.GeneratedPropertyConfigurer; +import org.apache.camel.util.CaseInsensitiveMap; +import org.apache.camel.support.component.PropertyConfigurerSupport; + +/** + * Generated by camel build tools - do NOT edit this file! + */ +@SuppressWarnings("unchecked") +public class MilvusComponentConfigurer extends PropertyConfigurerSupport implements GeneratedPropertyConfigurer, PropertyConfigurerGetter { + + private org.apache.camel.component.milvus.MilvusConfiguration getOrCreateConfiguration(MilvusComponent target) { + if (target.getConfiguration() == null) { + target.setConfiguration(new org.apache.camel.component.milvus.MilvusConfiguration()); + } + return target.getConfiguration(); + } + + @Override + public boolean configure(CamelContext camelContext, Object obj, String name, Object value, boolean ignoreCase) { + MilvusComponent target = (MilvusComponent) obj; + switch (ignoreCase ? name.toLowerCase() : name) { + case "autowiredenabled": + case "autowiredEnabled": target.setAutowiredEnabled(property(camelContext, boolean.class, value)); return true; + case "configuration": target.setConfiguration(property(camelContext, org.apache.camel.component.milvus.MilvusConfiguration.class, value)); return true; + case "host": getOrCreateConfiguration(target).setHost(property(camelContext, java.lang.String.class, value)); return true; + case "lazystartproducer": + case "lazyStartProducer": target.setLazyStartProducer(property(camelContext, boolean.class, value)); return true; + case "port": getOrCreateConfiguration(target).setPort(property(camelContext, int.class, value)); return true; + case "timeout": getOrCreateConfiguration(target).setTimeout(property(camelContext, long.class, value)); return true; + case "token": getOrCreateConfiguration(target).setToken(property(camelContext, java.lang.String.class, value)); return true; + default: return false; + } + } + + @Override + public Class<?> getOptionType(String name, boolean ignoreCase) { + switch (ignoreCase ? name.toLowerCase() : name) { + case "autowiredenabled": + case "autowiredEnabled": return boolean.class; + case "configuration": return org.apache.camel.component.milvus.MilvusConfiguration.class; + case "host": return java.lang.String.class; + case "lazystartproducer": + case "lazyStartProducer": return boolean.class; + case "port": return int.class; + case "timeout": return long.class; + case "token": return java.lang.String.class; + default: return null; + } + } + + @Override + public Object getOptionValue(Object obj, String name, boolean ignoreCase) { + MilvusComponent target = (MilvusComponent) obj; + switch (ignoreCase ? name.toLowerCase() : name) { + case "autowiredenabled": + case "autowiredEnabled": return target.isAutowiredEnabled(); + case "configuration": return target.getConfiguration(); + case "host": return getOrCreateConfiguration(target).getHost(); + case "lazystartproducer": + case "lazyStartProducer": return target.isLazyStartProducer(); + case "port": return getOrCreateConfiguration(target).getPort(); + case "timeout": return getOrCreateConfiguration(target).getTimeout(); + case "token": return getOrCreateConfiguration(target).getToken(); + default: return null; + } + } +} + diff --git a/components/camel-milvus/src/generated/java/org/apache/camel/component/milvus/MilvusConfigurationConfigurer.java b/components/camel-milvus/src/generated/java/org/apache/camel/component/milvus/MilvusConfigurationConfigurer.java new file mode 100644 index 00000000000..1cbd71181f7 --- /dev/null +++ b/components/camel-milvus/src/generated/java/org/apache/camel/component/milvus/MilvusConfigurationConfigurer.java @@ -0,0 +1,73 @@ +/* Generated by camel build tools - do NOT edit this file! */ +package org.apache.camel.component.milvus; + +import java.util.Map; + +import org.apache.camel.CamelContext; +import org.apache.camel.spi.ExtendedPropertyConfigurerGetter; +import org.apache.camel.spi.PropertyConfigurerGetter; +import org.apache.camel.spi.ConfigurerStrategy; +import org.apache.camel.spi.GeneratedPropertyConfigurer; +import org.apache.camel.util.CaseInsensitiveMap; +import org.apache.camel.component.milvus.MilvusConfiguration; + +/** + * Generated by camel build tools - do NOT edit this file! + */ +@SuppressWarnings("unchecked") +public class MilvusConfigurationConfigurer extends org.apache.camel.support.component.PropertyConfigurerSupport implements GeneratedPropertyConfigurer, PropertyConfigurerGetter { + + @Override + public boolean configure(CamelContext camelContext, Object obj, String name, Object value, boolean ignoreCase) { + org.apache.camel.component.milvus.MilvusConfiguration target = (org.apache.camel.component.milvus.MilvusConfiguration) obj; + switch (ignoreCase ? name.toLowerCase() : name) { + case "client": + case "Client": target.setClient(property(camelContext, io.milvus.client.MilvusClient.class, value)); return true; + case "host": + case "Host": target.setHost(property(camelContext, java.lang.String.class, value)); return true; + case "port": + case "Port": target.setPort(property(camelContext, int.class, value)); return true; + case "timeout": + case "Timeout": target.setTimeout(property(camelContext, long.class, value)); return true; + case "token": + case "Token": target.setToken(property(camelContext, java.lang.String.class, value)); return true; + default: return false; + } + } + + @Override + public Class<?> getOptionType(String name, boolean ignoreCase) { + switch (ignoreCase ? name.toLowerCase() : name) { + case "client": + case "Client": return io.milvus.client.MilvusClient.class; + case "host": + case "Host": return java.lang.String.class; + case "port": + case "Port": return int.class; + case "timeout": + case "Timeout": return long.class; + case "token": + case "Token": return java.lang.String.class; + default: return null; + } + } + + @Override + public Object getOptionValue(Object obj, String name, boolean ignoreCase) { + org.apache.camel.component.milvus.MilvusConfiguration target = (org.apache.camel.component.milvus.MilvusConfiguration) obj; + switch (ignoreCase ? name.toLowerCase() : name) { + case "client": + case "Client": return target.getClient(); + case "host": + case "Host": return target.getHost(); + case "port": + case "Port": return target.getPort(); + case "timeout": + case "Timeout": return target.getTimeout(); + case "token": + case "Token": return target.getToken(); + default: return null; + } + } +} + diff --git a/components/camel-milvus/src/generated/java/org/apache/camel/component/milvus/MilvusEndpointConfigurer.java b/components/camel-milvus/src/generated/java/org/apache/camel/component/milvus/MilvusEndpointConfigurer.java new file mode 100644 index 00000000000..208b66c195c --- /dev/null +++ b/components/camel-milvus/src/generated/java/org/apache/camel/component/milvus/MilvusEndpointConfigurer.java @@ -0,0 +1,61 @@ +/* Generated by camel build tools - do NOT edit this file! */ +package org.apache.camel.component.milvus; + +import java.util.Map; + +import org.apache.camel.CamelContext; +import org.apache.camel.spi.ExtendedPropertyConfigurerGetter; +import org.apache.camel.spi.PropertyConfigurerGetter; +import org.apache.camel.spi.ConfigurerStrategy; +import org.apache.camel.spi.GeneratedPropertyConfigurer; +import org.apache.camel.util.CaseInsensitiveMap; +import org.apache.camel.support.component.PropertyConfigurerSupport; + +/** + * Generated by camel build tools - do NOT edit this file! + */ +@SuppressWarnings("unchecked") +public class MilvusEndpointConfigurer extends PropertyConfigurerSupport implements GeneratedPropertyConfigurer, PropertyConfigurerGetter { + + @Override + public boolean configure(CamelContext camelContext, Object obj, String name, Object value, boolean ignoreCase) { + MilvusEndpoint target = (MilvusEndpoint) obj; + switch (ignoreCase ? name.toLowerCase() : name) { + case "host": target.getConfiguration().setHost(property(camelContext, java.lang.String.class, value)); return true; + case "lazystartproducer": + case "lazyStartProducer": target.setLazyStartProducer(property(camelContext, boolean.class, value)); return true; + case "port": target.getConfiguration().setPort(property(camelContext, int.class, value)); return true; + case "timeout": target.getConfiguration().setTimeout(property(camelContext, long.class, value)); return true; + case "token": target.getConfiguration().setToken(property(camelContext, java.lang.String.class, value)); return true; + default: return false; + } + } + + @Override + public Class<?> getOptionType(String name, boolean ignoreCase) { + switch (ignoreCase ? name.toLowerCase() : name) { + case "host": return java.lang.String.class; + case "lazystartproducer": + case "lazyStartProducer": return boolean.class; + case "port": return int.class; + case "timeout": return long.class; + case "token": return java.lang.String.class; + default: return null; + } + } + + @Override + public Object getOptionValue(Object obj, String name, boolean ignoreCase) { + MilvusEndpoint target = (MilvusEndpoint) obj; + switch (ignoreCase ? name.toLowerCase() : name) { + case "host": return target.getConfiguration().getHost(); + case "lazystartproducer": + case "lazyStartProducer": return target.isLazyStartProducer(); + case "port": return target.getConfiguration().getPort(); + case "timeout": return target.getConfiguration().getTimeout(); + case "token": return target.getConfiguration().getToken(); + default: return null; + } + } +} + diff --git a/components/camel-milvus/src/generated/java/org/apache/camel/component/milvus/MilvusEndpointUriFactory.java b/components/camel-milvus/src/generated/java/org/apache/camel/component/milvus/MilvusEndpointUriFactory.java new file mode 100644 index 00000000000..bec8e537b69 --- /dev/null +++ b/components/camel-milvus/src/generated/java/org/apache/camel/component/milvus/MilvusEndpointUriFactory.java @@ -0,0 +1,75 @@ +/* Generated by camel build tools - do NOT edit this file! */ +package org.apache.camel.component.milvus; + +import java.net.URISyntaxException; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +import org.apache.camel.spi.EndpointUriFactory; + +/** + * Generated by camel build tools - do NOT edit this file! + */ +public class MilvusEndpointUriFactory extends org.apache.camel.support.component.EndpointUriFactorySupport implements EndpointUriFactory { + + private static final String BASE = ":collection"; + + private static final Set<String> PROPERTY_NAMES; + private static final Set<String> SECRET_PROPERTY_NAMES; + private static final Set<String> MULTI_VALUE_PREFIXES; + static { + Set<String> props = new HashSet<>(6); + props.add("collection"); + props.add("host"); + props.add("lazyStartProducer"); + props.add("port"); + props.add("timeout"); + props.add("token"); + PROPERTY_NAMES = Collections.unmodifiableSet(props); + Set<String> secretProps = new HashSet<>(1); + secretProps.add("token"); + SECRET_PROPERTY_NAMES = Collections.unmodifiableSet(secretProps); + MULTI_VALUE_PREFIXES = Collections.emptySet(); + } + + @Override + public boolean isEnabled(String scheme) { + return "milvus".equals(scheme); + } + + @Override + public String buildUri(String scheme, Map<String, Object> properties, boolean encode) throws URISyntaxException { + String syntax = scheme + BASE; + String uri = syntax; + + Map<String, Object> copy = new HashMap<>(properties); + + uri = buildPathParameter(syntax, uri, "collection", null, true, copy); + uri = buildQueryParameters(uri, copy, encode); + return uri; + } + + @Override + public Set<String> propertyNames() { + return PROPERTY_NAMES; + } + + @Override + public Set<String> secretPropertyNames() { + return SECRET_PROPERTY_NAMES; + } + + @Override + public Set<String> multiValuePrefixes() { + return MULTI_VALUE_PREFIXES; + } + + @Override + public boolean isLenientProperties() { + return false; + } +} + diff --git a/components/camel-milvus/src/generated/resources/META-INF/org/apache/camel/component/milvus/milvus.json b/components/camel-milvus/src/generated/resources/META-INF/org/apache/camel/component/milvus/milvus.json new file mode 100644 index 00000000000..a27466d4406 --- /dev/null +++ b/components/camel-milvus/src/generated/resources/META-INF/org/apache/camel/component/milvus/milvus.json @@ -0,0 +1,53 @@ +{ + "component": { + "kind": "component", + "name": "milvus", + "title": "Milvus", + "description": "Perform operations on the Milvus Vector Database.", + "deprecated": false, + "firstVersion": "4.5.0", + "label": "database,ai", + "javaType": "org.apache.camel.component.milvus.MilvusComponent", + "supportLevel": "Preview", + "groupId": "org.apache.camel", + "artifactId": "camel-milvus", + "version": "4.5.0-SNAPSHOT", + "scheme": "milvus", + "extendsScheme": "", + "syntax": "milvus:collection", + "async": false, + "api": false, + "consumerOnly": false, + "producerOnly": true, + "lenientProperties": false, + "remote": true + }, + "componentProperties": { + "configuration": { "index": 0, "kind": "property", "displayName": "Configuration", "group": "producer", "label": "", "required": false, "type": "object", "javaType": "org.apache.camel.component.milvus.MilvusConfiguration", "deprecated": false, "autowired": false, "secret": false, "description": "The configuration;" }, + "host": { "index": 1, "kind": "property", "displayName": "Host", "group": "producer", "label": "", "required": false, "type": "string", "javaType": "java.lang.String", "deprecated": false, "deprecationNote": "", "autowired": false, "secret": false, "defaultValue": "localhost", "configurationClass": "org.apache.camel.component.milvus.MilvusConfiguration", "configurationField": "configuration", "description": "The host to connect to." }, + "lazyStartProducer": { "index": 2, "kind": "property", "displayName": "Lazy Start Producer", "group": "producer", "label": "producer", "required": false, "type": "boolean", "javaType": "boolean", "deprecated": false, "autowired": false, "secret": false, "defaultValue": false, "description": "Whether the producer should be started lazy (on the first message). By starting lazy you can use this to allow CamelContext and routes to startup in situations where a producer may otherwise fail [...] + "port": { "index": 3, "kind": "property", "displayName": "Port", "group": "producer", "label": "", "required": false, "type": "integer", "javaType": "int", "deprecated": false, "deprecationNote": "", "autowired": false, "secret": false, "defaultValue": 19530, "configurationClass": "org.apache.camel.component.milvus.MilvusConfiguration", "configurationField": "configuration", "description": "The port to connect to." }, + "timeout": { "index": 4, "kind": "property", "displayName": "Timeout", "group": "producer", "label": "", "required": false, "type": "integer", "javaType": "long", "deprecated": false, "autowired": false, "secret": false, "configurationClass": "org.apache.camel.component.milvus.MilvusConfiguration", "configurationField": "configuration", "description": "Sets a default timeout for all requests" }, + "token": { "index": 5, "kind": "property", "displayName": "Token", "group": "producer", "label": "", "required": false, "type": "string", "javaType": "java.lang.String", "deprecated": false, "deprecationNote": "", "autowired": false, "secret": true, "configurationClass": "org.apache.camel.component.milvus.MilvusConfiguration", "configurationField": "configuration", "description": "Sets the API key to use for authentication" }, + "autowiredEnabled": { "index": 6, "kind": "property", "displayName": "Autowired Enabled", "group": "advanced", "label": "advanced", "required": false, "type": "boolean", "javaType": "boolean", "deprecated": false, "autowired": false, "secret": false, "defaultValue": true, "description": "Whether autowiring is enabled. This is used for automatic autowiring options (the option must be marked as autowired) by looking up in the registry to find if there is a single instance of matching t [...] + }, + "headers": { + "CamelMilvusAction": { "index": 0, "kind": "header", "displayName": "", "group": "producer", "label": "", "required": false, "javaType": "String", "enum": [ "UPSERT", "RETRIEVE", "DELETE" ], "deprecated": false, "deprecationNote": "", "autowired": false, "secret": false, "description": "The action to be performed.", "constantName": "org.apache.camel.component.milvus.Milvus$Headers#ACTION" }, + "CamelMilvusPointsPayloadSelector": { "index": 1, "kind": "header", "displayName": "", "group": "producer", "label": "", "required": false, "javaType": "io.qdrant.client.grpc.Points$WithPayloadSelector", "deprecated": false, "deprecationNote": "", "autowired": false, "secret": false, "description": "Payload Selector.", "constantName": "org.apache.camel.component.milvus.Milvus$Headers#PAYLOAD_SELECTOR" }, + "CamelMilvusOperationID": { "index": 2, "kind": "header", "displayName": "", "group": "producer", "label": "", "required": false, "javaType": "long", "deprecated": false, "deprecationNote": "", "autowired": false, "secret": false, "description": "Operation ID.", "constantName": "org.apache.camel.component.milvus.Milvus$Headers#OPERATION_ID" }, + "CamelMilvusOperationStatus": { "index": 3, "kind": "header", "displayName": "", "group": "producer", "label": "", "required": false, "javaType": "String", "deprecated": false, "deprecationNote": "", "autowired": false, "secret": false, "description": "Operation Status.", "constantName": "org.apache.camel.component.milvus.Milvus$Headers#OPERATION_STATUS" }, + "CamelMilvusOperationStatusValue": { "index": 4, "kind": "header", "displayName": "", "group": "producer", "label": "", "required": false, "javaType": "int", "deprecated": false, "deprecationNote": "", "autowired": false, "secret": false, "description": "Operation Status Value.", "constantName": "org.apache.camel.component.milvus.Milvus$Headers#OPERATION_STATUS_VALUE" }, + "CamelMilvusReadConsistency": { "index": 5, "kind": "header", "displayName": "", "group": "producer", "label": "", "required": false, "javaType": "io.qdrant.client.grpc.Points$ReadConsistency", "deprecated": false, "deprecationNote": "", "autowired": false, "secret": false, "description": "Read Consistency.", "constantName": "org.apache.camel.component.milvus.Milvus$Headers#READ_CONSISTENCY" }, + "CamelMilvusWithPayload": { "index": 6, "kind": "header", "displayName": "", "group": "producer", "label": "", "required": false, "javaType": "boolean", "deprecated": false, "deprecationNote": "", "autowired": false, "secret": false, "defaultValue": "true", "description": "Include Payload.", "constantName": "org.apache.camel.component.milvus.Milvus$Headers#INCLUDE_PAYLOAD" }, + "CamelMilvusWithVectors": { "index": 7, "kind": "header", "displayName": "", "group": "producer", "label": "", "required": false, "javaType": "boolean", "deprecated": false, "deprecationNote": "", "autowired": false, "secret": false, "defaultValue": "false", "description": "Include Vectors.", "constantName": "org.apache.camel.component.milvus.Milvus$Headers#INCLUDE_VECTORS" }, + "CamelMilvusSize": { "index": 8, "kind": "header", "displayName": "", "group": "producer", "label": "", "required": false, "javaType": "int", "deprecated": false, "deprecationNote": "", "autowired": false, "secret": false, "description": "The number of elements.", "constantName": "org.apache.camel.component.milvus.Milvus$Headers#SIZE" } + }, + "properties": { + "collection": { "index": 0, "kind": "path", "displayName": "Collection", "group": "producer", "label": "", "required": true, "type": "string", "javaType": "java.lang.String", "deprecated": false, "deprecationNote": "", "autowired": false, "secret": false, "description": "The collection Name" }, + "host": { "index": 1, "kind": "parameter", "displayName": "Host", "group": "producer", "label": "", "required": false, "type": "string", "javaType": "java.lang.String", "deprecated": false, "deprecationNote": "", "autowired": false, "secret": false, "defaultValue": "localhost", "configurationClass": "org.apache.camel.component.milvus.MilvusConfiguration", "configurationField": "configuration", "description": "The host to connect to." }, + "port": { "index": 2, "kind": "parameter", "displayName": "Port", "group": "producer", "label": "", "required": false, "type": "integer", "javaType": "int", "deprecated": false, "deprecationNote": "", "autowired": false, "secret": false, "defaultValue": 19530, "configurationClass": "org.apache.camel.component.milvus.MilvusConfiguration", "configurationField": "configuration", "description": "The port to connect to." }, + "timeout": { "index": 3, "kind": "parameter", "displayName": "Timeout", "group": "producer", "label": "", "required": false, "type": "integer", "javaType": "long", "deprecated": false, "autowired": false, "secret": false, "configurationClass": "org.apache.camel.component.milvus.MilvusConfiguration", "configurationField": "configuration", "description": "Sets a default timeout for all requests" }, + "token": { "index": 4, "kind": "parameter", "displayName": "Token", "group": "producer", "label": "", "required": false, "type": "string", "javaType": "java.lang.String", "deprecated": false, "deprecationNote": "", "autowired": false, "secret": true, "configurationClass": "org.apache.camel.component.milvus.MilvusConfiguration", "configurationField": "configuration", "description": "Sets the API key to use for authentication" }, + "lazyStartProducer": { "index": 5, "kind": "parameter", "displayName": "Lazy Start Producer", "group": "producer (advanced)", "label": "producer,advanced", "required": false, "type": "boolean", "javaType": "boolean", "deprecated": false, "autowired": false, "secret": false, "defaultValue": false, "description": "Whether the producer should be started lazy (on the first message). By starting lazy you can use this to allow CamelContext and routes to startup in situations where a produc [...] + } +} diff --git a/components/camel-milvus/src/generated/resources/META-INF/services/org/apache/camel/component.properties b/components/camel-milvus/src/generated/resources/META-INF/services/org/apache/camel/component.properties new file mode 100644 index 00000000000..a013fa59295 --- /dev/null +++ b/components/camel-milvus/src/generated/resources/META-INF/services/org/apache/camel/component.properties @@ -0,0 +1,7 @@ +# Generated by camel build tools - do NOT edit this file! +components=milvus +groupId=org.apache.camel +artifactId=camel-milvus +version=4.5.0-SNAPSHOT +projectName=Camel :: Milvus +projectDescription=Camel Milvus support diff --git a/components/camel-milvus/src/generated/resources/META-INF/services/org/apache/camel/component/milvus b/components/camel-milvus/src/generated/resources/META-INF/services/org/apache/camel/component/milvus new file mode 100644 index 00000000000..db784999a21 --- /dev/null +++ b/components/camel-milvus/src/generated/resources/META-INF/services/org/apache/camel/component/milvus @@ -0,0 +1,2 @@ +# Generated by camel build tools - do NOT edit this file! +class=org.apache.camel.component.milvus.MilvusComponent diff --git a/components/camel-milvus/src/generated/resources/META-INF/services/org/apache/camel/configurer/milvus-component b/components/camel-milvus/src/generated/resources/META-INF/services/org/apache/camel/configurer/milvus-component new file mode 100644 index 00000000000..df42a884122 --- /dev/null +++ b/components/camel-milvus/src/generated/resources/META-INF/services/org/apache/camel/configurer/milvus-component @@ -0,0 +1,2 @@ +# Generated by camel build tools - do NOT edit this file! +class=org.apache.camel.component.milvus.MilvusComponentConfigurer diff --git a/components/camel-milvus/src/generated/resources/META-INF/services/org/apache/camel/configurer/milvus-endpoint b/components/camel-milvus/src/generated/resources/META-INF/services/org/apache/camel/configurer/milvus-endpoint new file mode 100644 index 00000000000..b7865c613f4 --- /dev/null +++ b/components/camel-milvus/src/generated/resources/META-INF/services/org/apache/camel/configurer/milvus-endpoint @@ -0,0 +1,2 @@ +# Generated by camel build tools - do NOT edit this file! +class=org.apache.camel.component.milvus.MilvusEndpointConfigurer diff --git a/components/camel-milvus/src/generated/resources/META-INF/services/org/apache/camel/configurer/org.apache.camel.component.milvus.MilvusConfiguration b/components/camel-milvus/src/generated/resources/META-INF/services/org/apache/camel/configurer/org.apache.camel.component.milvus.MilvusConfiguration new file mode 100644 index 00000000000..d46fe2053cd --- /dev/null +++ b/components/camel-milvus/src/generated/resources/META-INF/services/org/apache/camel/configurer/org.apache.camel.component.milvus.MilvusConfiguration @@ -0,0 +1,2 @@ +# Generated by camel build tools - do NOT edit this file! +class=org.apache.camel.component.milvus.MilvusConfigurationConfigurer diff --git a/components/camel-milvus/src/generated/resources/META-INF/services/org/apache/camel/urifactory/milvus-endpoint b/components/camel-milvus/src/generated/resources/META-INF/services/org/apache/camel/urifactory/milvus-endpoint new file mode 100644 index 00000000000..bcf6d6fc51b --- /dev/null +++ b/components/camel-milvus/src/generated/resources/META-INF/services/org/apache/camel/urifactory/milvus-endpoint @@ -0,0 +1,2 @@ +# Generated by camel build tools - do NOT edit this file! +class=org.apache.camel.component.milvus.MilvusEndpointUriFactory diff --git a/components/camel-milvus/src/main/docs/qdrant-component.adoc b/components/camel-milvus/src/main/docs/qdrant-component.adoc new file mode 100644 index 00000000000..35a089ca5ad --- /dev/null +++ b/components/camel-milvus/src/main/docs/qdrant-component.adoc @@ -0,0 +1,134 @@ += Qdrant Component +:doctitle: Qdrant +:shortname: qdrant +:artifactid: camel-qdrant +:description: Perform operations on the Qdrant Vector Database. +:since: 4.5 +:supportlevel: Preview +:tabs-sync-option: +:component-header: Only producer is supported +//Manually maintained attributes +:camel-spring-boot-name: qdrant + +*Since Camel {since}* + +*{component-header}* + +The Qdrant Component provides support for interacting with the https://qdrant.tech[Qdrant Vector Database]. + +== URI format + +[source] +---- +qdrant:collection[?options] +---- + +Where *collection* represents a named set of points (vectors with a payload) defined in your database. + + +// component-configure options: START + +// component-configure options: END + +// component options: START +include::partial$component-configure-options.adoc[] +include::partial$component-endpoint-options.adoc[] +// component options: END + +// endpoint options: START + +// endpoint options: END + + +== Collection Samples + +In the route below, we use the qdrant component to create a collection named _myCollection_ with the given parameters: + +[tabs] +==== +Java:: ++ +[source,java] +---- +from("direct:in") + .setHeader(Qdrant.Headers.ACTION) + .constant(QdrantAction.CREATE_COLLECTION) + .setBody() + .constant( + Collections.VectorParams.newBuilder() + .setSize(2) + .setDistance(Collections.Distance.Cosine).build()) + .to("qdrant:myCollection"); +---- +==== + +== Points Samples + +=== Upsert + +In the route below we use the qdrant component to perform insert + updates (upsert) on points in the collection named _myCollection_: + +[tabs] +==== +Java:: ++ +[source,java] +---- +from("direct:in") + .setHeader(Qdrant.Headers.ACTION) + .constant(QdrantAction.UPSERT) + .setBody() + .constant( + Points.PointStruct.newBuilder() + .setId(id(8)) + .setVectors(VectorsFactory.vectors(List.of(3.5f, 4.5f))) + .putAllPayload(Map.of( + "foo", value("hello"), + "bar", value(1))) + .build()) + .to("qdrant:myCollection"); +---- +==== + + +=== Retrieve + +In the route below, we use the qdrant component to retrieve information of a single point by id from the collection named _myCollection_: + +[tabs] +==== +Java:: ++ +[source,java] +---- +from("direct:in") + .setHeader(Qdrant.Headers.ACTION) + .constant(QdrantAction.RETRIEVE) + .setBody() + .constant(PointIdFactory.id(8)) + .to("qdrant:myCollection"); +---- +==== + + + +=== Delete + +In the route below, we use the qdrant component to delete points from the collection named `myCollection` according to a criteria: + +[tabs] +==== +Java:: ++ +[source,java] +---- +from("direct:in") + .setHeader(Qdrant.Headers.ACTION) + .constant(QdrantAction.DELETE) + .setBody() + .constant(ConditionFactory.matchKeyword("foo", "hello")) + .to("qdrant:myCollection"); +---- +==== + +include::spring-boot:partial$starter.adoc[] diff --git a/components/camel-milvus/src/main/java/org/apache/camel/component/milvus/Milvus.java b/components/camel-milvus/src/main/java/org/apache/camel/component/milvus/Milvus.java new file mode 100644 index 00000000000..ec15ec893e1 --- /dev/null +++ b/components/camel-milvus/src/main/java/org/apache/camel/component/milvus/Milvus.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.camel.component.milvus; + +import org.apache.camel.spi.Metadata; + +public class Milvus { + public static final String SCHEME = "milvus"; + + private Milvus() { + } + + public static class Headers { + @Metadata(description = "The action to be performed.", javaType = "String", enums = "UPSERT,RETRIEVE,DELETE") + public static final String ACTION = "CamelMilvusAction"; + + @Metadata(description = "Payload Selector.", javaType = "io.qdrant.client.grpc.Points$WithPayloadSelector") + public static final String PAYLOAD_SELECTOR = "CamelMilvusPointsPayloadSelector"; + + @Metadata(description = "Operation ID.", javaType = "long") + public static final String OPERATION_ID = "CamelMilvusOperationID"; + + @Metadata(description = "Operation Status.", javaType = "String") + public static final String OPERATION_STATUS = "CamelMilvusOperationStatus"; + + @Metadata(description = "Operation Status Value.", javaType = "int") + public static final String OPERATION_STATUS_VALUE = "CamelMilvusOperationStatusValue"; + + @Metadata(description = "Read Consistency.", javaType = "io.qdrant.client.grpc.Points$ReadConsistency") + public static final String READ_CONSISTENCY = "CamelMilvusReadConsistency"; + + @Metadata(description = "Include Payload.", javaType = "boolean", defaultValue = "true") + public static final String INCLUDE_PAYLOAD = "CamelMilvusWithPayload"; + public static final boolean DEFAULT_INCLUDE_PAYLOAD = true; + + @Metadata(description = "Include Vectors.", javaType = "boolean", defaultValue = "false") + public static final String INCLUDE_VECTORS = "CamelMilvusWithVectors"; + public static final boolean DEFAULT_INCLUDE_VECTORS = false; + + @Metadata(description = "The number of elements.", javaType = "int") + public static final String SIZE = "CamelMilvusSize"; + } +} diff --git a/components/camel-milvus/src/main/java/org/apache/camel/component/milvus/MilvusAction.java b/components/camel-milvus/src/main/java/org/apache/camel/component/milvus/MilvusAction.java new file mode 100644 index 00000000000..3ba9a290ac7 --- /dev/null +++ b/components/camel-milvus/src/main/java/org/apache/camel/component/milvus/MilvusAction.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.camel.component.milvus; + +public enum MilvusAction { + CREATE_COLLECTION, + CREATE_INDEX, + UPSERT, + INSERT, + SEARCH, + DELETE +} diff --git a/components/camel-milvus/src/main/java/org/apache/camel/component/milvus/MilvusComponent.java b/components/camel-milvus/src/main/java/org/apache/camel/component/milvus/MilvusComponent.java new file mode 100644 index 00000000000..878a543b81b --- /dev/null +++ b/components/camel-milvus/src/main/java/org/apache/camel/component/milvus/MilvusComponent.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.camel.component.milvus; + +import java.util.Map; + +import org.apache.camel.CamelContext; +import org.apache.camel.Endpoint; +import org.apache.camel.spi.Metadata; +import org.apache.camel.spi.annotations.Component; +import org.apache.camel.support.DefaultComponent; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@Component(Milvus.SCHEME) +public class MilvusComponent extends DefaultComponent { + private static final Logger LOGGER = LoggerFactory.getLogger(MilvusComponent.class); + + @Metadata + private MilvusConfiguration configuration; + + public MilvusComponent() { + this(null); + } + + public MilvusComponent(CamelContext context) { + super(context); + + this.configuration = new MilvusConfiguration(); + } + + public MilvusConfiguration getConfiguration() { + return configuration; + } + + /** + * The configuration; + */ + public void setConfiguration(MilvusConfiguration configuration) { + this.configuration = configuration; + } + + @Override + protected Endpoint createEndpoint( + String uri, + String remaining, + Map<String, Object> parameters) + throws Exception { + + MilvusConfiguration configuration = this.configuration.copy(); + + MilvusEndpoint endpoint = new MilvusEndpoint(uri, this, remaining, configuration); + setProperties(endpoint, parameters); + + return endpoint; + } +} diff --git a/components/camel-milvus/src/main/java/org/apache/camel/component/milvus/MilvusConfiguration.java b/components/camel-milvus/src/main/java/org/apache/camel/component/milvus/MilvusConfiguration.java new file mode 100644 index 00000000000..802401b6e7a --- /dev/null +++ b/components/camel-milvus/src/main/java/org/apache/camel/component/milvus/MilvusConfiguration.java @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.camel.component.milvus; + +import io.milvus.client.MilvusClient; +import org.apache.camel.RuntimeCamelException; +import org.apache.camel.spi.Configurer; +import org.apache.camel.spi.Metadata; +import org.apache.camel.spi.UriParam; +import org.apache.camel.spi.UriParams; + +@Configurer +@UriParams +public class MilvusConfiguration implements Cloneable { + + @Metadata(defaultValue = "localhost") + @UriParam + private String host = "localhost"; + + @Metadata(defaultValue = "19530") + @UriParam + private int port = 19530; + + @Metadata(secret = true) + @UriParam + private String token; + + @UriParam + private long timeout = 10000L; + + @Metadata(autowired = true) + private MilvusClient client; + + public String getHost() { + return host; + } + + /** + * The host to connect to. + */ + public void setHost(String host) { + this.host = host; + } + + public int getPort() { + return port; + } + + /** + * The port to connect to. + */ + public void setPort(int port) { + this.port = port; + } + + public String getToken() { + return token; + } + + /** + * Sets the API key to use for authentication + */ + public void setToken(String token) { + this.token = token; + } + + public long getTimeout() { + return timeout; + } + + /** + * Sets a default timeout for all requests + */ + public void setTimeout(long timeout) { + this.timeout = timeout; + } + + public MilvusClient getClient() { + return client; + } + + /** + * Reference to a `io.milvus.client.MilvusClient`. + */ + public void setClient(MilvusClient client) { + this.client = client; + } + + // ************************ + // + // Clone + // + // ************************ + + public MilvusConfiguration copy() { + try { + return (MilvusConfiguration) super.clone(); + } catch (CloneNotSupportedException e) { + throw new RuntimeCamelException(e); + } + } +} diff --git a/components/camel-milvus/src/main/java/org/apache/camel/component/milvus/MilvusEndpoint.java b/components/camel-milvus/src/main/java/org/apache/camel/component/milvus/MilvusEndpoint.java new file mode 100644 index 00000000000..908e72380f6 --- /dev/null +++ b/components/camel-milvus/src/main/java/org/apache/camel/component/milvus/MilvusEndpoint.java @@ -0,0 +1,140 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.camel.component.milvus; + +import java.util.concurrent.TimeUnit; + +import io.milvus.client.MilvusClient; +import io.milvus.client.MilvusServiceClient; +import io.milvus.param.ConnectParam; +import org.apache.camel.Category; +import org.apache.camel.Component; +import org.apache.camel.Consumer; +import org.apache.camel.Processor; +import org.apache.camel.Producer; +import org.apache.camel.spi.Metadata; +import org.apache.camel.spi.UriEndpoint; +import org.apache.camel.spi.UriParam; +import org.apache.camel.spi.UriPath; +import org.apache.camel.support.DefaultEndpoint; + +/** + * Perform operations on the Milvus Vector Database. + */ +@UriEndpoint( + firstVersion = "4.5.0", + scheme = Milvus.SCHEME, + title = "Milvus", + syntax = "milvus:collection", + producerOnly = true, + category = { + Category.DATABASE, + Category.AI + }, + headersClass = Milvus.Headers.class) +public class MilvusEndpoint extends DefaultEndpoint { + + @Metadata(required = true) + @UriPath(description = "The collection Name") + private final String collection; + + @UriParam + private MilvusConfiguration configuration; + + private final Object lock; + + private volatile boolean closeClient; + private volatile MilvusClient client; + + public MilvusEndpoint( + String endpointUri, + Component component, + String collection, + MilvusConfiguration configuration) { + + super(endpointUri, component); + + this.collection = collection; + this.configuration = configuration; + + this.lock = new Object(); + } + + public MilvusConfiguration getConfiguration() { + return configuration; + } + + public String getCollection() { + return collection; + } + + public synchronized MilvusClient getClient() { + if (this.client == null) { + synchronized (this.lock) { + if (this.client == null) { + this.client = this.configuration.getClient(); + this.closeClient = false; + + if (this.client == null) { + this.client = createClient(); + this.closeClient = true; + } + } + } + } + + return this.client; + } + + @Override + public Producer createProducer() throws Exception { + return new MilvusProducer(this); + } + + @Override + public Consumer createConsumer(Processor processor) throws Exception { + throw new UnsupportedOperationException("Consumer is not implemented for this component"); + } + + @Override + public void doStart() throws Exception { + super.doStart(); + } + + @Override + public void doStop() throws Exception { + super.doStop(); + + if (this.client != null && this.closeClient) { + this.client.close(); + this.client = null; + this.closeClient = false; + } + } + + private MilvusClient createClient() { + + ConnectParam.Builder parameters = ConnectParam.newBuilder().withHost(configuration.getHost()) + .withPort(configuration.getPort()).withConnectTimeout(configuration.getTimeout(), TimeUnit.MILLISECONDS); + + if (configuration.getToken() != null) { + parameters.withToken(configuration.getToken()); + } + + return new MilvusServiceClient(parameters.build()); + } +} diff --git a/components/camel-milvus/src/main/java/org/apache/camel/component/milvus/MilvusProducer.java b/components/camel-milvus/src/main/java/org/apache/camel/component/milvus/MilvusProducer.java new file mode 100644 index 00000000000..8a3e2a8b78b --- /dev/null +++ b/components/camel-milvus/src/main/java/org/apache/camel/component/milvus/MilvusProducer.java @@ -0,0 +1,192 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.camel.component.milvus; + +import java.util.List; +import java.util.concurrent.ExecutorService; + +import io.milvus.client.MilvusClient; +import io.milvus.grpc.MutationResult; +import io.milvus.param.R; +import io.milvus.param.RpcStatus; +import io.milvus.param.collection.CreateCollectionParam; +import io.milvus.param.collection.LoadCollectionParam; +import io.milvus.param.dml.DeleteParam; +import io.milvus.param.dml.InsertParam; +import io.milvus.param.dml.UpsertParam; +import io.milvus.param.highlevel.dml.SearchSimpleParam; +import io.milvus.param.highlevel.dml.response.SearchResponse; +import io.milvus.param.index.CreateIndexParam; +import org.apache.camel.CamelContext; +import org.apache.camel.Exchange; +import org.apache.camel.Message; +import org.apache.camel.NoSuchHeaderException; +import org.apache.camel.support.DefaultProducer; + +public class MilvusProducer extends DefaultProducer { + private MilvusClient client; + private ExecutorService executor; + + public MilvusProducer(MilvusEndpoint endpoint) { + super(endpoint); + } + + @Override + public MilvusEndpoint getEndpoint() { + return (MilvusEndpoint) super.getEndpoint(); + } + + @Override + public void doStart() throws Exception { + super.doStart(); + + this.client = getEndpoint().getClient(); + } + + @Override + public void process(Exchange exchange) { + final Message in = exchange.getMessage(); + final MilvusAction action = in.getHeader(Milvus.Headers.ACTION, MilvusAction.class); + + try { + if (action == null) { + throw new NoSuchHeaderException("The action is a required header", exchange, Milvus.Headers.ACTION); + } + + switch (action) { + case CREATE_COLLECTION: + createCollection(exchange); + break; + case CREATE_INDEX: + createIndex(exchange); + break; + case UPSERT: + upsert(exchange); + break; + case INSERT: + insert(exchange); + break; + case SEARCH: + search(exchange); + break; + case DELETE: + delete(exchange); + break; + default: + throw new UnsupportedOperationException("Unsupported action: " + action.name()); + } + } catch (Exception e) { + exchange.setException(e); + } + } + + // *************************************** + // + // Actions + // + // *************************************** + + @SuppressWarnings({ "unchecked" }) + private void upsert(Exchange exchange) throws Exception { + final Message in = exchange.getMessage(); + final UpsertParam upsert = in.getMandatoryBody(UpsertParam.class); + + R<MutationResult> result = this.client.upsert(upsert); + + handleResponseStatus(result); + populateResponse(result, exchange); + } + + @SuppressWarnings({ "unchecked" }) + private void insert(Exchange exchange) throws Exception { + final Message in = exchange.getMessage(); + final InsertParam insert = in.getMandatoryBody(InsertParam.class); + + R<MutationResult> result = this.client.insert(insert); + + handleResponseStatus(result); + populateResponse(result, exchange); + } + + private void createCollection(Exchange exchange) throws Exception { + final Message in = exchange.getMessage(); + final CreateCollectionParam body = in.getMandatoryBody(CreateCollectionParam.class); + + R<RpcStatus> result = this.client.createCollection(body); + + handleResponseStatus(result); + populateResponse(result, exchange); + + } + + + private void createIndex(Exchange exchange) throws Exception { + final Message in = exchange.getMessage(); + final CreateIndexParam body = in.getMandatoryBody(CreateIndexParam.class); + + R<RpcStatus> result = this.client.createIndex(body); + + handleResponseStatus(result); + populateResponse(result, exchange); + + } + + private void search(Exchange exchange) throws Exception { + final Message in = exchange.getMessage(); + final SearchSimpleParam body = in.getMandatoryBody(SearchSimpleParam.class); + + this.client.loadCollection(LoadCollectionParam.newBuilder().withCollectionName(getEndpoint().getCollection()).withSyncLoad(true).build()); + R<SearchResponse> result = this.client.search(body); + + handleResponseStatus(result); + populateResponse(result, exchange); + + } + + private void delete(Exchange exchange) throws Exception { + final Message in = exchange.getMessage(); + final DeleteParam body = in.getMandatoryBody(DeleteParam.class); + + R<MutationResult> result = this.client.delete(body); + + handleResponseStatus(result); + populateResponse(result, exchange); + + } + + // *************************************** + // + // Helpers + // + // *************************************** + + private CamelContext getCamelContext() { + return getEndpoint().getCamelContext(); + } + + private void handleResponseStatus(R<?> r) { + if (r.getStatus() != R.Status.Success.getCode()) { + throw new RuntimeException(r.getMessage()); + } + } + + private void populateResponse(R<?> r, Exchange exchange) { + Message out = exchange.getMessage(); + out.setHeader(Milvus.Headers.OPERATION_STATUS, r.getStatus()); + out.setBody(r.getData()); + } +} diff --git a/components/camel-milvus/src/test/java/org/apache/camel/component/milvus/MilvusCreateCollectionTest.java b/components/camel-milvus/src/test/java/org/apache/camel/component/milvus/MilvusCreateCollectionTest.java new file mode 100644 index 00000000000..10ed923f5d2 --- /dev/null +++ b/components/camel-milvus/src/test/java/org/apache/camel/component/milvus/MilvusCreateCollectionTest.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.camel.component.milvus; + +import io.milvus.grpc.DataType; +import io.milvus.param.collection.CreateCollectionParam; +import io.milvus.param.collection.FieldType; +import org.apache.camel.Exchange; +import org.apache.camel.NoSuchHeaderException; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +public class MilvusCreateCollectionTest extends MilvusTestSupport { + + @DisplayName("Tests that trying to create a collection without passing the action name triggers a failure") + @Test + public void createCollectionWithoutRequiredParameters() { + FieldType fieldType1 = FieldType.newBuilder() + .withName("userID") + .withDescription("user identification") + .withDataType(DataType.Int64) + .withPrimaryKey(true) + .withAutoID(true) + .build(); + + FieldType fieldType2 = FieldType.newBuilder() + .withName("userFace") + .withDescription("face embedding") + .withDataType(DataType.FloatVector) + .withDimension(64) + .build(); + + FieldType fieldType3 = FieldType.newBuilder() + .withName("userAge") + .withDescription("user age") + .withDataType(DataType.Int8) + .build(); + + CreateCollectionParam createCollectionReq = CreateCollectionParam.newBuilder() + .withCollectionName("test") + .withDescription("customer info") + .withShardsNum(2) + .withEnableDynamicField(false) + .addFieldType(fieldType1) + .addFieldType(fieldType2) + .addFieldType(fieldType3) + .build(); + + Exchange result = fluentTemplate.to("milvus:createCollection") + .withBody( + createCollectionReq) + .request(Exchange.class); + + assertThat(result).isNotNull(); + assertThat(result.getException()).isInstanceOf(NoSuchHeaderException.class); + } +} diff --git a/components/camel-milvus/src/test/java/org/apache/camel/component/milvus/MilvusTestSupport.java b/components/camel-milvus/src/test/java/org/apache/camel/component/milvus/MilvusTestSupport.java new file mode 100644 index 00000000000..417354f8e1c --- /dev/null +++ b/components/camel-milvus/src/test/java/org/apache/camel/component/milvus/MilvusTestSupport.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.camel.component.milvus; + +import org.apache.camel.CamelContext; +import org.apache.camel.test.infra.milvus.services.MilvusService; +import org.apache.camel.test.infra.milvus.services.MilvusServiceFactory; +import org.apache.camel.test.junit5.CamelTestSupport; +import org.junit.jupiter.api.TestInstance; +import org.junit.jupiter.api.extension.RegisterExtension; + +import java.net.MalformedURLException; +import java.net.URL; + +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +public class MilvusTestSupport extends CamelTestSupport { + @RegisterExtension + static MilvusService MILVUS = MilvusServiceFactory.createSingletonService(); + + @Override + protected CamelContext createCamelContext() throws Exception { + CamelContext context = super.createCamelContext(); + + URL url = new URL(MILVUS.getMilvusEndpointUrl()); + MilvusComponent component = context.getComponent(Milvus.SCHEME, MilvusComponent.class); + component.getConfiguration().setHost(url.getHost()); + component.getConfiguration().setPort(url.getPort()); + + return context; + } +} diff --git a/components/camel-milvus/src/test/java/org/apache/camel/component/milvus/MilvusUpsertTest.java b/components/camel-milvus/src/test/java/org/apache/camel/component/milvus/MilvusUpsertTest.java new file mode 100644 index 00000000000..f16ee8b9798 --- /dev/null +++ b/components/camel-milvus/src/test/java/org/apache/camel/component/milvus/MilvusUpsertTest.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.camel.component.milvus; + +import org.apache.camel.Exchange; +import org.apache.camel.NoSuchHeaderException; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; + +public class MilvusUpsertTest extends MilvusTestSupport { + + @DisplayName("Tests that trying to upsert without passing the action name triggers a failure") + @Test + public void upsertWithoutRequiredParameters() { + Exchange result = fluentTemplate.to("milvus:upsert") + .withBody(null) + .request(Exchange.class); + + assertThat(result).isNotNull(); + assertThat(result.getException()).isInstanceOf(NoSuchHeaderException.class); + } +} diff --git a/components/camel-milvus/src/test/java/org/apache/camel/component/milvus/it/MilvusComponentIT.java b/components/camel-milvus/src/test/java/org/apache/camel/component/milvus/it/MilvusComponentIT.java new file mode 100644 index 00000000000..95ee6b323e2 --- /dev/null +++ b/components/camel-milvus/src/test/java/org/apache/camel/component/milvus/it/MilvusComponentIT.java @@ -0,0 +1,269 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.camel.component.milvus.it; + +import io.milvus.common.clientenum.ConsistencyLevelEnum; +import io.milvus.grpc.DataType; +import io.milvus.param.IndexType; +import io.milvus.param.MetricType; +import io.milvus.param.collection.CreateCollectionParam; +import io.milvus.param.collection.FieldType; +import io.milvus.param.dml.DeleteParam; +import io.milvus.param.dml.InsertParam; +import io.milvus.param.dml.UpsertParam; +import io.milvus.param.highlevel.dml.SearchSimpleParam; +import io.milvus.param.highlevel.dml.response.SearchResponse; +import io.milvus.param.index.CreateIndexParam; +import org.apache.camel.Exchange; +import org.apache.camel.component.milvus.Milvus; +import org.apache.camel.component.milvus.MilvusAction; +import org.apache.camel.component.milvus.MilvusTestSupport; +import org.assertj.core.util.Lists; +import org.junit.jupiter.api.*; + +import java.util.ArrayList; +import java.util.List; +import java.util.Random; + +import static org.assertj.core.api.Assertions.assertThat; + +@TestMethodOrder(MethodOrderer.OrderAnnotation.class) +public class MilvusComponentIT extends MilvusTestSupport { + @Test + @Order(1) + public void createCollection() { + FieldType fieldType1 = FieldType.newBuilder() + .withName("userID") + .withDescription("user identification") + .withDataType(DataType.Int64) + .withPrimaryKey(true) + .withAutoID(true) + .build(); + + FieldType fieldType2 = FieldType.newBuilder() + .withName("userFace") + .withDescription("face embedding") + .withDataType(DataType.FloatVector) + .withDimension(64) + .build(); + + FieldType fieldType3 = FieldType.newBuilder() + .withName("userAge") + .withDescription("user age") + .withDataType(DataType.Int8) + .build(); + + CreateCollectionParam createCollectionReq = CreateCollectionParam.newBuilder() + .withCollectionName("test") + .withDescription("customer info") + .withShardsNum(2) + .withEnableDynamicField(false) + .addFieldType(fieldType1) + .addFieldType(fieldType2) + .addFieldType(fieldType3) + .build(); + + Exchange result = fluentTemplate.to("milvus:test") + .withHeader(Milvus.Headers.ACTION, MilvusAction.CREATE_COLLECTION) + .withBody( + createCollectionReq) + .request(Exchange.class); + + assertThat(result).isNotNull(); + assertThat(result.getException()).isNull(); + } + + @Test + @Order(2) + public void createIndex() { + CreateIndexParam createAgeIndexParam = CreateIndexParam.newBuilder() + .withCollectionName("test") + .withFieldName("userAge") + .withIndexType(IndexType.STL_SORT) + .withSyncMode(Boolean.TRUE) + .build(); + + Exchange result = fluentTemplate.to("milvus:test") + .withHeader(Milvus.Headers.ACTION, MilvusAction.CREATE_INDEX) + .withBody( + createAgeIndexParam) + .request(Exchange.class); + + assertThat(result).isNotNull(); + assertThat(result.getException()).isNull(); + + CreateIndexParam createVectorIndexParam = CreateIndexParam.newBuilder() + .withCollectionName("test") + .withFieldName("userFace") + .withIndexName("userFaceIndex") + .withIndexType(IndexType.IVF_FLAT) + .withMetricType(MetricType.L2) + .withExtraParam("{\"nlist\":128}") + .withSyncMode(Boolean.TRUE) + .build(); + + result = fluentTemplate.to("milvus:test") + .withHeader(Milvus.Headers.ACTION, MilvusAction.CREATE_INDEX) + .withBody( + createVectorIndexParam) + .request(Exchange.class); + + assertThat(result).isNotNull(); + assertThat(result.getException()).isNull(); + } + + @Test + @Order(3) + public void insert() { + Random ran = new Random(); + List<Integer> ages = new ArrayList<>(); + for (long i = 0L; i < 2; ++i) { + ages.add(ran.nextInt(99)); + } + List<InsertParam.Field> fields = new ArrayList<>(); + fields.add(new InsertParam.Field("userAge", ages)); + fields.add(new InsertParam.Field("userFace", generateFloatVectors(2))); + + InsertParam insertParam = InsertParam.newBuilder() + .withCollectionName("test") + .withFields(fields) + .build(); + + Exchange result = fluentTemplate.to("milvus:test") + .withHeader(Milvus.Headers.ACTION, MilvusAction.INSERT) + .withBody( + insertParam) + .request(Exchange.class); + + assertThat(result).isNotNull(); + assertThat(result.getException()).isNull(); + } + + + @Test + @Order(4) + public void upsert() { + Random ran = new Random(); + List<Integer> ages = new ArrayList<>(); + for (long i = 0L; i < 2; ++i) { + ages.add(ran.nextInt(99)); + } + List<UpsertParam.Field> fields = new ArrayList<>(); + fields.add(new UpsertParam.Field("userAge", ages)); + fields.add(new UpsertParam.Field("userFace", generateFloatVectors(2))); + + UpsertParam upsertParam = UpsertParam.newBuilder() + .withCollectionName("test") + .withFields(fields) + .build(); + + Exchange result = fluentTemplate.to("milvus:test") + .withHeader(Milvus.Headers.ACTION, MilvusAction.UPSERT) + .withBody( + upsertParam) + .request(Exchange.class); + + assertThat(result).isNotNull(); + assertThat(result.getException()).message().isEqualTo("Upsert don't support autoID==True, collection: test"); + } + + @Test + @Order(5) + public void search() { + SearchSimpleParam searchSimpleParam = SearchSimpleParam.newBuilder() + .withCollectionName("test") + .withVectors(generateFloatVector()) + .withFilter("userAge>0") + .withLimit(100L) + .withOffset(0L) + .withOutputFields(Lists.newArrayList("userAge")) + .withConsistencyLevel(ConsistencyLevelEnum.STRONG) + .build(); + + Exchange result = fluentTemplate.to("milvus:test") + .withHeader(Milvus.Headers.ACTION, MilvusAction.SEARCH) + .withBody( + searchSimpleParam) + .request(Exchange.class); + + assertThat(result).isNotNull(); + assertThat(result.getException()).isNull(); + assertThat(result.getMessage().getBody(SearchResponse.class).rowRecords.size() == 2); + } + + @Test + @Order(6) + public void delete() { + DeleteParam delete = DeleteParam.newBuilder() + .withCollectionName("test") + .withExpr("userAge>0") + .build(); + + Exchange result = fluentTemplate.to("milvus:test") + .withHeader(Milvus.Headers.ACTION, MilvusAction.DELETE) + .withBody( + delete) + .request(Exchange.class); + + assertThat(result).isNotNull(); + assertThat(result.getException()).isNull(); + + SearchSimpleParam searchSimpleParam = SearchSimpleParam.newBuilder() + .withCollectionName("test") + .withVectors(generateFloatVector()) + .withFilter("userAge>0") + .withLimit(100L) + .withOffset(0L) + .withOutputFields(Lists.newArrayList("userAge")) + .withConsistencyLevel(ConsistencyLevelEnum.STRONG) + .build(); + + result = fluentTemplate.to("milvus:test") + .withHeader(Milvus.Headers.ACTION, MilvusAction.SEARCH) + .withBody( + searchSimpleParam) + .request(Exchange.class); + + assertThat(result).isNotNull(); + assertThat(result.getException()).isNull(); + assertThat(result.getMessage().getBody(SearchResponse.class).rowRecords.size() == 0); + } + + private List<List<Float>> generateFloatVectors(int count) { + Random ran = new Random(); + List<List<Float>> vectors = new ArrayList<>(); + for (int n = 0; n < count; ++n) { + List<Float> vector = new ArrayList<>(); + for (int i = 0; i < 64; ++i) { + vector.add(ran.nextFloat()); + } + vectors.add(vector); + } + + return vectors; + } + + private List<Float> generateFloatVector() { + Random ran = new Random(); + List<Float> vector = new ArrayList<>(); + for (int i = 0; i < 64; ++i) { + vector.add(ran.nextFloat()); + } + return vector; + } + +} diff --git a/components/camel-milvus/src/test/resources/log4j2.properties b/components/camel-milvus/src/test/resources/log4j2.properties new file mode 100644 index 00000000000..c8f8c126dd8 --- /dev/null +++ b/components/camel-milvus/src/test/resources/log4j2.properties @@ -0,0 +1,35 @@ +## --------------------------------------------------------------------------- +## Licensed to the Apache Software Foundation (ASF) under one or more +## contributor license agreements. See the NOTICE file distributed with +## this work for additional information regarding copyright ownership. +## The ASF licenses this file to You under the Apache License, Version 2.0 +## (the "License"); you may not use this file except in compliance with +## the License. You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## --------------------------------------------------------------------------- + +appender.file.type = File +appender.file.name = file +appender.file.fileName = target/camel-milvus-test.log +appender.file.layout.type = PatternLayout +appender.file.layout.pattern = %d [%-15.15t] %-5p %-30.30c{1} - %m%n + +appender.out.type = Console +appender.out.name = out +appender.out.layout.type = PatternLayout +appender.out.layout.pattern = [%30.30t] %-30.30c{1} %-5p %m%n + +rootLogger.level = INFO +rootLogger.appenderRef.file.ref = file +# To log to stdout +#rootLogger.appenderRef.out.ref = out + +logger.camel-milvus.name = org.apache.camel.component.milvus +logger.camel-milvus.level = DEBUG diff --git a/parent/pom.xml b/parent/pom.xml index 3c5cda1a4d9..67a73651342 100644 --- a/parent/pom.xml +++ b/parent/pom.xml @@ -1806,6 +1806,11 @@ <artifactId>camel-microprofile-health</artifactId> <version>${project.version}</version> </dependency> + <dependency> + <groupId>org.apache.camel</groupId> + <artifactId>camel-milvus</artifactId> + <version>${project.version}</version> + </dependency> <dependency> <groupId>org.apache.camel</groupId> <artifactId>camel-mina</artifactId> diff --git a/test-infra/camel-test-infra-milvus/pom.xml b/test-infra/camel-test-infra-milvus/pom.xml new file mode 100644 index 00000000000..2b4ab452d8f --- /dev/null +++ b/test-infra/camel-test-infra-milvus/pom.xml @@ -0,0 +1,55 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +--> +<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> + <parent> + <artifactId>camel-test-infra-parent</artifactId> + <groupId>org.apache.camel</groupId> + <relativePath>../camel-test-infra-parent/pom.xml</relativePath> + <version>4.5.0-SNAPSHOT</version> + </parent> + + <modelVersion>4.0.0</modelVersion> + + <artifactId>camel-test-infra-milvus</artifactId> + <name>Camel :: Test Infra :: Milvus</name> + + <dependencies> + <dependency> + <groupId>org.apache.camel</groupId> + <artifactId>camel-test-infra-common</artifactId> + <version>${project.version}</version> + <type>test-jar</type> + </dependency> + + <dependency> + <groupId>org.testcontainers</groupId> + <artifactId>testcontainers</artifactId> + <version>${testcontainers-version}</version> + </dependency> + + <dependency> + <groupId>org.testcontainers</groupId> + <artifactId>milvus</artifactId> + <version>${testcontainers-version}</version> + </dependency> + </dependencies> + + +</project> diff --git a/test-infra/camel-test-infra-milvus/src/main/resources/META-INF/MANIFEST.MF b/test-infra/camel-test-infra-milvus/src/main/resources/META-INF/MANIFEST.MF new file mode 100644 index 00000000000..e69de29bb2d diff --git a/test-infra/camel-test-infra-milvus/src/test/java/org/apache/camel/test/infra/milvus/common/MilvusProperties.java b/test-infra/camel-test-infra-milvus/src/test/java/org/apache/camel/test/infra/milvus/common/MilvusProperties.java new file mode 100644 index 00000000000..b4e82b6ea1f --- /dev/null +++ b/test-infra/camel-test-infra-milvus/src/test/java/org/apache/camel/test/infra/milvus/common/MilvusProperties.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.camel.test.infra.milvus.common; + +public final class MilvusProperties { + public static final String MILVUS_ENDPOINT_URL = "milvus.endpoint.url"; + public static final String MILVUS_CONTAINER = "milvus.container"; + + private MilvusProperties() { + + } +} diff --git a/test-infra/camel-test-infra-milvus/src/test/java/org/apache/camel/test/infra/milvus/services/MilvusLocalContainerService.java b/test-infra/camel-test-infra-milvus/src/test/java/org/apache/camel/test/infra/milvus/services/MilvusLocalContainerService.java new file mode 100644 index 00000000000..3f0a5a087eb --- /dev/null +++ b/test-infra/camel-test-infra-milvus/src/test/java/org/apache/camel/test/infra/milvus/services/MilvusLocalContainerService.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.camel.test.infra.milvus.services; + +import java.time.Duration; + +import org.apache.camel.test.infra.common.LocalPropertyResolver; +import org.apache.camel.test.infra.common.services.ContainerService; +import org.apache.camel.test.infra.milvus.common.MilvusProperties; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testcontainers.milvus.MilvusContainer; +import org.testcontainers.utility.DockerImageName; + +public class MilvusLocalContainerService implements MilvusService, ContainerService<MilvusContainer> { + + private static final Logger LOG = LoggerFactory.getLogger(MilvusLocalContainerService.class); + + private final MilvusContainer container; + + public MilvusLocalContainerService() { + this(LocalPropertyResolver.getProperty(MilvusLocalContainerService.class, MilvusProperties.MILVUS_CONTAINER)); + } + + public MilvusLocalContainerService(String imageName) { + container = initContainer(imageName); + } + + public MilvusLocalContainerService(MilvusContainer container) { + this.container = container; + } + + protected MilvusContainer initContainer(String imageName) { + return new MilvusContainer(DockerImageName.parse(imageName)) + .withStartupTimeout(Duration.ofMinutes(3L)); + } + + @Override + public void registerProperties() { + System.setProperty(MilvusProperties.MILVUS_ENDPOINT_URL, getMilvusEndpointUrl()); + } + + @Override + public void initialize() { + LOG.info("Trying to start the Milvus container"); + container.start(); + + registerProperties(); + LOG.info("Milvus instance running at {}", getMilvusEndpointUrl()); + } + + @Override + public void shutdown() { + LOG.info("Stopping the Milvus container"); + container.stop(); + } + + @Override + public MilvusContainer getContainer() { + return container; + } + + @Override + public String getMilvusEndpointUrl() { + return container.getEndpoint(); + } +} diff --git a/test-infra/camel-test-infra-milvus/src/test/java/org/apache/camel/test/infra/milvus/services/MilvusRemoteService.java b/test-infra/camel-test-infra-milvus/src/test/java/org/apache/camel/test/infra/milvus/services/MilvusRemoteService.java new file mode 100644 index 00000000000..34b76ae5558 --- /dev/null +++ b/test-infra/camel-test-infra-milvus/src/test/java/org/apache/camel/test/infra/milvus/services/MilvusRemoteService.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.camel.test.infra.milvus.services; + +import org.apache.camel.test.infra.milvus.common.MilvusProperties; + +public class MilvusRemoteService implements MilvusService { + + @Override + public void registerProperties() { + // NO-OP + } + + @Override + public void initialize() { + registerProperties(); + } + + @Override + public void shutdown() { + // NO-OP + } + + @Override + public String getMilvusEndpointUrl() { + return System.getProperty(MilvusProperties.MILVUS_ENDPOINT_URL); + } +} diff --git a/test-infra/camel-test-infra-milvus/src/test/java/org/apache/camel/test/infra/milvus/services/MilvusService.java b/test-infra/camel-test-infra-milvus/src/test/java/org/apache/camel/test/infra/milvus/services/MilvusService.java new file mode 100644 index 00000000000..d61d0791f93 --- /dev/null +++ b/test-infra/camel-test-infra-milvus/src/test/java/org/apache/camel/test/infra/milvus/services/MilvusService.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.camel.test.infra.milvus.services; + +import org.apache.camel.test.infra.common.services.TestService; + +/** + * Test infra service for Milvus + */ +public interface MilvusService extends TestService { + + String getMilvusEndpointUrl(); +} diff --git a/test-infra/camel-test-infra-milvus/src/test/java/org/apache/camel/test/infra/milvus/services/MilvusServiceFactory.java b/test-infra/camel-test-infra-milvus/src/test/java/org/apache/camel/test/infra/milvus/services/MilvusServiceFactory.java new file mode 100644 index 00000000000..51ca99ad080 --- /dev/null +++ b/test-infra/camel-test-infra-milvus/src/test/java/org/apache/camel/test/infra/milvus/services/MilvusServiceFactory.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.camel.test.infra.milvus.services; + +import org.apache.camel.test.infra.common.services.SimpleTestServiceBuilder; +import org.apache.camel.test.infra.common.services.SingletonService; + +public final class MilvusServiceFactory { + private MilvusServiceFactory() { + + } + + public static class SingletonMilvusService extends SingletonService<MilvusService> implements MilvusService { + + public SingletonMilvusService(MilvusService service, String name) { + super(service, name); + } + + @Override + public String getMilvusEndpointUrl() { + return getService().getMilvusEndpointUrl(); + } + } + + public static SimpleTestServiceBuilder<MilvusService> builder() { + return new SimpleTestServiceBuilder<>("milvus"); + } + + public static MilvusService createService() { + return builder() + .addLocalMapping(MilvusLocalContainerService::new) + .addRemoteMapping(MilvusRemoteService::new) + .build(); + } + + public static MilvusService createSingletonService() { + return builder() + .addLocalMapping(() -> new SingletonMilvusService(new MilvusLocalContainerService(), "milvus")) + .build(); + } +} diff --git a/test-infra/camel-test-infra-milvus/src/test/resources/org/apache/camel/test/infra/milvus/services/container.properties b/test-infra/camel-test-infra-milvus/src/test/resources/org/apache/camel/test/infra/milvus/services/container.properties new file mode 100644 index 00000000000..50a0dea030a --- /dev/null +++ b/test-infra/camel-test-infra-milvus/src/test/resources/org/apache/camel/test/infra/milvus/services/container.properties @@ -0,0 +1,17 @@ +## --------------------------------------------------------------------------- +## Licensed to the Apache Software Foundation (ASF) under one or more +## contributor license agreements. See the NOTICE file distributed with +## this work for additional information regarding copyright ownership. +## The ASF licenses this file to You under the Apache License, Version 2.0 +## (the "License"); you may not use this file except in compliance with +## the License. You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## --------------------------------------------------------------------------- +milvus.container=milvusdb/milvus:v2.3.9 \ No newline at end of file diff --git a/test-infra/pom.xml b/test-infra/pom.xml index fcd9be09cf2..e4e224cd174 100644 --- a/test-infra/pom.xml +++ b/test-infra/pom.xml @@ -79,5 +79,6 @@ <module>camel-test-infra-cli</module> <module>camel-test-infra-qdrant</module> <module>camel-test-infra-ollama</module> + <module>camel-test-infra-milvus</module> </modules> </project>