This is an automated email from the ASF dual-hosted git repository. yasith pushed a commit to branch AIRAVATA-3981/integration-health-check in repository https://gitbox.apache.org/repos/asf/airavata.git
commit 05cba470b7789e786a539b144092df2a4874bd65 Author: yasithdev <[email protected]> AuthorDate: Thu Mar 26 15:32:34 2026 -0500 feat: add integration-tests Maven module scaffold --- integration-tests/pom.xml | 99 +++++++ modules/ide-integration/.gitignore | 2 - modules/ide-integration/README.md | 294 --------------------- modules/ide-integration/pom.xml | 61 ----- .../airavata/ide/integration/APIServerStarter.java | 34 --- .../airavata/ide/integration/JobEngineStarter.java | 72 ----- .../ide/integration/JobMonitorStarter.java | 33 --- pom.xml | 2 +- 8 files changed, 100 insertions(+), 497 deletions(-) diff --git a/integration-tests/pom.xml b/integration-tests/pom.xml new file mode 100644 index 0000000000..df6504700d --- /dev/null +++ b/integration-tests/pom.xml @@ -0,0 +1,99 @@ +<!-- +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +--> +<project xmlns="http://maven.apache.org/POM/4.0.0"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd";> + + <modelVersion>4.0.0</modelVersion> + + <parent> + <groupId>org.apache.airavata</groupId> + <artifactId>airavata</artifactId> + <version>0.21-SNAPSHOT</version> + <relativePath>../pom.xml</relativePath> + </parent> + + <artifactId>airavata-integration-tests</artifactId> + <packaging>jar</packaging> + <name>Airavata Integration Tests</name> + + <dependencies> + <!-- Airavata API (thrift clients, model classes) --> + <dependency> + <groupId>org.apache.airavata</groupId> + <artifactId>airavata-api</artifactId> + <version>${project.version}</version> + <scope>test</scope> + </dependency> + + <!-- Thrift runtime --> + <dependency> + <groupId>org.apache.thrift</groupId> + <artifactId>libthrift</artifactId> + <scope>test</scope> + </dependency> + + <!-- MariaDB JDBC driver --> + <dependency> + <groupId>org.mariadb.jdbc</groupId> + <artifactId>mariadb-java-client</artifactId> + <scope>test</scope> + </dependency> + + <!-- RabbitMQ AMQP client --> + <dependency> + <groupId>com.rabbitmq</groupId> + <artifactId>amqp-client</artifactId> + <scope>test</scope> + </dependency> + + <!-- Kafka admin client --> + <dependency> + <groupId>org.apache.kafka</groupId> + <artifactId>kafka-clients</artifactId> + <scope>test</scope> + </dependency> + + <!-- JSON parsing for health endpoint responses --> + <dependency> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-databind</artifactId> + <scope>test</scope> + </dependency> + + <!-- JUnit 5 --> + <dependency> + <groupId>org.junit.jupiter</groupId> + <artifactId>junit-jupiter</artifactId> + <scope>test</scope> + </dependency> + </dependencies> + + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-surefire-plugin</artifactId> + <configuration> + <excludedGroups>integration</excludedGroups> + </configuration> + </plugin> + </plugins> + </build> +</project> diff --git a/modules/ide-integration/.gitignore b/modules/ide-integration/.gitignore deleted file mode 100644 index 5bd7a3df36..0000000000 --- a/modules/ide-integration/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -database_data/ -src/main/containers/pga/airavata-php-gateway/vendor diff --git a/modules/ide-integration/README.md b/modules/ide-integration/README.md deleted file mode 100644 index 40ab393d7d..0000000000 --- a/modules/ide-integration/README.md +++ /dev/null @@ -1,294 +0,0 @@ -# Apache Airavata - IDE Integration Setup - -<div align="center"> - <h3>🚀 Complete Development Environment Setup for IntelliJ IDEA</h3> - <p>Set up a full Airavata installation inside IntelliJ IDEA for seamless development</p> -</div> - ---- - -## 📋 Prerequisites - -Before starting, ensure you have the following installed on your system: - -| Tool | Version | Purpose | Installation Link | -|------|---------|---------|-------------------| -| 🐳 **Docker & Docker Compose** | Latest | Container orchestration | [Get Docker](https://docs.docker.com/compose/) | -| 💡 **IntelliJ IDEA** | Latest | IDE with Java 17+ | [Download IDEA](https://www.jetbrains.com/idea/download/) | -| ☕ **Java JDK** | 17+ | Runtime environment | [OpenJDK 17](https://openjdk.org/projects/jdk/17/) | -| 🔧 **Apache Maven** | 3.8+ | Build tool | [Install Maven](https://maven.apache.org/install.html) | -| 📝 **Git** | Latest | Version control | [Install Git](https://git-scm.com/downloads) | -| 🐍 **Python** | 3.8+ | Django portal | [Python.org](https://www.python.org/downloads/) | -| 📦 **Node.js & npm** | Latest LTS | Frontend build tools | [Node.js](https://nodejs.org/) | - -## 🏗️ Development Environment Setup - -### 1️⃣ Clone and Prepare Repository - -```bash -# Clone the main repository -git clone https://github.com/apache/airavata.git -cd airavata - -# Build the project (this may take a few minutes) -mvn clean install -DskipTests -``` - -### 2️⃣ Open in IntelliJ IDEA - -1. **Launch IntelliJ IDEA** -2. **Open Project** → Navigate to your cloned `airavata` directory -3. **Navigate to:** `modules` → `ide-integration` module - -## 🐳 Backend Services Setup - -### 3️⃣ Configure Host Resolution - -Add the following entry to your system's hosts file: - -**Linux/macOS:** `/etc/hosts` -**Windows:** `C:\Windows\System32\drivers\etc\hosts` - -```bash -127.0.0.1 airavata.host -``` - -### 4️⃣ Start Backend Services - -Navigate to the containers directory and start all required services: - -```bash -cd .devcontainer -docker-compose up -d -``` - -**Services Started:** -- 🗄️ **MySQL Database** -- 🔐 **Keycloak** (Authentication) -- 📨 **Apache Kafka** (Messaging) -- 🐰 **RabbitMQ** (Message Queue) -- 🔒 **SSHD Server** (Secure connections) - -### 5️⃣ Initialize Database - -Apply database migrations: - -```bash -cd .devcontainer -cat ./database_scripts/init/*-migrations.sql | docker exec -i containers-db-1 mysql -p123456 -``` - -## 🖥️ Starting Airavata Components - -### 6️⃣ Start API Server - -1. **Navigate to:** `org.apache.airavata.ide.integration.APIServerStarter` -2. **Right-click** in the editor -3. **Select:** `Run 'APIServerStarter.main()'` - -> 💡 **JDK 17+ Note:** Add this JVM argument in your run configuration: -> ``` -> --add-opens java.base/java.lang=ALL-UNNAMED -> ``` - -### 7️⃣ Start Job Execution Engine - -1. **Navigate to:** `org.apache.airavata.ide.integration.JobEngineStarter` -2. **Right-click** and select **Run** - -**Components Started:** -- 🔄 Helix Controller -- 👥 Helix Participant -- ⚙️ Pre Workflow Manager -- 📋 Post Workflow Manager - -### 8️⃣ Start Job Monitoring - -#### **Setup Email Monitor (One-Time Setup)** - -1. **Create a Gmail Account** [https://accounts.google.com/signup](https://accounts.google.com/signup) - -2. **Enable 2-Step Verification** [https://myaccount.google.com/security](https://myaccount.google.com/security) - -3. **Go to App Passwords** [https://myaccount.google.com/apppasswords](https://myaccount.google.com/apppasswords) - *(Make sure you're logged in and have already enabled 2-Step Verification.)* - -4. **Generate App Password:** - - Enter the name **"Airavata"** and click **"Generate"**. - -5. **Copy the Generated App Password** - - A 16 character password will appear **copy and save it immediately**, as it will not be shown again. - -5. **Update Configuration:** - Edit `src/main/resources/airavata-server.properties`: - ```properties - [email protected] - email.based.monitor.password=your-app-password - ``` - -6. **Start Monitor:** - - Navigate to: `org.apache.airavata.ide.integration.JobMonitorStarter` - - Right-click and **Run** - -## 🌐 User Portal Setup (Django) - -### 9️⃣ Django Portal Installation - -**You can create and launch experiments and manage credentials using this portal.** - -```bash -# Navigate outside the Airavata directory -cd .. - -# Clone the Django portal repository -git clone https://github.com/apache/airavata-portals.git -cd airavata-portals/airavata-django-portal - -# Create a virtual environment -python3 -m venv venv - -# Activate the virtual environment -source venv/bin/activate # For Windows: venv\Scripts\activate - -# Install Python dependencies -pip install -r requirements.txt - -### 🔟 Configure Django Portal - -```bash -# Create local settings -cp django_airavata/settings_local.py.ide django_airavata/settings_local.py - -# Run database migrations -python3 manage.py migrate - -# Build JavaScript components -./build_js.sh - -# Load default CMS pages -python3 manage.py load_default_gateway - -# Start development server -python3 manage.py runserver -``` - -### 🌍 Access User Portal - -- **URL:** [http://localhost:8000/auth/login](http://localhost:8000/auth/login) -- **Username:** `default-admin` -- **Password:** `123456` - -## 🛠️ Admin Portal Setup (Optional) - -For registering compute resources and storage resources: - -### 1️⃣ Starting Super Admin Portal (PGA) - -**This portal is required when registering new compute or storage resources into the gateway.** - -```bash -cd .devcontainer/pga -docker-compose up -d -``` - -### 2️⃣ Configure Host Resolution - -**Get host machine IP:** - -**macOS:** -```bash -docker-compose exec pga getent hosts docker.for.mac.host.internal | awk '{ print $1 }' -``` - -**Windows:** -```bash -docker-compose exec pga getent hosts host.docker.internal -``` - -**Update container hosts:** -*Replace <host-machine-ip> with the actual IP* -```bash -docker-compose exec pga /bin/sh -c "echo '<host-machine-ip> airavata.host' >> /etc/hosts" -``` - -### 3️⃣ Access Admin Portal - -- **URL:** [http://airavata.host:8008](http://airavata.host:8008) -- **Username:** `default-admin` -- **Password:** `123456` - -## 🛑 Cleanup & Troubleshooting - -### Stop All Services - -```bash -# In each docker-compose directory, run: -docker-compose down -docker-compose rm -f - -# Remove unused containers and networks -docker system prune -``` - -### 🔐 Certificate Renewal (If Expired) - -Only needed when Keycloak certificates expire: - -```bash -cd modules/ide-integration/src/main/resources/keystores - -# Remove old keystore -rm airavata.p12 - -# Generate new keystore (airavata.p12) -keytool -genkey -keyalg RSA -alias selfsigned -keystore airavata.p12 \ - -storetype pkcs12 -storepass airavata -validity 360 -keysize 2048 \ - -dname "CN=airavata.host,OU=airavata.host,O=airavata.host,L=airavata.host,ST=airavata.host,C=airavata.host" - -# Generate self-signed key-pair (for TLS) -openssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout server.key -out server.crt \ - -subj "/CN=airavata.host/OU=airavata.host/O=airavata.host/L=airavata.host/ST=airavata.host/C=airavata.host" \ - -addext "subjectAltName=DNS:airavata.host" -``` - -## 📊 Service Status Overview - -| Service | Port | Status Check | Purpose | -|---------|------|-------------|---------| -| 🗄️ **MySQL** | 3306 | `docker ps` | Database | -| 🔐 **Keycloak** | 8443 | [airavata.host:8443](http://airavata.host:8443) | Authentication | -| 📨 **Kafka** | 9092 | Internal | Messaging | -| 🐰 **RabbitMQ** | 5672 | Internal | Message Queue | -| 🌐 **Django Portal** | 8000 | [localhost:8000](http://localhost:8000) | User Interface | -| 🛠️ **PGA Admin** | 8008 | [airavata.host:8008](http://airavata.host:8008) | Admin Portal | - -## 🆘 Common Issues - -**Port Conflicts:** -```bash -# Check what's using a port -lsof -i :8000 -netstat -tulpn | grep :8000 -``` - -**Docker Issues:** -```bash -# Reset Docker -docker system prune -a -docker-compose down --volumes -``` - -**Build Failures:** -```bash -# Clean Maven cache -mvn clean -rm -rf ~/.m2/repository/org/apache/airavata -``` - ---- - -<div align="center"> - <strong>🎉 Happy Developing with Apache Airavata!</strong> - <br> - <em>Need help? Check our <a href="https://airavata.apache.org/mailing-list.html";>mailing lists</a></em> -</div> \ No newline at end of file diff --git a/modules/ide-integration/pom.xml b/modules/ide-integration/pom.xml deleted file mode 100644 index 4da3035247..0000000000 --- a/modules/ide-integration/pom.xml +++ /dev/null @@ -1,61 +0,0 @@ -<!-- -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. ---> -<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> - <parent> - <artifactId>airavata</artifactId> - <groupId>org.apache.airavata</groupId> - <version>0.21-SNAPSHOT</version> - <relativePath>../../pom.xml</relativePath> - </parent> - <modelVersion>4.0.0</modelVersion> - - <artifactId>ide-integration</artifactId> - <name>Airavata IDE Integration</name> - <packaging>jar</packaging> - - <dependencies> - <dependency> - <groupId>org.apache.airavata</groupId> - <artifactId>airavata-api</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>org.apache.airavata</groupId> - <artifactId>airavata-thrift-server</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>org.mariadb.jdbc</groupId> - <artifactId>mariadb-java-client</artifactId> - </dependency> - </dependencies> - - <build> - <resources> - <resource> - <directory>../../keystores</directory> - <targetPath>keystores</targetPath> - <includes> - <include>*.jks</include> - </includes> - </resource> - </resources> - </build> - -</project> diff --git a/modules/ide-integration/src/main/java/org/apache/airavata/ide/integration/APIServerStarter.java b/modules/ide-integration/src/main/java/org/apache/airavata/ide/integration/APIServerStarter.java deleted file mode 100644 index b27f84141f..0000000000 --- a/modules/ide-integration/src/main/java/org/apache/airavata/ide/integration/APIServerStarter.java +++ /dev/null @@ -1,34 +0,0 @@ -/** -* -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, -* software distributed under the License is distributed on an -* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -* KIND, either express or implied. See the License for the -* specific language governing permissions and limitations -* under the License. -*/ -package org.apache.airavata.ide.integration; - -import org.apache.airavata.api.server.AiravataServer; - -/** - * IDE convenience launcher that starts the consolidated Airavata Thrift server. - * The AiravataServer hosts all services (API, Registry, Orchestrator, Profile, etc.) - * on a single multiplexed port and manages all background services internally. - */ -public class APIServerStarter { - - public static void main(String[] args) throws Exception { - new AiravataServer().start(); - } -} diff --git a/modules/ide-integration/src/main/java/org/apache/airavata/ide/integration/JobEngineStarter.java b/modules/ide-integration/src/main/java/org/apache/airavata/ide/integration/JobEngineStarter.java deleted file mode 100644 index c9945f18c0..0000000000 --- a/modules/ide-integration/src/main/java/org/apache/airavata/ide/integration/JobEngineStarter.java +++ /dev/null @@ -1,72 +0,0 @@ -/** -* -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, -* software distributed under the License is distributed on an -* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -* KIND, either express or implied. See the License for the -* specific language governing permissions and limitations -* under the License. -*/ -package org.apache.airavata.ide.integration; - -import java.util.ArrayList; -import org.apache.airavata.common.utils.ServerSettings; -import org.apache.airavata.helix.core.AbstractTask; -import org.apache.airavata.helix.impl.controller.HelixController; -import org.apache.airavata.helix.impl.participant.GlobalParticipant; -import org.apache.airavata.helix.impl.workflow.PostWorkflowManager; -import org.apache.airavata.helix.impl.workflow.PreWorkflowManager; -import org.apache.helix.manager.zk.ZKHelixAdmin; -import org.apache.helix.manager.zk.ZNRecordSerializer; -import org.apache.helix.manager.zk.ZkClient; - -/** - * IDE convenience launcher for the Airavata job execution engine components. - * IServer implementations run via Runnable.run() — each is started in its own thread. - */ -public class JobEngineStarter { - - public static void main(String args[]) throws Exception { - - ZkClient zkClient = new ZkClient( - ServerSettings.getZookeeperConnection(), - ZkClient.DEFAULT_SESSION_TIMEOUT, - ZkClient.DEFAULT_CONNECTION_TIMEOUT, - new ZNRecordSerializer()); - ZKHelixAdmin zkHelixAdmin = new ZKHelixAdmin(zkClient); - - zkHelixAdmin.addCluster(ServerSettings.getSetting("helix.cluster.name"), true); - - System.out.println("Starting Helix Controller ......."); - HelixController controller = new HelixController(); - new Thread(controller, "helix-controller").start(); - - ArrayList<Class<? extends AbstractTask>> taskClasses = new ArrayList<>(); - - for (String taskClassName : GlobalParticipant.TASK_CLASS_NAMES) { - taskClasses.add(Class.forName(taskClassName).asSubclass(AbstractTask.class)); - } - - System.out.println("Starting Helix Participant ......."); - GlobalParticipant participant = new GlobalParticipant(taskClasses, null); - new Thread(participant, "helix-participant").start(); - - System.out.println("Starting Pre Workflow Manager ......."); - PreWorkflowManager preWorkflowManager = new PreWorkflowManager(); - new Thread(preWorkflowManager, "pre-workflow-manager").start(); - - System.out.println("Starting Post Workflow Manager ......."); - PostWorkflowManager postWorkflowManager = new PostWorkflowManager(); - new Thread(postWorkflowManager, "post-workflow-manager").start(); - } -} diff --git a/modules/ide-integration/src/main/java/org/apache/airavata/ide/integration/JobMonitorStarter.java b/modules/ide-integration/src/main/java/org/apache/airavata/ide/integration/JobMonitorStarter.java deleted file mode 100644 index a863b18440..0000000000 --- a/modules/ide-integration/src/main/java/org/apache/airavata/ide/integration/JobMonitorStarter.java +++ /dev/null @@ -1,33 +0,0 @@ -/** -* -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, -* software distributed under the License is distributed on an -* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -* KIND, either express or implied. See the License for the -* specific language governing permissions and limitations -* under the License. -*/ -package org.apache.airavata.ide.integration; - -import org.apache.airavata.monitor.email.EmailBasedMonitor; - -/** - * IDE convenience launcher for the email-based job monitor. - * EmailBasedMonitor implements IServer (Runnable) — started in its own thread. - */ -public class JobMonitorStarter { - public static void main(String args[]) throws Exception { - EmailBasedMonitor emailBasedMonitor = new EmailBasedMonitor(); - new Thread(emailBasedMonitor, "email-monitor").start(); - } -} diff --git a/pom.xml b/pom.xml index 957f82c16a..461450bf24 100644 --- a/pom.xml +++ b/pom.xml @@ -70,11 +70,11 @@ under the License. <modules> <module>airavata-api</module> <module>airavata-thrift-server</module> + <module>integration-tests</module> <module>modules/file-server</module> <module>modules/agent-framework/agent-service</module> <module>modules/research-framework/research-service</module> <module>modules/restproxy</module> - <module>modules/ide-integration</module> </modules> <properties>
