This is an automated email from the ASF dual-hosted git repository.
zihaoxiang pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git
The following commit(s) were added to refs/heads/dev by this push:
new c0f70b88cc [DSIP-38] Support upload/download plugins from maven repo
(#16654)
c0f70b88cc is described below
commit c0f70b88cc4d22dc9ce1abc30c06af538984e0e2
Author: xiangzihao <[email protected]>
AuthorDate: Thu Sep 26 14:39:14 2024 +0800
[DSIP-38] Support upload/download plugins from maven repo (#16654)
---
.github/workflows/api-test.yml | 6 +-
.github/workflows/backend.yml | 2 +-
.github/workflows/e2e-k8s.yml | 2 +-
.github/workflows/e2e.yml | 6 +-
.github/workflows/publish-nexus.yaml | 75 +++++++++++++
.../publish-nexus/release-asf-settings.xml | 34 ++++++
.../publish-nexus/snapshot-asf-settings.xml | 29 +++++
.licenserc.yaml | 1 +
config/plugins_config | 110 ++++++++++++++++++
docs/docs/en/guide/installation/pseudo-cluster.md | 19 +++-
docs/docs/en/guide/installation/standalone.md | 4 +
docs/docs/en/guide/upgrade/incompatible.md | 8 +-
docs/docs/en/guide/upgrade/upgrade.md | 3 +-
docs/docs/zh/guide/installation/pseudo-cluster.md | 19 +++-
docs/docs/zh/guide/installation/standalone.md | 4 +
docs/docs/zh/guide/upgrade/incompatible.md | 8 +-
docs/docs/zh/guide/upgrade/upgrade.md | 3 +-
.../src/main/bin/start.sh | 4 +-
dolphinscheduler-api/src/main/bin/start.sh | 4 +-
dolphinscheduler-dist/pom.xml | 125 +++++++++++----------
.../src/main/assembly/assembly-plugins.sh | 18 ++-
.../src/main/assembly/dolphinscheduler-bin.xml | 15 +++
.../src/main/docker/alert-server.dockerfile | 6 +-
.../src/main/docker/api-server.dockerfile | 6 +-
.../src/main/docker/master-server.dockerfile | 6 +-
.../src/main/docker/standalone-server.dockerfile | 6 +-
.../src/main/docker/tools.dockerfile | 6 +-
.../src/main/docker/worker-server.dockerfile | 6 +-
dolphinscheduler-master/src/main/bin/start.sh | 4 +-
.../src/main/bin/start.sh | 4 +-
.../src/main/resources/application.yaml | 1 +
.../{banner.txt => standalone-banner.txt} | 0
dolphinscheduler-ui/pom.xml | 89 +++++++--------
dolphinscheduler-worker/src/main/bin/start.sh | 4 +-
pom.xml | 33 +++++-
script/install-plugins.sh | 66 +++++++++++
36 files changed, 590 insertions(+), 146 deletions(-)
diff --git a/.github/workflows/api-test.yml b/.github/workflows/api-test.yml
index e9923fd59c..6c6c2a85f6 100644
--- a/.github/workflows/api-test.yml
+++ b/.github/workflows/api-test.yml
@@ -83,7 +83,7 @@ jobs:
-Dmaven.test.skip=true \
-Dmaven.javadoc.skip=true \
-Dspotless.skip=true \
- -Pdocker,release -Ddocker.tag=ci
+ -Pdocker,staging -Ddocker.tag=ci
- name: Export Docker Images
run: |
docker save apache/dolphinscheduler-standalone-server:ci -o
/tmp/standalone-image.tar \
@@ -91,7 +91,7 @@ jobs:
- uses: actions/upload-artifact@v4
name: Upload Docker Images
with:
- name: standalone-image
+ name: standalone-image-api-test
path: /tmp/standalone-image.tar
retention-days: 1
api-test:
@@ -140,7 +140,7 @@ jobs:
- uses: actions/download-artifact@v4
name: Download Docker Images
with:
- name: standalone-image
+ name: standalone-image-api-test
path: /tmp
- name: Load Docker Images
run: |
diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml
index 268c313fe7..389dcf4f32 100644
--- a/.github/workflows/backend.yml
+++ b/.github/workflows/backend.yml
@@ -89,7 +89,7 @@ jobs:
- name: Build and Package on ${{ matrix.java }}
run: |
./mvnw -B clean install \
- -Prelease \
+ -Pstaging \
-Dmaven.test.skip=true \
-Dspotless.skip=true
- name: Check dependency license
diff --git a/.github/workflows/e2e-k8s.yml b/.github/workflows/e2e-k8s.yml
index e5ad17ff86..d78d785773 100644
--- a/.github/workflows/e2e-k8s.yml
+++ b/.github/workflows/e2e-k8s.yml
@@ -73,7 +73,7 @@ jobs:
-Dmaven.test.skip=true \
-Dmaven.javadoc.skip=true \
-Dspotless.skip=true \
- -Pdocker,release -Ddocker.tag=ci
+ -Pdocker,staging -Ddocker.tag=ci
- name: Create k8s Kind Cluster
run: |
# install kubectl
diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml
index 8dc83efdc9..c88899a4dc 100644
--- a/.github/workflows/e2e.yml
+++ b/.github/workflows/e2e.yml
@@ -85,7 +85,7 @@ jobs:
-Dmaven.test.skip=true \
-Dmaven.javadoc.skip=true \
-Dspotless.skip=true \
- -Pdocker,release -Ddocker.tag=ci
+ -Pdocker,staging -Ddocker.tag=ci
- name: Export Docker Images
run: |
docker save apache/dolphinscheduler-standalone-server:ci -o
/tmp/standalone-image.tar \
@@ -93,7 +93,7 @@ jobs:
- uses: actions/upload-artifact@v4
name: Upload Docker Images
with:
- name: standalone-image
+ name: standalone-image-e2e
path: /tmp/standalone-image.tar
retention-days: 1
e2e:
@@ -168,7 +168,7 @@ jobs:
- uses: actions/download-artifact@v4
name: Download Docker Images
with:
- name: standalone-image
+ name: standalone-image-e2e
path: /tmp
- name: Load Docker Images
run: |
diff --git a/.github/workflows/publish-nexus.yaml
b/.github/workflows/publish-nexus.yaml
new file mode 100644
index 0000000000..e430bb036f
--- /dev/null
+++ b/.github/workflows/publish-nexus.yaml
@@ -0,0 +1,75 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+name: Publish Snapshot
+
+on:
+ release:
+ types:
+ - released
+
+jobs:
+ publish-snapshot:
+ if: github.repository == 'apache/dolphinscheduler'
+ runs-on: ubuntu-latest
+ steps:
+ - name: Checkout repository
+ uses: actions/checkout@v4
+ with:
+ submodules: true
+ - name: Maximize runner space
+ uses: ./.github/actions/maximize-build-space
+ with:
+ root-reserve-mb: 30720
+ temp-reserve-mb: 10240
+ remove-dotnet: 'true'
+ remove-android: 'true'
+ remove-haskell: 'true'
+ remove-codeql: 'true'
+ remove-docker-images: 'true'
+ - uses: actions/checkout@v4
+ - name: Cache local Maven repository
+ uses: actions/cache@v4
+ with:
+ path: ~/.m2/repository
+ key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-backend
+ restore-keys: ${{ runner.os }}-maven-
+ - name: Setup JDK 8
+ uses: actions/setup-java@v4
+ with:
+ distribution: 'adopt'
+ java-version: 8
+ - name: Set environment variables
+ run: |
+ if [[ ${{ github.event_name }} == "schedule" ]]; then
+ echo
"SETTINGS_PATH=./.github/workflows/publish-nexus/snapshot-asf-settings.xml" >>
$SETTINGS_PATH
+ elif [[ ${{ github.event_name }} == "release" ]]; then
+ echo
"SETTINGS_PATH=./.github/workflows/publish-nexus/release-asf-settings.xml" >>
$SETTINGS_PATH
+ else
+ echo "unknown event name: ${{ github.event_name }}"
+ exit 2
+ fi
+ - name: Publish snapshot with dev branch
+ env:
+ ASF_USERNAME: ${{ secrets.NEXUS_USER }}
+ ASF_PASSWORD: ${{ secrets.NEXUS_PW }}
+ run: |
+ ./mvnw clean deploy \
+ -s ${{ env.SETTINGS_PATH }} \
+ -Dmaven.test.skip=true \
+ -Dspotless.skip=true \
+ -Pstaging
diff --git a/.github/workflows/publish-nexus/release-asf-settings.xml
b/.github/workflows/publish-nexus/release-asf-settings.xml
new file mode 100644
index 0000000000..b6388f7e69
--- /dev/null
+++ b/.github/workflows/publish-nexus/release-asf-settings.xml
@@ -0,0 +1,34 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<settings xmlns="http://maven.apache.org/SETTINGS/1.0.0";
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
+ xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0
https://maven.apache.org/xsd/settings-1.0.0.xsd";>
+ <servers>
+ <server>
+ <id>apache.snapshots.https</id>
+ <username>${env.ASF_USERNAME}</username>
+ <password>${env.ASF_PASSWORD}</password>
+ </server>
+ <server>
+ <id>apache.releases.https</id>
+ <username>${env.ASF_USERNAME}</username>
+ <password>${env.ASF_PASSWORD}</password>
+ </server>
+ </servers>
+</settings>
diff --git a/.github/workflows/publish-nexus/snapshot-asf-settings.xml
b/.github/workflows/publish-nexus/snapshot-asf-settings.xml
new file mode 100644
index 0000000000..6529e98f06
--- /dev/null
+++ b/.github/workflows/publish-nexus/snapshot-asf-settings.xml
@@ -0,0 +1,29 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<settings xmlns="http://maven.apache.org/SETTINGS/1.0.0";
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
+ xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0
https://maven.apache.org/xsd/settings-1.0.0.xsd";>
+ <servers>
+ <server>
+ <id>apache.snapshots.https</id>
+ <username>${env.ASF_USERNAME}</username>
+ <password>${env.ASF_PASSWORD}</password>
+ </server>
+ </servers>
+</settings>
diff --git a/.licenserc.yaml b/.licenserc.yaml
index 1f0b43e502..6697c2cb73 100644
--- a/.licenserc.yaml
+++ b/.licenserc.yaml
@@ -48,6 +48,7 @@ header:
- 'org.mockito.plugins.MockMaker'
- tools/dependencies/known-dependencies.txt
- '**/banner.txt'
+ - '**/*-banner.txt'
- '.terraform.lock.hcl'
- deploy/kubernetes/dolphinscheduler/README.md.gotmpl
- .idea/vcs.xml
diff --git a/config/plugins_config b/config/plugins_config
new file mode 100644
index 0000000000..6fac612b01
--- /dev/null
+++ b/config/plugins_config
@@ -0,0 +1,110 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# This mapping is used to resolve the Jar package name without version (or
call artifactId)
+#
+# corresponding to the module in the user Config, helping Dolphinscheduler to
load the correct Jar package.
+# Don't modify the delimiter " -- ", just select the plugin you need
+
+--alert-plugins--
+dolphinscheduler-alert-aliyunVoice
+dolphinscheduler-alert-dingtalk
+dolphinscheduler-alert-email
+dolphinscheduler-alert-feishu
+dolphinscheduler-alert-http
+dolphinscheduler-alert-pagerduty
+dolphinscheduler-alert-prometheus
+dolphinscheduler-alert-script
+dolphinscheduler-alert-slack
+dolphinscheduler-alert-telegram
+dolphinscheduler-alert-webexteams
+dolphinscheduler-alert-wechat
+--end--
+
+--datasource-plugins--
+dolphinscheduler-datasource-aliyunserverlessspark
+dolphinscheduler-datasource-athena
+dolphinscheduler-datasource-azure-sql
+dolphinscheduler-datasource-clickhouse
+dolphinscheduler-datasource-dameng
+dolphinscheduler-datasource-databend
+dolphinscheduler-datasource-db2
+dolphinscheduler-datasource-doris
+dolphinscheduler-datasource-hana
+dolphinscheduler-datasource-hive
+dolphinscheduler-datasource-k8s
+dolphinscheduler-datasource-kyuubi
+dolphinscheduler-datasource-mysql
+dolphinscheduler-datasource-oceanbase
+dolphinscheduler-datasource-oracle
+dolphinscheduler-datasource-postgresql
+dolphinscheduler-datasource-presto
+dolphinscheduler-datasource-redshift
+dolphinscheduler-datasource-sagemaker
+dolphinscheduler-datasource-snowflake
+dolphinscheduler-datasource-spark
+dolphinscheduler-datasource-sqlserver
+dolphinscheduler-datasource-ssh
+dolphinscheduler-datasource-starrocks
+dolphinscheduler-datasource-trino
+dolphinscheduler-datasource-vertica
+dolphinscheduler-datasource-zeppelin
+--end--
+
+--storage-plugins--
+dolphinscheduler-storage-abs
+dolphinscheduler-storage-gcs
+dolphinscheduler-storage-hdfs
+dolphinscheduler-storage-obs
+dolphinscheduler-storage-oss
+dolphinscheduler-storage-s3
+--end--
+
+--task-plugins--
+dolphinscheduler-task-aliyunserverlessspark
+dolphinscheduler-task-chunjun
+dolphinscheduler-task-datafactory
+dolphinscheduler-task-dataquality
+dolphinscheduler-task-datasync
+dolphinscheduler-task-datax
+dolphinscheduler-task-dinky
+dolphinscheduler-task-dms
+dolphinscheduler-task-dvc
+dolphinscheduler-task-emr
+dolphinscheduler-task-flink
+dolphinscheduler-task-flink-stream
+dolphinscheduler-task-hivecli
+dolphinscheduler-task-http
+dolphinscheduler-task-java
+dolphinscheduler-task-jupyter
+dolphinscheduler-task-k8s
+dolphinscheduler-task-kubeflow
+dolphinscheduler-task-linkis
+dolphinscheduler-task-mlflow
+dolphinscheduler-task-mr
+dolphinscheduler-task-openmldb
+dolphinscheduler-task-procedure
+dolphinscheduler-task-python
+dolphinscheduler-task-pytorch
+dolphinscheduler-task-remoteshell
+dolphinscheduler-task-sagemaker
+dolphinscheduler-task-seatunnel
+dolphinscheduler-task-shell
+dolphinscheduler-task-spark
+dolphinscheduler-task-sql
+dolphinscheduler-task-sqoop
+dolphinscheduler-task-zeppelin
+--end--
diff --git a/docs/docs/en/guide/installation/pseudo-cluster.md
b/docs/docs/en/guide/installation/pseudo-cluster.md
index 6561759a9f..851b0501a1 100644
--- a/docs/docs/en/guide/installation/pseudo-cluster.md
+++ b/docs/docs/en/guide/installation/pseudo-cluster.md
@@ -16,10 +16,27 @@ Pseudo-cluster deployment of DolphinScheduler requires
external software support
- `pstree` for macOS
- `psmisc` for Fedora/Red/Hat/CentOS/Ubuntu/Debian
-> **_Note:_** DolphinScheduler itself does not depend on Hadoop, Hive, Spark,
but if you need to run tasks that depend on them, you need to have the
corresponding environment support.
+## Download Plugins Dependencies
+
+Starting from version 3.3.0, the binary package no longer provides plugin
dependencies, and users need to download them by themselves. The plugin
dependency package download address: [Plugin Dependency
Package](https://repo.maven.apache.org/maven2/org/apache/dolphinscheduler)
+You can also execute the following command to install plugin dependencies:
+
+```shell
+bash ./bin/install-plugins.sh 3.3.0
+```
+
+Usually, you do not need all connector plugins, you can specify the plugins
you need by configuring `conf/plugins_config`. For example, if you only need
the `dolphinscheduler-task-shell` plugin, you can modify the configuration file
as follows:
+
+```
+--task-plugins--
+dolphinscheduler-task-shell
+--end--
+```
## DolphinScheduler Startup Environment
+> **_Note:_** DolphinScheduler itself does not depend on Hadoop, Hive, Spark,
but if you need to run tasks that depend on them, you need to have the
corresponding environment support.
+
### Configure User Exemption and Permissions
Create a deployment user, and make sure to configure `sudo` without password.
Here make an example to create user `dolphinscheduler`:
diff --git a/docs/docs/en/guide/installation/standalone.md
b/docs/docs/en/guide/installation/standalone.md
index bf47c3b9a5..646c1d48b0 100644
--- a/docs/docs/en/guide/installation/standalone.md
+++ b/docs/docs/en/guide/installation/standalone.md
@@ -14,6 +14,10 @@ If you want to deploy DolphinScheduler in production, we
recommend you follow [c
- JDK：download [JDK][jdk] (1.8 or 11), install and configure environment
variable `JAVA_HOME` and append `bin` dir (included in `JAVA_HOME`) to `PATH`
variable. You can skip this step if it already exists in your environment.
- Binary package: download the DolphinScheduler binary package at [download
page](https://dolphinscheduler.apache.org/en-us/download/<version>). <!--
markdown-link-check-disable-line -->
+## Download Plugin Dependencies
+
+Please refer to the [Download Plugin
Dependencies](../installation/pseudo-cluster.md) in pseudo-cluster deployment.
+
### Configure User Exemption and Permissions
Create a deployment user, and make sure to configure `sudo` without password.
Here make an example to create user `dolphinscheduler`:
diff --git a/docs/docs/en/guide/upgrade/incompatible.md
b/docs/docs/en/guide/upgrade/incompatible.md
index 62852e2463..c0549badd2 100644
--- a/docs/docs/en/guide/upgrade/incompatible.md
+++ b/docs/docs/en/guide/upgrade/incompatible.md
@@ -28,8 +28,8 @@ This document records the incompatible updates between each
version. You need to
## 3.3.0
-* Remove the `udf-manage` function from the `resource center` ([#16209])
-* Remove the `Pigeon` from the `Task Plugin` ([#16218])
-* Uniformly name `process` in code as `workflow` ([#16515])
-* Deprecated upgrade code of 1.x and 2.x in 3.3.0-release ([#16543])
+* Remove the `udf-manage` function from the `resource center`
([#16209])(https://github.com/apache/dolphinscheduler/pull/16209)
+* Remove the `Pigeon` from the `Task Plugin`
([#16218])(https://github.com/apache/dolphinscheduler/pull/16218)
+* Uniformly name `process` in code as `workflow`
([#16515])(https://github.com/apache/dolphinscheduler/pull/16515)
+* Deprecated upgrade code of 1.x and 2.x in 3.3.0-release
([#16543])(https://github.com/apache/dolphinscheduler/pull/16543)
diff --git a/docs/docs/en/guide/upgrade/upgrade.md
b/docs/docs/en/guide/upgrade/upgrade.md
index 074b1ca442..dfb3016a07 100644
--- a/docs/docs/en/guide/upgrade/upgrade.md
+++ b/docs/docs/en/guide/upgrade/upgrade.md
@@ -72,7 +72,8 @@ Execution result:
#### Upgrade version restriction
-After version 3.3.X and later, we only support upgrading from 3.0.0. For
versions lower than this, please download the historical version and upgrade to
3.0.0.
+- After version 3.3.X and later, we only support upgrading from 3.0.0. For
versions lower than this, please download the historical version and upgrade to
3.0.0.
+- After version 3.3.X and later, binary packages no longer provide plugins
dependencies by default, so when you use them for the first time, you need to
download and install them yourself. For more information, please refer to
[Pseudo-Cluster](../installation/pseudo-cluster.md).
#### Precautions after the upgrade
diff --git a/docs/docs/zh/guide/installation/pseudo-cluster.md
b/docs/docs/zh/guide/installation/pseudo-cluster.md
index 5c910ee759..0449951d4d 100644
--- a/docs/docs/zh/guide/installation/pseudo-cluster.md
+++ b/docs/docs/zh/guide/installation/pseudo-cluster.md
@@ -16,10 +16,27 @@
- macOS 安装`pstree`
- Fedora/Red/Hat/CentOS/Ubuntu/Debian 安装`psmisc`
-> **_注意:_** DolphinScheduler 本身不依赖
Hadoop、Hive、Spark，但如果你运行的任务需要依赖他们，就需要有对应的环境支持
+## 下载插件依赖
+
+从 3.3.0
版本开始，二进制包不再提供插件依赖，需要用户自行下载。插件依赖包下载地址：[插件依赖包](https://repo.maven.apache.org/maven2/org/apache/dolphinscheduler)
+你也可以执行以下命令来安装插件依赖:
+
+```shell
+bash ./bin/install-plugins.sh 3.3.0
+```
+
+通常你并不需要所有的连接器插件，可以通过配置 `conf/plugins_config` 来指定你所需要的插件，例如，你只需要
`dolphinscheduler-task-shell` 插件，那么您可以修改配置文件如下：
+
+```
+--task-plugins--
+dolphinscheduler-task-shell
+--end--
+```
## 准备 DolphinScheduler 启动环境
+> **_注意:_** DolphinScheduler 本身不依赖
Hadoop、Hive、Spark，但如果你运行的任务需要依赖他们，就需要有对应的环境支持
+
### 配置用户免密及权限
创建部署用户，并且一定要配置 `sudo` 免密。以创建 dolphinscheduler 用户为例
diff --git a/docs/docs/zh/guide/installation/standalone.md
b/docs/docs/zh/guide/installation/standalone.md
index d669db4727..bc280d9ed7 100644
--- a/docs/docs/zh/guide/installation/standalone.md
+++ b/docs/docs/zh/guide/installation/standalone.md
@@ -12,6 +12,10 @@ Standalone 仅适用于 DolphinScheduler 的快速体验.
- JDK：下载[JDK][jdk] (1.8 or 11)，安装并配置 `JAVA_HOME` 环境变量，并将其下的 `bin` 目录追加到 `PATH`
环境变量中。如果你的环境中已存在，可以跳过这步。
- 二进制包：在[下载页面](https://dolphinscheduler.apache.org/en-us/download/<version>)下载
DolphinScheduler 二进制包 <!-- markdown-link-check-disable-line -->
+## 下载插件依赖
+
+请参考伪集群部署的[下载插件依赖](../installation/pseudo-cluster.md#下载插件依赖)
+
## 配置用户免密及权限
创建部署用户，并且一定要配置 `sudo` 免密。以创建 dolphinscheduler 用户为例
diff --git a/docs/docs/zh/guide/upgrade/incompatible.md
b/docs/docs/zh/guide/upgrade/incompatible.md
index 9dd689f0ae..098992294d 100644
--- a/docs/docs/zh/guide/upgrade/incompatible.md
+++ b/docs/docs/zh/guide/upgrade/incompatible.md
@@ -26,8 +26,8 @@
## 3.3.0
-* 从 `资源中心` 中移除了 `udf-manage` 功能 ([#16209])
-* 从 `任务插件` 中移除了 `Pigeon` 类型 ([#16218])
-* 统一代码中的 `process` 为 `workflow` ([#16515])
-* 在 3.3.0-release 中废弃了从 1.x 至 2.x 的升级代码 ([#16543])
+* 从 `资源中心` 中移除了 `udf-manage` 功能
([#16209])(https://github.com/apache/dolphinscheduler/pull/16209)
+* 从 `任务插件` 中移除了 `Pigeon` 类型
([#16218])(https://github.com/apache/dolphinscheduler/pull/16218)
+* 统一代码中的 `process` 为 `workflow`
([#16515])(https://github.com/apache/dolphinscheduler/pull/16515)
+* 在 3.3.0-release 中废弃了从 1.x 至 2.x 的升级代码
([#16543])(https://github.com/apache/dolphinscheduler/pull/16543)
diff --git a/docs/docs/zh/guide/upgrade/upgrade.md
b/docs/docs/zh/guide/upgrade/upgrade.md
index c2cec6783a..19ccad73ca 100644
--- a/docs/docs/zh/guide/upgrade/upgrade.md
+++ b/docs/docs/zh/guide/upgrade/upgrade.md
@@ -73,7 +73,8 @@ jar 包 并添加到 `./tools/libs` 目录下，设置以下环境变量
#### 升级版本限制
-在 3.3.X 以及之后的版本，我们仅支持从 3.0.0 开始进行升级，低于此版本的请下载历史版本升级至 3.0.0。
+- 在 3.3.X 以及之后的版本，我们仅支持从 3.0.0 开始进行升级，低于此版本的请下载历史版本升级至 3.0.0。
+- 在 3.3.X
以及之后的版本，二进制包不再默认提供插件依赖，因此第一次使用时，需要自行下载安装。具体请参考请参照[伪集群部署(Pseudo-Cluster)](../installation/pseudo-cluster.md)
#### 升级后的注意事项
diff --git
a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/bin/start.sh
b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/bin/start.sh
index 56a493d9b7..5d34faee25 100644
--- a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/bin/start.sh
+++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/bin/start.sh
@@ -58,7 +58,9 @@ alert-plugins
)
for plugin in ${PLUGINS_PATH[@]}; do
- CP=$CP:"$DOLPHINSCHEDULER_HOME/plugins/$plugin/*"
+ if [ -d "$DOLPHINSCHEDULER_HOME/plugins/$plugin" ]; then
+ CP=$CP:"$DOLPHINSCHEDULER_HOME/plugins/$plugin/*"
+ fi
done
diff --git a/dolphinscheduler-api/src/main/bin/start.sh
b/dolphinscheduler-api/src/main/bin/start.sh
index eb60455c53..e5d2a5b620 100644
--- a/dolphinscheduler-api/src/main/bin/start.sh
+++ b/dolphinscheduler-api/src/main/bin/start.sh
@@ -60,7 +60,9 @@ task-plugins
)
for plugin in ${PLUGINS_PATH[@]}; do
- CP=$CP:"$DOLPHINSCHEDULER_HOME/plugins/$plugin/*"
+ if [ -d "$DOLPHINSCHEDULER_HOME/plugins/$plugin" ]; then
+ CP=$CP:"$DOLPHINSCHEDULER_HOME/plugins/$plugin/*"
+ fi
done
$JAVA_HOME/bin/java $JAVA_OPTS \
diff --git a/dolphinscheduler-dist/pom.xml b/dolphinscheduler-dist/pom.xml
index 046b533518..c9b5814456 100644
--- a/dolphinscheduler-dist/pom.xml
+++ b/dolphinscheduler-dist/pom.xml
@@ -27,6 +27,10 @@
<artifactId>dolphinscheduler-dist</artifactId>
<name>${project.artifactId}</name>
+ <properties>
+ <assembly.skipAssembly>${build.assembly.skip}</assembly.skipAssembly>
+ </properties>
+
<dependencyManagement>
<dependencies>
<dependency>
@@ -73,71 +77,70 @@
<build>
<finalName>apache-dolphinscheduler-${project.version}</finalName>
- </build>
+ <plugins>
+ <plugin>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>dolphinscheduler-bin</id>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ <phase>package</phase>
- <profiles>
- <profile>
- <id>release</id>
- <build>
- <plugins>
- <plugin>
- <artifactId>maven-assembly-plugin</artifactId>
- <executions>
- <execution>
- <id>dolphinscheduler-bin</id>
- <goals>
- <goal>single</goal>
- </goals>
- <phase>package</phase>
+ <configuration>
+ <descriptors>
+
<descriptor>src/main/assembly/dolphinscheduler-bin.xml</descriptor>
+ </descriptors>
+ <appendAssemblyId>true</appendAssemblyId>
+ </configuration>
+ </execution>
- <configuration>
- <descriptors>
-
<descriptor>src/main/assembly/dolphinscheduler-bin.xml</descriptor>
- </descriptors>
- <appendAssemblyId>true</appendAssemblyId>
- </configuration>
- </execution>
+ <execution>
+ <id>src</id>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ <phase>package</phase>
+ <configuration>
+ <descriptors>
+
<descriptor>src/main/assembly/dolphinscheduler-src.xml</descriptor>
+ </descriptors>
+ <appendAssemblyId>true</appendAssemblyId>
+ </configuration>
+ </execution>
- <execution>
- <id>src</id>
- <goals>
- <goal>single</goal>
- </goals>
- <phase>package</phase>
- <configuration>
- <descriptors>
-
<descriptor>src/main/assembly/dolphinscheduler-src.xml</descriptor>
- </descriptors>
- <appendAssemblyId>true</appendAssemblyId>
- </configuration>
- </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>exec-maven-plugin</artifactId>
+ <version>${exec-maven-plugin.version}</version>
+ <configuration>
+ <skip>${build.assembly.skip}</skip>
+ </configuration>
+ <executions>
+ <execution>
+ <id>assembly-plugins</id>
+ <goals>
+ <goal>exec</goal>
+ </goals>
+ <phase>package</phase>
+ <configuration>
+ <executable>bash</executable>
+
<workingDirectory>${project.basedir}</workingDirectory>
+ <arguments>
+
<argument>src/main/assembly/assembly-plugins.sh</argument>
+ <argument>${build.plugins.skip}</argument>
+ </arguments>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>exec-maven-plugin</artifactId>
- <version>${exec-maven-plugin.version}</version>
- <executions>
- <execution>
- <id>assembly-plugins</id>
- <goals>
- <goal>exec</goal>
- </goals>
- <phase>package</phase>
- <configuration>
- <executable>bash</executable>
-
<workingDirectory>${project.basedir}</workingDirectory>
- <arguments>
-
<argument>src/main/assembly/assembly-plugins.sh</argument>
- </arguments>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
- </profile>
+ <profiles>
<profile>
<id>docker</id>
<build>
diff --git a/dolphinscheduler-dist/src/main/assembly/assembly-plugins.sh
b/dolphinscheduler-dist/src/main/assembly/assembly-plugins.sh
index 36cb430df2..80f7dd6860 100755
--- a/dolphinscheduler-dist/src/main/assembly/assembly-plugins.sh
+++ b/dolphinscheduler-dist/src/main/assembly/assembly-plugins.sh
@@ -17,6 +17,8 @@
#
set -xeo pipefail
+PLUGINS_ASSEMBLY_SKIP=$1
+
DIST_DIR="$(pwd)/target"
BIN_TAR_FILE="$DIST_DIR/apache-dolphinscheduler-*-bin.tar.gz"
if [ ! -f $BIN_TAR_FILE ]; then
@@ -36,12 +38,16 @@ storage-plugins
task-plugins
)
-for plugin_path in ${PLUGINS_PATH[@]}
-do
- cd $BIN_DIR/plugins/$plugin_path
- find ./* -name "*.jar" | xargs -I {} mv {} ./
- ls -d */ | xargs -I {} rm -rf {}
-done
+if [ $PLUGINS_ASSEMBLY_SKIP == "true" ]; then
+ rm -rf $BIN_DIR/plugins/*
+else
+ for plugin_path in ${PLUGINS_PATH[@]}
+ do
+ cd $BIN_DIR/plugins/$plugin_path
+ find ./* -name "*.jar" | xargs -I {} mv {} ./
+ ls -d */ | xargs -I {} rm -rf {}
+ done
+fi
# move *-server/libs/*.jar to libs/ and create symbolic link in *-server/libs/
MODULES_PATH=(
diff --git a/dolphinscheduler-dist/src/main/assembly/dolphinscheduler-bin.xml
b/dolphinscheduler-dist/src/main/assembly/dolphinscheduler-bin.xml
index 72018cfb9e..1dba381630 100644
--- a/dolphinscheduler-dist/src/main/assembly/dolphinscheduler-bin.xml
+++ b/dolphinscheduler-dist/src/main/assembly/dolphinscheduler-bin.xml
@@ -107,6 +107,21 @@
<directoryMode>0755</directoryMode>
</fileSet>
+ <fileSet>
+ <directory>${basedir}/../config</directory>
+ <outputDirectory>conf</outputDirectory>
+ </fileSet>
+
+ <fileSet>
+ <directory>${basedir}/.././</directory>
+ <includes>
+ <include>mvnw</include>
+ <include>mvnw.cmd</include>
+ </includes>
+ <fileMode>0755</fileMode>
+ <outputDirectory>.</outputDirectory>
+ </fileSet>
+
<fileSet>
<directory>${basedir}/release-docs</directory>
<outputDirectory>.</outputDirectory>
diff --git a/dolphinscheduler-dist/src/main/docker/alert-server.dockerfile
b/dolphinscheduler-dist/src/main/docker/alert-server.dockerfile
index cc8d0646c4..26e39c5447 100644
--- a/dolphinscheduler-dist/src/main/docker/alert-server.dockerfile
+++ b/dolphinscheduler-dist/src/main/docker/alert-server.dockerfile
@@ -27,7 +27,11 @@ RUN apt update ; \
WORKDIR $DOLPHINSCHEDULER_HOME
-ADD ./target/apache-dolphinscheduler-*-bin $DOLPHINSCHEDULER_HOME
+COPY ./target/apache-dolphinscheduler-*-bin.tar.gz $DOLPHINSCHEDULER_HOME
+RUN tar -zxvf apache-dolphinscheduler-*-bin.tar.gz ; \
+ rm -rf apache-dolphinscheduler-*-bin.tar.gz ; \
+ mv apache-dolphinscheduler-*-bin/* . ; \
+ rm -rf apache-dolphinscheduler-*-bin
EXPOSE 12345 25333
diff --git a/dolphinscheduler-dist/src/main/docker/api-server.dockerfile
b/dolphinscheduler-dist/src/main/docker/api-server.dockerfile
index a0201810d2..941a46e79a 100644
--- a/dolphinscheduler-dist/src/main/docker/api-server.dockerfile
+++ b/dolphinscheduler-dist/src/main/docker/api-server.dockerfile
@@ -27,7 +27,11 @@ RUN apt update ; \
WORKDIR $DOLPHINSCHEDULER_HOME
-ADD ./target/apache-dolphinscheduler-*-bin $DOLPHINSCHEDULER_HOME
+COPY ./target/apache-dolphinscheduler-*-bin.tar.gz $DOLPHINSCHEDULER_HOME
+RUN tar -zxvf apache-dolphinscheduler-*-bin.tar.gz ; \
+ rm -rf apache-dolphinscheduler-*-bin.tar.gz ; \
+ mv apache-dolphinscheduler-*-bin/* . ; \
+ rm -rf apache-dolphinscheduler-*-bin
EXPOSE 12345 25333
diff --git a/dolphinscheduler-dist/src/main/docker/master-server.dockerfile
b/dolphinscheduler-dist/src/main/docker/master-server.dockerfile
index fbcc8bf621..a89b6348b0 100644
--- a/dolphinscheduler-dist/src/main/docker/master-server.dockerfile
+++ b/dolphinscheduler-dist/src/main/docker/master-server.dockerfile
@@ -27,7 +27,11 @@ RUN apt update ; \
WORKDIR $DOLPHINSCHEDULER_HOME
-ADD ./target/apache-dolphinscheduler-*-bin $DOLPHINSCHEDULER_HOME
+COPY ./target/apache-dolphinscheduler-*-bin.tar.gz $DOLPHINSCHEDULER_HOME
+RUN tar -zxvf apache-dolphinscheduler-*-bin.tar.gz ; \
+ rm -rf apache-dolphinscheduler-*-bin.tar.gz ; \
+ mv apache-dolphinscheduler-*-bin/* . ; \
+ rm -rf apache-dolphinscheduler-*-bin
EXPOSE 12345 25333
diff --git a/dolphinscheduler-dist/src/main/docker/standalone-server.dockerfile
b/dolphinscheduler-dist/src/main/docker/standalone-server.dockerfile
index b0cad0632a..0abd547ea2 100644
--- a/dolphinscheduler-dist/src/main/docker/standalone-server.dockerfile
+++ b/dolphinscheduler-dist/src/main/docker/standalone-server.dockerfile
@@ -27,7 +27,11 @@ RUN apt update ; \
WORKDIR $DOLPHINSCHEDULER_HOME
-ADD ./target/apache-dolphinscheduler-*-bin $DOLPHINSCHEDULER_HOME
+COPY ./target/apache-dolphinscheduler-*-bin.tar.gz $DOLPHINSCHEDULER_HOME
+RUN tar -zxvf apache-dolphinscheduler-*-bin.tar.gz ; \
+ rm -rf apache-dolphinscheduler-*-bin.tar.gz ; \
+ mv apache-dolphinscheduler-*-bin/* . ; \
+ rm -rf apache-dolphinscheduler-*-bin
EXPOSE 12345 25333
diff --git a/dolphinscheduler-dist/src/main/docker/tools.dockerfile
b/dolphinscheduler-dist/src/main/docker/tools.dockerfile
index c795a2b507..c50d91bc92 100644
--- a/dolphinscheduler-dist/src/main/docker/tools.dockerfile
+++ b/dolphinscheduler-dist/src/main/docker/tools.dockerfile
@@ -28,6 +28,10 @@ RUN apt update ; \
WORKDIR $DOLPHINSCHEDULER_HOME
# see doc:
https://dolphinscheduler.apache.org/en-us/docs/dev/user_doc/guide/upgrade.html
-ADD ./target/apache-dolphinscheduler-*-bin $DOLPHINSCHEDULER_HOME
+COPY ./target/apache-dolphinscheduler-*-bin.tar.gz $DOLPHINSCHEDULER_HOME
+RUN tar -zxvf apache-dolphinscheduler-*-bin.tar.gz ; \
+ rm -rf apache-dolphinscheduler-*-bin.tar.gz ; \
+ mv apache-dolphinscheduler-*-bin/* . ; \
+ rm -rf apache-dolphinscheduler-*-bin
ENTRYPOINT [ "/bin/bash" ]
diff --git a/dolphinscheduler-dist/src/main/docker/worker-server.dockerfile
b/dolphinscheduler-dist/src/main/docker/worker-server.dockerfile
index c8ce0b61c9..4f4dfdb7cc 100644
--- a/dolphinscheduler-dist/src/main/docker/worker-server.dockerfile
+++ b/dolphinscheduler-dist/src/main/docker/worker-server.dockerfile
@@ -27,7 +27,11 @@ RUN apt update ; \
WORKDIR $DOLPHINSCHEDULER_HOME
-ADD ./target/apache-dolphinscheduler-*-bin $DOLPHINSCHEDULER_HOME
+COPY ./target/apache-dolphinscheduler-*-bin.tar.gz $DOLPHINSCHEDULER_HOME
+RUN tar -zxvf apache-dolphinscheduler-*-bin.tar.gz ; \
+ rm -rf apache-dolphinscheduler-*-bin.tar.gz ; \
+ mv apache-dolphinscheduler-*-bin/* . ; \
+ rm -rf apache-dolphinscheduler-*-bin
EXPOSE 12345 25333
diff --git a/dolphinscheduler-master/src/main/bin/start.sh
b/dolphinscheduler-master/src/main/bin/start.sh
index 74b7ba1112..de933f464a 100644
--- a/dolphinscheduler-master/src/main/bin/start.sh
+++ b/dolphinscheduler-master/src/main/bin/start.sh
@@ -60,7 +60,9 @@ task-plugins
)
for plugin in ${PLUGINS_PATH[@]}; do
- CP=$CP:"$DOLPHINSCHEDULER_HOME/plugins/$plugin/*"
+ if [ -d "$DOLPHINSCHEDULER_HOME/plugins/$plugin" ]; then
+ CP=$CP:"$DOLPHINSCHEDULER_HOME/plugins/$plugin/*"
+ fi
done
$JAVA_HOME/bin/java $JAVA_OPTS \
diff --git a/dolphinscheduler-standalone-server/src/main/bin/start.sh
b/dolphinscheduler-standalone-server/src/main/bin/start.sh
index 39596fc69c..47b73afdf0 100755
--- a/dolphinscheduler-standalone-server/src/main/bin/start.sh
+++ b/dolphinscheduler-standalone-server/src/main/bin/start.sh
@@ -65,7 +65,9 @@ task-plugins
)
for plugin in ${PLUGINS_PATH[@]}; do
- CP=$CP:"$DOLPHINSCHEDULER_HOME/plugins/$plugin/*"
+ if [ -d "$DOLPHINSCHEDULER_HOME/plugins/$plugin" ]; then
+ CP=$CP:"$DOLPHINSCHEDULER_HOME/plugins/$plugin/*"
+ fi
done
for jar in $(find $STANDALONE_HOME/libs/* -name "*.jar"); do
diff --git
a/dolphinscheduler-standalone-server/src/main/resources/application.yaml
b/dolphinscheduler-standalone-server/src/main/resources/application.yaml
index 3304d8ad5f..97b9b6d22d 100644
--- a/dolphinscheduler-standalone-server/src/main/resources/application.yaml
+++ b/dolphinscheduler-standalone-server/src/main/resources/application.yaml
@@ -23,6 +23,7 @@ spring:
date-format: "yyyy-MM-dd HH:mm:ss"
banner:
charset: UTF-8
+ location: classpath:standalone-banner.txt
sql:
init:
schema-locations: classpath:sql/dolphinscheduler_h2.sql
diff --git a/dolphinscheduler-standalone-server/src/main/resources/banner.txt
b/dolphinscheduler-standalone-server/src/main/resources/standalone-banner.txt
similarity index 100%
rename from dolphinscheduler-standalone-server/src/main/resources/banner.txt
rename to
dolphinscheduler-standalone-server/src/main/resources/standalone-banner.txt
diff --git a/dolphinscheduler-ui/pom.xml b/dolphinscheduler-ui/pom.xml
index 2e61184ef5..117af4ed9f 100644
--- a/dolphinscheduler-ui/pom.xml
+++ b/dolphinscheduler-ui/pom.xml
@@ -35,54 +35,49 @@
<frontend-maven-plugin.version>1.12.1</frontend-maven-plugin.version>
</properties>
- <profiles>
- <profile>
- <id>release</id>
- <build>
- <plugins>
- <plugin>
- <groupId>com.github.eirslett</groupId>
- <artifactId>frontend-maven-plugin</artifactId>
- <version>${frontend-maven-plugin.version}</version>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>com.github.eirslett</groupId>
+ <artifactId>frontend-maven-plugin</artifactId>
+ <version>${frontend-maven-plugin.version}</version>
+ <configuration>
+ <skip>${build.ui.skip}</skip>
+
<pnpmInheritsProxyConfigFromMaven>false</pnpmInheritsProxyConfigFromMaven>
+ </configuration>
+ <executions>
+ <execution>
+ <id>install node and pnpm</id>
+ <goals>
+ <goal>install-node-and-pnpm</goal>
+ </goals>
<configuration>
- <skip>${build.ui.skip}</skip>
-
<pnpmInheritsProxyConfigFromMaven>false</pnpmInheritsProxyConfigFromMaven>
+ <nodeVersion>${node.version}</nodeVersion>
+ <pnpmVersion>${pnpm.version}</pnpmVersion>
</configuration>
- <executions>
- <execution>
- <id>install node and pnpm</id>
- <goals>
- <goal>install-node-and-pnpm</goal>
- </goals>
- <configuration>
- <nodeVersion>${node.version}</nodeVersion>
- <pnpmVersion>${pnpm.version}</pnpmVersion>
- </configuration>
- </execution>
- <execution>
- <id>pnpm install</id>
- <goals>
- <goal>pnpm</goal>
- </goals>
- <phase>generate-resources</phase>
- <configuration>
- <arguments>install</arguments>
- </configuration>
- </execution>
- <execution>
- <id>pnpm run build:prod</id>
- <goals>
- <goal>pnpm</goal>
- </goals>
- <configuration>
- <arguments>run build:prod</arguments>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
+ </execution>
+ <execution>
+ <id>pnpm install</id>
+ <goals>
+ <goal>pnpm</goal>
+ </goals>
+ <phase>generate-resources</phase>
+ <configuration>
+ <arguments>install</arguments>
+ </configuration>
+ </execution>
+ <execution>
+ <id>pnpm run build:prod</id>
+ <goals>
+ <goal>pnpm</goal>
+ </goals>
+ <configuration>
+ <arguments>run build:prod</arguments>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
- </build>
- </profile>
- </profiles>
+ </build>
</project>
diff --git a/dolphinscheduler-worker/src/main/bin/start.sh
b/dolphinscheduler-worker/src/main/bin/start.sh
index 0a1b9dd832..4930338c5a 100644
--- a/dolphinscheduler-worker/src/main/bin/start.sh
+++ b/dolphinscheduler-worker/src/main/bin/start.sh
@@ -60,7 +60,9 @@ task-plugins
)
for plugin in ${PLUGINS_PATH[@]}; do
- CP=$CP:"$DOLPHINSCHEDULER_HOME/plugins/$plugin/*"
+ if [ -d "$DOLPHINSCHEDULER_HOME/plugins/$plugin" ]; then
+ CP=$CP:"$DOLPHINSCHEDULER_HOME/plugins/$plugin/*"
+ fi
done
$JAVA_HOME/bin/java $JAVA_OPTS \
diff --git a/pom.xml b/pom.xml
index f4e553e17b..2dd7aba16b 100755
--- a/pom.xml
+++ b/pom.xml
@@ -95,7 +95,9 @@
<docker.build.skip>true</docker.build.skip>
<docker.push.skip>true</docker.push.skip>
<skipDepCheck>true</skipDepCheck>
- <build.ui.skip>false</build.ui.skip>
+ <build.ui.skip>true</build.ui.skip>
+ <build.plugins.skip>false</build.plugins.skip>
+ <build.assembly.skip>true</build.assembly.skip>
<spotless.skip>false</spotless.skip>
<skipUT>false</skipUT>
@@ -777,7 +779,36 @@
<tag>HEAD</tag>
</scm>
+ <distributionManagement>
+ <repository>
+ <id>apache.releases.https</id>
+ <name>Apache Release Distribution Repository</name>
+
<url>https://repository.apache.org/service/local/staging/deploy/maven2</url>
+ </repository>
+ <snapshotRepository>
+ <id>apache.snapshots.https</id>
+ <name>Apache Development Snapshot Repository</name>
+
<url>https://repository.apache.org/content/repositories/snapshots</url>
+ </snapshotRepository>
+ </distributionManagement>
+
<profiles>
+ <profile>
+ <id>release</id>
+ <properties>
+ <build.ui.skip>false</build.ui.skip>
+ <build.plugins.skip>true</build.plugins.skip>
+ <build.assembly.skip>false</build.assembly.skip>
+ </properties>
+ </profile>
+ <profile>
+ <id>staging</id>
+ <properties>
+ <build.ui.skip>false</build.ui.skip>
+ <build.plugins.skip>false</build.plugins.skip>
+ <build.assembly.skip>false</build.assembly.skip>
+ </properties>
+ </profile>
<profile>
<id>docker</id>
<properties>
diff --git a/script/install-plugins.sh b/script/install-plugins.sh
new file mode 100755
index 0000000000..67bb05d16d
--- /dev/null
+++ b/script/install-plugins.sh
@@ -0,0 +1,66 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+set -eo pipefail
+
+# This script is used to download the plugins required during the running
process.
+# All are downloaded by default. You can also choose what you need.
+# You only need to configure the plug-in name in config/plugin_config.
+
+# get ds home
+DOLPHINSCHEDULER_HOME=$(cd $(dirname $0);cd ../;pwd)
+
+# plugins default version is 3.3.0, you can also choose a custom version. eg:
3.3.0: bash install-plugins.sh 3.3.0
+version=3.3.0
+
+if [ -n "$1" ]; then
+ if [ "$1" == "dev" ]; then
+ version="dev-SNAPSHOT"
+ else
+ version="$1"
+ fi
+fi
+
+echo "Install Dolphinscheduler plugins, usage version is ${version}"
+
+# create the plugins directory
+if [ ! -d ${DOLPHINSCHEDULER_HOME}/plugins ]; then
+ mkdir -p ${DOLPHINSCHEDULER_HOME}/plugins
+fi
+
+plugin_dir=""
+while read line; do
+ if [ -z "$line" ]; then
+ continue
+ fi
+
+ start_char=$(echo "$line" | cut -c 1)
+
+ if [ "$start_char" == "-" ]; then
+ plugin_dir=$(echo ${line//--/})
+ if [ "$plugin_dir" != "end" ]; then
+ mkdir -p ${DOLPHINSCHEDULER_HOME}/plugins/${plugin_dir}
+ fi
+ fi
+
+ if [ "$start_char" != "-" ] && [ "$start_char" != "#" ]; then
+ echo "installing plugin: " $line
+ ${DOLPHINSCHEDULER_HOME}/mvnw dependency:get
-DgroupId=org.apache.dolphinscheduler -DartifactId=${line} -Dversion=${version}
-Ddest=${DOLPHINSCHEDULER_HOME}/plugins/${plugin_dir}
+ fi
+
+done < ${DOLPHINSCHEDULER_HOME}/conf/plugins_config
+
