This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new 1f8faaa4525 branch-3.0: [opt](hive docker)Exit on creating table
failed #47390 (#47452)
1f8faaa4525 is described below
commit 1f8faaa4525a8ffdd14e5f54294908c423265a8c
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Sun Jan 26 16:02:52 2025 +0800
branch-3.0: [opt](hive docker)Exit on creating table failed #47390 (#47452)
Cherry-picked from #47390
Co-authored-by: Thearas <gaozif...@selectdb.com>
---
.../docker-compose/hive/hive-2x.yaml.tpl | 3 +++
.../docker-compose/hive/hive-3x.yaml.tpl | 3 +++
.../partition_location_1/create_table.hql | 3 ++-
.../partition_location_2/create_table.hql | 1 +
.../data/regression/crdmm_data/create_table.hql | 2 +-
.../docker-compose/hive/scripts/hive-metastore.sh | 28 +++++++++++-----------
6 files changed, 24 insertions(+), 16 deletions(-)
diff --git a/docker/thirdparties/docker-compose/hive/hive-2x.yaml.tpl
b/docker/thirdparties/docker-compose/hive/hive-2x.yaml.tpl
index 10285d7e405..88a9597032b 100644
--- a/docker/thirdparties/docker-compose/hive/hive-2x.yaml.tpl
+++ b/docker/thirdparties/docker-compose/hive/hive-2x.yaml.tpl
@@ -21,6 +21,7 @@ version: "3.8"
services:
namenode:
image: bde2020/hadoop-namenode:2.0.0-hadoop2.7.4-java8
+ restart: always
environment:
- CLUSTER_NAME=test
env_file:
@@ -38,6 +39,7 @@ services:
datanode:
image: bde2020/hadoop-datanode:2.0.0-hadoop2.7.4-java8
+ restart: always
env_file:
- ./hadoop-hive-2x.env
environment:
@@ -82,6 +84,7 @@ services:
command: /bin/bash /mnt/scripts/hive-metastore.sh
environment:
SERVICE_PRECONDITION: "${IP_HOST}:50070 ${IP_HOST}:50075
${IP_HOST}:${PG_PORT}"
+ HMS_PORT: "${HMS_PORT}"
container_name: ${CONTAINER_UID}hive2-metastore
expose:
- "${HMS_PORT}"
diff --git a/docker/thirdparties/docker-compose/hive/hive-3x.yaml.tpl
b/docker/thirdparties/docker-compose/hive/hive-3x.yaml.tpl
index 7fdac05f71c..84d4a653554 100644
--- a/docker/thirdparties/docker-compose/hive/hive-3x.yaml.tpl
+++ b/docker/thirdparties/docker-compose/hive/hive-3x.yaml.tpl
@@ -21,6 +21,7 @@ version: "3.8"
services:
namenode:
image: bde2020/hadoop-namenode:2.0.0-hadoop3.2.1-java8
+ restart: always
environment:
- CLUSTER_NAME=test
env_file:
@@ -38,6 +39,7 @@ services:
datanode:
image: bde2020/hadoop-datanode:2.0.0-hadoop3.2.1-java8
+ restart: always
env_file:
- ./hadoop-hive-3x.env
environment:
@@ -82,6 +84,7 @@ services:
command: /bin/bash /mnt/scripts/hive-metastore.sh
environment:
SERVICE_PRECONDITION: "${IP_HOST}:9870 ${IP_HOST}:9864
${IP_HOST}:${PG_PORT}"
+ HMS_PORT: "${HMS_PORT}"
container_name: ${CONTAINER_UID}hive3-metastore
expose:
- "${HMS_PORT}"
diff --git
a/docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/partition_location_1/create_table.hql
b/docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/partition_location_1/create_table.hql
index a477daf78ea..79ba42ac23f 100644
---
a/docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/partition_location_1/create_table.hql
+++
b/docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/partition_location_1/create_table.hql
@@ -17,6 +17,7 @@ TBLPROPERTIES (
'transient_lastDdlTime'='1682405696');
ALTER TABLE partition_location_1 ADD PARTITION (part='part1') LOCATION
'/user/doris/suites/multi_catalog/partition_location_1/part=part1';
-ALTER TABLE partition_location_1 ADD PARTITION (part='part2') LOCATION
'/user/doris/suites/multi_catalog/partition_location_1/20230425';
+ALTER TABLE partition_location_1 ADD PARTITION (part='part2') LOCATION
'/user/doris/suites/multi_catalog/partition_location_1/part=part2';
+set hive.msck.path.validation=ignore;
msck repair table partition_location_1;
diff --git
a/docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/partition_location_2/create_table.hql
b/docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/partition_location_2/create_table.hql
index bc40eb85f1c..9c3c025954a 100644
---
a/docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/partition_location_2/create_table.hql
+++
b/docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/partition_location_2/create_table.hql
@@ -20,4 +20,5 @@ TBLPROPERTIES (
ALTER TABLE partition_location_2 ADD PARTITION (part1='part1_1',
part2='part2_1') LOCATION
'/user/doris/suites/multi_catalog/partition_location_2/part1=part1_1/part2=part2_1';
ALTER TABLE partition_location_2 ADD PARTITION (part1='part1_2',
part2='part2_2') LOCATION
'/user/doris/suites/multi_catalog/partition_location_2/20230425';
+set hive.msck.path.validation=ignore;
msck repair table partition_location_2;
diff --git
a/docker/thirdparties/docker-compose/hive/scripts/data/regression/crdmm_data/create_table.hql
b/docker/thirdparties/docker-compose/hive/scripts/data/regression/crdmm_data/create_table.hql
index 0abba278308..e38d7c2cad1 100644
---
a/docker/thirdparties/docker-compose/hive/scripts/data/regression/crdmm_data/create_table.hql
+++
b/docker/thirdparties/docker-compose/hive/scripts/data/regression/crdmm_data/create_table.hql
@@ -158,5 +158,5 @@ LOCATION
TBLPROPERTIES (
'transient_lastDdlTime'='1685331029');
-msck repair crdmm_data;
+msck repair table crdmm_data;
diff --git a/docker/thirdparties/docker-compose/hive/scripts/hive-metastore.sh
b/docker/thirdparties/docker-compose/hive/scripts/hive-metastore.sh
index 6681a513066..3a6fda081c3 100755
--- a/docker/thirdparties/docker-compose/hive/scripts/hive-metastore.sh
+++ b/docker/thirdparties/docker-compose/hive/scripts/hive-metastore.sh
@@ -22,26 +22,26 @@ parallel=$(getconf _NPROCESSORS_ONLN)
nohup /opt/hive/bin/hive --service metastore &
-# wait metastore start
-sleep 10s
-
-# create tables for other cases
-# new cases should use separate dir
-hadoop fs -mkdir -p /user/doris/suites/
-
-lockfile1="/mnt/scripts/run-data.lock"
-
# wait lockfile
+lockfile1="/mnt/scripts/run-data.lock"
while [[ -f "${lockfile1}" ]]; do
sleep 10
done
-
touch "${lockfile1}"
+# wait metastore start
+while ! $(nc -z localhost "${HMS_PORT:-9083}"); do
+ sleep 5s
+done
+
+# create tables for other cases
+# new cases should use separate dir
+hadoop fs -mkdir -p /user/doris/suites/
+
DATA_DIR="/mnt/scripts/data/"
-find "${DATA_DIR}" -type f -name "run.sh" -print0 | xargs -0 -n 1 -P
"${parallel}" -I {} sh -c '
+find "${DATA_DIR}" -type f -name "run.sh" -print0 | xargs -0 -n 1 -P
"${parallel}" -I {} bash -ec '
START_TIME=$(date +%s)
- chmod +x "{}" && "{}"
+ bash -e "{}" || (echo "Failed to executing script: {}" && exit 1)
END_TIME=$(date +%s)
EXECUTION_TIME=$((END_TIME - START_TIME))
echo "Script: {} executed in $EXECUTION_TIME seconds"
@@ -145,9 +145,9 @@ if [[ -z "$(hadoop fs -ls /user/doris/tvf_data)" ]]; then
fi
# create tables
-ls /mnt/scripts/create_preinstalled_scripts/*.hql | xargs -n 1 -P
"${parallel}" -I {} bash -c '
+ls /mnt/scripts/create_preinstalled_scripts/*.hql | xargs -n 1 -P
"${parallel}" -I {} bash -ec '
START_TIME=$(date +%s)
- hive -f {}
+ hive -f {} || (echo "Failed to executing hql: {}" && exit 1)
END_TIME=$(date +%s)
EXECUTION_TIME=$((END_TIME - START_TIME))
echo "Script: {} executed in $EXECUTION_TIME seconds"
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
