This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 74bba4bdaf [enhancement](regression-test) Add routine load case
(#24536)
74bba4bdaf is described below
commit 74bba4bdaf7e2dc9c1801ff357dca5a85e98e1af
Author: HHoflittlefish777 <77738092+hhoflittlefish...@users.noreply.github.com>
AuthorDate: Fri Sep 22 14:55:01 2023 +0800
[enhancement](regression-test) Add routine load case (#24536)
---
.../kafka/scripts/basic_array_data.csv | 20 +
.../kafka/scripts/basic_array_data_timezone.csv | 20 +
.../kafka/scripts/basic_array_data_with_errors.csv | 20 +
.../docker-compose/kafka/scripts/basic_data.csv | 20 +
.../kafka/scripts/basic_data_timezone.csv | 20 +
.../kafka/scripts/basic_data_with_errors.csv | 20 +
.../docker-compose/kafka/scripts/test.csv | 1 -
docker/thirdparties/run-thirdparties-docker.sh | 20 +-
.../load_p0/routine_load/test_routine_load.out | 19 +
.../routine_load/ddl/agg_tbl_basic_create.sql | 59 +++
.../routine_load/ddl/agg_tbl_basic_drop.sql | 1 +
.../routine_load/ddl/dup_tbl_array_create.sql | 42 ++
.../routine_load/ddl/dup_tbl_array_drop.sql | 1 +
.../routine_load/ddl/dup_tbl_basic_create.sql | 65 +++
.../routine_load/ddl/dup_tbl_basic_drop.sql | 1 +
.../routine_load/ddl/mow_tbl_array_create.sql | 44 ++
.../routine_load/ddl/mow_tbl_array_drop.sql | 1 +
.../routine_load/ddl/mow_tbl_basic_create.sql | 67 +++
.../routine_load/ddl/mow_tbl_basic_drop.sql | 1 +
.../routine_load/ddl/uniq_tbl_array_create.sql | 43 ++
.../routine_load/ddl/uniq_tbl_array_drop.sql | 1 +
.../routine_load/ddl/uniq_tbl_basic_create.sql | 57 +++
.../routine_load/ddl/uniq_tbl_basic_drop.sql | 1 +
.../load_p0/routine_load/test_routine_load.groovy | 499 +++++++++++++++++++--
24 files changed, 999 insertions(+), 44 deletions(-)
diff --git
a/docker/thirdparties/docker-compose/kafka/scripts/basic_array_data.csv
b/docker/thirdparties/docker-compose/kafka/scripts/basic_array_data.csv
new file mode 100644
index 0000000000..d25ef6489b
--- /dev/null
+++ b/docker/thirdparties/docker-compose/kafka/scripts/basic_array_data.csv
@@ -0,0 +1,20 @@
+55|[1, 1, 1, 1, 1, 1]|[65, 65, 65, 65, 65, 65]|[18805, 18805, 18805, 18805,
18805, 18805]|[229725878, 229725878, 229725878, 229725878, 229725878,
229725878]|[2742856458318615325, 2742856458318615325, 2742856458318615325,
2742856458318615325, 2742856458318615325,
2742856458318615325]|[5907702768956232371, 5907702768956232371,
5907702768956232371, 5907702768956232371, 5907702768956232371,
5907702768956232371]|[12354.624, 12354.624, 12354.624, 12354.624,
12354.624]|[1697579881.947477, 16975 [...]
+84|[0, 0, 0, 0, 0, 0]|[-10, -10, -10, -10, -10, -10]|[9493, 9493, 9493, 9493,
9493, 9493]|[-547874696, -547874696, -547874696, -547874696, -547874696,
-547874696]|[-115057683458952756, -115057683458952756, -115057683458952756,
-115057683458952756, -115057683458952756,
-115057683458952756]|[4473017779279230085, 4473017779279230085,
4473017779279230085, 4473017779279230085, 4473017779279230085,
4473017779279230085]|[13718.372, 13718.372, 13718.372, 13718.372,
13718.372]|[-978213266.02697, [...]
+48|[0, 0, 0, 0, 0, 0]|[88, 88, 88, 88, 88, 88]|[-18899, -18899, -18899,
-18899, -18899, -18899]|[1953750640, 1953750640, 1953750640, 1953750640,
1953750640, 1953750640]|[-6083034186246180312, -6083034186246180312,
-6083034186246180312, -6083034186246180312, -6083034186246180312,
-6083034186246180312]|[7861718260607212662, 7861718260607212662,
7861718260607212662, 7861718260607212662, 7861718260607212662,
7861718260607212662]|[20562.791, 20562.791, 20562.791, 20562.791,
20562.791]|[-15979 [...]
+44|[0, 0, 0, 0, 0, 0]|[61, 61, 61, 61, 61, 61]|[-23419, -23419, -23419,
-23419, -23419, -23419]|[378600280, 378600280, 378600280, 378600280, 378600280,
378600280]|[6788166268039991679, 6788166268039991679, 6788166268039991679,
6788166268039991679, 6788166268039991679,
6788166268039991679]|[-2814786606977504852, -2814786606977504852,
-2814786606977504852, -2814786606977504852, -2814786606977504852,
-2814786606977504852]|[-20151.432, -20151.432, -20151.432, -20151.432,
-20151.432]|[-124822 [...]
+4|[0, 0, 0, 0, 0, 0]|[63, 63, 63, 63, 63, 63]|[11399, 11399, 11399, 11399,
11399, 11399]|[-583523026, -583523026, -583523026, -583523026, -583523026,
-583523026]|[8801164674137231293, 8801164674137231293, 8801164674137231293,
8801164674137231293, 8801164674137231293,
8801164674137231293]|[-8287675635310193906, -8287675635310193906,
-8287675635310193906, -8287675635310193906, -8287675635310193906,
-8287675635310193906]|[23243.16, 23243.16, 23243.16, 23243.16,
23243.16]|[716719993.249115, [...]
+4|[0, 0, 0, 0, 0, 0]|[-67, -67, -67, -67, -67, -67]|[-30372, -30372, -30372,
-30372, -30372, -30372]|[181502941, 181502941, 181502941, 181502941, 181502941,
181502941]|[-2062236823576972800, -2062236823576972800, -2062236823576972800,
-2062236823576972800, -2062236823576972800,
-2062236823576972800]|[6357002962400127842, 6357002962400127842,
6357002962400127842, 6357002962400127842, 6357002962400127842,
6357002962400127842]|[21235.783, 21235.783, 21235.783, 21235.783,
21235.783]|[-110169 [...]
+61|[1, 1, 1, 1, 1, 1]|[121, 121, 121, 121, 121, 121]|[31806, 31806, 31806,
31806, 31806, 31806]|[-1410915562, -1410915562, -1410915562, -1410915562,
-1410915562, -1410915562]|[-250403393155768717, -250403393155768717,
-250403393155768717, -250403393155768717, -250403393155768717,
-250403393155768717]|[4301573778529723431, 4301573778529723431,
4301573778529723431, 4301573778529723431, 4301573778529723431,
4301573778529723431]|[10719.892, 10719.892, 10719.892, 10719.892,
10719.892]|[107378 [...]
+62|[0, 0, 0, 0, 0, 0]|[-126, -126, -126, -126, -126, -126]|[-3813, -3813,
-3813, -3813, -3813, -3813]|[1950651540, 1950651540, 1950651540, 1950651540,
1950651540, 1950651540]|[-7509418841468966217, -7509418841468966217,
-7509418841468966217, -7509418841468966217, -7509418841468966217,
-7509418841468966217]|[5935288575051629397, 5935288575051629397,
5935288575051629397, 5935288575051629397, 5935288575051629397,
5935288575051629397]|[-17121.195, -17121.195, -17121.195, -17121.195, -17121.1
[...]
+88|[1, 1, 1, 1, 1, 1]|[23, 23, 23, 23, 23, 23]|[14833, 14833, 14833, 14833,
14833, 14833]|[2017339015, 2017339015, 2017339015, 2017339015, 2017339015,
2017339015]|[7437673973249797930, 7437673973249797930, 7437673973249797930,
7437673973249797930, 7437673973249797930,
7437673973249797930]|[-5508026584283164347, -5508026584283164347,
-5508026584283164347, -5508026584283164347, -5508026584283164347,
-5508026584283164347]|[-29817.77, -29817.77, -29817.77, -29817.77,
-29817.77]|[-1646754251. [...]
+27|[0, 0, 0, 0, 0, 0]|[-7, -7, -7, -7, -7, -7]|[18307, 18307, 18307, 18307,
18307, 18307]|[76399879, 76399879, 76399879, 76399879, 76399879,
76399879]|[8050764818738996699, 8050764818738996699, 8050764818738996699,
8050764818738996699, 8050764818738996699,
8050764818738996699]|[2402219865213589999, 2402219865213589999,
2402219865213589999, 2402219865213589999, 2402219865213589999,
2402219865213589999]|[-16678.924, -16678.924, -16678.924, -16678.924,
-16678.924]|[-2053879544.844726, -2053 [...]
+8|[1, 1, 1, 1, 1, 1]|[-11, -11, -11, -11, -11, -11]|[-9648, -9648, -9648,
-9648, -9648, -9648]|[-505356927, -505356927, -505356927, -505356927,
-505356927, -505356927]|[7604760670442035037, 7604760670442035037,
7604760670442035037, 7604760670442035037, 7604760670442035037,
7604760670442035037]|[1634770507625165798, 1634770507625165798,
1634770507625165798, 1634770507625165798, 1634770507625165798,
1634770507625165798]|[10822.962, 10822.962, 10822.962, 10822.962,
10822.962]|[1987551048.86 [...]
+76|[1, 1, 1, 1, 1, 1]|[-63, -63, -63, -63, -63, -63]|[25799, 25799, 25799,
25799, 25799, 25799]|[-1387912656, -1387912656, -1387912656, -1387912656,
-1387912656, -1387912656]|[8967926767558546181, 8967926767558546181,
8967926767558546181, 8967926767558546181, 8967926767558546181,
8967926767558546181]|[-3537865898119184476, -3537865898119184476,
-3537865898119184476, -3537865898119184476, -3537865898119184476,
-3537865898119184476]|[5311.188, 5311.188, 5311.188, 5311.188, 5311.188]|[17362
[...]
+52|[0, 0, 0, 0, 0, 0]|[-7, -7, -7, -7, -7, -7]|[-6985, -6985, -6985, -6985,
-6985, -6985]|[826683531, 826683531, 826683531, 826683531, 826683531,
826683531]|[-8966681855246736361, -8966681855246736361, -8966681855246736361,
-8966681855246736361, -8966681855246736361,
-8966681855246736361]|[4814686163176635446, 4814686163176635446,
4814686163176635446, 4814686163176635446, 4814686163176635446,
4814686163176635446]|[-6490.247, -6490.247, -6490.247, -6490.247,
-6490.247]|[1076976372.033826, [...]
+3|[0, 0, 0, 0, 0, 0]|[65, 65, 65, 65, 65, 65]|[-4963, -4963, -4963, -4963,
-4963, -4963]|[-1415431954, -1415431954, -1415431954, -1415431954, -1415431954,
-1415431954]|[-3804309860450207000, -3804309860450207000, -3804309860450207000,
-3804309860450207000, -3804309860450207000,
-3804309860450207000]|[8209240008557215376, 8209240008557215376,
8209240008557215376, 8209240008557215376, 8209240008557215376,
8209240008557215376]|[-5058.13, -5058.13, -5058.13, -5058.13,
-5058.13]|[1034763010.6 [...]
+49|[0, 0, 0, 0, 0, 0]|[126, 126, 126, 126, 126, 126]|[31661, 31661, 31661,
31661, 31661, 31661]|[359703581, 359703581, 359703581, 359703581, 359703581,
359703581]|[-2399575246807057939, -2399575246807057939, -2399575246807057939,
-2399575246807057939, -2399575246807057939,
-2399575246807057939]|[7684667782059034391, 7684667782059034391,
7684667782059034391, 7684667782059034391, 7684667782059034391,
7684667782059034391]|[-504.68152, -504.68152, -504.68152, -504.68152,
-504.68152]|[2121528 [...]
+41|[0, 0, 0, 0, 0, 0]|[-25, -25, -25, -25, -25, -25]|[28704, 28704, 28704,
28704, 28704, 28704]|[-437867812, -437867812, -437867812, -437867812,
-437867812, -437867812]|[5508042206505207079, 5508042206505207079,
5508042206505207079, 5508042206505207079, 5508042206505207079,
5508042206505207079]|[462527544684407597, 462527544684407597,
462527544684407597, 462527544684407597, 462527544684407597,
462527544684407597]|[13629.614, 13629.614, 13629.614, 13629.614,
13629.614]|[213722401.337962, [...]
+16|[1, 1, 1, 1, 1, 1]|[-7, -7, -7, -7, -7, -7]|[18655, 18655, 18655, 18655,
18655, 18655]|[6240356, 6240356, 6240356, 6240356, 6240356,
6240356]|[4552179257266841393, 4552179257266841393, 4552179257266841393,
4552179257266841393, 4552179257266841393,
4552179257266841393]|[323868824766329978, 323868824766329978,
323868824766329978, 323868824766329978, 323868824766329978,
323868824766329978]|[2972.2478, 2972.2478, 2972.2478, 2972.2478,
2972.2478]|[-1177167334.995008, -1177167334.995008, -1 [...]
+16|[0, 0, 0, 0, 0, 0]|[-43, -43, -43, -43, -43, -43]|[13560, 13560, 13560,
13560, 13560, 13560]|[-1743686513, -1743686513, -1743686513, -1743686513,
-1743686513, -1743686513]|[7234719406392208769, 7234719406392208769,
7234719406392208769, 7234719406392208769, 7234719406392208769,
7234719406392208769]|[-3871745630024229413, -3871745630024229413,
-3871745630024229413, -3871745630024229413, -3871745630024229413,
-3871745630024229413]|[12225.427, 12225.427, 12225.427, 12225.427, 12225.427]|[
[...]
+17|[1, 1, 1, 1, 1, 1]|[126, 126, 126, 126, 126, 126]|[28165, 28165, 28165,
28165, 28165, 28165]|[2032059721, 2032059721, 2032059721, 2032059721,
2032059721, 2032059721]|[-2686776977990574879, -2686776977990574879,
-2686776977990574879, -2686776977990574879, -2686776977990574879,
-2686776977990574879]|[-8498149444423102876, -8498149444423102876,
-8498149444423102876, -8498149444423102876, -8498149444423102876,
-8498149444423102876]|[-21681.223, -21681.223, -21681.223, -21681.223, -21681.2
[...]
+2|[0, 0, 0, 0, 0, 0]|[117, 117, 117, 117, 117, 117]|[-4744, -4744, -4744,
-4744, -4744, -4744]|[-1593211961, -1593211961, -1593211961, -1593211961,
-1593211961, -1593211961]|[-3869640069299678780, -3869640069299678780,
-3869640069299678780, -3869640069299678780, -3869640069299678780,
-3869640069299678780]|[8491817458398170567, 8491817458398170567,
8491817458398170567, 8491817458398170567, 8491817458398170567,
8491817458398170567]|[-30948.857, -30948.857, -30948.857, -30948.857, -30948.85
[...]
diff --git
a/docker/thirdparties/docker-compose/kafka/scripts/basic_array_data_timezone.csv
b/docker/thirdparties/docker-compose/kafka/scripts/basic_array_data_timezone.csv
new file mode 100644
index 0000000000..a08b92360c
--- /dev/null
+++
b/docker/thirdparties/docker-compose/kafka/scripts/basic_array_data_timezone.csv
@@ -0,0 +1,20 @@
+[1, 1, 1, 1, 1, 1]|[65, 65, 65, 65, 65, 65]|[18805, 18805, 18805, 18805,
18805, 18805]|[229725878, 229725878, 229725878, 229725878, 229725878,
229725878]|[2742856458318615325, 2742856458318615325, 2742856458318615325,
2742856458318615325, 2742856458318615325,
2742856458318615325]|[5907702768956232371, 5907702768956232371,
5907702768956232371, 5907702768956232371, 5907702768956232371,
5907702768956232371]|[12354.624, 12354.624, 12354.624, 12354.624,
12354.624]|[1697579881.947477, 16975798 [...]
+[0, 0, 0, 0, 0, 0]|[-10, -10, -10, -10, -10, -10]|[9493, 9493, 9493, 9493,
9493, 9493]|[-547874696, -547874696, -547874696, -547874696, -547874696,
-547874696]|[-115057683458952756, -115057683458952756, -115057683458952756,
-115057683458952756, -115057683458952756,
-115057683458952756]|[4473017779279230085, 4473017779279230085,
4473017779279230085, 4473017779279230085, 4473017779279230085,
4473017779279230085]|[13718.372, 13718.372, 13718.372, 13718.372,
13718.372]|[-978213266.02697, -97 [...]
+[0, 0, 0, 0, 0, 0]|[88, 88, 88, 88, 88, 88]|[-18899, -18899, -18899, -18899,
-18899, -18899]|[1953750640, 1953750640, 1953750640, 1953750640, 1953750640,
1953750640]|[-6083034186246180312, -6083034186246180312, -6083034186246180312,
-6083034186246180312, -6083034186246180312,
-6083034186246180312]|[7861718260607212662, 7861718260607212662,
7861718260607212662, 7861718260607212662, 7861718260607212662,
7861718260607212662]|[20562.791, 20562.791, 20562.791, 20562.791,
20562.791]|[-15979946 [...]
+[0, 0, 0, 0, 0, 0]|[61, 61, 61, 61, 61, 61]|[-23419, -23419, -23419, -23419,
-23419, -23419]|[378600280, 378600280, 378600280, 378600280, 378600280,
378600280]|[6788166268039991679, 6788166268039991679, 6788166268039991679,
6788166268039991679, 6788166268039991679,
6788166268039991679]|[-2814786606977504852, -2814786606977504852,
-2814786606977504852, -2814786606977504852, -2814786606977504852,
-2814786606977504852]|[-20151.432, -20151.432, -20151.432, -20151.432,
-20151.432]|[-124822900 [...]
+[0, 0, 0, 0, 0, 0]|[63, 63, 63, 63, 63, 63]|[11399, 11399, 11399, 11399,
11399, 11399]|[-583523026, -583523026, -583523026, -583523026, -583523026,
-583523026]|[8801164674137231293, 8801164674137231293, 8801164674137231293,
8801164674137231293, 8801164674137231293,
8801164674137231293]|[-8287675635310193906, -8287675635310193906,
-8287675635310193906, -8287675635310193906, -8287675635310193906,
-8287675635310193906]|[23243.16, 23243.16, 23243.16, 23243.16,
23243.16]|[716719993.249115, 71 [...]
+[0, 0, 0, 0, 0, 0]|[-67, -67, -67, -67, -67, -67]|[-30372, -30372, -30372,
-30372, -30372, -30372]|[181502941, 181502941, 181502941, 181502941, 181502941,
181502941]|[-2062236823576972800, -2062236823576972800, -2062236823576972800,
-2062236823576972800, -2062236823576972800,
-2062236823576972800]|[6357002962400127842, 6357002962400127842,
6357002962400127842, 6357002962400127842, 6357002962400127842,
6357002962400127842]|[21235.783, 21235.783, 21235.783, 21235.783,
21235.783]|[-11016947 [...]
+[1, 1, 1, 1, 1, 1]|[121, 121, 121, 121, 121, 121]|[31806, 31806, 31806, 31806,
31806, 31806]|[-1410915562, -1410915562, -1410915562, -1410915562, -1410915562,
-1410915562]|[-250403393155768717, -250403393155768717, -250403393155768717,
-250403393155768717, -250403393155768717,
-250403393155768717]|[4301573778529723431, 4301573778529723431,
4301573778529723431, 4301573778529723431, 4301573778529723431,
4301573778529723431]|[10719.892, 10719.892, 10719.892, 10719.892,
10719.892]|[107378059 [...]
+[0, 0, 0, 0, 0, 0]|[-126, -126, -126, -126, -126, -126]|[-3813, -3813, -3813,
-3813, -3813, -3813]|[1950651540, 1950651540, 1950651540, 1950651540,
1950651540, 1950651540]|[-7509418841468966217, -7509418841468966217,
-7509418841468966217, -7509418841468966217, -7509418841468966217,
-7509418841468966217]|[5935288575051629397, 5935288575051629397,
5935288575051629397, 5935288575051629397, 5935288575051629397,
5935288575051629397]|[-17121.195, -17121.195, -17121.195, -17121.195,
-17121.195] [...]
+[1, 1, 1, 1, 1, 1]|[23, 23, 23, 23, 23, 23]|[14833, 14833, 14833, 14833,
14833, 14833]|[2017339015, 2017339015, 2017339015, 2017339015, 2017339015,
2017339015]|[7437673973249797930, 7437673973249797930, 7437673973249797930,
7437673973249797930, 7437673973249797930,
7437673973249797930]|[-5508026584283164347, -5508026584283164347,
-5508026584283164347, -5508026584283164347, -5508026584283164347,
-5508026584283164347]|[-29817.77, -29817.77, -29817.77, -29817.77,
-29817.77]|[-1646754251.102 [...]
+[0, 0, 0, 0, 0, 0]|[-7, -7, -7, -7, -7, -7]|[18307, 18307, 18307, 18307,
18307, 18307]|[76399879, 76399879, 76399879, 76399879, 76399879,
76399879]|[8050764818738996699, 8050764818738996699, 8050764818738996699,
8050764818738996699, 8050764818738996699,
8050764818738996699]|[2402219865213589999, 2402219865213589999,
2402219865213589999, 2402219865213589999, 2402219865213589999,
2402219865213589999]|[-16678.924, -16678.924, -16678.924, -16678.924,
-16678.924]|[-2053879544.844726, -2053879 [...]
+[1, 1, 1, 1, 1, 1]|[-11, -11, -11, -11, -11, -11]|[-9648, -9648, -9648, -9648,
-9648, -9648]|[-505356927, -505356927, -505356927, -505356927, -505356927,
-505356927]|[7604760670442035037, 7604760670442035037, 7604760670442035037,
7604760670442035037, 7604760670442035037,
7604760670442035037]|[1634770507625165798, 1634770507625165798,
1634770507625165798, 1634770507625165798, 1634770507625165798,
1634770507625165798]|[10822.962, 10822.962, 10822.962, 10822.962,
10822.962]|[1987551048.8630 [...]
+[1, 1, 1, 1, 1, 1]|[-63, -63, -63, -63, -63, -63]|[25799, 25799, 25799, 25799,
25799, 25799]|[-1387912656, -1387912656, -1387912656, -1387912656, -1387912656,
-1387912656]|[8967926767558546181, 8967926767558546181, 8967926767558546181,
8967926767558546181, 8967926767558546181,
8967926767558546181]|[-3537865898119184476, -3537865898119184476,
-3537865898119184476, -3537865898119184476, -3537865898119184476,
-3537865898119184476]|[5311.188, 5311.188, 5311.188, 5311.188,
5311.188]|[17362874 [...]
+[0, 0, 0, 0, 0, 0]|[-7, -7, -7, -7, -7, -7]|[-6985, -6985, -6985, -6985,
-6985, -6985]|[826683531, 826683531, 826683531, 826683531, 826683531,
826683531]|[-8966681855246736361, -8966681855246736361, -8966681855246736361,
-8966681855246736361, -8966681855246736361,
-8966681855246736361]|[4814686163176635446, 4814686163176635446,
4814686163176635446, 4814686163176635446, 4814686163176635446,
4814686163176635446]|[-6490.247, -6490.247, -6490.247, -6490.247,
-6490.247]|[1076976372.033826, 10 [...]
+[0, 0, 0, 0, 0, 0]|[65, 65, 65, 65, 65, 65]|[-4963, -4963, -4963, -4963,
-4963, -4963]|[-1415431954, -1415431954, -1415431954, -1415431954, -1415431954,
-1415431954]|[-3804309860450207000, -3804309860450207000, -3804309860450207000,
-3804309860450207000, -3804309860450207000,
-3804309860450207000]|[8209240008557215376, 8209240008557215376,
8209240008557215376, 8209240008557215376, 8209240008557215376,
8209240008557215376]|[-5058.13, -5058.13, -5058.13, -5058.13,
-5058.13]|[1034763010.616 [...]
+[0, 0, 0, 0, 0, 0]|[126, 126, 126, 126, 126, 126]|[31661, 31661, 31661, 31661,
31661, 31661]|[359703581, 359703581, 359703581, 359703581, 359703581,
359703581]|[-2399575246807057939, -2399575246807057939, -2399575246807057939,
-2399575246807057939, -2399575246807057939,
-2399575246807057939]|[7684667782059034391, 7684667782059034391,
7684667782059034391, 7684667782059034391, 7684667782059034391,
7684667782059034391]|[-504.68152, -504.68152, -504.68152, -504.68152,
-504.68152]|[2121528178 [...]
+[0, 0, 0, 0, 0, 0]|[-25, -25, -25, -25, -25, -25]|[28704, 28704, 28704, 28704,
28704, 28704]|[-437867812, -437867812, -437867812, -437867812, -437867812,
-437867812]|[5508042206505207079, 5508042206505207079, 5508042206505207079,
5508042206505207079, 5508042206505207079,
5508042206505207079]|[462527544684407597, 462527544684407597,
462527544684407597, 462527544684407597, 462527544684407597,
462527544684407597]|[13629.614, 13629.614, 13629.614, 13629.614,
13629.614]|[213722401.337962, 213 [...]
+[1, 1, 1, 1, 1, 1]|[-7, -7, -7, -7, -7, -7]|[18655, 18655, 18655, 18655,
18655, 18655]|[6240356, 6240356, 6240356, 6240356, 6240356,
6240356]|[4552179257266841393, 4552179257266841393, 4552179257266841393,
4552179257266841393, 4552179257266841393,
4552179257266841393]|[323868824766329978, 323868824766329978,
323868824766329978, 323868824766329978, 323868824766329978,
323868824766329978]|[2972.2478, 2972.2478, 2972.2478, 2972.2478,
2972.2478]|[-1177167334.995008, -1177167334.995008, -1177 [...]
+[0, 0, 0, 0, 0, 0]|[-43, -43, -43, -43, -43, -43]|[13560, 13560, 13560, 13560,
13560, 13560]|[-1743686513, -1743686513, -1743686513, -1743686513, -1743686513,
-1743686513]|[7234719406392208769, 7234719406392208769, 7234719406392208769,
7234719406392208769, 7234719406392208769,
7234719406392208769]|[-3871745630024229413, -3871745630024229413,
-3871745630024229413, -3871745630024229413, -3871745630024229413,
-3871745630024229413]|[12225.427, 12225.427, 12225.427, 12225.427,
12225.427]|[-13 [...]
+[1, 1, 1, 1, 1, 1]|[126, 126, 126, 126, 126, 126]|[28165, 28165, 28165, 28165,
28165, 28165]|[2032059721, 2032059721, 2032059721, 2032059721, 2032059721,
2032059721]|[-2686776977990574879, -2686776977990574879, -2686776977990574879,
-2686776977990574879, -2686776977990574879,
-2686776977990574879]|[-8498149444423102876, -8498149444423102876,
-8498149444423102876, -8498149444423102876, -8498149444423102876,
-8498149444423102876]|[-21681.223, -21681.223, -21681.223, -21681.223,
-21681.223] [...]
+[0, 0, 0, 0, 0, 0]|[117, 117, 117, 117, 117, 117]|[-4744, -4744, -4744, -4744,
-4744, -4744]|[-1593211961, -1593211961, -1593211961, -1593211961, -1593211961,
-1593211961]|[-3869640069299678780, -3869640069299678780, -3869640069299678780,
-3869640069299678780, -3869640069299678780,
-3869640069299678780]|[8491817458398170567, 8491817458398170567,
8491817458398170567, 8491817458398170567, 8491817458398170567,
8491817458398170567]|[-30948.857, -30948.857, -30948.857, -30948.857,
-30948.857] [...]
diff --git
a/docker/thirdparties/docker-compose/kafka/scripts/basic_array_data_with_errors.csv
b/docker/thirdparties/docker-compose/kafka/scripts/basic_array_data_with_errors.csv
new file mode 100644
index 0000000000..b2d8b3736c
--- /dev/null
+++
b/docker/thirdparties/docker-compose/kafka/scripts/basic_array_data_with_errors.csv
@@ -0,0 +1,20 @@
+55|[, 1, 1, 1, 1, 1]|[65, 65, 65, 65, 65, 65]|[18805, 18805, 18805, 18805,
18805, 18805]|[229725878, 229725878, 229725878, 229725878, 229725878,
229725878]|[2742856458318615325, 2742856458318615325, 2742856458318615325,
2742856458318615325, 2742856458318615325,
2742856458318615325]|[5907702768956232371, 5907702768956232371,
5907702768956232371, 5907702768956232371, 5907702768956232371,
5907702768956232371]|[12354.624, 12354.624, 12354.624, 12354.624,
12354.624]|[1697579881.947477, 169757 [...]
+84|[0, 0, 0, 0, 0, 0]|[-10, -10, -10, -10, -10, -10|[9493, 9493, 9493, 9493,
9493, 9493]|[-547874696, -547874696, -547874696, -547874696, -547874696,
-547874696]|[-115057683458952756, -115057683458952756, -115057683458952756,
-115057683458952756, -115057683458952756,
-115057683458952756]|[4473017779279230085, 4473017779279230085,
4473017779279230085, 4473017779279230085, 4473017779279230085,
4473017779279230085]|[13718.372, 13718.372, 13718.372, 13718.372,
13718.372]|[-978213266.02697, - [...]
+48|[0, 0, 0, 0, 0, 2]|[88, 88, 88, 88, 88, 88]|[-18899, -18899, -18899,
-18899, -18899, -18899]|[1953750640, 1953750640, 1953750640, 1953750640,
1953750640, 1953750640]|[-6083034186246180312, -6083034186246180312,
-6083034186246180312, -6083034186246180312, -6083034186246180312,
-6083034186246180312]|[7861718260607212662, 7861718260607212662,
7861718260607212662, 7861718260607212662, 7861718260607212662,
7861718260607212662]|[20562.791, 20562.791, 20562.791, 20562.791,
20562.791]|[-15979 [...]
+44|[0, 0, 0, 0, 0, 0]|[61, 61, 61, 61, 61, 61999999999999999999]|[-23419,
-23419, -23419, -23419, -23419, -23419]|[378600280, 378600280, 378600280,
378600280, 378600280, 378600280]|[6788166268039991679, 6788166268039991679,
6788166268039991679, 6788166268039991679, 6788166268039991679,
6788166268039991679]|[-2814786606977504852, -2814786606977504852,
-2814786606977504852, -2814786606977504852, -2814786606977504852,
-2814786606977504852]|[-20151.432, -20151.432, -20151.432, -20151.432, -2 [...]
+4|[0, 0, 0, 0, 0, 0]|[63, 63, 63, 63, 63, 63]|[11399, 11399, 11399, 11399,
11399, 11399]|[-583523026, -583523026, -583523026, -583523026, -583523026,
-583523026]|[8801164674137231293, 8801164674137231293, 8801164674137231293,
8801164674137231293, 8801164674137231293,
8801164674137231293]|[-8287675635310193906, -8287675635310193906,
-8287675635310193906, -8287675635310193906, -8287675635310193906,
-8287675635310193906]|[23243.16, 23243.16, 23243.16, 23243.16,
23243.16]|[716719993.249115, [...]
+4|[0, 0, 0, 0, 0, 0]|[-67, -67, -67, -67, -67, -67]|[-30372, -30372, -30372,
-30372, -30372, -30372]|[181502941, 181502941, 181502941, 181502941, 181502941,
181502941]|[-2062236823576972800, -2062236823576972800, -2062236823576972800,
-2062236823576972800, -2062236823576972800,
-2062236823576972800]|[6357002962400127842, 6357002962400127842,
6357002962400127842, 6357002962400127842, 6357002962400127842,
6357002962400127842]|[21235.783, 21235.783, 21235.783, 21235.783,
21235.783]|[-110169 [...]
+61|[1, 1, 1, 1, 1, 1]|[121, 121, 121, 121, 121, 121]|[31806, 31806, 31806,
31806, 31806, 31806]|[-1410915562, -1410915562, -1410915562, -1410915562,
-1410915562, -1410915562]|[-250403393155768717, -250403393155768717,
-250403393155768717, -250403393155768717, -250403393155768717,
-250403393155768717]|[4301573778529723431, 4301573778529723431,
4301573778529723431, 4301573778529723431, 4301573778529723431,
4301573778529723431]|[10719.892, 10719.892, 10719.892, 10719.892,
10719.892]|[107378 [...]
+62|[0, 0, 0, 0, 0, 0]|[-126, -126, -126, -126, -126, -126]|[-3813, -3813,
-3813, -3813, -3813, -3813]|[1950651540, 1950651540, 1950651540, 1950651540,
1950651540, 1950651540]|[-7509418841468966217, -7509418841468966217,
-7509418841468966217, -7509418841468966217, -7509418841468966217,
-7509418841468966217]|[5935288575051629397, 5935288575051629397,
5935288575051629397, 5935288575051629397, 5935288575051629397,
5935288575051629397]|[-17121.195, -17121.195, -17121.195, -17121.195, -17121.1
[...]
+88|[1, 1, 1, 1, 1, 1]|[23, 23, 23, 23, 23, 23]|[14833, 14833, 14833, 14833,
14833, 14833]|[2017339015, 2017339015, 2017339015, 2017339015, 2017339015,
2017339015]|[7437673973249797930, 7437673973249797930, 7437673973249797930,
7437673973249797930, 7437673973249797930,
7437673973249797930]|[-5508026584283164347, -5508026584283164347,
-5508026584283164347, -5508026584283164347, -5508026584283164347,
-5508026584283164347]|[-29817.77, -29817.77, -29817.77, -29817.77,
-29817.77]|[-1646754251. [...]
+27|[0, 0, 0, 0, 0, 0]|[-7, -7, -7, -7, -7, -7]|[18307, 18307, 18307, 18307,
18307, 18307]|[76399879, 76399879, 76399879, 76399879, 76399879,
76399879]|[8050764818738996699, 8050764818738996699, 8050764818738996699,
8050764818738996699, 8050764818738996699,
8050764818738996699]|[2402219865213589999, 2402219865213589999,
2402219865213589999, 2402219865213589999, 2402219865213589999,
2402219865213589999]|[-16678.924, -16678.924, -16678.924, -16678.924,
-16678.924]|[-2053879544.844726, -2053 [...]
+8|[1, 1, 1, 1, 1, 1]|[-11, -11, -11, -11, -11, -11]|[-9648, -9648, -9648,
-9648, -9648, -9648]|[-505356927, -505356927, -505356927, -505356927,
-505356927, -505356927]|[7604760670442035037, 7604760670442035037,
7604760670442035037, 7604760670442035037, 7604760670442035037,
7604760670442035037]|[1634770507625165798, 1634770507625165798,
1634770507625165798, 1634770507625165798, 1634770507625165798,
1634770507625165798]|[10822.962, 10822.962, 10822.962, 10822.962,
10822.962]|[1987551048.86 [...]
+76|[1, 1, 1, 1, 1, 1]|[-63, -63, -63, -63, -63, -63]|[25799, 25799, 25799,
25799, 25799, 25799]|[-1387912656, -1387912656, -1387912656, -1387912656,
-1387912656, -1387912656]|[8967926767558546181, 8967926767558546181,
8967926767558546181, 8967926767558546181, 8967926767558546181,
8967926767558546181]|[-3537865898119184476, -3537865898119184476,
-3537865898119184476, -3537865898119184476, -3537865898119184476,
-3537865898119184476]|[5311.188, 5311.188, 5311.188, 5311.188, 5311.188]|[17362
[...]
+52|[0, 0, 0, 0, 0, 0]|[-7, -7, -7, -7, -7, -7]|[-6985, -6985, -6985, -6985,
-6985, -6985]|[826683531, 826683531, 826683531, 826683531, 826683531,
826683531]|[-8966681855246736361, -8966681855246736361, -8966681855246736361,
-8966681855246736361, -8966681855246736361,
-8966681855246736361]|[4814686163176635446, 4814686163176635446,
4814686163176635446, 4814686163176635446, 4814686163176635446,
4814686163176635446]|[-6490.247, -6490.247, -6490.247, -6490.247,
-6490.247]|[1076976372.033826, [...]
+3|[0, 0, 0, 0, 0, 0]|[65, 65, 65, 65, 65, 65]|[-4963, -4963, -4963, -4963,
-4963, -4963]|[-1415431954, -1415431954, -1415431954, -1415431954, -1415431954,
-1415431954]|[-3804309860450207000, -3804309860450207000, -3804309860450207000,
-3804309860450207000, -3804309860450207000,
-3804309860450207000]|[8209240008557215376, 8209240008557215376,
8209240008557215376, 8209240008557215376, 8209240008557215376,
8209240008557215376]|[-5058.13, -5058.13, -5058.13, -5058.13,
-5058.13]|[1034763010.6 [...]
+49|[0, 0, 0, 0, 0, 0]|[126, 126, 126, 126, 126, 126]|[31661, 31661, 31661,
31661, 31661, 31661]|[359703581, 359703581, 359703581, 359703581, 359703581,
359703581]|[-2399575246807057939, -2399575246807057939, -2399575246807057939,
-2399575246807057939, -2399575246807057939,
-2399575246807057939]|[7684667782059034391, 7684667782059034391,
7684667782059034391, 7684667782059034391, 7684667782059034391,
7684667782059034391]|[-504.68152, -504.68152, -504.68152, -504.68152,
-504.68152]|[2121528 [...]
+41|[0, 0, 0, 0, 0, 0]|[-25, -25, -25, -25, -25, -25]|[28704, 28704, 28704,
28704, 28704, 28704]|[-437867812, -437867812, -437867812, -437867812,
-437867812, -437867812]|[5508042206505207079, 5508042206505207079,
5508042206505207079, 5508042206505207079, 5508042206505207079,
5508042206505207079]|[462527544684407597, 462527544684407597,
462527544684407597, 462527544684407597, 462527544684407597,
462527544684407597]|[13629.614, 13629.614, 13629.614, 13629.614,
13629.614]|[213722401.337962, [...]
+16|[1, 1, 1, 1, 1, 1]|[-7, -7, -7, -7, -7, -7]|[18655, 18655, 18655, 18655,
18655, 18655]|[6240356, 6240356, 6240356, 6240356, 6240356,
6240356]|[4552179257266841393, 4552179257266841393, 4552179257266841393,
4552179257266841393, 4552179257266841393,
4552179257266841393]|[323868824766329978, 323868824766329978,
323868824766329978, 323868824766329978, 323868824766329978,
323868824766329978]|[2972.2478, 2972.2478, 2972.2478, 2972.2478,
2972.2478]|[-1177167334.995008, -1177167334.995008, -1 [...]
+16|[0, 0, 0, 0, 0, 0]|[-43, -43, -43, -43, -43, -43]|[13560, 13560, 13560,
13560, 13560, 13560]|[-1743686513, -1743686513, -1743686513, -1743686513,
-1743686513, -1743686513]|[7234719406392208769, 7234719406392208769,
7234719406392208769, 7234719406392208769, 7234719406392208769,
7234719406392208769]|[-3871745630024229413, -3871745630024229413,
-3871745630024229413, -3871745630024229413, -3871745630024229413,
-3871745630024229413]|[12225.427, 12225.427, 12225.427, 12225.427, 12225.427]|[
[...]
+17|[1, 1, 1, 1, 1, 1]|[126, 126, 126, 126, 126, 126]|[28165, 28165, 28165,
28165, 28165, 28165]|[2032059721, 2032059721, 2032059721, 2032059721,
2032059721, 2032059721]|[-2686776977990574879, -2686776977990574879,
-2686776977990574879, -2686776977990574879, -2686776977990574879,
-2686776977990574879]|[-8498149444423102876, -8498149444423102876,
-8498149444423102876, -8498149444423102876, -8498149444423102876,
-8498149444423102876]|[-21681.223, -21681.223, -21681.223, -21681.223, -21681.2
[...]
+2|[0, 0, 0, 0, 0, 0]|[117, 117, 117, 117, 117, 117]|[-4744, -4744, -4744,
-4744, -4744, -4744]|[-1593211961, -1593211961, -1593211961, -1593211961,
-1593211961, -1593211961]|[-3869640069299678780, -3869640069299678780,
-3869640069299678780, -3869640069299678780, -3869640069299678780,
-3869640069299678780]|[8491817458398170567, 8491817458398170567,
8491817458398170567, 8491817458398170567, 8491817458398170567,
8491817458398170567]|[-30948.857, -30948.857, -30948.857, -30948.857, -30948.85
[...]
diff --git a/docker/thirdparties/docker-compose/kafka/scripts/basic_data.csv
b/docker/thirdparties/docker-compose/kafka/scripts/basic_data.csv
new file mode 100644
index 0000000000..daa9ebd355
--- /dev/null
+++ b/docker/thirdparties/docker-compose/kafka/scripts/basic_data.csv
@@ -0,0 +1,20 @@
+57|2023-08-19|TRUE|2|-25462|-74112029|6458082754318544493|-7910671781690629051|-15205.859375|-306870797.484914|759730669.0|-628556336.0|2023-07-10
18:39:10|2023-02-12|2023-01-27
07:26:06|y||Xi9nDVrLv8m6AwEpUxmtzFAuK48sQ|{"name": "John", "age": 25, "city":
"New York"}
+49|2023-08-08|FALSE|\N|16275|-2144851675|-2303421957908954634|-46526938720058765|-13141.142578|-686632233.230200|229942298.0|-152553823.0|2022-09-01
00:16:01|2023-03-25|2022-09-07 14:59:03|s||yvuILR2iNxfe8RRml|{"student": true,
"name": "Alice", "grade": 9, "subjects": ["math", "science", "history"]}
+66|2023-08-15|TRUE|-91|28378|609923317|4872185586197131212|1207709464099378591|\N|-1863683325.985123|-783792012.0|-708986976.0|2022-09-24
10:39:23|2022-09-24|2022-10-16
18:36:43|Y|z|AI1BSPQdKiHJiQH1kguyLSWsDXkC7zwy7PwgWnyGSaa9tBKRex8vHBdxg2QSKZKL2mV2lHz7iI1PnsTd4MXDcIKhqiHyPuQPt2tEtgt0UgF6|{"book":
{"title": "The Great Gatsby", "author": "F. Scott Fitzgerald"}, "year": 1925}
+91|2023-08-27|TRUE|90|2465|702240964|6373830997821598984|305860046137409400|15991.356445|1599972327.386147|-165530947.0|\N|2023-04-26
19:31:10|2023-07-21|\N|2||B7YKYBYT8w0YC926bZ8Yz1VzyiWw2NWDAiTlEoPVyz9AXGti2Npg1FxWqWk4hEaALw0ZBSuiAIPj41lq36g5QRpPmAjNPK|{"fruit":
"apple", "color": "red", "qty": 5, "price": 2.5}
+80|2023-08-18|FALSE|-18|-8971|679027874|6535956962935330265|3960889045799757165|-13219.759766|1187161924.505394|-526615878.0|-947410627.0|2023-03-11
07:40:00|2022-11-29|2023-01-14
07:24:07|\N|D|3Nhx6xX1qdwaq7lxwLRSKMtJFbC03swWv12mpySSVysH3igGZTiGPuKMsYW7HAkf6CWc7c0nzqDsjuH3FYVMNCWRmfxMrmY8rykQCC4Ve|{"car":
"BMW", "model": "X5", "year": 2020, "color": "black"}
+85|2023-08-11|TRUE|-7|24304|-2043877415|-2024144417867729183|\N|5363.024414|-578615669.042831|-378574346.0|-810302932.0|2023-07-15
01:07:41|2023-08-13|2023-01-20 11:57:48|i||WQ9dh9ajPu0y|{"country": "France",
"capital": "Paris", "population": 67081000}
+31|2023-08-27|FALSE|17|-18849|1728109133|3266501886640700374|527195452623418935|-24062.328125|-1514348021.262435|-322205854.0|-278237157.0|2022-10-07
03:24:23|2022-09-25|\N|0|8|yKMiAntORoRa8svnMfcxlOPwwND1m5s2fdS26Xu6cfs6HK5SAibqIp9h8sZcpjHy4|{"team":
"Manchester United", "players": ["Ronaldo", "Rooney", "Giggs"], "coach": "Ole
Gunnar Solskjaer"}
+20|2023-08-17|FALSE|-5|18158|784479801|1485484354598941738|-6632681928222776815|9708.430664|-330432620.706069|-816424174.0|571112646.0|2022-09-15
21:40:55|2023-02-23|2023-08-13
21:31:54|O|X|2pYmX2vAhfEEHZZYPsgAmda1G7otnwx5TmUC879FPhDeIjvWI79ksBZpfFG2gp7jhCSbpZiecKGklB5SvG8tm31i5SUqe1xrWgLt4HSq7lMJWp75tx2kxD7pRIOpn|{"name":
"Sarah", "age": 30, "city": "London", "isMarried": false}
+90|2023-08-27|TRUE|22|16456|-1476824962|-3279894870153540825|8990195191470116763|26651.906250|206860148.942546|-580959198.0|-210329147.0|2022-10-07
03:11:03|2023-03-18|2023-04-15
00:38:33|T|L|QW0GQ3GoMtHgxPQOWGfVaveynahNpsNs09siMFA1OtO6QEDBQTdivmGyq7bFzejAqwbbVQQpREAmeLjcFSXLnQuou2KbwYD|{"company":
"Apple", "products": [{"name": "iPhone", "price": 1000}, {"name": "MacBook",
"price": 1500}]}
+8|2023-08-14|TRUE|109|-31573|-1362465190|3990845741226497177|2732763251146840270|-25698.552734|1312831962.567818|771983879.0|173937916.0|2023-03-07
14:13:19|2022-10-18|2023-07-16
05:03:13|D||PBn1wa6X8WneZYLMac11zzyhGl7tPXB5XgjmOV8L6uav9ja5oY433ktb2yhyQQIqBveZPkme|{"animal":
"lion", "weight": 200, "habitat": ["savannah", "grassland"]}
+65|2023-08-09|FALSE|94|31514|814994517|-297697460695940343|734910652450318597|-13061.891602|62750847.041706|-9808654.0|\N|2023-08-14
22:01:27|2023-05-19|2022-11-13
13:44:28|V||aGeMsI24O12chGlP5ak0AHghAz7bu5MargJBStHnt0yMnChH0JnfYhsfH1u59XIHkJKMsHYktBqORkGlovu8V47E74KeFpaqxn5yLyXfDbhhzUKf|{"language":
"Python", "version": 3.9, "frameworks": ["Django", "Flask"]}
+62|2023-08-21|FALSE|81|20302|-200761532|6365479976421007608|\N|-29916.533203|1709141750.828478|549873536.0|-119205359.0|2023-05-04
01:14:51|2022-09-17|2022-12-04
19:30:09|d|v|BKWy9dTNg1aZW7ancEJAmEDOPK5TwFsNSHbI78emu9gymeIlx5NoLmyii0QAqdzRvSQPZKiqKkwInGCTIBnK1yYkK7zD|{"username":
"user123", "password": "pass123", "email": "user...@example.com"}
+50|2023-08-06|TRUE|109|-6330|1479023892|-8630800697573159428|-1645095773540208759|17880.960938|-1453844792.013949|-158871820.0|-862940384.0|2022-09-22
02:03:21|2023-05-14|2023-03-25
02:18:34|m||JKnIgXvGVidGiWl9YRSi3mFI7wHKt1sBpWSadKF8VX3LAuElm4sdc9gtxREaUr57oikSYlU8We8h1MWqQlYNiJObl|{"city":
"Tokyo", "temperature": 20.5, "humidity": 75}
+58|2023-08-22|\N|0|-18231|1832867360|6997858407575297145|2480714305422728023|-5450.488770|1475901032.138386|-893480655.0|-607891858.0|2023-02-02
05:13:24|2022-09-18|2023-04-23
10:51:15|k||LdFXF7Kmfzgmnn2R6zLsXdmi3A2cLBLq4G4WDVNDhxvH7dYH8Kga2WA47uSIxp6NSrwPSdw0ssB1TS8RFJTDJAB0Uba3e05NL2Aiw0ja|{"restaurant":
"Pizza Hut", "menu": ["pizza", "pasta", "salad"]}
+60|2023-08-27|FALSE|-52|-2338|-757056972|1047567408607120856|6541476642780646552|6614.089355|-1204448798.517855|236657733.0|731515433.0|2022-12-29
14:47:30|2022-09-24|2023-08-01
12:41:59|O|F|RM4F1Ke7lkcnuxF2nK0j9VBW3MDcgyHR4pseBjtFnqS6GUkVFuzF6u3Cp9Nv7ab0O6UYrpP4DhU|{"game":
"Chess", "players": 2, "time": "1 hour"}
+68|2023-08-23|TRUE|-73|20117|1737338128|795638676048937749|-5551546237562433901|-30627.039062|68589475.684545|585022347.0|513722420.0|2022-12-28
20:26:51|2022-10-04|2023-07-30
00:20:06|y||keZ3JlWWpdnPBejf0cuiCQCVBBTd5gjvO08NVdcAFewqL7nRT4N9lnvSU6pWmletA5VbPQCeQapJdcnQCHfZUDCf4ulCnczyqr7SGrbGRT0XYcd7iktKM|{"country":
"Brazil", "continent": "South America", "population": 211049527}
+50|2023-08-24|TRUE|15|14403|\N|-6418906115745394180|9205303779366462513|-4331.548828|-615112179.557648|367305015.0|-551652958.0|2022-12-29
02:27:20|2023-06-01|2023-08-12
04:50:04|a||eCl38sztIvBQvGvGKyYZmyMXy9vIJx197iu3JwP9doJGcrYUl9Uova0rz4iCCgrjlAiZU18Fs9YtCq830nhM|{"band":
"The Beatles", "members": ["John Lennon", "Paul McCartney", "George Harrison",
"Ringo Starr"]}
+81|2023-08-23|FALSE|106|11492|-667795397|4480250461471356146|-5346660566234294101|9082.750000|385167225.902608|-717553011.0|649146853.0|2023-03-20
03:33:16|2022-11-24|2023-02-16
18:29:41|G|9|Lk3eNVQNjucbekD1rZmUlGPiXS5JvcWr2LQzRU8GSGIbSag|{"flower": "rose",
"color": "red", "fragrance": true}
+41|2023-08-27|TRUE|-104|22750|\N|8527773271030840740|5554497317268279215|-5296.828125|-1715646888.013040|-306075962.0|897769189.0|2022-12-02
17:56:44|2022-10-12|2023-02-19
07:02:54|V|\N|E9GzQdTwX1ITUQz27IVznAs6Ca4WwprKk6Odjs6SH75D2F1089QiY3HQ52LXRD1V6xAWjhLE2hWgW3EdHuAOnUDVrb5V|{"food":
"Sushi", "price": 10, "restaurant": "Sushi King"}
+21|2023-08-18|FALSE|63|-27847|-35409596|8638201997392767650|4919963231735304178|-23382.541016|-1803403621.426313|-22009767.0|661750756.0|2023-03-31
10:56:14|2023-01-20|2023-02-18
13:37:52|N|T|PSiFwUEx3eVFNtjlnQ70YkgZNvKrGmQ2DN5K9yYHiSdFWeEDB1UpL3Frt8z1kEAIWRDWqXZuyi|{"city":
"Sydney", "population": 5312000, "area": 2058.7}
diff --git
a/docker/thirdparties/docker-compose/kafka/scripts/basic_data_timezone.csv
b/docker/thirdparties/docker-compose/kafka/scripts/basic_data_timezone.csv
new file mode 100644
index 0000000000..d4fbc5967d
--- /dev/null
+++ b/docker/thirdparties/docker-compose/kafka/scripts/basic_data_timezone.csv
@@ -0,0 +1,20 @@
+2023-08-19|TRUE|2|-25462|-74112029|6458082754318544493|-7910671781690629051|-15205.859375|-306870797.484914|759730669.0|-628556336.0|2023-07-10
18:39:10|2023-02-12|2023-01-27
07:26:06|y||Xi9nDVrLv8m6AwEpUxmtzFAuK48sQ|{"name": "John", "age": 25, "city":
"New York"}
+2023-08-08|FALSE|\N|16275|-2144851675|-2303421957908954634|-46526938720058765|-13141.142578|-686632233.230200|229942298.0|-152553823.0|2022-09-01
00:16:01|2023-03-25|2022-09-07 14:59:03|s||yvuILR2iNxfe8RRml|{"student": true,
"name": "Alice", "grade": 9, "subjects": ["math", "science", "history"]}
+2023-08-15|TRUE|-91|28378|609923317|4872185586197131212|1207709464099378591|\N|-1863683325.985123|-783792012.0|-708986976.0|2022-09-24
10:39:23|2022-09-24|2022-10-16
18:36:43|Y|z|AI1BSPQdKiHJiQH1kguyLSWsDXkC7zwy7PwgWnyGSaa9tBKRex8vHBdxg2QSKZKL2mV2lHz7iI1PnsTd4MXDcIKhqiHyPuQPt2tEtgt0UgF6|{"book":
{"title": "The Great Gatsby", "author": "F. Scott Fitzgerald"}, "year": 1925}
+2023-08-27|TRUE|90|2465|702240964|6373830997821598984|305860046137409400|15991.356445|1599972327.386147|-165530947.0|\N|2023-04-26
19:31:10|2023-07-21|\N|2||B7YKYBYT8w0YC926bZ8Yz1VzyiWw2NWDAiTlEoPVyz9AXGti2Npg1FxWqWk4hEaALw0ZBSuiAIPj41lq36g5QRpPmAjNPK|{"fruit":
"apple", "color": "red", "qty": 5, "price": 2.5}
+2023-08-18|FALSE|-18|-8971|679027874|6535956962935330265|3960889045799757165|-13219.759766|1187161924.505394|-526615878.0|-947410627.0|2023-03-11
07:40:00|2022-11-29|2023-01-14
07:24:07|\N|D|3Nhx6xX1qdwaq7lxwLRSKMtJFbC03swWv12mpySSVysH3igGZTiGPuKMsYW7HAkf6CWc7c0nzqDsjuH3FYVMNCWRmfxMrmY8rykQCC4Ve|{"car":
"BMW", "model": "X5", "year": 2020, "color": "black"}
+2023-08-11|TRUE|-7|24304|-2043877415|-2024144417867729183|\N|5363.024414|-578615669.042831|-378574346.0|-810302932.0|2023-07-15
01:07:41|2023-08-13|2023-01-20 11:57:48|i||WQ9dh9ajPu0y|{"country": "France",
"capital": "Paris", "population": 67081000}
+2023-08-27|FALSE|17|-18849|1728109133|3266501886640700374|527195452623418935|-24062.328125|-1514348021.262435|-322205854.0|-278237157.0|2022-10-07
03:24:23|2022-09-25|\N|0|8|yKMiAntORoRa8svnMfcxlOPwwND1m5s2fdS26Xu6cfs6HK5SAibqIp9h8sZcpjHy4|{"team":
"Manchester United", "players": ["Ronaldo", "Rooney", "Giggs"], "coach": "Ole
Gunnar Solskjaer"}
+2023-08-17|FALSE|-5|18158|784479801|1485484354598941738|-6632681928222776815|9708.430664|-330432620.706069|-816424174.0|571112646.0|2022-09-15
21:40:55|2023-02-23|2023-08-13
21:31:54|O|X|2pYmX2vAhfEEHZZYPsgAmda1G7otnwx5TmUC879FPhDeIjvWI79ksBZpfFG2gp7jhCSbpZiecKGklB5SvG8tm31i5SUqe1xrWgLt4HSq7lMJWp75tx2kxD7pRIOpn|{"name":
"Sarah", "age": 30, "city": "London", "isMarried": false}
+2023-08-27|TRUE|22|16456|-1476824962|-3279894870153540825|8990195191470116763|26651.906250|206860148.942546|-580959198.0|-210329147.0|2022-10-07
03:11:03|2023-03-18|2023-04-15
00:38:33|T|L|QW0GQ3GoMtHgxPQOWGfVaveynahNpsNs09siMFA1OtO6QEDBQTdivmGyq7bFzejAqwbbVQQpREAmeLjcFSXLnQuou2KbwYD|{"company":
"Apple", "products": [{"name": "iPhone", "price": 1000}, {"name": "MacBook",
"price": 1500}]}
+2023-08-14|TRUE|109|-31573|-1362465190|3990845741226497177|2732763251146840270|-25698.552734|1312831962.567818|771983879.0|173937916.0|2023-03-07
14:13:19|2022-10-18|2023-07-16
05:03:13|D||PBn1wa6X8WneZYLMac11zzyhGl7tPXB5XgjmOV8L6uav9ja5oY433ktb2yhyQQIqBveZPkme|{"animal":
"lion", "weight": 200, "habitat": ["savannah", "grassland"]}
+2023-08-09|FALSE|94|31514|814994517|-297697460695940343|734910652450318597|-13061.891602|62750847.041706|-9808654.0|\N|2023-08-14
22:01:27|2023-05-19|2022-11-13
13:44:28|V||aGeMsI24O12chGlP5ak0AHghAz7bu5MargJBStHnt0yMnChH0JnfYhsfH1u59XIHkJKMsHYktBqORkGlovu8V47E74KeFpaqxn5yLyXfDbhhzUKf|{"language":
"Python", "version": 3.9, "frameworks": ["Django", "Flask"]}
+2023-08-21|FALSE|81|20302|-200761532|6365479976421007608|\N|-29916.533203|1709141750.828478|549873536.0|-119205359.0|2023-05-04
01:14:51|2022-09-17|2022-12-04
19:30:09|d|v|BKWy9dTNg1aZW7ancEJAmEDOPK5TwFsNSHbI78emu9gymeIlx5NoLmyii0QAqdzRvSQPZKiqKkwInGCTIBnK1yYkK7zD|{"username":
"user123", "password": "pass123", "email": "user...@example.com"}
+2023-08-06|TRUE|109|-6330|1479023892|-8630800697573159428|-1645095773540208759|17880.960938|-1453844792.013949|-158871820.0|-862940384.0|2022-09-22
02:03:21|2023-05-14|2023-03-25
02:18:34|m||JKnIgXvGVidGiWl9YRSi3mFI7wHKt1sBpWSadKF8VX3LAuElm4sdc9gtxREaUr57oikSYlU8We8h1MWqQlYNiJObl|{"city":
"Tokyo", "temperature": 20.5, "humidity": 75}
+2023-08-22|\N|0|-18231|1832867360|6997858407575297145|2480714305422728023|-5450.488770|1475901032.138386|-893480655.0|-607891858.0|2023-02-02
05:13:24|2022-09-18|2023-04-23
10:51:15|k||LdFXF7Kmfzgmnn2R6zLsXdmi3A2cLBLq4G4WDVNDhxvH7dYH8Kga2WA47uSIxp6NSrwPSdw0ssB1TS8RFJTDJAB0Uba3e05NL2Aiw0ja|{"restaurant":
"Pizza Hut", "menu": ["pizza", "pasta", "salad"]}
+2023-08-27|FALSE|-52|-2338|-757056972|1047567408607120856|6541476642780646552|6614.089355|-1204448798.517855|236657733.0|731515433.0|2022-12-29
14:47:30|2022-09-24|2023-08-01
12:41:59|O|F|RM4F1Ke7lkcnuxF2nK0j9VBW3MDcgyHR4pseBjtFnqS6GUkVFuzF6u3Cp9Nv7ab0O6UYrpP4DhU|{"game":
"Chess", "players": 2, "time": "1 hour"}
+2023-08-23|TRUE|-73|20117|1737338128|795638676048937749|-5551546237562433901|-30627.039062|68589475.684545|585022347.0|513722420.0|2022-12-28
20:26:51|2022-10-04|2023-07-30
00:20:06|y||keZ3JlWWpdnPBejf0cuiCQCVBBTd5gjvO08NVdcAFewqL7nRT4N9lnvSU6pWmletA5VbPQCeQapJdcnQCHfZUDCf4ulCnczyqr7SGrbGRT0XYcd7iktKM|{"country":
"Brazil", "continent": "South America", "population": 211049527}
+2023-08-24|TRUE|15|14403|\N|-6418906115745394180|9205303779366462513|-4331.548828|-615112179.557648|367305015.0|-551652958.0|2022-12-29
02:27:20|2023-06-01|2023-08-12
04:50:04|a||eCl38sztIvBQvGvGKyYZmyMXy9vIJx197iu3JwP9doJGcrYUl9Uova0rz4iCCgrjlAiZU18Fs9YtCq830nhM|{"band":
"The Beatles", "members": ["John Lennon", "Paul McCartney", "George Harrison",
"Ringo Starr"]}
+2023-08-23|FALSE|106|11492|-667795397|4480250461471356146|-5346660566234294101|9082.750000|385167225.902608|-717553011.0|649146853.0|2023-03-20
03:33:16|2022-11-24|2023-02-16
18:29:41|G|9|Lk3eNVQNjucbekD1rZmUlGPiXS5JvcWr2LQzRU8GSGIbSag|{"flower": "rose",
"color": "red", "fragrance": true}
+2023-08-27|TRUE|-104|22750|\N|8527773271030840740|5554497317268279215|-5296.828125|-1715646888.013040|-306075962.0|897769189.0|2022-12-02
17:56:44|2022-10-12|2023-02-19
07:02:54|V|\N|E9GzQdTwX1ITUQz27IVznAs6Ca4WwprKk6Odjs6SH75D2F1089QiY3HQ52LXRD1V6xAWjhLE2hWgW3EdHuAOnUDVrb5V|{"food":
"Sushi", "price": 10, "restaurant": "Sushi King"}
+2023-08-18|FALSE|63|-27847|-35409596|8638201997392767650|4919963231735304178|-23382.541016|-1803403621.426313|-22009767.0|661750756.0|2023-03-31
10:56:14|2023-01-20|2023-02-18
13:37:52|N|T|PSiFwUEx3eVFNtjlnQ70YkgZNvKrGmQ2DN5K9yYHiSdFWeEDB1UpL3Frt8z1kEAIWRDWqXZuyi|{"city":
"Sydney", "population": 5312000, "area": 2058.7}
diff --git
a/docker/thirdparties/docker-compose/kafka/scripts/basic_data_with_errors.csv
b/docker/thirdparties/docker-compose/kafka/scripts/basic_data_with_errors.csv
new file mode 100644
index 0000000000..8452e9513f
--- /dev/null
+++
b/docker/thirdparties/docker-compose/kafka/scripts/basic_data_with_errors.csv
@@ -0,0 +1,20 @@
+|2023-08-19|TRUE|2|-25462|-74112029|6458082754318544493|-7910671781690629051|-15205.859375|-306870797.484914|759730669.0|-628556336.0|2023-07-10
18:39:10|2023-02-12|2023-01-27
07:26:06|y||Xi9nDVrLv8m6AwEpUxmtzFAuK48sQ|{"name": "John", "age": 25, "city":
"New York"}
+49|2023-08-08|FALSE|16275|-2144851675|-2303421957908954634|-46526938720058765|-13141.142578|-686632233.230200|229942298.0|-152553823.0|2022-09-01
00:16:01|2023-03-25|2022-09-07 14:59:03|s||yvuILR2iNxfe8RRml|{"student": true,
"name": "Alice", "grade": 9, "subjects": ["math", "science", "history"]}
+66|2023-08-15|TRUE|redundant|-91|28378|609923317|4872185586197131212|1207709464099378591|\N|-1863683325.985123|-783792012.0|-708986976.0|2022-09-24
10:39:23|2022-09-24|2022-10-16
18:36:43|Y|z|AI1BSPQdKiHJiQH1kguyLSWsDXkC7zwy7PwgWnyGSaa9tBKRex8vHBdxg2QSKZKL2mV2lHz7iI1PnsTd4MXDcIKhqiHyPuQPt2tEtgt0UgF6|{"book":
{"title": "The Great Gatsby", "author": "F. Scott Fitzgerald"}, "year": 1925}
+91|2023-08-27|TRUE|90000000000000|2465|702240964|6373830997821598984|305860046137409400|15991.356445|1599972327.386147|-165530947.0|\N|2023-04-26
19:31:10|2023-07-21|\N|2||B7YKYBYT8w0YC926bZ8Yz1VzyiWw2NWDAiTlEoPVyz9AXGti2Npg1FxWqWk4hEaALw0ZBSuiAIPj41lq36g5QRpPmAjNPK|{"fruit":
"apple", "color": "red", "qty": 5, "price": 2.5}
+80|2023-08-18|FALSE|-18|-8971|679027874|6535956962935330265|3960889045799757165|-13219.759766|1187161924.505394|-526615878.0|-947410627.0|2023-03-11
07:40:00|2022-11-29|2023-01-14
07:24:07|\N|D|3Nhx6xX1qdwaq7lxwLRSKMtJFbC03swWv12mpySSVysH3igGZTiGPuKMsYW7HAkf6CWc7c0nzqDsjuH3FYVMNCWRmfxMrmY8rykQCC4Ve|{"car":
"BMW", "model": "X5", "year": 2020, "color": "black"}
+85|2023-08-11
10:10:10|TRUE|-7|24304|-2043877415|-2024144417867729183|\N|5363.024414|-578615669.042831|-378574346.0|-810302932.0|2023-07-15
01:07:41|2023-08-13|2023-01-20 11:57:48|i||WQ9dh9ajPu0y|{"country": "France",
"capital": "Paris", "population": 67081000}
+31|2023-08-27|FALSE|17|-18849|1728109133|3266501886640700374|527195452623418935|-24062.328125|-1514348021.262435|-322205854.0|-278237157.0|2022-10-07
03:24:23|2022-09-25|\N|0|8|yKMiAntORoRa8svnMfcxlOPwwND1m5s2fdS26Xu6cfs6HK5SAibqIp9h8sZcpjHy4|{"team":
"Manchester United", "players": ["Ronaldo", "Rooney", "Giggs"], "coach": "Ole
Gunnar Solskjaer"}
+20|2023-08-17|FALSE|-5|18158|784479801|1485484354598941738|-6632681928222776815|9708.430664|-330432620.706069|-816424174.0|571112646.00000|2022-09-15
21:40:55|2023-02-23|2023-08-13
21:31:54|O|X|2pYmX2vAhfEEHZZYPsgAmda1G7otnwx5TmUC879FPhDeIjvWI79ksBZpfFG2gp7jhCSbpZiecKGklB5SvG8tm31i5SUqe1xrWgLt4HSq7lMJWp75tx2kxD7pRIOpn|{"name":
"Sarah", "age": 30, "city": "London", "isMarried": false}
+90|2023-08-27|TRUE|22|16456|-1476824962|-3279894870153540825|8990195191470116763e|26651.906250|206860148.942546|-580959198.0|-210329147.0|2022-10-07
03:11:03|2023-03-18|2023-04-15
00:38:33|T|L|QW0GQ3GoMtHgxPQOWGfVaveynahNpsNs09siMFA1OtO6QEDBQTdivmGyq7bFzejAqwbbVQQpREAmeLjcFSXLnQuou2KbwYD|{"company":
"Apple", "products": [{"name": "iPhone", "price": 1000}, {"name": "MacBook",
"price": 1500}]}
+8|2023-08-14|TRUE|109|-31573|-1362465190|3990845741226497177|2732763251146840270|-25698.552734|1312831962.567818|771983879.0|173937916.0|2023-03-07
14:13:19|2022-10-18|2023-07-16
05:03:13|D||PBn1wa6X8WneZYLMac11zzyhGl7tPXB5XgjmOV8L6uav9ja5oY433ktb2yhyQQIqBveZPkme|{"animal":
"lion", "weight": 200, "habitat": ["savannah", "grassland"]}
+65|2023-08-09|FALSE|94|31514|814994517|-297697460695940343|734910652450318597|-13061.891602|62750847.041706|-9808654.0|\N|2023-08-14
22:01:27|2023-05-19|2022-11-13
13:44:28|V||aGeMsI24O12chGlP5ak0AHghAz7bu5MargJBStHnt0yMnChH0JnfYhsfH1u59XIHkJKMsHYktBqORkGlovu8V47E74KeFpaqxn5yLyXfDbhhzUKf|{"language":
"Python" "version": 3.9, "frameworks": ["Django", "Flask"]}
+62|2023-08-21|FALSE|81|20302|-200761532|6365479976421007608|\N|-29916.533203|1709141750.828478|549873536.0|-119205359.0|2023-05-04
01:14:51|2022-09-17|2022-12-04
19:30:09|d|v|BKWy9dTNg1aZW7ancEJAmEDOPK5TwFsNSHbI78emu9gymeIlx5NoLmyii0QAqdzRvSQPZKiqKkwInGCTIBnK1yYkK7zD|{"username":
"user123", "password": "pass123", "email": "user...@example.com"
+50|2023-08-06|TRUE|109|-6330|1479023892|-8630800697573159428|-1645095773540208759|17880.960938|-1453844792.013949|-158871820.0|-862940384.0|2022-09-22
02:03:21|2023-05-14|2023-03-25
02:18:34|m||JKnIgXvGVidGiWl9YRSi3mFI7wHKt1sBpWSadKF8VX3LAuElm4sdc9gtxREaUr57oikSYlU8We8h1MWqQlYNiJObl|{"city":
"Tokyo", "temperature": 20.5, "humidity": 75}
+58|2023-08-22|\N|0|-18231|1832867360|6997858407575297145|2480714305422728023|-5450.488770|1475901032.138386|-893480655.0|-607891858.0|2023-02-02
05:13:24|2022-09-18|2023-04-23
10:51:15|k||LdFXF7Kmfzgmnn2R6zLsXdmi3A2cLBLq4G4WDVNDhxvH7dYH8Kga2WA47uSIxp6NSrwPSdw0ssB1TS8RFJTDJAB0Uba3e05NL2Aiw0ja|{"restaurant":
"Pizza Hut", ["pizza", "pasta", "salad"]}
+60|2023-08-27|FALSE|-52|-2338|-757056972|1047567408607120856|6541476642780646552|6614.089355|-1204448798.517855|236657733.0|731515433.0|2022-12-29
14:47:30|2022-09-24|2023-08-01
12:41:59|O|F|RM4F1Ke7lkcnuxF2nK0j9VBW3MDcgyHR4pseBjtFnqS6GUkVFuzF6u3Cp9Nv7ab0O6UYrpP4DhU|{"game":
"Chess", "players": 2, "time": "1 hour"}
+68|2023-08-23|TRUE|-73|20117|1737338128|795638676048937749|-5551546237562433901|-30627.039062|68589475.684545|585022347.0|513722420.0|2022-12-28
20:26:51|2022-10-04|2023-07-30
00:20:06|y||keZ3JlWWpdnPBejf0cuiCQCVBBTd5gjvO08NVdcAFewqL7nRT4N9lnvSU6pWmletA5VbPQCeQapJdcnQCHfZUDCf4ulCnczyqr7SGrbGRT0XYcd7iktKM|{"country":
"Brazil", "continent": "South America", "population": 211049527}
+50|2023-08-24|TRUE|15|14403|\N|-6418906115745394180|9205303779366462513|-4331.548828|-615112179.557648|367305015.0|-551652958.0|2022-12-29
02:27:20|2023-06-01|2023-08-12
04:50:04|a||eCl38sztIvBQvGvGKyYZmyMXy9vIJx197iu3JwP9doJGcrYUl9Uova0rz4iCCgrjlAiZU18Fs9YtCq830nhM|{"band":
"The Beatles", "members": ["John Lennon", "Paul McCartney", "George Harrison",
"Ringo Starr"]}
+81|2023-08-23|FALSE|106|11492|-667795397|4480250461471356146|-5346660566234294101|9082.750000|385167225.902608|-717553011.0|649146853.0|2023-03-20
03:33:16|2022-11-24|2023-02-16
18:29:41|G|9|Lk3eNVQNjucbekD1rZmUlGPiXS5JvcWr2LQzRU8GSGIbSag|{"flower": "rose",
"color": "red", "fragrance": true}
+41|2023-08-27|TRUE|-104|22750|\N|8527773271030840740|5554497317268279215|-5296.828125|-1715646888.013040|-306075962.0|897769189.0|2022-12-02
17:56:44|2022-10-12|2023-02-19
07:02:54|V|\N|E9GzQdTwX1ITUQz27IVznAs6Ca4WwprKk6Odjs6SH75D2F1089QiY3HQ52LXRD1V6xAWjhLE2hWgW3EdHuAOnUDVrb5V|{"food":
"Sushi", "price": 10, "restaurant": "Sushi King"}
+21|2023-08-18|FALSE|63|-27847|-35409596|8638201997392767650|4919963231735304178|-23382.541016|-1803403621.426313|-22009767.0|661750756.0|2023-03-31
10:56:14|2023-01-20|2023-02-18
13:37:52|N|T|PSiFwUEx3eVFNtjlnQ70YkgZNvKrGmQ2DN5K9yYHiSdFWeEDB1UpL3Frt8z1kEAIWRDWqXZuyi|{"city":
"Sydney", "population": 5312000, "area": 2058.7}
diff --git a/docker/thirdparties/docker-compose/kafka/scripts/test.csv
b/docker/thirdparties/docker-compose/kafka/scripts/test.csv
deleted file mode 100644
index 3fde4e2023..0000000000
--- a/docker/thirdparties/docker-compose/kafka/scripts/test.csv
+++ /dev/null
@@ -1 +0,0 @@
-1,2,3
\ No newline at end of file
diff --git a/docker/thirdparties/run-thirdparties-docker.sh
b/docker/thirdparties/run-thirdparties-docker.sh
index c7d376a348..e52eb8b5d0 100755
--- a/docker/thirdparties/run-thirdparties-docker.sh
+++ b/docker/thirdparties/run-thirdparties-docker.sh
@@ -251,14 +251,24 @@ if [[ "${RUN_KAFKA}" -eq 1 ]]; then
sed -i "s/doris--/${CONTAINER_UID}/g"
"${ROOT}"/docker-compose/kafka/kafka.yaml
sed -i "s/localhost/${IP_HOST}/g" "${ROOT}"/docker-compose/kafka/kafka.yaml
sudo docker compose -f "${ROOT}"/docker-compose/kafka/kafka.yaml
--env-file "${ROOT}"/docker-compose/kafka/kafka.env down
+ start_kafka_producers() {
+ local container_id="$1"
+ local ip_host="$2"
+
+ declare -a topics=("basic_data" "basic_array_data"
"basic_data_with_errors" "basic_array_data_with_errors" "basic_data_timezone"
"basic_array_data_timezone")
+
+ for topic in "${topics[@]}"; do
+ while IFS= read -r line; do
+ docker exec -i "${container_id}" bash -c "echo '$line' |
/opt/kafka/bin/kafka-console-producer.sh --broker-list '${ip_host}:19193'
--topic '${topic}'"
+ sleep 1
+ done < "${ROOT}/docker-compose/kafka/scripts/${topic}.csv"
+ done
+ }
+
if [[ "${STOP}" -ne 1 ]]; then
sudo docker compose -f "${ROOT}"/docker-compose/kafka/kafka.yaml
--env-file "${ROOT}"/docker-compose/kafka/kafka.env up --build --remove-orphans
-d
sleep 30s
- while IFS= read -r line
- do
- docker exec -i ${KAFKA_CONTAINER_ID} bash -c "echo "$line" |
/opt/kafka/bin/kafka-console-producer.sh --broker-list '${IP_HOST}:19193'
--topic test"
- sleep 1
- done < ""${ROOT}"/docker-compose/kafka/scripts/test.csv"
+ start_kafka_producers "${KAFKA_CONTAINER_ID}" "${IP_HOST}"
fi
fi
diff --git a/regression-test/data/load_p0/routine_load/test_routine_load.out
b/regression-test/data/load_p0/routine_load/test_routine_load.out
new file mode 100644
index 0000000000..602d22c130
--- /dev/null
+++ b/regression-test/data/load_p0/routine_load/test_routine_load.out
@@ -0,0 +1,19 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !sql_timezone_shanghai --
+
+-- !sql_timezone_shanghai --
+
+-- !sql_timezone_shanghai --
+57 2023-08-19 true 2 -25462 -74112029
6458082754318544493 -7910671781690629051 -15205.859
-3.06870797484914E8 \N \N 2023-07-10T18:39:10 2023-02-12
2023-01-27T07:26:06 y Xi9nDVrLv8m6AwEpUxmtzFAuK48sQ
{"name":"John","age":25,"city":"New York"} true 1 2 3
4 5 6.0 7.0 888888888 999999999 2023-08-24
2023-08-24T12:00 2023-08-24 2023-08-24T12:00 我能吞下玻璃而不伤身体
我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 \N
+
+-- !sql_timezone_shanghai --
+
+-- !sql_timezone_shanghai --
+55 [1, 1, 1, 1, 1, 1] [65, 65, 65, 65, 65, 65] [18805, 18805,
18805, 18805, 18805, 18805] [229725878, 229725878, 229725878, 229725878,
229725878, 229725878] [2742856458318615325, 2742856458318615325,
2742856458318615325, 2742856458318615325, 2742856458318615325,
2742856458318615325] [5907702768956232371, 5907702768956232371,
5907702768956232371, 5907702768956232371, 5907702768956232371,
5907702768956232371] [12354.624, 12354.624, 12354.624, 12354.624, 12354.624]
[1697579881.9474771, 1697 [...]
+
+-- !sql_timezone_shanghai --
+55 [1, 1, 1, 1, 1, 1] [65, 65, 65, 65, 65, 65] [18805, 18805,
18805, 18805, 18805, 18805] [229725878, 229725878, 229725878, 229725878,
229725878, 229725878] [2742856458318615325, 2742856458318615325,
2742856458318615325, 2742856458318615325, 2742856458318615325,
2742856458318615325] [5907702768956232371, 5907702768956232371,
5907702768956232371, 5907702768956232371, 5907702768956232371,
5907702768956232371] [12354.624, 12354.624, 12354.624, 12354.624, 12354.624]
[1697579881.9474771, 1697 [...]
+
+-- !sql_timezone_shanghai --
+55 [1, 1, 1, 1, 1, 1] [65, 65, 65, 65, 65, 65] [18805, 18805,
18805, 18805, 18805, 18805] [229725878, 229725878, 229725878, 229725878,
229725878, 229725878] [2742856458318615325, 2742856458318615325,
2742856458318615325, 2742856458318615325, 2742856458318615325,
2742856458318615325] [5907702768956232371, 5907702768956232371,
5907702768956232371, 5907702768956232371, 5907702768956232371,
5907702768956232371] [12354.624, 12354.624, 12354.624, 12354.624, 12354.624]
[1697579881.9474771, 1697 [...]
+
diff --git
a/regression-test/suites/load_p0/routine_load/ddl/agg_tbl_basic_create.sql
b/regression-test/suites/load_p0/routine_load/ddl/agg_tbl_basic_create.sql
new file mode 100644
index 0000000000..a001299748
--- /dev/null
+++ b/regression-test/suites/load_p0/routine_load/ddl/agg_tbl_basic_create.sql
@@ -0,0 +1,59 @@
+CREATE TABLE routine_load_agg_tbl_basic
+(
+ k00 INT NOT NULL,
+ k01 DATE NOT NULL,
+ k02 BOOLEAN REPLACE NULL,
+ k03 TINYINT SUM NULL,
+ k04 SMALLINT SUM NULL,
+ k05 INT SUM NULL,
+ k06 BIGINT SUM NULL,
+ k07 LARGEINT SUM NULL,
+ k08 FLOAT SUM NULL,
+ k09 DOUBLE SUM NULL,
+ k10 DECIMAL(9,1) SUM NULL,
+ k11 DECIMALV3(9,1) SUM NULL,
+ k12 DATETIME REPLACE NULL,
+ k13 DATEV2 REPLACE NULL,
+ k14 DATETIMEV2 REPLACE NULL,
+ k15 CHAR(300) REPLACE NULL,
+ k16 VARCHAR(300) REPLACE NULL,
+ k17 STRING REPLACE NULL,
+ k18 JSON REPLACE NULL,
+ k19 BITMAP BITMAP_UNION NULL,
+ k20 HLL HLL_UNION NULL,
+ k21 QUANTILE_STATE QUANTILE_UNION NULL,
+ kd01 BOOLEAN REPLACE NOT NULL DEFAULT "TRUE",
+ kd02 TINYINT SUM NOT NULL DEFAULT "1",
+ kd03 SMALLINT SUM NOT NULL DEFAULT "2",
+ kd04 INT SUM NOT NULL DEFAULT "3",
+ kd05 BIGINT SUM NOT NULL DEFAULT "4",
+ kd06 LARGEINT SUM NOT NULL DEFAULT "5",
+ kd07 FLOAT SUM NOT NULL DEFAULT "6.0",
+ kd08 DOUBLE SUM NOT NULL DEFAULT "7.0",
+ kd09 DECIMAL SUM NOT NULL DEFAULT "888888888",
+ kd10 DECIMALV3 SUM NOT NULL DEFAULT "999999999",
+ kd11 DATE REPLACE NOT NULL DEFAULT "2023-08-24",
+ kd12 DATETIME REPLACE NOT NULL DEFAULT "2023-08-24 12:00:00",
+ kd13 DATEV2 REPLACE NOT NULL DEFAULT "2023-08-24",
+ kd14 DATETIMEV2 REPLACE NOT NULL DEFAULT "2023-08-24 12:00:00",
+ kd15 CHAR(300) REPLACE NOT NULL DEFAULT "我能吞下玻璃而不伤身体",
+ kd16 VARCHAR(300) REPLACE NOT NULL DEFAULT "我能吞下玻璃而不伤身体",
+ kd17 STRING REPLACE NOT NULL DEFAULT "我能吞下玻璃而不伤身体",
+ kd18 JSON REPLACE NULL,
+ kd19 BITMAP BITMAP_UNION NULL,
+ kd20 HLL HLL_UNION NULL,
+ kd21 QUANTILE_STATE QUANTILE_UNION NULL,
+
+ INDEX idx_bitmap_k104 (`k01`) USING BITMAP
+)
+AGGREGATE KEY(k00,k01)
+PARTITION BY RANGE(k01)
+(
+ PARTITION p1 VALUES [('2023-08-01'), ('2023-08-11')),
+ PARTITION p2 VALUES [('2023-08-11'), ('2023-08-21')),
+ PARTITION p3 VALUES [('2023-08-21'), ('2023-09-01'))
+)
+DISTRIBUTED BY HASH(k00) BUCKETS 32
+PROPERTIES (
+ "replication_num" = "1"
+);
\ No newline at end of file
diff --git
a/regression-test/suites/load_p0/routine_load/ddl/agg_tbl_basic_drop.sql
b/regression-test/suites/load_p0/routine_load/ddl/agg_tbl_basic_drop.sql
new file mode 100644
index 0000000000..81a4954fa1
--- /dev/null
+++ b/regression-test/suites/load_p0/routine_load/ddl/agg_tbl_basic_drop.sql
@@ -0,0 +1 @@
+DROP TABLE IF EXISTS routine_load_agg_tbl_basic;
\ No newline at end of file
diff --git
a/regression-test/suites/load_p0/routine_load/ddl/dup_tbl_array_create.sql
b/regression-test/suites/load_p0/routine_load/ddl/dup_tbl_array_create.sql
new file mode 100644
index 0000000000..b0c3a2bd7f
--- /dev/null
+++ b/regression-test/suites/load_p0/routine_load/ddl/dup_tbl_array_create.sql
@@ -0,0 +1,42 @@
+CREATE TABLE routine_load_dup_tbl_array
+(
+ k00 INT NOT NULL,
+ k01 array<BOOLEAN> NULL,
+ k02 array<TINYINT> NULL,
+ k03 array<SMALLINT> NULL,
+ k04 array<INT> NULL,
+ k05 array<BIGINT> NULL,
+ k06 array<LARGEINT> NULL,
+ k07 array<FLOAT> NULL,
+ k08 array<DOUBLE> NULL,
+ k09 array<DECIMAL> NULL,
+ k10 array<DECIMALV3> NULL,
+ k11 array<DATE> NULL,
+ k12 array<DATETIME> NULL,
+ k13 array<DATEV2> NULL,
+ k14 array<DATETIMEV2> NULL,
+ k15 array<CHAR> NULL,
+ k16 array<VARCHAR> NULL,
+ k17 array<STRING> NULL,
+ kd01 array<BOOLEAN> NOT NULL DEFAULT "[]",
+ kd02 array<TINYINT> NOT NULL DEFAULT "[]",
+ kd03 array<SMALLINT> NOT NULL DEFAULT "[]",
+ kd04 array<INT> NOT NULL DEFAULT "[]",
+ kd05 array<BIGINT> NOT NULL DEFAULT "[]",
+ kd06 array<LARGEINT> NOT NULL DEFAULT "[]",
+ kd07 array<FLOAT> NOT NULL DEFAULT "[]",
+ kd08 array<DOUBLE> NOT NULL DEFAULT "[]",
+ kd09 array<DECIMAL> NOT NULL DEFAULT "[]",
+ kd10 array<DECIMALV3> NOT NULL DEFAULT "[]",
+ kd11 array<DATE> NOT NULL DEFAULT "[]",
+ kd12 array<DATETIME> NOT NULL DEFAULT "[]",
+ kd13 array<DATEV2> NOT NULL DEFAULT "[]",
+ kd14 array<DATETIMEV2> NOT NULL DEFAULT "[]",
+ kd15 array<CHAR> NOT NULL DEFAULT "[]",
+ kd16 array<VARCHAR> NOT NULL DEFAULT "[]",
+)
+DUPLICATE KEY(k00)
+DISTRIBUTED BY HASH(k00) BUCKETS 32
+PROPERTIES (
+ "replication_num" = "1"
+);
\ No newline at end of file
diff --git
a/regression-test/suites/load_p0/routine_load/ddl/dup_tbl_array_drop.sql
b/regression-test/suites/load_p0/routine_load/ddl/dup_tbl_array_drop.sql
new file mode 100644
index 0000000000..63f253219f
--- /dev/null
+++ b/regression-test/suites/load_p0/routine_load/ddl/dup_tbl_array_drop.sql
@@ -0,0 +1 @@
+DROP TABLE IF EXISTS routine_load_dup_tbl_array;
\ No newline at end of file
diff --git
a/regression-test/suites/load_p0/routine_load/ddl/dup_tbl_basic_create.sql
b/regression-test/suites/load_p0/routine_load/ddl/dup_tbl_basic_create.sql
new file mode 100644
index 0000000000..85fa379018
--- /dev/null
+++ b/regression-test/suites/load_p0/routine_load/ddl/dup_tbl_basic_create.sql
@@ -0,0 +1,65 @@
+CREATE TABLE routine_load_dup_tbl_basic
+(
+ k00 INT NOT NULL,
+ k01 DATE NOT NULL,
+ k02 BOOLEAN NULL,
+ k03 TINYINT NULL,
+ k04 SMALLINT NULL,
+ k05 INT NULL,
+ k06 BIGINT NULL,
+ k07 LARGEINT NULL,
+ k08 FLOAT NULL,
+ k09 DOUBLE NULL,
+ k10 DECIMAL(9,1) NULL,
+ k11 DECIMALV3(9,1) NULL,
+ k12 DATETIME NULL,
+ k13 DATEV2 NULL,
+ k14 DATETIMEV2 NULL,
+ k15 CHAR NULL,
+ k16 VARCHAR NULL,
+ k17 STRING NULL,
+ k18 JSON NULL,
+ kd01 BOOLEAN NOT NULL DEFAULT "TRUE",
+ kd02 TINYINT NOT NULL DEFAULT "1",
+ kd03 SMALLINT NOT NULL DEFAULT "2",
+ kd04 INT NOT NULL DEFAULT "3",
+ kd05 BIGINT NOT NULL DEFAULT "4",
+ kd06 LARGEINT NOT NULL DEFAULT "5",
+ kd07 FLOAT NOT NULL DEFAULT "6.0",
+ kd08 DOUBLE NOT NULL DEFAULT "7.0",
+ kd09 DECIMAL NOT NULL DEFAULT "888888888",
+ kd10 DECIMALV3 NOT NULL DEFAULT "999999999",
+ kd11 DATE NOT NULL DEFAULT "2023-08-24",
+ kd12 DATETIME NOT NULL DEFAULT "2023-08-24 12:00:00",
+ kd13 DATEV2 NOT NULL DEFAULT "2023-08-24",
+ kd14 DATETIMEV2 NOT NULL DEFAULT "2023-08-24 12:00:00",
+ kd15 CHAR(300) NOT NULL DEFAULT "我能吞下玻璃而不伤身体",
+ kd16 VARCHAR(300) NOT NULL DEFAULT "我能吞下玻璃而不伤身体",
+ kd17 STRING NOT NULL DEFAULT "我能吞下玻璃而不伤身体",
+ kd18 JSON NULL,
+
+ INDEX idx_inverted_k104 (`k05`) USING INVERTED,
+ INDEX idx_inverted_k110 (`k11`) USING INVERTED,
+ INDEX idx_inverted_k113 (`k13`) USING INVERTED,
+ INDEX idx_inverted_k114 (`k14`) USING INVERTED,
+ INDEX idx_inverted_k117 (`k17`) USING INVERTED PROPERTIES("parser" =
"english"),
+ INDEX idx_ngrambf_k115 (`k15`) USING NGRAM_BF PROPERTIES("gram_size"="3",
"bf_size"="256"),
+ INDEX idx_ngrambf_k116 (`k16`) USING NGRAM_BF PROPERTIES("gram_size"="3",
"bf_size"="256"),
+ INDEX idx_ngrambf_k117 (`k17`) USING NGRAM_BF PROPERTIES("gram_size"="3",
"bf_size"="256"),
+
+ INDEX idx_bitmap_k104 (`k02`) USING BITMAP,
+ INDEX idx_bitmap_k110 (`kd01`) USING BITMAP,
+
+)
+DUPLICATE KEY(k00)
+PARTITION BY RANGE(k01)
+(
+ PARTITION p1 VALUES [('2023-08-01'), ('2023-08-11')),
+ PARTITION p2 VALUES [('2023-08-11'), ('2023-08-21')),
+ PARTITION p3 VALUES [('2023-08-21'), ('2023-09-01'))
+)
+DISTRIBUTED BY HASH(k00) BUCKETS 32
+PROPERTIES (
+ "bloom_filter_columns"="k05",
+ "replication_num" = "1"
+);
\ No newline at end of file
diff --git
a/regression-test/suites/load_p0/routine_load/ddl/dup_tbl_basic_drop.sql
b/regression-test/suites/load_p0/routine_load/ddl/dup_tbl_basic_drop.sql
new file mode 100644
index 0000000000..0a023935b7
--- /dev/null
+++ b/regression-test/suites/load_p0/routine_load/ddl/dup_tbl_basic_drop.sql
@@ -0,0 +1 @@
+DROP TABLE IF EXISTS routine_load_dup_tbl_basic;
\ No newline at end of file
diff --git
a/regression-test/suites/load_p0/routine_load/ddl/mow_tbl_array_create.sql
b/regression-test/suites/load_p0/routine_load/ddl/mow_tbl_array_create.sql
new file mode 100644
index 0000000000..14e62bb7ac
--- /dev/null
+++ b/regression-test/suites/load_p0/routine_load/ddl/mow_tbl_array_create.sql
@@ -0,0 +1,44 @@
+CREATE TABLE routine_load_mow_tbl_array
+(
+ k00 INT NOT NULL,
+ k01 array<BOOLEAN> NULL,
+ k02 array<TINYINT> NULL,
+ k03 array<SMALLINT> NULL,
+ k04 array<INT> NULL,
+ k05 array<BIGINT> NULL,
+ k06 array<LARGEINT> NULL,
+ k07 array<FLOAT> NULL,
+ k08 array<DOUBLE> NULL,
+ k09 array<DECIMAL> NULL,
+ k10 array<DECIMALV3> NULL,
+ k11 array<DATE> NULL,
+ k12 array<DATETIME> NULL,
+ k13 array<DATEV2> NULL,
+ k14 array<DATETIMEV2> NULL,
+ k15 array<CHAR> NULL,
+ k16 array<VARCHAR> NULL,
+ k17 array<STRING> NULL,
+ kd01 array<BOOLEAN> NOT NULL DEFAULT "[]",
+ kd02 array<TINYINT> NOT NULL DEFAULT "[]",
+ kd03 array<SMALLINT> NOT NULL DEFAULT "[]",
+ kd04 array<INT> NOT NULL DEFAULT "[]",
+ kd05 array<BIGINT> NOT NULL DEFAULT "[]",
+ kd06 array<LARGEINT> NOT NULL DEFAULT "[]",
+ kd07 array<FLOAT> NOT NULL DEFAULT "[]",
+ kd08 array<DOUBLE> NOT NULL DEFAULT "[]",
+ kd09 array<DECIMAL> NOT NULL DEFAULT "[]",
+ kd10 array<DECIMALV3> NOT NULL DEFAULT "[]",
+ kd11 array<DATE> NOT NULL DEFAULT "[]",
+ kd12 array<DATETIME> NOT NULL DEFAULT "[]",
+ kd13 array<DATEV2> NOT NULL DEFAULT "[]",
+ kd14 array<DATETIMEV2> NOT NULL DEFAULT "[]",
+ kd15 array<CHAR> NOT NULL DEFAULT "[]",
+ kd16 array<VARCHAR> NOT NULL DEFAULT "[]",
+ kd17 array<STRING> NOT NULL DEFAULT "[]"
+)
+UNIQUE KEY(k00)
+DISTRIBUTED BY HASH(k00) BUCKETS 32
+PROPERTIES (
+ "replication_num" = "1",
+ "enable_unique_key_merge_on_write" = "true"
+);
\ No newline at end of file
diff --git
a/regression-test/suites/load_p0/routine_load/ddl/mow_tbl_array_drop.sql
b/regression-test/suites/load_p0/routine_load/ddl/mow_tbl_array_drop.sql
new file mode 100644
index 0000000000..6c56c68507
--- /dev/null
+++ b/regression-test/suites/load_p0/routine_load/ddl/mow_tbl_array_drop.sql
@@ -0,0 +1 @@
+DROP TABLE IF EXISTS routine_load_mow_tbl_array;
\ No newline at end of file
diff --git
a/regression-test/suites/load_p0/routine_load/ddl/mow_tbl_basic_create.sql
b/regression-test/suites/load_p0/routine_load/ddl/mow_tbl_basic_create.sql
new file mode 100644
index 0000000000..2c9b472cda
--- /dev/null
+++ b/regression-test/suites/load_p0/routine_load/ddl/mow_tbl_basic_create.sql
@@ -0,0 +1,67 @@
+CREATE TABLE routine_load_mow_tbl_basic
+(
+ k00 INT NOT NULL,
+ k01 DATE NULL,
+ k02 BOOLEAN NULL,
+ k03 TINYINT NULL,
+ k04 SMALLINT NULL,
+ k05 INT NULL,
+ k06 BIGINT NULL,
+ k07 LARGEINT NULL,
+ k08 FLOAT NULL,
+ k09 DOUBLE NULL,
+ k10 DECIMAL(9,1) NULL,
+ k11 DECIMALV3(9,1) NULL,
+ k12 DATETIME NULL,
+ k13 DATEV2 NULL,
+ k14 DATETIMEV2 NULL,
+ k15 CHAR NULL,
+ k16 VARCHAR NULL,
+ k17 STRING NULL,
+ k18 JSON NULL,
+ kd01 BOOLEAN NOT NULL DEFAULT "TRUE",
+ kd02 TINYINT NOT NULL DEFAULT "1",
+ kd03 SMALLINT NOT NULL DEFAULT "2",
+ kd04 INT NOT NULL DEFAULT "3",
+ kd05 BIGINT NOT NULL DEFAULT "4",
+ kd06 LARGEINT NOT NULL DEFAULT "5",
+ kd07 FLOAT NOT NULL DEFAULT "6.0",
+ kd08 DOUBLE NOT NULL DEFAULT "7.0",
+ kd09 DECIMAL NOT NULL DEFAULT "888888888",
+ kd10 DECIMALV3 NOT NULL DEFAULT "999999999",
+ kd11 DATE NOT NULL DEFAULT "2023-08-24",
+ kd12 DATETIME NOT NULL DEFAULT "2023-08-24 12:00:00",
+ kd13 DATEV2 NOT NULL DEFAULT "2023-08-24",
+ kd14 DATETIMEV2 NOT NULL DEFAULT "2023-08-24 12:00:00",
+ kd15 CHAR(300) NOT NULL DEFAULT "我能吞下玻璃而不伤身体",
+ kd16 VARCHAR(300) NOT NULL DEFAULT "我能吞下玻璃而不伤身体",
+ kd17 STRING NOT NULL DEFAULT "我能吞下玻璃而不伤身体",
+ kd18 JSON NULL,
+
+ INDEX idx_inverted_k104 (`k05`) USING INVERTED,
+ INDEX idx_inverted_k110 (`k11`) USING INVERTED,
+ INDEX idx_inverted_k113 (`k13`) USING INVERTED,
+ INDEX idx_inverted_k114 (`k14`) USING INVERTED,
+ INDEX idx_inverted_k117 (`k17`) USING INVERTED PROPERTIES("parser" =
"english"),
+ INDEX idx_bitmap_k104 (`k05`) USING BITMAP,
+ INDEX idx_bitmap_k110 (`k11`) USING BITMAP,
+ INDEX idx_bitmap_k113 (`k13`) USING BITMAP,
+ INDEX idx_bitmap_k114 (`k14`) USING BITMAP,
+ INDEX idx_bitmap_k117 (`k17`) USING BITMAP,
+ INDEX idx_ngrambf_k115 (`k15`) USING NGRAM_BF PROPERTIES("gram_size"="3",
"bf_size"="256"),
+ INDEX idx_ngrambf_k116 (`k16`) USING NGRAM_BF PROPERTIES("gram_size"="3",
"bf_size"="256"),
+ INDEX idx_ngrambf_k117 (`k17`) USING NGRAM_BF PROPERTIES("gram_size"="3",
"bf_size"="256")
+)
+UNIQUE KEY(k00,k01)
+PARTITION BY RANGE(k01)
+(
+ PARTITION p1 VALUES [('2023-08-01'), ('2023-08-11')),
+ PARTITION p2 VALUES [('2023-08-11'), ('2023-08-21')),
+ PARTITION p3 VALUES [('2023-08-21'), ('2023-09-01'))
+)
+DISTRIBUTED BY HASH(k00) BUCKETS 32
+PROPERTIES (
+ "bloom_filter_columns"="k05",
+ "replication_num" = "1",
+ "enable_unique_key_merge_on_write" = "true"
+);
\ No newline at end of file
diff --git
a/regression-test/suites/load_p0/routine_load/ddl/mow_tbl_basic_drop.sql
b/regression-test/suites/load_p0/routine_load/ddl/mow_tbl_basic_drop.sql
new file mode 100644
index 0000000000..e0949e81fe
--- /dev/null
+++ b/regression-test/suites/load_p0/routine_load/ddl/mow_tbl_basic_drop.sql
@@ -0,0 +1 @@
+DROP TABLE IF EXISTS routine_load_mow_tbl_basic;
\ No newline at end of file
diff --git
a/regression-test/suites/load_p0/routine_load/ddl/uniq_tbl_array_create.sql
b/regression-test/suites/load_p0/routine_load/ddl/uniq_tbl_array_create.sql
new file mode 100644
index 0000000000..c5b4f07ff8
--- /dev/null
+++ b/regression-test/suites/load_p0/routine_load/ddl/uniq_tbl_array_create.sql
@@ -0,0 +1,43 @@
+CREATE TABLE routine_load_uniq_tbl_array
+(
+ k00 INT NOT NULL,
+ k01 array<BOOLEAN> NULL,
+ k02 array<TINYINT> NULL,
+ k03 array<SMALLINT> NULL,
+ k04 array<INT> NULL,
+ k05 array<BIGINT> NULL,
+ k06 array<LARGEINT> NULL,
+ k07 array<FLOAT> NULL,
+ k08 array<DOUBLE> NULL,
+ k09 array<DECIMAL> NULL,
+ k10 array<DECIMALV3> NULL,
+ k11 array<DATE> NULL,
+ k12 array<DATETIME> NULL,
+ k13 array<DATEV2> NULL,
+ k14 array<DATETIMEV2> NULL,
+ k15 array<CHAR> NULL,
+ k16 array<VARCHAR> NULL,
+ k17 array<STRING> NULL,
+ kd01 array<BOOLEAN> NOT NULL DEFAULT "[]",
+ kd02 array<TINYINT> NOT NULL DEFAULT "[]",
+ kd03 array<SMALLINT> NOT NULL DEFAULT "[]",
+ kd04 array<INT> NOT NULL DEFAULT "[]",
+ kd05 array<BIGINT> NOT NULL DEFAULT "[]",
+ kd06 array<LARGEINT> NOT NULL DEFAULT "[]",
+ kd07 array<FLOAT> NOT NULL DEFAULT "[]",
+ kd08 array<DOUBLE> NOT NULL DEFAULT "[]",
+ kd09 array<DECIMAL> NOT NULL DEFAULT "[]",
+ kd10 array<DECIMALV3> NOT NULL DEFAULT "[]",
+ kd11 array<DATE> NOT NULL DEFAULT "[]",
+ kd12 array<DATETIME> NOT NULL DEFAULT "[]",
+ kd13 array<DATEV2> NOT NULL DEFAULT "[]",
+ kd14 array<DATETIMEV2> NOT NULL DEFAULT "[]",
+ kd15 array<CHAR> NOT NULL DEFAULT "[]",
+ kd16 array<VARCHAR> NOT NULL DEFAULT "[]",
+ kd17 array<STRING> NOT NULL DEFAULT "[]",
+)
+UNIQUE KEY(k00)
+DISTRIBUTED BY HASH(k00) BUCKETS 32
+PROPERTIES (
+ "replication_num" = "1"
+);
\ No newline at end of file
diff --git
a/regression-test/suites/load_p0/routine_load/ddl/uniq_tbl_array_drop.sql
b/regression-test/suites/load_p0/routine_load/ddl/uniq_tbl_array_drop.sql
new file mode 100644
index 0000000000..e8193bf390
--- /dev/null
+++ b/regression-test/suites/load_p0/routine_load/ddl/uniq_tbl_array_drop.sql
@@ -0,0 +1 @@
+DROP TABLE IF EXISTS routine_load_uniq_tbl_array;
\ No newline at end of file
diff --git
a/regression-test/suites/load_p0/routine_load/ddl/uniq_tbl_basic_create.sql
b/regression-test/suites/load_p0/routine_load/ddl/uniq_tbl_basic_create.sql
new file mode 100644
index 0000000000..97b8cfd147
--- /dev/null
+++ b/regression-test/suites/load_p0/routine_load/ddl/uniq_tbl_basic_create.sql
@@ -0,0 +1,57 @@
+CREATE TABLE routine_load_uniq_tbl_basic
+(
+ k00 INT NOT NULL,
+ k01 DATE NOT NULL,
+ k02 BOOLEAN NULL,
+ k03 TINYINT NULL,
+ k04 SMALLINT NULL,
+ k05 INT NULL,
+ k06 BIGINT NULL,
+ k07 LARGEINT NULL,
+ k08 FLOAT NULL,
+ k09 DOUBLE NULL,
+ k10 DECIMAL(9,1) NULL,
+ k11 DECIMALV3(9,1) NULL,
+ k12 DATETIME NULL,
+ k13 DATEV2 NULL,
+ k14 DATETIMEV2 NULL,
+ k15 CHAR NULL,
+ k16 VARCHAR NULL,
+ k17 STRING NULL,
+ k18 JSON NULL,
+ kd01 BOOLEAN NOT NULL DEFAULT "TRUE",
+ kd02 TINYINT NOT NULL DEFAULT "1",
+ kd03 SMALLINT NOT NULL DEFAULT "2",
+ kd04 INT NOT NULL DEFAULT "3",
+ kd05 BIGINT NOT NULL DEFAULT "4",
+ kd06 LARGEINT NOT NULL DEFAULT "5",
+ kd07 FLOAT NOT NULL DEFAULT "6.0",
+ kd08 DOUBLE NOT NULL DEFAULT "7.0",
+ kd09 DECIMAL NOT NULL DEFAULT "888888888",
+ kd10 DECIMALV3 NOT NULL DEFAULT "999999999",
+ kd11 DATE NOT NULL DEFAULT "2023-08-24",
+ kd12 DATETIME NOT NULL DEFAULT "2023-08-24 12:00:00",
+ kd13 DATEV2 NOT NULL DEFAULT "2023-08-24",
+ kd14 DATETIMEV2 NOT NULL DEFAULT "2023-08-24 12:00:00",
+ kd15 CHAR(300) NOT NULL DEFAULT "我能吞下玻璃而不伤身体",
+ kd16 VARCHAR(300) NOT NULL DEFAULT "我能吞下玻璃而不伤身体",
+ kd17 STRING NOT NULL DEFAULT "我能吞下玻璃而不伤身体",
+ kd18 JSON NULL,
+
+ INDEX idx_bitmap_k104 (`k02`) USING BITMAP,
+ INDEX idx_bitmap_k110 (`kd01`) USING BITMAP,
+ INDEX idx_bitmap_k113 (`k13`) USING BITMAP,
+ INDEX idx_bitmap_k114 (`k14`) USING BITMAP,
+ INDEX idx_bitmap_k117 (`k17`) USING BITMAP
+)
+UNIQUE KEY(k00,k01)
+PARTITION BY RANGE(k01)
+(
+ PARTITION p1 VALUES [('2023-08-01'), ('2023-08-11')),
+ PARTITION p2 VALUES [('2023-08-11'), ('2023-08-21')),
+ PARTITION p3 VALUES [('2023-08-21'), ('2023-09-01'))
+)
+DISTRIBUTED BY HASH(k00) BUCKETS 32
+PROPERTIES (
+ "replication_num" = "1"
+);
\ No newline at end of file
diff --git
a/regression-test/suites/load_p0/routine_load/ddl/uniq_tbl_basic_drop.sql
b/regression-test/suites/load_p0/routine_load/ddl/uniq_tbl_basic_drop.sql
new file mode 100644
index 0000000000..5d1a486db7
--- /dev/null
+++ b/regression-test/suites/load_p0/routine_load/ddl/uniq_tbl_basic_drop.sql
@@ -0,0 +1 @@
+DROP TABLE IF EXISTS routine_load_uniq_tbl_basic;
diff --git
a/regression-test/suites/load_p0/routine_load/test_routine_load.groovy
b/regression-test/suites/load_p0/routine_load/test_routine_load.groovy
index 751a8d0280..95122e2e1f 100644
--- a/regression-test/suites/load_p0/routine_load/test_routine_load.groovy
+++ b/regression-test/suites/load_p0/routine_load/test_routine_load.groovy
@@ -16,46 +16,469 @@
// under the License.
suite("test_routine_load","p0,external,external_docker,external_docker_routine_load")
{
- // todo: test routine load, need kafka
- // sql "show routine load"
- // define a sql table
- def testTable = "tbl_test_routine_load_basic"
-
- def result1 = sql """
- CREATE TABLE IF NOT EXISTS ${testTable} (
- `k1` int(20) NULL COMMENT "",
- `k2` int(20) NULL COMMENT "",
- `k3` int(20) NULL COMMENT "",
- ) ENGINE=OLAP
- DUPLICATE KEY(`k1`)
- DISTRIBUTED BY HASH(`k1`) BUCKETS 1
- PROPERTIES (
- "replication_allocation" = "tag.location.default: 1",
- "storage_format" = "V2"
- )
- """
+
+ def tables = [
+ "dup_tbl_basic",
+ "uniq_tbl_basic",
+ "mow_tbl_basic",
+ "agg_tbl_basic",
+ "dup_tbl_array",
+ "uniq_tbl_array",
+ "mow_tbl_array",
+ ]
+
+ def jobs = [
+ "dup_tbl_basic_job",
+ "uniq_tbl_basic_job",
+ "mow_tbl_basic_job",
+ "agg_tbl_basic_job",
+ "dup_tbl_array_job",
+ "uniq_tbl_array_job",
+ "mow_tbl_array_job",
+ ]
+
+ def topics = [
+ "basic_data",
+ "basic_data",
+ "basic_data",
+ "basic_data",
+ "basic_array_data",
+ "basic_array_data",
+ "basic_array_data",
+ ]
+
+ def errorTopics = [
+ "basic_data_with_errors",
+ "basic_data_with_errors",
+ "basic_data_with_errors",
+ "basic_data_with_errors",
+ "basic_array_data_with_errors",
+ "basic_array_data_with_errors",
+ "basic_array_data_with_errors",
+ ]
+
+ def timezoneTopics = [
+ "basic_data_timezone",
+ "basic_data_timezone",
+ "basic_data_timezone",
+ "basic_data_timezone",
+ "basic_array_data_timezone",
+ "basic_array_data_timezone",
+ "basic_array_data_timezone",
+ ]
+
+ def columns = [
+
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18",
+
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18",
+
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18",
+
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18,k19=to_bitmap(k04),k20=HLL_HASH(k04),k21=TO_QUANTILE_STATE(k04,1.0),kd19=to_bitmap(k05),kd20=HLL_HASH(k05),kd21=TO_QUANTILE_STATE(k05,1.0)",
+
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17",
+
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17",
+
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17",
+ ]
+
+ def timezoneColumns =
+ [
+ "k00=unix_timestamp('2007-11-30
10:30:19'),k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18",
+ "k00=unix_timestamp('2007-11-30
10:30:19'),k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18",
+ "k00=unix_timestamp('2007-11-30
10:30:19'),k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18",
+ "k00=unix_timestamp('2007-11-30
10:30:19'),k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18,k19=to_bitmap(k04),k20=HLL_HASH(k04),k21=TO_QUANTILE_STATE(k04,1.0),kd19=to_bitmap(k05),kd20=HLL_HASH(k05),kd21=TO_QUANTILE_STATE(k05,1.0)",
+ "k00=unix_timestamp('2007-11-30
10:30:19'),k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17",
+ "k00=unix_timestamp('2007-11-30
10:30:19'),k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17",
+ "k00=unix_timestamp('2007-11-30
10:30:19'),k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17",
+ ]
+
+ def loadedRows = [0,0,0,0,17,17,17]
+
+ def filteredRows = [20,20,20,20,3,3,3]
+
+ def maxFilterRatio = [1,1,1,1,0.15,0.15,0.15]
String enabled = context.config.otherConfigs.get("enableKafkaTest")
+ String kafka_port = context.config.otherConfigs.get("kafka_port")
+ String externalEnvIp = context.config.otherConfigs.get("externalEnvIp")
+
+ // exec_mem_limit
+ def i = 0
+ if (enabled != null && enabled.equalsIgnoreCase("true")) {
+ try {
+ for (String tableName in tables) {
+ sql new
File("""${context.file.parent}/ddl/${tableName}_drop.sql""").text
+ sql new
File("""${context.file.parent}/ddl/${tableName}_create.sql""").text
+
+ def name = "routine_load_" + tableName
+ sql """
+ CREATE ROUTINE LOAD ${jobs[i]} ON ${name}
+ COLUMNS(${columns[i]}),
+ COLUMNS TERMINATED BY "|"
+ PROPERTIES
+ (
+ "exec_mem_limit" = "1",
+ "max_batch_interval" = "1",
+ "max_batch_rows" = "300000",
+ "max_batch_size" = "209715200"
+ )
+ FROM KAFKA
+ (
+ "kafka_broker_list" = "${externalEnvIp}:${kafka_port}",
+ "kafka_topic" = "${topics[i]}",
+ "property.kafka_default_offsets" = "OFFSET_BEGINNING"
+ );
+ """
+ sql "sync"
+ i++
+ }
+
+ i = 0
+ for (String tableName in tables) {
+ while (true) {
+ sleep(1000)
+ def res = sql "show routine load for ${jobs[i]}"
+ def state = res[0][8].toString()
+ if (state == "NEED_SCHEDULE") {
+ continue;
+ }
+ assertEquals(res[0][8].toString(), "RUNNING")
+ break;
+ }
+ sql "stop routine load for ${jobs[i]}"
+ i++
+ }
+ } finally {
+ for (String tableName in tables) {
+ sql new
File("""${context.file.parent}/ddl/${tableName}_drop.sql""").text
+ }
+ }
+ }
+
+ // timezone
+ i = 0
+ if (enabled != null && enabled.equalsIgnoreCase("true")) {
+ try {
+ for (String tableName in tables) {
+ sql new
File("""${context.file.parent}/ddl/${tableName}_drop.sql""").text
+ sql new
File("""${context.file.parent}/ddl/${tableName}_create.sql""").text
+
+ def name = "routine_load_" + tableName
+ sql """
+ CREATE ROUTINE LOAD ${jobs[i]} ON ${name}
+ COLUMNS(${columns[i]}),
+ COLUMNS TERMINATED BY "|"
+ PROPERTIES
+ (
+ "timezone" = "Asia/Shanghai",
+ "max_batch_interval" = "1",
+ "max_batch_rows" = "300000",
+ "max_batch_size" = "209715200"
+ )
+ FROM KAFKA
+ (
+ "kafka_broker_list" = "${externalEnvIp}:${kafka_port}",
+ "kafka_topic" = "${topics[i]}",
+ "property.kafka_default_offsets" = "OFFSET_BEGINNING"
+ );
+ """
+ sql "sync"
+ i++
+ }
+
+ i = 0
+ for (String tableName in tables) {
+ while (true) {
+ sleep(1000)
+ def res = sql "show routine load for ${jobs[i]}"
+ def state = res[0][8].toString()
+ if (state == "NEED_SCHEDULE") {
+ continue;
+ }
+ assertEquals(res[0][8].toString(), "RUNNING")
+ break;
+ }
+
+ def tableName1 = "routine_load_" + tableName
+ if (i <= 3) {
+ qt_sql_timezone_shanghai "select * from ${tableName1}
order by k00,k01"
+ } else {
+ qt_sql_timezone_shanghai "select * from ${tableName1}
order by k00"
+ }
+
+ sql "stop routine load for ${jobs[i]}"
+ i++
+ }
+ } finally {
+ for (String tableName in tables) {
+ sql new
File("""${context.file.parent}/ddl/${tableName}_drop.sql""").text
+ }
+ }
+ }
+
+ //strict_mode
+ i = 0
+ if (enabled != null && enabled.equalsIgnoreCase("true")) {
+ try {
+ for (String tableName in tables) {
+ sql new
File("""${context.file.parent}/ddl/${tableName}_drop.sql""").text
+ sql new
File("""${context.file.parent}/ddl/${tableName}_create.sql""").text
+
+ def name = "routine_load_" + tableName
+ sql """
+ CREATE ROUTINE LOAD ${jobs[i]} ON ${name}
+ COLUMNS(${columns[i]}),
+ COLUMNS TERMINATED BY "|"
+ PROPERTIES
+ (
+ "strict_mode" = "true",
+ "max_batch_interval" = "1",
+ "max_batch_rows" = "300000",
+ "max_batch_size" = "209715200"
+ )
+ FROM KAFKA
+ (
+ "kafka_broker_list" = "${externalEnvIp}:${kafka_port}",
+ "kafka_topic" = "${errorTopics[i]}",
+ "property.kafka_default_offsets" = "OFFSET_BEGINNING"
+ );
+ """
+ sql "sync"
+ i++
+ }
+
+ i = 0
+ for (String tableName in tables) {
+ while (true) {
+ sleep(1000)
+ def res = sql "show routine load for ${jobs[i]}"
+ def state = res[0][8].toString()
+ if (state == "NEED_SCHEDULE") {
+ continue;
+ }
+ assertEquals(res[0][8].toString(), "PAUSED")
+ break;
+ }
+
+ sql "stop routine load for ${jobs[i]}"
+ i++
+ }
+ } finally {
+ for (String tableName in tables) {
+ sql new
File("""${context.file.parent}/ddl/${tableName}_drop.sql""").text
+ }
+ }
+ }
+
+ // max_error_number
+ i = 0
if (enabled != null && enabled.equalsIgnoreCase("true")) {
- String kafka_port = context.config.otherConfigs.get("kafka_port")
- String externalEnvIp = context.config.otherConfigs.get("externalEnvIp")
-
- def topic1 = "test"
- sql """
- CREATE ROUTINE LOAD ${topic1} ON ${testTable}
- COLUMNS TERMINATED BY ","
- PROPERTIES
- (
- "max_batch_interval" = "5",
- "max_batch_rows" = "200000",
- "max_batch_size" = "209715200"
- )
- FROM KAFKA
- (
- "kafka_broker_list" = "${externalEnvIp}:${kafka_port}",
- "kafka_topic" = "${topic1}",
- "property.kafka_default_offsets" = "OFFSET_BEGINNING"
- );
- """
+ try {
+ for (String tableName in tables) {
+ sql new
File("""${context.file.parent}/ddl/${tableName}_drop.sql""").text
+ sql new
File("""${context.file.parent}/ddl/${tableName}_create.sql""").text
+
+ def name = "routine_load_" + tableName
+ sql """
+ CREATE ROUTINE LOAD ${jobs[i]} ON ${name}
+ COLUMNS(${columns[i]}),
+ COLUMNS TERMINATED BY "|"
+ PROPERTIES
+ (
+ "max_error_number" = "${filteredRows[i]}",
+ "max_batch_interval" = "1",
+ "max_batch_rows" = "300000",
+ "max_batch_size" = "209715200"
+ )
+ FROM KAFKA
+ (
+ "kafka_broker_list" = "${externalEnvIp}:${kafka_port}",
+ "kafka_topic" = "${errorTopics[i]}",
+ "property.kafka_default_offsets" = "OFFSET_BEGINNING"
+ );
+ """
+ sql "sync"
+ i++
+ }
+
+ i = 0
+ for (String tableName in tables) {
+ while (true) {
+ sleep(1000)
+ def res = sql "show routine load for ${jobs[i]}"
+ def state = res[0][8].toString()
+ if (state == "NEED_SCHEDULE") {
+ continue;
+ }
+ assertEquals(res[0][8].toString(), "RUNNING")
+ break;
+ }
+
+ sql "stop routine load for ${jobs[i]}"
+ i++
+ }
+ } finally {
+ for (String tableName in tables) {
+ sql new
File("""${context.file.parent}/ddl/${tableName}_drop.sql""").text
+ }
+ }
+ }
+
+ // max_filter_ratio
+ i = 0
+ if (enabled != null && enabled.equalsIgnoreCase("true")) {
+ try {
+ for (String tableName in tables) {
+ sql new
File("""${context.file.parent}/ddl/${tableName}_drop.sql""").text
+ sql new
File("""${context.file.parent}/ddl/${tableName}_create.sql""").text
+
+ def name = "routine_load_" + tableName
+ sql """
+ CREATE ROUTINE LOAD ${jobs[i]} ON ${name}
+ COLUMNS(${columns[i]}),
+ COLUMNS TERMINATED BY "|"
+ PROPERTIES
+ (
+ "max_filter_ratio" = "${maxFilterRatio[i]}",
+ "max_error_number" = "1000",
+ "max_batch_interval" = "1",
+ "max_batch_rows" = "300000",
+ "max_batch_size" = "209715200"
+ )
+ FROM KAFKA
+ (
+ "kafka_broker_list" = "${externalEnvIp}:${kafka_port}",
+ "kafka_topic" = "${errorTopics[i]}",
+ "property.kafka_default_offsets" = "OFFSET_BEGINNING"
+ );
+ """
+ sql "sync"
+ i++
+ }
+
+ i = 0
+ for (String tableName in tables) {
+ while (true) {
+ sleep(1000)
+ def res = sql "show routine load for ${jobs[i]}"
+ def state = res[0][8].toString()
+ if (state == "NEED_SCHEDULE") {
+ continue;
+ }
+ assertEquals(res[0][8].toString(), "RUNNING")
+ break;
+ }
+
+ sql "stop routine load for ${jobs[i]}"
+ i++
+ }
+ } finally {
+ for (String tableName in tables) {
+ sql new
File("""${context.file.parent}/ddl/${tableName}_drop.sql""").text
+ }
+ }
+ }
+
+ // load_to_single_tablet
+ // todo need verify
+ i = 0
+ if (enabled != null && enabled.equalsIgnoreCase("true")) {
+ try {
+ for (String tableName in tables) {
+ sql new
File("""${context.file.parent}/ddl/${tableName}_drop.sql""").text
+ sql new
File("""${context.file.parent}/ddl/${tableName}_create.sql""").text
+
+ def name = "routine_load_" + tableName
+ sql """
+ CREATE ROUTINE LOAD ${jobs[i]} ON ${name}
+ COLUMNS(${columns[i]}),
+ COLUMNS TERMINATED BY "|"
+ PROPERTIES
+ (
+ "load_to_single_tablet" = "true",
+ "max_batch_interval" = "1",
+ "max_batch_rows" = "300000",
+ "max_batch_size" = "209715200"
+ )
+ FROM KAFKA
+ (
+ "kafka_broker_list" = "${externalEnvIp}:${kafka_port}",
+ "kafka_topic" = "${topics[i]}",
+ "property.kafka_default_offsets" = "OFFSET_BEGINNING"
+ );
+ """
+ sql "sync"
+ i++
+ }
+
+ i = 0
+ for (String tableName in tables) {
+ while (true) {
+ sleep(1000)
+ def res = sql "show routine load for ${jobs[i]}"
+ def state = res[0][8].toString()
+ if (state == "NEED_SCHEDULE") {
+ continue;
+ }
+ assertEquals(res[0][8].toString(), "RUNNING")
+ break;
+ }
+
+ sql "stop routine load for ${jobs[i]}"
+ i++
+ }
+ } finally {
+ for (String tableName in tables) {
+ sql new
File("""${context.file.parent}/ddl/${tableName}_drop.sql""").text
+ }
+ }
+ }
+
+ // column_separator
+ i = 0
+ if (enabled != null && enabled.equalsIgnoreCase("true")) {
+ try {
+ for (String tableName in tables) {
+ sql new
File("""${context.file.parent}/ddl/${tableName}_drop.sql""").text
+ sql new
File("""${context.file.parent}/ddl/${tableName}_create.sql""").text
+
+ def name = "routine_load_" + tableName
+ sql """
+ CREATE ROUTINE LOAD ${jobs[i]} ON ${name}
+ COLUMNS(${columns[i]})
+ PROPERTIES
+ (
+ "max_batch_interval" = "1",
+ "max_batch_rows" = "300000",
+ "max_batch_size" = "209715200"
+ )
+ FROM KAFKA
+ (
+ "kafka_broker_list" = "${externalEnvIp}:${kafka_port}",
+ "kafka_topic" = "${topics[i]}",
+ "property.kafka_default_offsets" = "OFFSET_BEGINNING"
+ );
+ """
+ sql "sync"
+ i++
+ }
+
+ i = 0
+ for (String tableName in tables) {
+ while (true) {
+ sleep(1000)
+ def res = sql "show routine load for ${jobs[i]}"
+ def state = res[0][8].toString()
+ if (state == "NEED_SCHEDULE") {
+ continue;
+ }
+ assertEquals(res[0][8].toString(), "PAUSED")
+ break;
+ }
+
+ sql "stop routine load for ${jobs[i]}"
+ i++
+ }
+ } finally {
+ for (String tableName in tables) {
+ sql new
File("""${context.file.parent}/ddl/${tableName}_drop.sql""").text
+ }
+ }
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
