This is an automated email from the ASF dual-hosted git repository. panxiaolei pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push: new a6bc92edc58 [RegressionTest](test) Add fuzzy test config for column string overflow and p2 case (#34091) a6bc92edc58 is described below commit a6bc92edc5870481d9a7091f422054589d394150 Author: HappenLee <happen...@hotmail.com> AuthorDate: Thu Apr 25 20:15:04 2024 +0800 [RegressionTest](test) Add fuzzy test config for column string overflow and p2 case (#34091) Add fuzzy test config for column string overflow and p2 case --- be/src/common/config.cpp | 4 + be/src/common/config.h | 3 + be/src/vec/columns/column_string.cpp | 3 +- .../load_from_big_lateral_view.out | 721 +++++++++++++++++++++ .../string_over_flow/string_over_flow.groovy | 24 + 5 files changed, 753 insertions(+), 2 deletions(-) diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index de1458c240d..11a8b97842d 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -1216,6 +1216,8 @@ DEFINE_mBool(enable_injection_point, "false"); DEFINE_mBool(ignore_schema_change_check, "false"); +DEFINE_mInt64(string_overflow_size, "4294967295"); // std::numic_limits<uint32_t>::max() + // The min thread num for BufferedReaderPrefetchThreadPool DEFINE_Int64(num_buffered_reader_prefetch_thread_pool_min_thread, "16"); // The max thread num for BufferedReaderPrefetchThreadPool @@ -1663,6 +1665,8 @@ Status set_fuzzy_configs() { ((distribution(*generator) % 2) == 0) ? "true" : "false"; fuzzy_field_and_value["enable_shrink_memory"] = ((distribution(*generator) % 2) == 0) ? "true" : "false"; + fuzzy_field_and_value["string_overflow_size"] = + ((distribution(*generator) % 2) == 0) ? "10" : "4294967295"; fmt::memory_buffer buf; for (auto& it : fuzzy_field_and_value) { diff --git a/be/src/common/config.h b/be/src/common/config.h index 4139d76b6bc..fe7009c7691 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -1295,6 +1295,9 @@ DECLARE_mBool(enable_injection_point); DECLARE_mBool(ignore_schema_change_check); +/** Only use in fuzzy test **/ +DECLARE_mInt64(string_overflow_size); + // The min thread num for BufferedReaderPrefetchThreadPool DECLARE_Int64(num_buffered_reader_prefetch_thread_pool_min_thread); // The max thread num for BufferedReaderPrefetchThreadPool diff --git a/be/src/vec/columns/column_string.cpp b/be/src/vec/columns/column_string.cpp index 843b2fa72a7..514980cc8ba 100644 --- a/be/src/vec/columns/column_string.cpp +++ b/be/src/vec/columns/column_string.cpp @@ -608,8 +608,7 @@ ColumnPtr ColumnStr<T>::index(const IColumn& indexes, size_t limit) const { template <typename T> ColumnPtr ColumnStr<T>::convert_column_if_overflow() { - // TODO: Try to fuzzy the overflow size to test more case in CI - if (std::is_same_v<T, UInt32> && chars.size() > std::numeric_limits<UInt32>::max()) { + if (std::is_same_v<T, UInt32> && chars.size() > config::string_overflow_size) { auto new_col = ColumnStr<uint64_t>::create(); const auto length = offsets.size(); diff --git a/regression-test/data/query_p2/string_over_flow/load_from_big_lateral_view.out b/regression-test/data/query_p2/string_over_flow/load_from_big_lateral_view.out new file mode 100644 index 00000000000..8426d697d00 --- /dev/null +++ b/regression-test/data/query_p2/string_over_flow/load_from_big_lateral_view.out @@ -0,0 +1,721 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql -- + above the unusual accounts. slyl + according to the blith + accounts haggle blithely above the s + across the fu + along the slyly regula + alongside of + alongside of the carefully exp + among the ironic, final requests ma + asymptotes + beans. de + blithely + blithely bold forges above t + blithely regular deposits cajole fluff + blithely. quickly final deposits wa + bold accounts. fluffil + bold foxes sleep finally + boost blithely after the inst + cajole quickly regular, bold packa + cajole regular packages. special, ir + carefully even ideas cajole + carefully even packages are carefully pend + carefully regular ac + carefully regular instructions. regula + deposits run blithely furiously special re + deposits sleep furiously across the pendin + deposits. e + deposits. furiously close t + deposits. ironic, regular foxes throu + dolphins are furiously slyly regular depo + even deposi + even deposits. carefully ir + even foxes. final pinto beans sleep + even instructions according to the furio + even packages co + excuses. bl + express foxes + express, regular packages hag + final dep + final pinto + furiously blithely unusual requests. + furiously bold dolphins; slyly expr + furiously bold platel + furiously quickly bold frets. bra + furiously regular accounts cajole carefu + furiously regular pinto be + furiously stealthy deposits cajole around + haggle carefully. reg + ideas use blithely a + instructions sleep + ironic accounts. special theod + ironic packages. + ironic requests nag accord + ironic, even accounts. accounts use b + of the careful + on the furiously permanent dol + orbits haggle + packages a + packages according to the final, even p + packages sleep blithely fluffily final + packages. busy, + packages. car + packages. careful + packages. carefull + pending foxes cajole slyly carefull + pending ideas. quickly quiet requests i + pending, pending platelets nag e + pinto beans above the furiously + pinto beans are slyly + poach unusual foxes. fo + quickly even accounts. furiously i + quickly pending p + quickly regular ideas: express, final requ + quietly carefull + regular deposits cajole blithel + regular deposits unwind slyly al + regular packag + regular, even instructions lose s + requests cajole qu + requests. blithely ironic instructions sl + requests. quickly regula + silent orbits across the + sleep blithely at the slow packages! bli + slyly above th + slyly across the + slyly at + slyly even ideas. + slyly ironic accoun + slyly pending + slyly special theodolit + special excuses. carefully expres + special packages after the d + special, silent deposi + the ideas. + the regular, ironic excuses. furious + the unusual accounts hagg + theodolites. quickly final deposits + unusual instructions doze bol + use about the carefully pending deposits. +, express ideas. carefully regular wa +, express packages accord +, final platelets. quick theodolites us +, ironic accounts +, ironic instructions. iro +- carefully daring foxes wake furiously. +. blithely final decoy +. carefully regular packages nod +. fluffily final platelets play ca +. furiously bold theodolites +. furiously expre +. furiously ironic dependenc +. ironic i +above the quickly ironic dependenc +accounts ar +accounts boost sl +accounts haggle foxes. un +accounts. ironic accounts haggle aft +accounts? +ackages whitho +ackages. carefully special instructio +ackages. silent theodolites detect alo +across the furiousl +across the s +after the quickly +ag slyly pac +ages affix according to the express accoun +ages boost. even accounts haggle carefull +ages haggle slyly among the ironic, r +ages haggle. pending deposits kind +ages. slyly ironic +aggle acro +aggle carefu +aggle quickly requests. fluffily final +ainst the furiously +aintain idly ironic, express +ajole quickly. ironic accounts cajo +ake furiously. furiously bold deposits nag +ake slyly about the +ake. carefully bold deposits boost. i +ake. silent dependencies x-ray finally. pac +al accounts. slyly final foxes aff +al dolphins. so +al foxes caj +al foxes. slyly final theodolit +al ideas should have to integrate +al ideas sleep acc +al ideas. busy requests wake car +al packages use fluffily alon +al, express requests. blithely exp +alms sleep furiou +along the quickly final theodolite +althily regular requests. f +ans wake after the evenly p +ans. regular, special ideas ha +apades are alongside of the saut +ar pinto beans about the bold, unusua +ar pinto beans. carefully unusual pinto bea +ar theodolites along the final +are carefully deposits! furiously bold pac +arefully dogged, regular packages. +arefully final dependenc +arefully. care +arthogs lose at +arthogs lose quickly +as. slyly silent dolphins sublat +at the thin, ironic packages +beans use carefully abo +beans. even, ironic dep +blithely e +blithely ironi +blithely. furiou +bold foxes cajole blithely after the flu +bold, silent accounts detect carefully un +boost against the slyly regular pl +boost slyly +c packages. ironic accounts believe +c packages. slyly express shea +carefully f +carefully ir +carefully pending requ +carefully quiet packages wake blithely alon +carefully speci +ccounts along the ironic re +ccounts wake i +ccounts wake quickly. furiousl +ccounts. slyly iron +ch carefully excuses. carefully e +cies according to the fluffily ruthless +ckages affix. ca +ckages snooze never after the carefull +ckly above the quickly +ckly bold asymptotes cajole blit +ckly even, busy pinto beans. +ckly express pinto b +ckly ironic dolphins. +ckly pending foxes. deposits ha +counts according +counts across the special, unusual p +counts. blithe +cross the pending +d ideas. requests +d theodolites cajo +de of the br +deas was beside +deas. furiously bold packages alo +dencies. bold theodolites haggle. blit +dependencies among the packa +dependencies. p +deposits along the furiously special ac +deposits nag. carefully ironic depos +ding deposits above the c +doggedly ironic ideas +doubt among the ironic, regular +doubt ironic multipliers. quick +e above the blithely +e accounts. regular, even theodolites do en +e blithely bold foxes. accounts serve furi +e busily pending requests. fu +e carefull +e carefully regular foxes sleep furio +e carefully. bl +e fluffily bold de +e fluffily final account +e fluffily unusual hockey players. sl +e furiously final foxes. fur +e ironically even requests. sl +e pinto beans. quickly unusual +e quickly even requests sleep furiousl +e the blithely unusual packag +e the carefully final d +e. deposits alon +e. fluffy depths x-ray around the furious +e. furiously +ecial dependenc +ecial deposits. blithely even requests +ecial foxes affix furiously agains +ecial theodolites. carefully bol +eep blithely. ruthless pac +eep silently de +efully against +efully express requests. boldly silent theo +efully. quickly express deposits use. iro +efully: slyly e +egular accounts. unusual, special reques +egular deposits nag about t +egular ideas n +egular ideas. blit +egular packages integrate blithel +egular pearls integrate carefully. sl +egular pla +egularly regu +ely after the fluffily bold +ely silent foxes +en account +en deposits cajole slyly fin +en escapades +en excuses beside the furiously final acco +encies affix slyly: unusual deposits +encies snooz +endencies. furiously express depend +ending dependencie +ending deposits. fluffi +ending packages wake around the +eodolites. bold, brave patterns sleep. qui +ep ironic dependencies. regula +ep slyly. slyly brave deposits +ependencies. +eposits sleep blithely final id +eposits wake furiously unusual th +eposits. blithely unusual idea +equests. fina +equests. final +ernes use at +erve daringly quick packages. +es detect. deposits wa +es sleep. slyly even packages a +es use final, final dependencies. pl +ess packages. ironic, ironic +ests boost according +ests haggle. carefully pending +ests. blithely even deposit +ests. regular, final theodolites sleep thi +evenly special deposits. ir +express re +f the quickly pend +ffily bold +ffily bold +ffily bold accounts. final, regular +ffily busy packages haggle +fily final packages boost furiously sil +fily regular deposits. sl +final deposits wake furiously pen +final, final deposits. quickly regular pin +final, ironic accounts. regular +foxes wake quickly plat +foxes. fluffily ironic theodolites affi +fter the accounts. fluffily ironic packag +ful deposits. blithely +fully even i +fully final theodolites +furiously b +furiously final accounts. request +furiously quiet pinto beans boos +furiously. +g pinto beans. +g requests wake car +g the final, pending pinto be +g the ironic, regular +g to the regular, r +gage fluffily ru +gage. slyly pending platelets in pl +gainst the special request +ges are sometimes special dolphins. +ges boost along the fu +ges sleep. final deposits sleep +ges-- carefully ironic accounts +ges. furious +ges. regular theodolites +ges; regular theodolites cajo +gifts x-ray carefully +gle slyly final deposits. daringl +gle. packages haggle furiously a +gular pinto beans. finally fin +gularly expre +haggle furiously alo +haggle-- even foxes affix. de +have to caj +he fluffily silent accounts. careful +he furiously bold de +he ironic plat +he ironic, express realms. furiously ironic +he quickly bold ideas might +he slyly special +he slyly special asympt +hely pending reques +hely regular theodolites. bold p +hins nag furiously. regular theodolites +hins. ideas dazzle a +hlessly even accou +hogs. furiously regular accounts again +ial foxes. furiously final instructions abo +ic asymptotes cajole even packages. +ic theodolites. +icingly even accounts use +ick ideas. foxes along the special +ickly ironic deposits +ickly regula +ideas. furiously regular r +idly across the final, special +ies. special e +ilent accounts. even dependencies haggl +ily express pac +ily ironic +iments wake. slyly pending requests hinde +inal accounts. express, r +ince the furiously final a +ins sleep fluffily about t +inst the carefully bold instructions +instructions. slow +into beans alo +into beans cajole blithely about the +iously ironic deposits sleep blithely bl +ironic instruct +ites must have to inte +ites nag slyly alongside of th +ites. furio +ites. slyly special ideas wake furiously ag +ithely about the fluffily even requests. +ithely according to t +ithely among the slyly regular excus +ithely idle foxes nod alongside of the +ithely ironic accounts solve +ithely ironic requests! furiously regula +ithely pending f +ithely. sly +its doze furi +ix carefully silently final +jole quickly fluf +kages sleep. blithely regul +kages wake above the asymptotes. carefull +kages. blith +kages. blithely +ke carefully carefu +key players. sil +l accounts-- furi +l deposits sleep blithely about the qui +l packages are carefully above the +l packages. blithely permanent ideas +l requests. bl +l tithes. packages play quickly according +l, thin foxes boost slyly platelets +lar ideas nag alongside of the slyly ironi +lar pinto beans dazz +lar requests. blithely expr +le packages wake slyly among the +le quickly slyly regu +le slyly af +le. packages boost slyly. final, ex +lent foxes cajole fur +lent packages. silent, ironic asym +lently regular deposits. carefully unusua +lently spe +less warthogs wake fluff +lites against the carefully +lites use furiously alongside of the +lites. blithely silent +lites. carefully ironic courts alongsid +lithely bold packages sleep fluffily. f +lithely furiously +lithely slyly bold pinto beans. e +lly furious request +lly regular excuses. qui +lly unusual deposits impress. doggedly bo +long the c +long the carefully fin +longside of the quic +ls. busily regular reques +lve furiously am +ly about the ir +ly above the blithely final foxes. iro +ly according to the fluffily final packages +ly among the slyly even deposits. even ex +ly bold foxes. ironic, expres +ly even deposits are fluffily after t +ly even deposits. a +ly even packages. ironic packag +ly furiously +ly idly bold requests. evenl +ly ironic warthogs are slyly regular, +ly pending requests bo +ly quickly ev +ly regular instructions integrate carefu +ly regular requests. thinly bold deposi +ly regular theodolites wake th +ly silent deposits are bra +ly silent requests haggle f +ly special war +ly stealth +ly. carefully speci +ly. fluffily +lyly after the furiously regular instr +lyly ironic instructions run unusual pack +lyly slyly ironic pinto beans. foxes +mptotes. furiously final +must boost. qu +n requests. final, final packages haggle +n, final deposits wake slyly blithe +nal ideas integrate slyly furiously final +nal sauternes cajole. ironic pac +nal, express courts c +nal, regular deposits p +nal, unusual theodolites +ncies hang aroun +ndencies sleep fluffily +nding pearls. furiously even ep +nding request +nding requests wake above the carefully b +ng deposits d +ng ideas above the final instructi +ng the carefully sil +ng the quickly reg +ng the regular fo +ng the unusual, special inst +ng, even courts nod furiously. careful +ngside of the fu +ngside of the slyly express reque +nic depths. even deposits about the +nic foxes. ideas are. sly +nic frays hag +ns haggle against +ns. carefully pending requests us +ns. furiously even deposits integrat +nst the furiously r +nstructions slee +ntly ironic instructions u +nto beans. f +nts are. furiously enticing packages +nts detect quickly against the blit +nts sleep slyly ironic accounts. ironic pac +nts use bold, pending +nusual sheaves wake blithely am +o the slyly final packages. furiously fi +odolites. deposits haggle car +old ideas. blithely final depo +olites. ca +olites. daringly express req +olphins. blithe +ong the regular packages x-ray fluffi +ongside of the regular theodoli +onic foxes. quickly iron +onic packages cajole +ons. fluffily final foxes above the +oost busily ironic, ir +orges. unusual packages cajole furiou +osits. ideas hang after +otes wake enticing dep +ound the ideas haggle even, fluffy ideas. +ounts are carefull +ounts cajole closely regular reque +ounts must have to +ounts. ideas ar +ounts. slyly ruthless dugouts wake +ously bold courts. regular +ously ironic accounts. even dependencies +ously regular requests caj +ously special instructions among the +out the blithely +out the carefully ironic pa +ove the requests sleep above +p alongside of the ironic, unusual in +p slyly. theodolites alongside +p. furiously pending in +packages cajol +packages wake sometim +packages. packages integrate carefull +packages. slyly regular foxes boost. slyly +packages? slyly regular pin +pecial pinto beans ha +pecial pinto beans. regular packages +permanent courts detec +pinto beans haggle carefully bold req +pon the pending, ev +press, special +pths sleep along th +ptotes: th +quests are furiously express theodo +quests nag sly +quests. fluffi +quickly express ideas. u +r accounts. blithely regular deposits in +r instructions slee +r packages cajole sometimes +r pinto beans are fluffily a +r the carefully +r the regular, express requests are fur +rate. pending packages against the +ray. slyly regular accounts sleep slowly +rding to the depths. ironic foxes in +re slyly carefu +refully even deposits. accounts use +regular asymptotes are quickly regular pint +regular dolphins wake +regular frets serve furiously. +regular ideas. slyly express pack +regular packages. blithely final foxes wake +requests sleep +ress, express instructi +riously bold deposits sleep +riously express ideas. Tiresias at t +riously slyly ironic pinto be +riously special packages n +ronic packages sleep. blithely unus +round the exp +s cajole furiously express request +s cajole slyly even req +s except the furiously regular requ +s hinder fu +s integrate. slyly ironic foxes wake quick +s outside the permanent, even instructio +s sleep bravely +s sleep carefully. slyly special +s sleep. idle multipliers w +s the express, silent reque +s wake evenl +s wake furiously regular asymp +s wake quickly. slyly pending p +s. blithely final re +s. final ideas boost slyl +s. quickly regular accounts acros +s. slyly special accounts sleep blithely +s. special packages a +s; ironic, silent pain +s? final, ironic accounts cajole quickly +se quickly? +se theodolites integrate +side of the expr +side of the quickly exp +silent pack +sits cajole blithely after the regularly i +sits sleep furiously fin +sits. furiously even packages snooze across +sleep slyly iron +sly after the even accounts. blithe +sly final packages along the bold packag +sly theodolites haggle carefully quickly +slyly ironi +slyly slyly regular theodo +solve-- slyly silent instructio +ss the regular accounts. daringly fin +ss, bold courts boos +st the carefully regular ideas- +st the ironic dep +sts. ironi +sual pinto beans boost blithely re +sublate according to the unusual, f +symptotes breach carefully according to th +symptotes wake carefully pending i +t deposits boost blithely s +t the blithely unusual deposits. bu +t the quickly pen +tainments. +telets across the final sheaves h +ter the package +tes are silent +tes kindle fluffily blit +tes. final, express deposits against the +the carefully regular ideas cajo +the furiou +the furiously even acc +the furiously f +the ideas. caref +the instructions. carefully ironic pa +the quickly bold packages. carefu +the regular instructions. quickly even pack +the slyly +the slyly final requests: blithe packa +the warhorses nag enticingly at th +thely carefully silent dep +thely never even accounts. express, fi +thely regular deposits detect furiously. +times. furiously idle instr +tions affix. blithe +tions. blithely special foxes above the un +tructions about the quickly final +ts above the quickly unus +ts are quickl +ts haggle above the final as +ts nag. blithely +ts poach. carefully darin +ts use around +ts use furiousl +ts wake. blithely sil +ts. carefully regular th +ual packages haggl +ual, ironic ac +uests boost along the final, regular acc +uests need to wake fu +uests snooze blith +ugouts are blithely regu +uickly according +uickly alongside of the care +ular accounts nag blithely iron +ular accounts sublate careful +ular accounts! +ular excuses. fluffily s +ular packages. final p +ular pinto bean +ully bold packages. s +ully thin wa +und the final foxes. enticing p +und the packages. express exc +unts affix sl +unts nag blithely b +unts promise. i +unts. furiousl +unusual asymptotes cajole around the fu +unusual excuses. ironic, even account +unusual, regular foxes sno +uring the carefully eve +uriously at the pending, final frets. acco +uriously even pi +uriously even requests. slyly fi +uriously final accounts. furiously pending +uriously pending +uriously regular +uriously regular foxes. ev +use. quietl +usily final +usly regular +usly. regular p +usual accounts cajole a +usy packages. even, sly ideas sleep slyl +ve dolphins sleep permanentl +ve the careful +ve to nag furiously. fluffily iro +ven packages. even requests across the +ven, final ins +ven, special foxes. pending inst +wake acros +wake carefully +wake furio +wake furiously about the regular request +would detect ironic, even pinto be +x-ray slyly fluffily bus +xcuses thrash according to th +xpress accounts. regular, regular pint +xpress depos +xpress deposits. requests shall ha +xpress escapades. fl +y according to the express +y according to the fluf +y according to the regular, fluff +y against the quickly ironic pin +y bold pains? final pinto beans +y final accounts. carefully even theo +y final asymptotes. blithely fluffy req +y final foxe +y final foxes. fluffily fina +y final requests wake busily e +y final requests wake. blithely +y final sentiments. reg +y ironic braids during the +y ironic grouches. blithely even +y ironic instructions +y ironic packages boost permanentl +y pending instructions hagg +y regular requests nag furio +y! even accou +y. close accounts according to the final +y. regular pinto b +y. slyly fin +yly express instructions cajole +yly final instructions across th +yly final requests cajole furiou +yly ironic theodolites at the quickly iron +yly ruthless accounts wake carefull +ymptotes above the pend +ymptotes dazzle; bold, regular re +ys carefully regular instructi + diff --git a/regression-test/suites/query_p2/string_over_flow/string_over_flow.groovy b/regression-test/suites/query_p2/string_over_flow/string_over_flow.groovy new file mode 100644 index 00000000000..71f95493138 --- /dev/null +++ b/regression-test/suites/query_p2/string_over_flow/string_over_flow.groovy @@ -0,0 +1,24 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("string_over_flow") { + def dbName = "regression_test_tpch_sf100_p2" + sql "USE ${dbName}" + sql "set parallel_pipeline_task_num = 4;" + qt_sql """select b.l_comment from nation a join[broadcast] lineitem b on a.n_nationkey = b.l_partkey order by 1;""" +} + --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org