This is an automated email from the ASF dual-hosted git repository.
jakevin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new d5d82b7c31 [stats](nereids) fix bug for avg-size (#22421)
d5d82b7c31 is described below
commit d5d82b7c31a93dc9ad4443c35f9d7f3e9d380d1e
Author: minghong <[email protected]>
AuthorDate: Tue Aug 1 17:13:00 2023 +0800
[stats](nereids) fix bug for avg-size (#22421)
---
.../doris/nereids/stats/FilterEstimation.java | 2 +-
.../apache/doris/statistics/ColumnStatistic.java | 4 +-
.../nereids_tpcds_shape_sf100_p0/shape/query65.out | 59 ++++++++++----------
.../nereids_tpcds_shape_sf100_p0/shape/query72.out | 65 +++++++++++-----------
.../nereids_tpch_shape_sf1000_p0/shape/q22.out | 33 ++++++-----
.../data/nereids_tpch_shape_sf500_p0/shape/q22.out | 33 ++++++-----
6 files changed, 96 insertions(+), 100 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/FilterEstimation.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/FilterEstimation.java
index 44e9a97a8b..653471e98c 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/FilterEstimation.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/FilterEstimation.java
@@ -469,7 +469,7 @@ public class FilterEstimation extends
ExpressionVisitor<Statistics, EstimationCo
ColumnStatistic rightColumnStatistic = new
ColumnStatisticBuilder(rightStats)
.setMinValue(Math.max(leftRange.getLow(), rightRange.getLow()))
.setMaxValue(rightRange.getHigh())
- .setAvgSizeByte(rightStats.ndv *
(rightAlwaysGreaterRangeFraction + rightOverlappingRangeFraction))
+ .setNdv(rightStats.ndv * (rightAlwaysGreaterRangeFraction +
rightOverlappingRangeFraction))
.setNumNulls(0)
.build();
double sel = leftAlwaysLessThanRightPercent
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/ColumnStatistic.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/ColumnStatistic.java
index 7e23c83e07..bfc5c33302 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/ColumnStatistic.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/ColumnStatistic.java
@@ -311,8 +311,8 @@ public class ColumnStatistic {
@Override
public String toString() {
- return isUnKnown ? "unknown" : String.format("ndv=%.4f, min=%f(%s),
max=%f(%s), count=%.4f",
- ndv, minValue, minExpr, maxValue, maxExpr, count);
+ return isUnKnown ? "unknown" : String.format("ndv=%.4f, min=%f(%s),
max=%f(%s), count=%.4f, avgSizeByte=%f",
+ ndv, minValue, minExpr, maxValue, maxExpr, count, avgSizeByte);
}
public JSONObject toJson() {
diff --git
a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query65.out
b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query65.out
index 2d74160159..ae54fc7949 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query65.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query65.out
@@ -5,42 +5,41 @@ PhysicalResultSink
----PhysicalDistribute
------PhysicalTopN
--------PhysicalProject
-----------hashJoin[INNER_JOIN](item.i_item_sk = sc.ss_item_sk)
-------------PhysicalDistribute
---------------PhysicalProject
-----------------hashJoin[INNER_JOIN](store.s_store_sk = sc.ss_store_sk)
-------------------hashJoin[INNER_JOIN](sb.ss_store_sk =
sc.ss_store_sk)(cast(revenue as DOUBLE) <= cast((0.1 * ave) as DOUBLE))
+----------hashJoin[INNER_JOIN](store.s_store_sk = sc.ss_store_sk)
+------------hashJoin[INNER_JOIN](item.i_item_sk = sc.ss_item_sk)
+--------------PhysicalDistribute
+----------------hashJoin[INNER_JOIN](sb.ss_store_sk =
sc.ss_store_sk)(cast(revenue as DOUBLE) <= cast((0.1 * ave) as DOUBLE))
+------------------hashAgg[GLOBAL]
+--------------------PhysicalDistribute
+----------------------hashAgg[LOCAL]
+------------------------PhysicalProject
+--------------------------hashJoin[INNER_JOIN](store_sales.ss_sold_date_sk =
date_dim.d_date_sk)
+----------------------------PhysicalProject
+------------------------------PhysicalOlapScan[store_sales]
+----------------------------PhysicalDistribute
+------------------------------PhysicalProject
+--------------------------------filter((date_dim.d_month_seq <=
1232)(date_dim.d_month_seq >= 1221))
+----------------------------------PhysicalOlapScan[date_dim]
+------------------PhysicalDistribute
--------------------hashAgg[GLOBAL]
----------------------PhysicalDistribute
------------------------hashAgg[LOCAL]
--------------------------PhysicalProject
-----------------------------hashJoin[INNER_JOIN](store_sales.ss_sold_date_sk =
date_dim.d_date_sk)
-------------------------------PhysicalProject
---------------------------------PhysicalOlapScan[store_sales]
+----------------------------hashAgg[GLOBAL]
------------------------------PhysicalDistribute
---------------------------------PhysicalProject
-----------------------------------filter((date_dim.d_month_seq <=
1232)(date_dim.d_month_seq >= 1221))
-------------------------------------PhysicalOlapScan[date_dim]
---------------------PhysicalDistribute
-----------------------hashAgg[GLOBAL]
-------------------------PhysicalDistribute
---------------------------hashAgg[LOCAL]
-----------------------------PhysicalProject
-------------------------------hashAgg[GLOBAL]
---------------------------------PhysicalDistribute
-----------------------------------hashAgg[LOCAL]
-------------------------------------PhysicalProject
---------------------------------------hashJoin[INNER_JOIN](store_sales.ss_sold_date_sk
= date_dim.d_date_sk)
+--------------------------------hashAgg[LOCAL]
+----------------------------------PhysicalProject
+------------------------------------hashJoin[INNER_JOIN](store_sales.ss_sold_date_sk
= date_dim.d_date_sk)
+--------------------------------------PhysicalProject
+----------------------------------------PhysicalOlapScan[store_sales]
+--------------------------------------PhysicalDistribute
----------------------------------------PhysicalProject
-------------------------------------------PhysicalOlapScan[store_sales]
-----------------------------------------PhysicalDistribute
-------------------------------------------PhysicalProject
---------------------------------------------filter((date_dim.d_month_seq >=
1221)(date_dim.d_month_seq <= 1232))
-----------------------------------------------PhysicalOlapScan[date_dim]
-------------------PhysicalDistribute
---------------------PhysicalProject
-----------------------PhysicalOlapScan[store]
+------------------------------------------filter((date_dim.d_month_seq >=
1221)(date_dim.d_month_seq <= 1232))
+--------------------------------------------PhysicalOlapScan[date_dim]
+--------------PhysicalDistribute
+----------------PhysicalProject
+------------------PhysicalOlapScan[item]
------------PhysicalDistribute
--------------PhysicalProject
-----------------PhysicalOlapScan[item]
+----------------PhysicalOlapScan[store]
diff --git
a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query72.out
b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query72.out
index 9571315ac7..90bf3501c5 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query72.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query72.out
@@ -19,42 +19,41 @@ PhysicalResultSink
----------------------------PhysicalProject
------------------------------hashJoin[INNER_JOIN](item.i_item_sk =
catalog_sales.cs_item_sk)
--------------------------------PhysicalProject
-----------------------------------hashJoin[INNER_JOIN](inventory.inv_date_sk =
d2.d_date_sk)(d1.d_week_seq = d2.d_week_seq)
-------------------------------------PhysicalProject
---------------------------------------hashJoin[INNER_JOIN](catalog_sales.cs_item_sk
= inventory.inv_item_sk)(inventory.inv_quantity_on_hand <
catalog_sales.cs_quantity)
-----------------------------------------PhysicalDistribute
+----------------------------------PhysicalOlapScan[item]
+--------------------------------PhysicalDistribute
+----------------------------------PhysicalProject
+------------------------------------hashJoin[INNER_JOIN](inventory.inv_date_sk
= d2.d_date_sk)(d1.d_week_seq = d2.d_week_seq)
+--------------------------------------PhysicalProject
+----------------------------------------hashJoin[INNER_JOIN](catalog_sales.cs_item_sk
= inventory.inv_item_sk)(inventory.inv_quantity_on_hand <
catalog_sales.cs_quantity)
------------------------------------------PhysicalOlapScan[inventory]
-----------------------------------------PhysicalDistribute
-------------------------------------------PhysicalProject
---------------------------------------------hashJoin[INNER_JOIN](catalog_sales.cs_ship_date_sk
= d3.d_date_sk)(d3.d_date > cast((cast(d_date as BIGINT) + 5) as DATEV2))
-----------------------------------------------PhysicalDistribute
-------------------------------------------------hashJoin[INNER_JOIN](catalog_sales.cs_bill_cdemo_sk
= customer_demographics.cd_demo_sk)
---------------------------------------------------PhysicalDistribute
-----------------------------------------------------hashJoin[INNER_JOIN](catalog_sales.cs_sold_date_sk
= d1.d_date_sk)
-------------------------------------------------------hashJoin[INNER_JOIN](catalog_sales.cs_bill_hdemo_sk
= household_demographics.hd_demo_sk)
---------------------------------------------------------PhysicalProject
-----------------------------------------------------------PhysicalOlapScan[catalog_sales]
+------------------------------------------PhysicalDistribute
+--------------------------------------------PhysicalProject
+----------------------------------------------hashJoin[INNER_JOIN](catalog_sales.cs_ship_date_sk
= d3.d_date_sk)(d3.d_date > cast((cast(d_date as BIGINT) + 5) as DATEV2))
+------------------------------------------------PhysicalDistribute
+--------------------------------------------------hashJoin[INNER_JOIN](catalog_sales.cs_bill_cdemo_sk
= customer_demographics.cd_demo_sk)
+----------------------------------------------------PhysicalDistribute
+------------------------------------------------------hashJoin[INNER_JOIN](catalog_sales.cs_sold_date_sk
= d1.d_date_sk)
+--------------------------------------------------------hashJoin[INNER_JOIN](catalog_sales.cs_bill_hdemo_sk
= household_demographics.hd_demo_sk)
+----------------------------------------------------------PhysicalProject
+------------------------------------------------------------PhysicalOlapScan[catalog_sales]
+----------------------------------------------------------PhysicalDistribute
+------------------------------------------------------------PhysicalProject
+--------------------------------------------------------------filter((cast(hd_buy_potential
as VARCHAR(*)) = '501-1000'))
+----------------------------------------------------------------PhysicalOlapScan[household_demographics]
--------------------------------------------------------PhysicalDistribute
----------------------------------------------------------PhysicalProject
-------------------------------------------------------------filter((cast(hd_buy_potential
as VARCHAR(*)) = '501-1000'))
---------------------------------------------------------------PhysicalOlapScan[household_demographics]
-------------------------------------------------------PhysicalDistribute
---------------------------------------------------------PhysicalProject
-----------------------------------------------------------filter((d1.d_year =
2002))
-------------------------------------------------------------PhysicalOlapScan[date_dim]
---------------------------------------------------PhysicalDistribute
-----------------------------------------------------PhysicalProject
-------------------------------------------------------filter((cast(cd_marital_status
as VARCHAR(*)) = 'W'))
---------------------------------------------------------PhysicalOlapScan[customer_demographics]
-----------------------------------------------PhysicalDistribute
-------------------------------------------------PhysicalProject
---------------------------------------------------PhysicalOlapScan[date_dim]
-------------------------------------PhysicalDistribute
---------------------------------------PhysicalProject
-----------------------------------------PhysicalOlapScan[date_dim]
---------------------------------PhysicalDistribute
-----------------------------------PhysicalProject
-------------------------------------PhysicalOlapScan[item]
+------------------------------------------------------------filter((d1.d_year
= 2002))
+--------------------------------------------------------------PhysicalOlapScan[date_dim]
+----------------------------------------------------PhysicalDistribute
+------------------------------------------------------PhysicalProject
+--------------------------------------------------------filter((cast(cd_marital_status
as VARCHAR(*)) = 'W'))
+----------------------------------------------------------PhysicalOlapScan[customer_demographics]
+------------------------------------------------PhysicalDistribute
+--------------------------------------------------PhysicalProject
+----------------------------------------------------PhysicalOlapScan[date_dim]
+--------------------------------------PhysicalDistribute
+----------------------------------------PhysicalProject
+------------------------------------------PhysicalOlapScan[date_dim]
----------------------------PhysicalDistribute
------------------------------PhysicalProject
--------------------------------PhysicalOlapScan[warehouse]
diff --git a/regression-test/data/nereids_tpch_shape_sf1000_p0/shape/q22.out
b/regression-test/data/nereids_tpch_shape_sf1000_p0/shape/q22.out
index 3525bb7168..09e3c287f6 100644
--- a/regression-test/data/nereids_tpch_shape_sf1000_p0/shape/q22.out
+++ b/regression-test/data/nereids_tpch_shape_sf1000_p0/shape/q22.out
@@ -8,22 +8,21 @@ PhysicalResultSink
----------PhysicalDistribute
------------hashAgg[LOCAL]
--------------PhysicalProject
-----------------NestedLoopJoin[INNER_JOIN](cast(c_acctbal as DECIMALV3(38, 4))
> avg(c_acctbal))
-------------------PhysicalProject
---------------------hashJoin[RIGHT_ANTI_JOIN](orders.o_custkey =
customer.c_custkey)
-----------------------PhysicalDistribute
-------------------------PhysicalProject
---------------------------PhysicalOlapScan[orders]
-----------------------PhysicalDistribute
-------------------------PhysicalProject
---------------------------filter(substring(c_phone, 1, 2) IN ('13', '31',
'23', '29', '30', '18', '17'))
-----------------------------PhysicalOlapScan[customer]
+----------------hashJoin[RIGHT_ANTI_JOIN](orders.o_custkey =
customer.c_custkey)
------------------PhysicalDistribute
---------------------PhysicalAssertNumRows
-----------------------hashAgg[GLOBAL]
-------------------------PhysicalDistribute
---------------------------hashAgg[LOCAL]
-----------------------------PhysicalProject
-------------------------------filter((customer.c_acctbal >
0.00)substring(c_phone, 1, 2) IN ('13', '31', '23', '29', '30', '18', '17'))
---------------------------------PhysicalOlapScan[customer]
+--------------------PhysicalProject
+----------------------PhysicalOlapScan[orders]
+------------------PhysicalDistribute
+--------------------NestedLoopJoin[INNER_JOIN](cast(c_acctbal as DECIMALV3(38,
4)) > avg(c_acctbal))
+----------------------PhysicalProject
+------------------------filter(substring(c_phone, 1, 2) IN ('13', '31', '23',
'29', '30', '18', '17'))
+--------------------------PhysicalOlapScan[customer]
+----------------------PhysicalDistribute
+------------------------PhysicalAssertNumRows
+--------------------------hashAgg[GLOBAL]
+----------------------------PhysicalDistribute
+------------------------------hashAgg[LOCAL]
+--------------------------------PhysicalProject
+----------------------------------filter((customer.c_acctbal >
0.00)substring(c_phone, 1, 2) IN ('13', '31', '23', '29', '30', '18', '17'))
+------------------------------------PhysicalOlapScan[customer]
diff --git a/regression-test/data/nereids_tpch_shape_sf500_p0/shape/q22.out
b/regression-test/data/nereids_tpch_shape_sf500_p0/shape/q22.out
index 3525bb7168..09e3c287f6 100644
--- a/regression-test/data/nereids_tpch_shape_sf500_p0/shape/q22.out
+++ b/regression-test/data/nereids_tpch_shape_sf500_p0/shape/q22.out
@@ -8,22 +8,21 @@ PhysicalResultSink
----------PhysicalDistribute
------------hashAgg[LOCAL]
--------------PhysicalProject
-----------------NestedLoopJoin[INNER_JOIN](cast(c_acctbal as DECIMALV3(38, 4))
> avg(c_acctbal))
-------------------PhysicalProject
---------------------hashJoin[RIGHT_ANTI_JOIN](orders.o_custkey =
customer.c_custkey)
-----------------------PhysicalDistribute
-------------------------PhysicalProject
---------------------------PhysicalOlapScan[orders]
-----------------------PhysicalDistribute
-------------------------PhysicalProject
---------------------------filter(substring(c_phone, 1, 2) IN ('13', '31',
'23', '29', '30', '18', '17'))
-----------------------------PhysicalOlapScan[customer]
+----------------hashJoin[RIGHT_ANTI_JOIN](orders.o_custkey =
customer.c_custkey)
------------------PhysicalDistribute
---------------------PhysicalAssertNumRows
-----------------------hashAgg[GLOBAL]
-------------------------PhysicalDistribute
---------------------------hashAgg[LOCAL]
-----------------------------PhysicalProject
-------------------------------filter((customer.c_acctbal >
0.00)substring(c_phone, 1, 2) IN ('13', '31', '23', '29', '30', '18', '17'))
---------------------------------PhysicalOlapScan[customer]
+--------------------PhysicalProject
+----------------------PhysicalOlapScan[orders]
+------------------PhysicalDistribute
+--------------------NestedLoopJoin[INNER_JOIN](cast(c_acctbal as DECIMALV3(38,
4)) > avg(c_acctbal))
+----------------------PhysicalProject
+------------------------filter(substring(c_phone, 1, 2) IN ('13', '31', '23',
'29', '30', '18', '17'))
+--------------------------PhysicalOlapScan[customer]
+----------------------PhysicalDistribute
+------------------------PhysicalAssertNumRows
+--------------------------hashAgg[GLOBAL]
+----------------------------PhysicalDistribute
+------------------------------hashAgg[LOCAL]
+--------------------------------PhysicalProject
+----------------------------------filter((customer.c_acctbal >
0.00)substring(c_phone, 1, 2) IN ('13', '31', '23', '29', '30', '18', '17'))
+------------------------------------PhysicalOlapScan[customer]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]