mongo360 opened a new pull request, #37661:
URL: https://github.com/apache/doris/pull/37661
## Proposed changes
Issue Number: close #xxx
Problem:
partition storage policy info lost when cluster restart (FE restart).
Example:
1. Create table with storage policy set on some partitions
```
CREATE TABLE `ad_effects` (
`pin_id` bigint(20) NOT NULL COMMENT '广告主pin_id',
`date_time` datetime NOT NULL COMMENT '点击时间',
`ad_plan_id` bigint(20) NOT NULL COMMENT '广告计划id',
`ad_group_id` bigint(20) NOT NULL COMMENT '广告单元id',
`loc` int(11) NOT NULL COMMENT '站内外标识 1表示站内, 2表示站外（中间页包含在站内）',
`impressions` bigint(20) SUM NULL DEFAULT "0" COMMENT '用户总展现',
`clicks` bigint(20) SUM NULL DEFAULT "0" COMMENT '用户总点击',
`cost` bigint(20) SUM NULL DEFAULT "0" COMMENT '广告花费'
) ENGINE=OLAP
AGGREGATE KEY(`pin_id`, `date_time`, `ad_plan_id`, `ad_group_id`, `loc`)
COMMENT 'OLAP'
PARTITION BY RANGE(`date_time`)
(PARTITION p202401 VALUES [('2024-01-01 00:00:00'), ('2024-02-01
00:00:00'))PROPERTIES ("STORAGE POLICY" = "chubaofs_ssd_storage_policy"),
PARTITION p202402 VALUES [('2024-02-01 00:00:00'), ('2024-03-01 00:00:00')),
PARTITION p202403 VALUES [('2024-03-01 00:00:00'), ('2024-04-01 00:00:00')))
DISTRIBUTED BY HASH(`pin_id`) BUCKETS 16
PROPERTIES (
"replication_allocation" = "tag.location.default: 3",
"is_being_synced" = "false",
"dynamic_partition.enable" = "false",
"dynamic_partition.time_unit" = "MONTH",
"dynamic_partition.time_zone" = "Asia/Shanghai",
"dynamic_partition.start" = "-2147483648",
"dynamic_partition.end" = "5",
"dynamic_partition.prefix" = "p",
"dynamic_partition.replication_allocation" = "tag.location.default: 3",
"dynamic_partition.buckets" = "32",
"dynamic_partition.create_history_partition" = "false",
"dynamic_partition.history_partition_num" = "-1",
"dynamic_partition.hot_partition_num" = "0",
"dynamic_partition.reserved_history_periods" = "NULL",
"dynamic_partition.storage_policy" = "",
"dynamic_partition.start_day_of_month" = "1",
"storage_format" = "V2",
"disable_auto_compaction" = "false",
"enable_single_replica_compaction" = "false"
);
```
2. Restart cluster (restart FEs)
3. show create table ad_effects
```
CREATE TABLE `ad_effects` (
`pin_id` bigint(20) NOT NULL COMMENT '广告主pin_id',
`date_time` datetime NOT NULL COMMENT '点击时间',
`ad_plan_id` bigint(20) NOT NULL COMMENT '广告计划id',
`ad_group_id` bigint(20) NOT NULL COMMENT '广告单元id',
`loc` int(11) NOT NULL COMMENT '站内外标识 1表示站内, 2表示站外（中间页包含在站内）',
`impressions` bigint(20) SUM NULL DEFAULT "0" COMMENT '用户总展现',
`clicks` bigint(20) SUM NULL DEFAULT "0" COMMENT '用户总点击',
`cost` bigint(20) SUM NULL DEFAULT "0" COMMENT '广告花费'
) ENGINE=OLAP
AGGREGATE KEY(`pin_id`, `date_time`, `ad_plan_id`, `ad_group_id`, `loc`)
COMMENT 'OLAP'
PARTITION BY RANGE(`date_time`)
(PARTITION p202401 VALUES [('2024-01-01 00:00:00'), ('2024-02-01 00:00:00')),
PARTITION p202402 VALUES [('2024-02-01 00:00:00'), ('2024-03-01 00:00:00')),
PARTITION p202403 VALUES [('2024-03-01 00:00:00'), ('2024-04-01 00:00:00')))
DISTRIBUTED BY HASH(`pin_id`) BUCKETS 16
PROPERTIES (
"replication_allocation" = "tag.location.default: 3",
"is_being_synced" = "false",
"dynamic_partition.enable" = "false",
"dynamic_partition.time_unit" = "MONTH",
"dynamic_partition.time_zone" = "Asia/Shanghai",
"dynamic_partition.start" = "-2147483648",
"dynamic_partition.end" = "5",
"dynamic_partition.prefix" = "p",
"dynamic_partition.replication_allocation" = "tag.location.default: 3",
"dynamic_partition.buckets" = "32",
"dynamic_partition.create_history_partition" = "false",
"dynamic_partition.history_partition_num" = "-1",
"dynamic_partition.hot_partition_num" = "0",
"dynamic_partition.reserved_history_periods" = "NULL",
"dynamic_partition.storage_policy" = "",
"dynamic_partition.start_day_of_month" = "1",
"storage_format" = "V2",
"disable_auto_compaction" = "false",
"enable_single_replica_compaction" = "false"
);
```
we find STORAGE POLICY of partition p202401 is missing.
Reason:
idToStoragePolicy in PartitionInfo which saved the storage policy info of
the partitions was not serialize to local mate disk。
Solved:
add storage policy info where serialize PartitionInfo in function write and
read storage policy info where init PartitionInfo with readFields when FE start.
add FeMetaVersion to compatible history version。
related to https://github.com/apache/doris/pull/32852
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
