morningman commented on a change in pull request #4039:
URL: https://github.com/apache/incubator-doris/pull/4039#discussion_r455059053
##########
File path: be/src/olap/tablet.cpp
##########
@@ -358,9 +383,102 @@ void Tablet::delete_expired_inc_rowsets() {
save_meta();
}
+void Tablet::delete_expired_snapshot_rowset() {
+
+ int64_t now = UnixSeconds();
+ vector<pair<Version, VersionHash>> expired_versions;
+ WriteLock wrlock(&_meta_lock);
+ // Compute the end time to delete rowsets, when a expired rowset
createtime less then this time, it will be deleted.
+ double expired_snapshot_sweep_endtime = ::difftime(now,
config::tablet_rowset_expired_snapshot_sweep_time);
+
+ std::vector<int64_t> path_version_vec;
+ // capture the path version to delete
+
_timestamped_version_tracker.capture_expired_paths(static_cast<int64_t>(expired_snapshot_sweep_endtime),
&path_version_vec);
+
+ if (path_version_vec.empty()) {
+ return;
+ }
+
+ // do check consistent operation
+ auto path_id_iter = path_version_vec.rbegin();
+
+ std::map<int64_t, PathVersionListSharedPtr>
expired_snapshot_version_path_map;
+ while (path_id_iter != path_version_vec.rend()) {
+
+ PathVersionListSharedPtr version_path =
_timestamped_version_tracker.fetch_and_delete_path_by_id(*path_id_iter);
+ const std::vector<TimestampedVersionSharedPtr>& to_delete_version =
version_path->timestamped_versions();
+
+ auto first_version = to_delete_version.front();
+ auto end_version = to_delete_version.back();
+ Version test_version = Version(first_version->version().first,
end_version->version().second);
+
+ OLAPStatus status = capture_consistent_versions(test_version, nullptr);
+ // When there is no consistent versions, we must reconstruct the
tracker.
+ if (status != OLAP_SUCCESS) {
+ LOG(WARNING) << "The consistent version check fails, there are
bugs. "
+ << "Reconstruct the tracker to recover versions in tablet=" <<
tablet_id();
+
+ _timestamped_version_tracker.reconstruct_versioned_tracker(
+ _tablet_meta->all_rs_metas(),
expired_snapshot_version_path_map);
+
+ // double check the consistent versions
+ status = capture_consistent_versions(test_version, nullptr);
+
+ if (status != OLAP_SUCCESS) {
+ if
(!config::ignore_rowset_expired_snapshot_unconsistent_delete) {
+ LOG(FATAL) << "rowset expired snapshot unconsistent
delete. tablet= " << tablet_id();
+ } else {
+ LOG(WARNING) << "rowset expired snapshot unconsistent
delete. tablet= " << tablet_id();
+ }
+ }
+ return;
+ }
+ expired_snapshot_version_path_map[*path_id_iter] = version_path;
+ path_id_iter++;
+ }
+
+ auto old_size = _expired_snapshot_rs_version_map.size();
+ auto old_meta_size = _tablet_meta->all_expired_snapshot_rs_metas().size();
+
+ // do delete operation
+ auto to_delete_iter = expired_snapshot_version_path_map.begin();
+ while (to_delete_iter != expired_snapshot_version_path_map.end()) {
+
+ std::vector<TimestampedVersionSharedPtr>& to_delete_version =
to_delete_iter->second->timestamped_versions();
+ for (auto& timestampedVersion : to_delete_version) {
+ auto it =
_expired_snapshot_rs_version_map.find(timestampedVersion->version());
+ if (it != _expired_snapshot_rs_version_map.end()) {
+ // delete rowset
+ StorageEngine::instance()->add_unused_rowset(it->second);
+ _expired_snapshot_rs_version_map.erase(it);
+ LOG(INFO) << "delete expired snapshot rowset tablet=" <<
full_name()
+ <<" version[" << timestampedVersion->version().first <<
"," << timestampedVersion->version().second
+ << "] move to unused_rowset success " << std::fixed <<
expired_snapshot_sweep_endtime;
+ } else {
+ LOG(WARNING) << "delete expired snapshot rowset tablet=" <<
full_name()
+ <<" version[" << timestampedVersion->version().first <<
"," << timestampedVersion->version().second
+ << "] not find in expired snapshot rs version ";
Review comment:
```suggestion
<< "] not find in expired snapshot rs version map";
```
##########
File path: be/src/olap/tablet.cpp
##########
@@ -358,9 +383,102 @@ void Tablet::delete_expired_inc_rowsets() {
save_meta();
}
+void Tablet::delete_expired_snapshot_rowset() {
+
+ int64_t now = UnixSeconds();
+ vector<pair<Version, VersionHash>> expired_versions;
+ WriteLock wrlock(&_meta_lock);
+ // Compute the end time to delete rowsets, when a expired rowset
createtime less then this time, it will be deleted.
+ double expired_snapshot_sweep_endtime = ::difftime(now,
config::tablet_rowset_expired_snapshot_sweep_time);
+
+ std::vector<int64_t> path_version_vec;
+ // capture the path version to delete
+
_timestamped_version_tracker.capture_expired_paths(static_cast<int64_t>(expired_snapshot_sweep_endtime),
&path_version_vec);
+
+ if (path_version_vec.empty()) {
+ return;
+ }
+
+ // do check consistent operation
+ auto path_id_iter = path_version_vec.rbegin();
+
+ std::map<int64_t, PathVersionListSharedPtr>
expired_snapshot_version_path_map;
+ while (path_id_iter != path_version_vec.rend()) {
+
+ PathVersionListSharedPtr version_path =
_timestamped_version_tracker.fetch_and_delete_path_by_id(*path_id_iter);
+ const std::vector<TimestampedVersionSharedPtr>& to_delete_version =
version_path->timestamped_versions();
+
+ auto first_version = to_delete_version.front();
+ auto end_version = to_delete_version.back();
+ Version test_version = Version(first_version->version().first,
end_version->version().second);
+
+ OLAPStatus status = capture_consistent_versions(test_version, nullptr);
+ // When there is no consistent versions, we must reconstruct the
tracker.
+ if (status != OLAP_SUCCESS) {
+ LOG(WARNING) << "The consistent version check fails, there are
bugs. "
+ << "Reconstruct the tracker to recover versions in tablet=" <<
tablet_id();
+
+ _timestamped_version_tracker.reconstruct_versioned_tracker(
+ _tablet_meta->all_rs_metas(),
expired_snapshot_version_path_map);
Review comment:
expired_snapshot_version_path_map does not contains the expired path you
just deleted.
##########
File path: be/src/olap/tablet.cpp
##########
@@ -358,9 +383,102 @@ void Tablet::delete_expired_inc_rowsets() {
save_meta();
}
+void Tablet::delete_expired_snapshot_rowset() {
+
+ int64_t now = UnixSeconds();
+ vector<pair<Version, VersionHash>> expired_versions;
+ WriteLock wrlock(&_meta_lock);
+ // Compute the end time to delete rowsets, when a expired rowset
createtime less then this time, it will be deleted.
+ double expired_snapshot_sweep_endtime = ::difftime(now,
config::tablet_rowset_expired_snapshot_sweep_time);
+
+ std::vector<int64_t> path_version_vec;
+ // capture the path version to delete
+
_timestamped_version_tracker.capture_expired_paths(static_cast<int64_t>(expired_snapshot_sweep_endtime),
&path_version_vec);
+
+ if (path_version_vec.empty()) {
+ return;
+ }
+
+ // do check consistent operation
+ auto path_id_iter = path_version_vec.rbegin();
+
+ std::map<int64_t, PathVersionListSharedPtr>
expired_snapshot_version_path_map;
+ while (path_id_iter != path_version_vec.rend()) {
+
+ PathVersionListSharedPtr version_path =
_timestamped_version_tracker.fetch_and_delete_path_by_id(*path_id_iter);
+ const std::vector<TimestampedVersionSharedPtr>& to_delete_version =
version_path->timestamped_versions();
+
+ auto first_version = to_delete_version.front();
+ auto end_version = to_delete_version.back();
+ Version test_version = Version(first_version->version().first,
end_version->version().second);
+
+ OLAPStatus status = capture_consistent_versions(test_version, nullptr);
+ // When there is no consistent versions, we must reconstruct the
tracker.
+ if (status != OLAP_SUCCESS) {
+ LOG(WARNING) << "The consistent version check fails, there are
bugs. "
+ << "Reconstruct the tracker to recover versions in tablet=" <<
tablet_id();
+
+ _timestamped_version_tracker.reconstruct_versioned_tracker(
+ _tablet_meta->all_rs_metas(),
expired_snapshot_version_path_map);
+
+ // double check the consistent versions
+ status = capture_consistent_versions(test_version, nullptr);
+
+ if (status != OLAP_SUCCESS) {
+ if
(!config::ignore_rowset_expired_snapshot_unconsistent_delete) {
+ LOG(FATAL) << "rowset expired snapshot unconsistent
delete. tablet= " << tablet_id();
+ } else {
+ LOG(WARNING) << "rowset expired snapshot unconsistent
delete. tablet= " << tablet_id();
+ }
+ }
+ return;
+ }
+ expired_snapshot_version_path_map[*path_id_iter] = version_path;
+ path_id_iter++;
+ }
+
+ auto old_size = _expired_snapshot_rs_version_map.size();
+ auto old_meta_size = _tablet_meta->all_expired_snapshot_rs_metas().size();
+
+ // do delete operation
+ auto to_delete_iter = expired_snapshot_version_path_map.begin();
+ while (to_delete_iter != expired_snapshot_version_path_map.end()) {
+
+ std::vector<TimestampedVersionSharedPtr>& to_delete_version =
to_delete_iter->second->timestamped_versions();
+ for (auto& timestampedVersion : to_delete_version) {
+ auto it =
_expired_snapshot_rs_version_map.find(timestampedVersion->version());
+ if (it != _expired_snapshot_rs_version_map.end()) {
+ // delete rowset
+ StorageEngine::instance()->add_unused_rowset(it->second);
+ _expired_snapshot_rs_version_map.erase(it);
+ LOG(INFO) << "delete expired snapshot rowset tablet=" <<
full_name()
+ <<" version[" << timestampedVersion->version().first <<
"," << timestampedVersion->version().second
+ << "] move to unused_rowset success " << std::fixed <<
expired_snapshot_sweep_endtime;
+ } else {
+ LOG(WARNING) << "delete expired snapshot rowset tablet=" <<
full_name()
+ <<" version[" << timestampedVersion->version().first <<
"," << timestampedVersion->version().second
+ << "] not find in expired snapshot rs version ";
+ }
+
_delete_expired_snapshot_rowset_by_version(timestampedVersion->version());
Review comment:
in `_delete_expired_snapshot_rowset_by_version`, we call
`_expired_snapshot_rs_version_map.erase()` again, which is duplicated.
##########
File path: be/src/olap/tablet.cpp
##########
@@ -329,6 +342,18 @@ void Tablet::_delete_inc_rowset_by_version(const Version&
version,
VLOG(3) << "delete incremental rowset. tablet=" << full_name() << ",
version=" << version;
}
+void Tablet::_delete_expired_snapshot_rowset_by_version(const Version&
version) {
+ // delete expired snapshot rowset from map
+ _expired_snapshot_rs_version_map.erase(version);
+
+ RowsetMetaSharedPtr rowset_meta =
_tablet_meta->acquire_expired_snapshot_rs_meta_by_version(version);
Review comment:
This check is meaningless. I think you can just call
`_tablet_meta->delete_expired_snapshot_rs_meta_by_version(version);`
##########
File path: be/src/olap/tablet_meta.cpp
##########
@@ -544,6 +548,27 @@ OLAPStatus TabletMeta::add_inc_rs_meta(const
RowsetMetaSharedPtr& rs_meta) {
return OLAP_SUCCESS;
}
+void TabletMeta::delete_expired_snapshot_rs_meta_by_version(const Version&
version) {
+ auto it = _expired_snapshot_rs_metas.begin();
+ while (it != _expired_snapshot_rs_metas.end()) {
+ if ((*it)->version() == version) {
+ _expired_snapshot_rs_metas.erase(it);
+ break;
Review comment:
Are you sure there will be only one version which equals to the
specified version?
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
