pengxiangyu commented on code in PR #15832:
URL: https://github.com/apache/doris/pull/15832#discussion_r1068878580
##########
be/src/olap/tablet.cpp:
##########
@@ -1729,18 +1757,118 @@ Status Tablet::cooldown() {
has_shutdown = tablet_state() == TABLET_SHUTDOWN;
if (!has_shutdown) {
modify_rowsets(to_add, to_delete);
- _self_owned_remote_rowsets.insert(to_add.front());
+ if (!config::cooldown_single_remote_file) {
+ _self_owned_remote_rowsets.insert(to_add.front());
+ }
save_meta();
}
}
- if (has_shutdown) {
+ if (has_shutdown && !config::cooldown_single_remote_file) {
record_unused_remote_rowset(new_rowset_id, dest_fs->resource_id(),
to_add.front()->num_segments());
return Status::Aborted("tablet {} has shutdown", tablet_id());
}
return Status::OK();
}
+Status Tablet::_read_remote_tablet_meta(FileSystemSPtr fs, TabletMetaPB*
tablet_meta_pb) {
+ std::string remote_meta_path =
+ BetaRowset::remote_tablet_meta_path(tablet_id(),
_tablet_meta->cooldown_replica_id());
+ bool exist = false;
+ RETURN_IF_ERROR(fs->exists(remote_meta_path, &exist));
+ if (exist) {
+ io::FileReaderSPtr tablet_meta_reader;
+ RETURN_IF_ERROR(fs->open_file(remote_meta_path, &tablet_meta_reader));
+ if (tablet_meta_reader == nullptr) {
+ return Status::InternalError("tablet_meta_reader is null");
+ }
+ auto file_size = tablet_meta_reader->size();
+ size_t bytes_read;
+ uint8_t* buf = new uint8_t[file_size];
+ Slice slice(buf, file_size);
+ IOContext io_ctx;
+ Status st = tablet_meta_reader->read_at(0, slice, io_ctx, &bytes_read);
+ if (!st.ok()) {
+ tablet_meta_reader->close();
+ return st;
+ }
+ tablet_meta_reader->close();
+ if (!tablet_meta_pb->ParseFromString(slice.to_string())) {
+ LOG(WARNING) << "parse tablet meta failed";
+ return Status::InternalError("parse tablet meta failed");
+ }
+ }
+ LOG(INFO) << "No tablet meta file founded, init needed. tablet_id: " <<
tablet_id();
+ return Status::InternalError("No tablet meta file founded, init needed.");
+}
+
+Status Tablet::_write_remote_tablet_meta(FileSystemSPtr fs, const
TabletMetaPB& tablet_meta_pb) {
+ std::string remote_meta_path =
+ BetaRowset::remote_tablet_meta_path(tablet_id(),
_tablet_meta->replica_id());
+ io::FileWriterPtr tablet_meta_writer;
+ RETURN_IF_ERROR(fs->create_file(remote_meta_path, &tablet_meta_writer));
+ if (tablet_meta_writer == nullptr) {
+ return Status::InternalError("tablet_meta_writer is null");
+ }
+ string value;
+ tablet_meta_pb.SerializeToString(&value);
+ uint8_t* buf = new uint8_t[value.size()];
+ memcpy(buf, value.c_str(), value.size());
+ Slice slice(buf, value.size());
+ Status st = tablet_meta_writer->appendv(&slice, 1);
+ if (!st.ok()) {
+ tablet_meta_writer->close();
+ return st;
+ }
+ tablet_meta_writer->close();
+ return Status::OK();
+}
+
+Status Tablet::_cooldown_use_remote_data() {
+ auto dest_fs = io::FileSystemMap::instance()->get(storage_policy());
+ if (!dest_fs) {
+ return Status::InternalError("storage_policy doesn't exist: " +
storage_policy());
+ }
+ DCHECK(dest_fs->type() == io::FileSystemType::S3);
+ TabletMetaPB remote_tablet_meta_pb;
+ RETURN_IF_ERROR(_read_remote_tablet_meta(dest_fs, &remote_tablet_meta_pb));
+ std::vector<RowsetSharedPtr> to_add;
+ std::vector<RowsetSharedPtr> to_delete;
+ int64_t max_version = -1;
+ for (auto& rowset_meta_pb : remote_tablet_meta_pb.rs_metas()) {
+ RowsetMetaSharedPtr rowset_meta(new RowsetMeta());
+ rowset_meta->init_from_pb(rowset_meta_pb);
+ RowsetSharedPtr new_rowset;
+ RowsetFactory::create_rowset(_schema, _tablet_path, rowset_meta,
&new_rowset);
+ to_add.push_back(new_rowset);
+ if (rowset_meta_pb.end_version() > max_version) {
+ max_version = rowset_meta_pb.end_version();
+ }
+ }
+
+ {
+ std::shared_lock meta_rlock(_meta_lock);
+ for (const auto& it : _rs_version_map) {
+ auto& rs = it.second;
+ if (rs->end_version() <= max_version) {
+ to_delete.push_back(rs);
+ }
+ }
+ }
+
+ bool has_shutdown = false;
+ {
+ std::unique_lock meta_wlock(_meta_lock);
+ has_shutdown = tablet_state() == TABLET_SHUTDOWN;
+ if (!has_shutdown) {
+ modify_rowsets(to_add, to_delete);
+ save_meta();
Review Comment:
save_meta(); failed will cause FATAL
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
