github-actions[bot] commented on code in PR #15832:
URL: https://github.com/apache/doris/pull/15832#discussion_r1068971124
##########
be/src/olap/tablet.cpp:
##########
@@ -1717,27 +1735,125 @@ Status Tablet::cooldown() {
new_rowset_meta->set_resource_id(dest_fs->resource_id());
new_rowset_meta->set_fs(dest_fs);
new_rowset_meta->set_creation_time(time(nullptr));
+
+ // write remote tablet meta
+ TabletMetaPB remote_tablet_meta_pb;
+ _tablet_meta->to_meta_pb(true, &remote_tablet_meta_pb);
+ new_rowset_meta->to_rowset_pb(remote_tablet_meta_pb.add_rs_metas());
+ // upload rowset_meta to remote fs.
+ RETURN_IF_ERROR(_write_remote_tablet_meta(dest_fs, remote_tablet_meta_pb));
+
RowsetSharedPtr new_rowset;
RowsetFactory::create_rowset(_schema, _tablet_path, new_rowset_meta,
&new_rowset);
std::vector to_add {std::move(new_rowset)};
std::vector to_delete {std::move(old_rowset)};
- bool has_shutdown = false;
{
std::unique_lock meta_wlock(_meta_lock);
- has_shutdown = tablet_state() == TABLET_SHUTDOWN;
- if (!has_shutdown) {
+ if (tablet_state() != TABLET_SHUTDOWN) {
modify_rowsets(to_add, to_delete);
- _self_owned_remote_rowsets.insert(to_add.front());
save_meta();
}
}
- if (has_shutdown) {
- record_unused_remote_rowset(new_rowset_id, dest_fs->resource_id(),
- to_add.front()->num_segments());
- return Status::Aborted("tablet {} has shutdown", tablet_id());
+ return Status::OK();
+}
+
+Status Tablet::_read_remote_tablet_meta(FileSystemSPtr fs, TabletMetaPB*
tablet_meta_pb) {
+ std::string remote_meta_path =
+ BetaRowset::remote_tablet_meta_path(tablet_id(),
_tablet_meta->cooldown_replica_id());
+ bool exist = false;
+ RETURN_IF_ERROR(fs->exists(remote_meta_path, &exist));
+ if (exist) {
+ IOContext io_ctx;
+ io::FileReaderSPtr tablet_meta_reader;
+ RETURN_IF_ERROR(fs->open_file(remote_meta_path, &tablet_meta_reader,
&io_ctx));
+ if (tablet_meta_reader == nullptr) {
+ return Status::InternalError("tablet_meta_reader is null");
+ }
+ auto file_size = tablet_meta_reader->size();
+ size_t bytes_read;
+ uint8_t* buf = new uint8_t[file_size];
+ Slice slice(buf, file_size);
+ Status st = tablet_meta_reader->read_at(0, slice, io_ctx, &bytes_read);
+ if (!st.ok()) {
+ tablet_meta_reader->close();
+ return st;
+ }
+ tablet_meta_reader->close();
+ if (!tablet_meta_pb->ParseFromString(slice.to_string())) {
+ LOG(WARNING) << "parse tablet meta failed";
+ return Status::InternalError("parse tablet meta failed");
+ }
+ }
+ LOG(INFO) << "No tablet meta file founded, init needed. tablet_id: " <<
tablet_id();
+ return Status::InternalError("No tablet meta file founded, init needed.");
+}
+
+Status Tablet::_write_remote_tablet_meta(FileSystemSPtr fs, const
TabletMetaPB& tablet_meta_pb) {
+ std::string remote_meta_path =
+ BetaRowset::remote_tablet_meta_path(tablet_id(),
_tablet_meta->replica_id());
+ io::FileWriterPtr tablet_meta_writer;
+ RETURN_IF_ERROR(fs->create_file(remote_meta_path, &tablet_meta_writer));
+ if (tablet_meta_writer == nullptr) {
+ return Status::InternalError("tablet_meta_writer is null");
+ }
+ string value;
+ tablet_meta_pb.SerializeToString(&value);
+ uint8_t* buf = new uint8_t[value.size()];
+ memcpy(buf, value.c_str(), value.size());
+ Slice slice(buf, value.size());
+ Status st = tablet_meta_writer->appendv(&slice, 1);
+ if (!st.ok()) {
+ tablet_meta_writer->close();
+ return st;
}
+ tablet_meta_writer->close();
+ return Status::OK();
+}
+
+Status Tablet::_follow_cooldowned_data() {
+ auto dest_fs = io::FileSystemMap::instance()->get(storage_policy());
+ if (!dest_fs) {
+ return Status::InternalError("storage_policy doesn't exist: " +
storage_policy());
+ }
+ DCHECK(dest_fs->type() == io::FileSystemType::S3);
+ TabletMetaPB remote_tablet_meta_pb;
+ RETURN_IF_ERROR(_read_remote_tablet_meta(dest_fs, &remote_tablet_meta_pb));
+ int64_t max_version = -1;
+ for (auto& rowset_meta_pb : remote_tablet_meta_pb.rs_metas()) {
+ if (max_version < rowset_meta_pb.end_version()) {
+ max_version = rowset_meta_pb.end_version();
+ }
+ }
+
+ std::vector<RowsetSharedPtr> to_add;
Review Comment:
warning: use of undeclared identifier 'rs' [clang-diagnostic-error]
```cpp
metas()) {
^
```
##########
be/src/olap/tablet.cpp:
##########
@@ -1717,27 +1735,125 @@
new_rowset_meta->set_resource_id(dest_fs->resource_id());
new_rowset_meta->set_fs(dest_fs);
new_rowset_meta->set_creation_time(time(nullptr));
+
+ // write remote tablet meta
+ TabletMetaPB remote_tablet_meta_pb;
+ _tablet_meta->to_meta_pb(true, &remote_tablet_meta_pb);
+ new_rowset_meta->to_rowset_pb(remote_tablet_meta_pb.add_rs_metas());
+ // upload rowset_meta to remote fs.
+ RETURN_IF_ERROR(_write_remote_tablet_meta(dest_fs, remote_tablet_meta_pb));
+
RowsetSharedPtr new_rowset;
RowsetFactory::create_rowset(_schema, _tablet_path, new_rowset_meta,
&new_rowset);
std::vector to_add {std::move(new_rowset)};
std::vector to_delete {std::move(old_rowset)};
- bool has_shutdown = false;
{
std::unique_lock meta_wlock(_meta_lock);
- has_shutdown = tablet_state() == TABLET_SHUTDOWN;
- if (!has_shutdown) {
+ if (tablet_state() != TABLET_SHUTDOWN) {
modify_rowsets(to_add, to_delete);
- _self_owned_remote_rowsets.insert(to_add.front());
save_meta();
}
}
- if (has_shutdown) {
- record_unused_remote_rowset(new_rowset_id, dest_fs->resource_id(),
- to_add.front()->num_segments());
- return Status::Aborted("tablet {} has shutdown", tablet_id());
+ return Status::OK();
+}
+
+Status Tablet::_read_remote_tablet_meta(FileSystemSPtr fs, TabletMetaPB*
tablet_meta_pb) {
+ std::string remote_meta_path =
+ BetaRowset::remote_tablet_meta_path(tablet_id(),
_tablet_meta->cooldown_replica_id());
+ bool exist = false;
+ RETURN_IF_ERROR(fs->exists(remote_meta_path, &exist));
+ if (exist) {
+ IOContext io_ctx;
+ io::FileReaderSPtr tablet_meta_reader;
+ RETURN_IF_ERROR(fs->open_file(remote_meta_path, &tablet_meta_reader,
&io_ctx));
+ if (tablet_meta_reader == nullptr) {
+ return Status::InternalError("tablet_meta_reader is null");
+ }
+ auto file_size = tablet_meta_reader->size();
+ size_t bytes_read;
+ uint8_t* buf = new uint8_t[file_size];
+ Slice slice(buf, file_size);
+ Status st = tablet_meta_reader->read_at(0, slice, io_ctx, &bytes_read);
+ if (!st.ok()) {
+ tablet_meta_reader->close();
+ return st;
+ }
+ tablet_meta_reader->close();
+ if (!tablet_meta_pb->ParseFromString(slice.to_string())) {
+ LOG(WARNING) << "parse tablet meta failed";
+ return Status::InternalError("parse tablet meta failed");
+ }
+ }
+ LOG(INFO) << "No tablet meta file founded, init needed. tablet_id: " <<
tablet_id();
+ return Status::InternalError("No tablet meta file founded, init needed.");
+}
+
+Status Tablet::_write_remote_tablet_meta(FileSystemSPtr fs, const
TabletMetaPB& tablet_meta_pb) {
+ std::string remote_meta_path =
+ BetaRowset::remote_tablet_meta_path(tablet_id(),
_tablet_meta->replica_id());
+ io::FileWriterPtr tablet_meta_writer;
+ RETURN_IF_ERROR(fs->create_file(remote_meta_path, &tablet_meta_writer));
+ if (tablet_meta_writer == nullptr) {
+ return Status::InternalError("tablet_meta_writer is null");
+ }
+ string value;
+ tablet_meta_pb.SerializeToString(&value);
+ uint8_t* buf = new uint8_t[value.size()];
+ memcpy(buf, value.c_str(), value.size());
+ Slice slice(buf, value.size());
+ Status st = tablet_meta_writer->appendv(&slice, 1);
+ if (!st.ok()) {
+ tablet_meta_writer->close();
+ return st;
}
+ tablet_meta_writer->close();
+ return Status::OK();
+}
+
+Status Tablet::_follow_cooldowned_data() {
+ auto dest_fs = io::FileSystemMap::instance()->get(storage_policy());
+ if (!dest_fs) {
+ return Status::InternalError("storage_policy doesn't exist: " +
storage_policy());
+ }
+ DCHECK(dest_fs->type() == io::FileSystemType::S3);
+ TabletMetaPB remote_tablet_meta_pb;
+ RETURN_IF_ERROR(_read_remote_tablet_meta(dest_fs, &remote_tablet_meta_pb));
+ int64_t max_version = -1;
+ for (auto& rowset_meta_pb : remote_tablet_meta_pb.rs_metas()) {
+ if (max_version < rowset_meta_pb.end_version()) {
+ max_version = rowset_meta_pb.end_version();
+ }
+ }
+
+ std::vector<RowsetSharedPtr> to_add;
+ std::vector<RowsetSharedPtr> to_delete;
Review Comment:
warning: use of undeclared identifier 'rs' [clang-diagnostic-error]
```cpp
end_version()) {
^
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
