fred1268 commented on code in PR #21825:
URL: https://github.com/apache/datafusion/pull/21825#discussion_r3159508156
##########
datafusion/datasource-parquet/src/file_format.rs:
##########
@@ -1175,6 +1181,70 @@ impl DisplayAs for ParquetSink {
}
}
+/// Wraps any [`Future`] and accumulates the time spent actively executing
+/// inside `poll()` calls into `elapsed_compute`. Time between polls (when
+/// the future is suspended waiting for I/O or an upstream channel) is not
+/// measured, so only CPU time is captured.
+///
+/// Used by the sequential write path so that `elapsed_compute` reflects
+/// pure Arrow→Parquet encoding time, automatically excluding object-store
+/// I/O latency without requiring a separate subtraction step.
+///
+/// Note: uses `pin-project` rather than `pin-project-lite` in order to
+/// support `PinnedDrop`, which ensures accumulated time is flushed even
+/// if the future is cancelled (dropped before completion).
+#[pin_project(PinnedDrop)]
+struct ElapsedComputeFuture<T> {
+ #[pin]
+ inner: T,
+ curr: Duration,
+ elapsed_compute: Time,
+}
+
+#[pinned_drop]
+impl<T> PinnedDrop for ElapsedComputeFuture<T> {
+ fn drop(self: Pin<&mut Self>) {
+ if self.curr > Duration::default() {
+ let self_projected = self.project();
+ self_projected
+ .elapsed_compute
+ .add_duration(*self_projected.curr);
+ }
+ }
+}
+
+impl<O, F: Future<Output = O>> Future for ElapsedComputeFuture<F> {
+ type Output = O;
+
+ fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
+ let self_projected = self.project();
+ let start = Instant::now();
+ let result = self_projected.inner.poll(cx);
+ *self_projected.curr += start.elapsed();
+ if result.is_ready() {
Review Comment:
I tracked this rationale as a comment within the code.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
