puchengy commented on issue #12792:
URL: https://github.com/apache/iceberg/issues/12792#issuecomment-2847442190
I have another issue that seems to be related as well
```
py4j.protocol.Py4JJavaError: An error occurred while calling
o4880.commitDataFrameWithFilter.
: org.apache.spark.SparkException: Writing job aborted
at
org.apache.spark.sql.errors.QueryExecutionErrors$.writingJobAbortedError(QueryExecutionErrors.scala:613)
at
org.apache.spark.sql.execution.datasources.v2.V2TableWriteExec.writeWithV2(WriteToDataSourceV2Exec.scala:386)
at
org.apache.spark.sql.execution.datasources.v2.V2TableWriteExec.writeWithV2$(WriteToDataSourceV2Exec.scala:330)
at
org.apache.spark.sql.execution.datasources.v2.OverwriteByExpressionExec.writeWithV2(WriteToDataSourceV2Exec.scala:254)
at
org.apache.spark.sql.execution.datasources.v2.V2ExistingTableWriteExec.run(WriteToDataSourceV2Exec.scala:309)
at
org.apache.spark.sql.execution.datasources.v2.V2ExistingTableWriteExec.run$(WriteToDataSourceV2Exec.scala:308)
at
org.apache.spark.sql.execution.datasources.v2.OverwriteByExpressionExec.run(WriteToDataSourceV2Exec.scala:254)
at
org.apache.spark.sql.execution.datasources.v2.V2CommandExec.result$lzycompute(V2CommandExec.scala:43)
at
org.apache.spark.sql.execution.datasources.v2.V2CommandExec.result(V2CommandExec.scala:43)
at
org.apache.spark.sql.execution.datasources.v2.V2CommandExec.executeCollect(V2CommandExec.scala:49)
at
org.apache.spark.sql.execution.QueryExecution$$anonfun$eagerlyExecuteCommands$1.$anonfun$applyOrElse$1(QueryExecution.scala:139)
at
org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$5(SQLExecution.scala:111)
at
org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:172)
at
org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$1(SQLExecution.scala:97)
at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:776)
at
org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:70)
at
org.apache.spark.sql.execution.QueryExecution$$anonfun$eagerlyExecuteCommands$1.applyOrElse(QueryExecution.scala:139)
at
org.apache.spark.sql.execution.QueryExecution$$anonfun$eagerlyExecuteCommands$1.applyOrElse(QueryExecution.scala:135)
at
org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$transformDownWithPruning$1(TreeNode.scala:481)
at
org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(TreeNode.scala:82)
at
org.apache.spark.sql.catalyst.trees.TreeNode.transformDownWithPruning(TreeNode.scala:481)
at
org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.org$apache$spark$sql$catalyst$plans$logical$AnalysisHelper$$super$transformDownWithPruning(LogicalPlan.scala:30)
at
org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.transformDownWithPruning(AnalysisHelper.scala:267)
at
org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.transformDownWithPruning$(AnalysisHelper.scala:263)
at
org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.transformDownWithPruning(LogicalPlan.scala:30)
at
org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.transformDownWithPruning(LogicalPlan.scala:30)
at
org.apache.spark.sql.catalyst.trees.TreeNode.transformDown(TreeNode.scala:457)
at
org.apache.spark.sql.execution.QueryExecution.eagerlyExecuteCommands(QueryExecution.scala:135)
at
org.apache.spark.sql.execution.QueryExecution.commandExecuted$lzycompute(QueryExecution.scala:122)
at
org.apache.spark.sql.execution.QueryExecution.commandExecuted(QueryExecution.scala:120)
at
org.apache.spark.sql.execution.QueryExecution.assertCommandExecuted(QueryExecution.scala:161)
at
org.apache.spark.sql.DataFrameWriterV2.runCommand(DataFrameWriterV2.scala:194)
at
org.apache.spark.sql.DataFrameWriterV2.overwrite(DataFrameWriterV2.scala:165)
at
com.pinterest.pintel.mldatasetstore.Committer.commitDataFrameWithFilter(Committer.scala:28)
at
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:566)
at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)
at py4j.Gateway.invoke(Gateway.java:282)
at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
at py4j.commands.CallCommand.execute(CallCommand.java:79)
at
py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)
at py4j.ClientServerConnection.run(ClientServerConnection.java:106)
at java.base/java.lang.Thread.run(Thread.java:829)
Caused by: org.apache.iceberg.exceptions.RESTException: Failed to convert
HTTP response body to string
at
org.apache.iceberg.rest.HTTPClient.extractResponseBodyAsString(HTTPClient.java:110)
at org.apache.iceberg.rest.HTTPClient.execute(HTTPClient.java:281)
at org.apache.iceberg.rest.HTTPClient.execute(HTTPClient.java:219)
at org.apache.iceberg.rest.HTTPClient.post(HTTPClient.java:330)
at org.apache.iceberg.rest.RESTClient.post(RESTClient.java:112)
at
org.apache.iceberg.rest.RESTTableOperations.commit(RESTTableOperations.java:144)
at
org.apache.iceberg.SnapshotProducer.lambda$commit$2(SnapshotProducer.java:394)
at
org.apache.iceberg.util.Tasks$Builder.runTaskWithRetry(Tasks.java:413)
at
org.apache.iceberg.util.Tasks$Builder.runSingleThreaded(Tasks.java:219)
at org.apache.iceberg.util.Tasks$Builder.run(Tasks.java:203)
at org.apache.iceberg.util.Tasks$Builder.run(Tasks.java:196)
at org.apache.iceberg.SnapshotProducer.commit(SnapshotProducer.java:368)
at
org.apache.iceberg.BaseOverwriteFiles.commit(BaseOverwriteFiles.java:31)
at
org.apache.iceberg.spark.source.SparkWrite.commitOperation(SparkWrite.java:210)
at
org.apache.iceberg.spark.source.SparkWrite.access$1300(SparkWrite.java:83)
at
org.apache.iceberg.spark.source.SparkWrite$OverwriteByFilter.commit(SparkWrite.java:350)
at
org.apache.spark.sql.execution.datasources.v2.V2TableWriteExec.writeWithV2(WriteToDataSourceV2Exec.scala:369)
... 44 more
Suppressed: org.apache.hc.core5.http.ConnectionClosedException:
Premature end of chunk coded message body: closing chunk expected
at
org.apache.hc.core5.http.impl.io.ChunkedInputStream.getChunkSize(ChunkedInputStream.java:263)
at
org.apache.hc.core5.http.impl.io.ChunkedInputStream.nextChunk(ChunkedInputStream.java:222)
at
org.apache.hc.core5.http.impl.io.ChunkedInputStream.read(ChunkedInputStream.java:147)
at
org.apache.hc.core5.http.impl.io.ChunkedInputStream.close(ChunkedInputStream.java:314)
at org.apache.hc.core5.io.Closer.close(Closer.java:48)
at
org.apache.hc.core5.http.impl.io.IncomingHttpEntity.close(IncomingHttpEntity.java:112)
at
org.apache.hc.core5.http.io.entity.HttpEntityWrapper.close(HttpEntityWrapper.java:120)
at
org.apache.hc.client5.http.impl.classic.ResponseEntityProxy.close(ResponseEntityProxy.java:180)
at org.apache.hc.core5.io.Closer.close(Closer.java:48)
at
org.apache.hc.core5.http.message.BasicClassicHttpResponse.close(BasicClassicHttpResponse.java:93)
at
org.apache.hc.client5.http.impl.classic.CloseableHttpResponse.close(CloseableHttpResponse.java:200)
at
org.apache.iceberg.rest.HTTPClient.execute(HTTPClient.java:267)
... 59 more
Caused by: org.apache.hc.core5.http.ConnectionClosedException: Premature end
of chunk coded message body: closing chunk expected
at
org.apache.hc.core5.http.impl.io.ChunkedInputStream.getChunkSize(ChunkedInputStream.java:263)
at
org.apache.hc.core5.http.impl.io.ChunkedInputStream.nextChunk(ChunkedInputStream.java:222)
at
org.apache.hc.core5.http.impl.io.ChunkedInputStream.read(ChunkedInputStream.java:183)
at
org.apache.hc.core5.http.io.EofSensorInputStream.read(EofSensorInputStream.java:135)
at java.base/sun.nio.cs.StreamDecoder.readBytes(StreamDecoder.java:295)
at java.base/sun.nio.cs.StreamDecoder.implRead(StreamDecoder.java:337)
at java.base/sun.nio.cs.StreamDecoder.read(StreamDecoder.java:179)
at java.base/java.io.InputStreamReader.read(InputStreamReader.java:181)
at java.base/java.io.Reader.read(Reader.java:229)
at
org.apache.hc.core5.http.io.entity.EntityUtils.toCharArrayBuffer(EntityUtils.java:179)
at
org.apache.hc.core5.http.io.entity.EntityUtils.toString(EntityUtils.java:222)
at
org.apache.hc.core5.http.io.entity.EntityUtils.toString(EntityUtils.java:283)
at
org.apache.hc.core5.http.io.entity.EntityUtils.toString(EntityUtils.java:325)
at
org.apache.hc.core5.http.io.entity.EntityUtils.toString(EntityUtils.java:303)
at
org.apache.iceberg.rest.HTTPClient.extractResponseBodyAsString(HTTPClient.java:108)
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
