OIiveirra commented on code in PR #68161:
URL: https://github.com/apache/doris/pull/68161#discussion_r4247432984
##########
fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonScanPlanProvider.java:
##########
@@ -1289,6 +1296,81 @@ public Map<String, String> getScanNodeProperties(
return props;
}
+ static boolean containsTimestampType(DataType type) {
+ if (type instanceof org.apache.paimon.types.TimestampType
+ || type instanceof
org.apache.paimon.types.LocalZonedTimestampType) {
+ return true;
+ }
+ if (type instanceof org.apache.paimon.types.RowType) {
+ return ((org.apache.paimon.types.RowType)
type).getFields().stream()
+ .anyMatch(field -> containsTimestampType(field.type()));
+ }
+ if (type instanceof org.apache.paimon.types.ArrayType) {
+ return containsTimestampType(((org.apache.paimon.types.ArrayType)
type).getElementType());
+ }
+ if (type instanceof org.apache.paimon.types.MapType) {
+ org.apache.paimon.types.MapType mapType =
(org.apache.paimon.types.MapType) type;
+ return containsTimestampType(mapType.getKeyType()) ||
containsTimestampType(mapType.getValueType());
+ }
+ return false;
+ }
+
+ private boolean requiresTimestampPrecisionRepair(Table table,
PaimonTableHandle handle) {
+ Table schemaTable = resolveSchemaDictTable(table, handle);
+ if (!(schemaTable instanceof FileStoreTable)) {
+ return false;
+ }
+ FileStoreTable fileStoreTable = (FileStoreTable) schemaTable;
+ RowType currentType = schemaTable.rowType();
+ SchemaManager schemaManager = fileStoreTable.schemaManager();
+ for (Long schemaId : schemaManager.listAllIds()) {
Review Comment:
Addressed in 7a66530e04a. Timestamp history now unwraps decorators and
checks both wrapped and fallback branches independently. Timestamp-dependent
predicates remain residual whenever either branch has narrowing history. Added
a real fallback-only p6-to-p4 leading-zero fixture test covering split planning
and serialized JNI predicates, plus reversed branch coverage. Focused FE/JNI
validation: 184 tests passed.
##########
fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonScanPlanProvider.java:
##########
@@ -1289,6 +1296,81 @@ public Map<String, String> getScanNodeProperties(
return props;
}
+ static boolean containsTimestampType(DataType type) {
+ if (type instanceof org.apache.paimon.types.TimestampType
+ || type instanceof
org.apache.paimon.types.LocalZonedTimestampType) {
+ return true;
+ }
+ if (type instanceof org.apache.paimon.types.RowType) {
+ return ((org.apache.paimon.types.RowType)
type).getFields().stream()
+ .anyMatch(field -> containsTimestampType(field.type()));
+ }
+ if (type instanceof org.apache.paimon.types.ArrayType) {
+ return containsTimestampType(((org.apache.paimon.types.ArrayType)
type).getElementType());
+ }
+ if (type instanceof org.apache.paimon.types.MapType) {
+ org.apache.paimon.types.MapType mapType =
(org.apache.paimon.types.MapType) type;
+ return containsTimestampType(mapType.getKeyType()) ||
containsTimestampType(mapType.getValueType());
+ }
+ return false;
+ }
+
+ private boolean requiresTimestampPrecisionRepair(Table table,
PaimonTableHandle handle) {
+ Table schemaTable = resolveSchemaDictTable(table, handle);
+ if (!(schemaTable instanceof FileStoreTable)) {
+ return false;
+ }
+ FileStoreTable fileStoreTable = (FileStoreTable) schemaTable;
+ RowType currentType = schemaTable.rowType();
+ SchemaManager schemaManager = fileStoreTable.schemaManager();
+ for (Long schemaId : schemaManager.listAllIds()) {
+ TableSchema historicalSchema = schemaManager.schema(schemaId);
Review Comment:
Addressed in 7a66530e04a. Predicate conversion first checks whether
referenced fields can contain timestamps; timestamp-free schemas and unrelated
id predicates skip history I/O. Necessary history results are shared between
both FE paths using a table/options key in the existing statement scope, not a
cross-query cache. Recording FileIO tests verify no history listing for
unrelated predicates and reuse across provider instances. Connector install and
focused tests passed.
##########
fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonColumnValue.java:
##########
@@ -195,11 +197,36 @@ public LocalDateTime getDateTime() {
@Override
public LocalDateTime getTimeStampTz() {
- Timestamp ts = record.getTimestamp(idx, dorisType.getPrecision());
+ Timestamp ts = truncateTimestampPrecision(
+ record.getTimestamp(idx, getPaimonTimestampPrecision()));
// Timestamp's local representation is identical to converting its
epoch instant in UTC.
return ts.toLocalDateTime();
}
+ private Timestamp truncateTimestampPrecision(Timestamp timestamp) {
+ if (!(dataType instanceof TimestampType) && !(dataType instanceof
LocalZonedTimestampType)) {
+ return timestamp;
+ }
+ int precision = dorisType.getPrecision();
+ long divisor = 1;
+ for (int i = precision; i < 9; ++i) {
+ divisor *= 10;
+ }
+ LocalDateTime value = timestamp.toLocalDateTime();
+ int truncatedNano = (int) (value.getNano() / divisor * divisor);
+ return Timestamp.fromLocalDateTime(value.withNano(truncatedNano));
+ }
Review Comment:
Addressed in 7a66530e04a. Aligned values return the original Timestamp
before any LocalDateTime conversion or reconstruction. FE passes an explicit
historical-precision-repair flag so ordinary scans do not widen timestamp read
types; absent flags retain the old safe behavior for compatibility. Added
object-identity fast-path tests and a real ordinary ORC scanner read-type test.
All 50 targeted JNI tests passed.
##########
be/src/format_v2/expr/cast.cpp:
##########
@@ -24,15 +24,69 @@
#include <ostream>
#include "common/status.h"
+#include "core/assert_cast.h"
#include "core/block/block.h"
#include "core/block/column_with_type_and_name.h"
#include "core/block/columns_with_type_and_name.h"
+#include "core/column/column_nullable.h"
+#include "core/data_type/data_type_date_or_datetime_v2.h"
+#include "core/data_type/data_type_timestamptz.h"
#include "exprs/function/simple_function_factory.h"
#include "exprs/vexpr_context.h"
#include "exprs/vliteral.h"
namespace doris::format {
+static bool can_truncate_datetimev2_precision(const DataTypePtr& source_type,
+ const DataTypePtr& target_type) {
+ const auto source = remove_nullable(source_type);
+ const auto target = remove_nullable(target_type);
+ const auto primitive = source->get_primitive_type();
+ return primitive == target->get_primitive_type() &&
+ (primitive == TYPE_DATETIMEV2 || primitive == TYPE_TIMESTAMPTZ) &&
+ source->get_scale() > target->get_scale();
+}
+
+static void truncate_datetimev2_precision(ColumnPtr* column, const
DataTypePtr& target_type) {
+ DORIS_CHECK(column != nullptr);
+ auto mutable_column = IColumn::mutate(std::move(*column));
+ IColumn* nested_column = mutable_column.get();
+ if (is_column_nullable(*nested_column)) {
+ nested_column =
static_cast<ColumnNullable*>(nested_column)->get_nested_column_ptr().get();
+ }
+ const auto scale = remove_nullable(target_type)->get_scale();
+ uint32_t divisor = 1;
+ for (uint32_t i = scale; i < 6; ++i) {
+ divisor *= 10;
+ }
+ if (remove_nullable(target_type)->get_primitive_type() == TYPE_DATETIMEV2)
{
+ auto& data = assert_cast<ColumnDateTimeV2&>(*nested_column).get_data();
+ for (auto& value : data) {
+
value.unchecked_set_time_unit<TimeUnit::MICROSECOND>(value.microsecond() /
divisor *
+ divisor);
Review Comment:
Addressed in 7a66530e04a by routing only ORC files whose read fields narrow
historical p7-p9 timestamp precision to JNI, preserving nanoseconds before
Doris truncation. Generic native ORC decoding and SQL CAST remain unchanged.
Added p6/p7/p9 routing tests, a real ORC scanner test, and generated p9-to-p0
TIMESTAMP/LTZ fixtures and snapshots covering V1, V2, forced JNI and equality
filters. The deployed ASAN personal-cluster paimon_timestamp_types regression
passed; previous snapshot results are unchanged. Negative TIMESTAMP is covered;
negative fractional LTZ is excluded because the Paimon reader already shifts it
before schema evolution.
##########
fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonScanPlanProvider.java:
##########
@@ -1289,6 +1296,81 @@ public Map<String, String> getScanNodeProperties(
return props;
}
+ static boolean containsTimestampType(DataType type) {
+ if (type instanceof org.apache.paimon.types.TimestampType
+ || type instanceof
org.apache.paimon.types.LocalZonedTimestampType) {
+ return true;
+ }
+ if (type instanceof org.apache.paimon.types.RowType) {
+ return ((org.apache.paimon.types.RowType)
type).getFields().stream()
+ .anyMatch(field -> containsTimestampType(field.type()));
+ }
+ if (type instanceof org.apache.paimon.types.ArrayType) {
+ return containsTimestampType(((org.apache.paimon.types.ArrayType)
type).getElementType());
+ }
+ if (type instanceof org.apache.paimon.types.MapType) {
+ org.apache.paimon.types.MapType mapType =
(org.apache.paimon.types.MapType) type;
+ return containsTimestampType(mapType.getKeyType()) ||
containsTimestampType(mapType.getValueType());
+ }
+ return false;
+ }
+
+ private boolean requiresTimestampPrecisionRepair(Table table,
PaimonTableHandle handle) {
+ Table schemaTable = resolveSchemaDictTable(table, handle);
+ if (!(schemaTable instanceof FileStoreTable)) {
+ return false;
+ }
+ FileStoreTable fileStoreTable = (FileStoreTable) schemaTable;
+ RowType currentType = schemaTable.rowType();
+ SchemaManager schemaManager = fileStoreTable.schemaManager();
+ for (Long schemaId : schemaManager.listAllIds()) {
+ TableSchema historicalSchema = schemaManager.schema(schemaId);
+ if (historicalSchema != null
Review Comment:
Addressed in 7a66530e04a. The complete schema-table resolution and history
lookup, including listAllIds and schema(id), now executes inside
context.executeAuthenticated at both FE entry points. ORC historical-schema
routing reads use the same authenticated wrapper. Recording FileIO tests assert
authentication during listing/reads, including a fresh $ro lookup. Connector
install and all focused tests passed; a real Kerberos-cluster end-to-end test
was not run.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
