This is an automated email from the ASF dual-hosted git repository.
xiangfu0 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 3790129aa15 Generate UUID sample data in the schema recommender
(#18973)
3790129aa15 is described below
commit 3790129aa15ec70d9870a5f61fa09b67d2a1fbc3
Author: Xiang Fu <[email protected]>
AuthorDate: Thu Jul 16 20:37:20 2026 +0200
Generate UUID sample data in the schema recommender (#18973)
---
.../data/generator/GeneratorFactory.java | 3 +
.../recommender/data/generator/UuidGenerator.java | 76 +++++++++++++++++++
.../data/generator/UuidGeneratorTest.java | 87 ++++++++++++++++++++++
.../recommender/data/writer/AvroWriterTest.java | 50 +++++++++++++
.../plugin/inputformat/avro/AvroSchemaUtil.java | 18 ++++-
.../inputformat/avro/AvroSchemaUtilTest.java | 14 +++-
6 files changed, 242 insertions(+), 6 deletions(-)
diff --git
a/pinot-controller/src/main/java/org/apache/pinot/controller/recommender/data/generator/GeneratorFactory.java
b/pinot-controller/src/main/java/org/apache/pinot/controller/recommender/data/generator/GeneratorFactory.java
index 4186543eec9..61c7614ad6a 100644
---
a/pinot-controller/src/main/java/org/apache/pinot/controller/recommender/data/generator/GeneratorFactory.java
+++
b/pinot-controller/src/main/java/org/apache/pinot/controller/recommender/data/generator/GeneratorFactory.java
@@ -42,6 +42,9 @@ public class GeneratorFactory {
if (type == DataType.BYTES) {
return new BytesGenerator(cardinality, entryLength);
}
+ if (type == DataType.UUID) {
+ return new UuidGenerator(cardinality, numberOfValuesPerEntry);
+ }
if (timeUnit != null) {
return new TimeGenerator(cardinality, type, timeUnit);
}
diff --git
a/pinot-controller/src/main/java/org/apache/pinot/controller/recommender/data/generator/UuidGenerator.java
b/pinot-controller/src/main/java/org/apache/pinot/controller/recommender/data/generator/UuidGenerator.java
new file mode 100644
index 00000000000..5ba4ad952f0
--- /dev/null
+++
b/pinot-controller/src/main/java/org/apache/pinot/controller/recommender/data/generator/UuidGenerator.java
@@ -0,0 +1,76 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.controller.recommender.data.generator;
+
+import com.google.common.annotations.VisibleForTesting;
+import java.util.Random;
+import java.util.UUID;
+
+
+/// Generates canonical, parseable UUID-format strings for a column of type
UUID. UUID is a logical type whose
+/// recommender Avro schema is a `string` annotated with `logicalType: uuid`
(see
+/// `AvroSchemaUtil#toAvroSchemaJsonObject`), so the generated value is the
canonical string form and the writer
+/// serializes it directly.
+///
+/// The number of distinct values is bounded by the requested cardinality: a
fixed random high-order half is combined
+/// with a low-order counter that cycles `[1, cardinality]`, mirroring
`StringGenerator`. These are deliberately not
+/// RFC-4122 version-4 UUIDs (the version/variant bits are not set); sample
data only needs distinct,
+/// canonically-formatted, parseable values.
+public class UuidGenerator implements Generator {
+ private static final double DEFAULT_NUMBER_OF_VALUES_PER_ENTRY = 1;
+
+ private final int _cardinality;
+ private final double _numberOfValuesPerEntry;
+ private final Random _random;
+ private final long _mostSignificantBits;
+ private long _counter;
+
+ public UuidGenerator(Integer cardinality, Double numberOfValuesPerEntry) {
+ this(cardinality, numberOfValuesPerEntry, new
Random(System.currentTimeMillis()));
+ }
+
+ @VisibleForTesting
+ UuidGenerator(Integer cardinality, Double numberOfValuesPerEntry, Random
random) {
+ _cardinality = cardinality;
+ _numberOfValuesPerEntry =
+ numberOfValuesPerEntry != null ? numberOfValuesPerEntry :
DEFAULT_NUMBER_OF_VALUES_PER_ENTRY;
+ _random = random;
+ _mostSignificantBits = random.nextLong();
+ }
+
+ @Override
+ public void init() {
+ }
+
+ @Override
+ public Object next() {
+ if (_numberOfValuesPerEntry == 1) {
+ return nextUuid();
+ }
+ return
MultiValueGeneratorHelper.generateMultiValueEntries(_numberOfValuesPerEntry,
_random, this::nextUuid);
+ }
+
+ private String nextUuid() {
+ if (_counter >= _cardinality) {
+ _counter = 0;
+ }
+ _counter++;
+ return new UUID(_mostSignificantBits, _counter).toString();
+ }
+}
diff --git
a/pinot-controller/src/test/java/org/apache/pinot/controller/recommender/data/generator/UuidGeneratorTest.java
b/pinot-controller/src/test/java/org/apache/pinot/controller/recommender/data/generator/UuidGeneratorTest.java
new file mode 100644
index 00000000000..9f6eccbbeb9
--- /dev/null
+++
b/pinot-controller/src/test/java/org/apache/pinot/controller/recommender/data/generator/UuidGeneratorTest.java
@@ -0,0 +1,87 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.controller.recommender.data.generator;
+
+import java.util.HashSet;
+import java.util.List;
+import java.util.Random;
+import java.util.Set;
+import java.util.UUID;
+import org.testng.annotations.Test;
+
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertTrue;
+
+
+public class UuidGeneratorTest {
+
+ /// Single-value generation must yield canonical, parseable UUID strings.
+ @Test
+ public void testSingleValueProducesCanonicalUuids() {
+ UuidGenerator generator = new UuidGenerator(1000, 1.0, new Random(42));
+ generator.init();
+ for (int i = 0; i < 100; i++) {
+ String value = (String) generator.next();
+ // Round-trips through UUID.fromString and is already in canonical
lowercase form.
+ assertEquals(UUID.fromString(value).toString(), value);
+ }
+ }
+
+ /// Distinct values must be bounded by the requested cardinality, and the
low-order counter must cycle.
+ @Test
+ public void testCardinalityBoundsAndCycles() {
+ int cardinality = 3;
+ UuidGenerator generator = new UuidGenerator(cardinality, 1.0, new
Random(42));
+ generator.init();
+
+ String first = (String) generator.next();
+ String second = (String) generator.next();
+ String third = (String) generator.next();
+ String fourth = (String) generator.next();
+
+ Set<String> distinct = new HashSet<>(List.of(first, second, third,
fourth));
+ assertEquals(distinct.size(), cardinality, "distinct values must not
exceed cardinality");
+ assertEquals(fourth, first, "the counter must cycle back to the first
value after cardinality entries");
+ }
+
+ /// A null numberOfValuesPerEntry must default to single-value generation (a
bare string, not a list).
+ @Test
+ public void testNullNumberOfValuesPerEntryDefaultsToSingleValue() {
+ UuidGenerator generator = new UuidGenerator(1000, null, new Random(42));
+ generator.init();
+ assertTrue(generator.next() instanceof String, "null
numberOfValuesPerEntry must yield a single string value");
+ }
+
+ /// Multi-value generation must return a list of parseable UUID strings.
+ @Test
+ public void testMultiValueReturnsListOfUuids() {
+ UuidGenerator generator = new UuidGenerator(1000, 3.0, new Random(42));
+ generator.init();
+
+ Object next = generator.next();
+ assertTrue(next instanceof List, "multi-value entry must be a List");
+ List<?> values = (List<?>) next;
+ // For an integer numberOfValuesPerEntry the count is deterministic (see
MultiValueGeneratorHelper).
+ assertEquals(values.size(), 3);
+ for (Object value : values) {
+ String uuid = (String) value;
+ assertEquals(UUID.fromString(uuid).toString(), uuid);
+ }
+ }
+}
diff --git
a/pinot-controller/src/test/java/org/apache/pinot/controller/recommender/data/writer/AvroWriterTest.java
b/pinot-controller/src/test/java/org/apache/pinot/controller/recommender/data/writer/AvroWriterTest.java
index 8362c362707..f8bfae6cd49 100644
---
a/pinot-controller/src/test/java/org/apache/pinot/controller/recommender/data/writer/AvroWriterTest.java
+++
b/pinot-controller/src/test/java/org/apache/pinot/controller/recommender/data/writer/AvroWriterTest.java
@@ -23,6 +23,7 @@ import java.nio.file.Files;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.UUID;
import java.util.concurrent.TimeUnit;
import org.apache.avro.Schema;
import org.apache.avro.file.DataFileReader;
@@ -117,6 +118,55 @@ public class AvroWriterTest {
assertEquals(count, totalDocs);
}
+ /// The recommender must support UUID columns end to end: the generated
schema declares a logicalType "uuid" string
+ /// and the values round-trip as canonical UUID strings. Before UUID
generation was wired up, writing a UUID column
+ /// threw because the number generator rejected the type.
+ @Test
+ public void testUuidRoundTrip()
+ throws Exception {
+ List<String> columns = List.of("uuidCol");
+ Map<String, DataType> dataTypes = new HashMap<>();
+ dataTypes.put("uuidCol", DataType.UUID);
+ Map<String, FieldType> fieldTypes = new HashMap<>();
+ Map<String, Integer> cardinality = new HashMap<>();
+ for (String column : columns) {
+ fieldTypes.put(column, FieldType.DIMENSION);
+ cardinality.put(column, 5);
+ }
+
+ DataGeneratorSpec spec =
+ new DataGeneratorSpec(columns, cardinality, new HashMap<String,
IntegerRange>(), new HashMap<>(),
+ new HashMap<>(), new HashMap<>(), dataTypes, fieldTypes, new
HashMap<String, TimeUnit>(), new HashMap<>(),
+ new HashMap<>());
+ DataGenerator generator = new DataGenerator();
+ generator.init(spec);
+
+ int totalDocs = 10;
+ AvroWriterSpec writerSpec = new AvroWriterSpec(generator, _baseDir,
totalDocs, 1, 0);
+ AvroWriter writer = new AvroWriter();
+ writer.init(writerSpec);
+ writer.write();
+
+ Schema avroSchema = AvroWriter.getAvroSchema(writerSpec.getSchema());
+ Schema uuidBranch = nonNullBranch(avroSchema, "uuidCol");
+ assertEquals(uuidBranch.getType(), Schema.Type.STRING);
+ assertEquals(uuidBranch.getProp("logicalType"), "uuid");
+
+ File avroFile = new File(_baseDir, "part-0.avro");
+ assertTrue(avroFile.exists());
+ int count = 0;
+ try (DataFileReader<GenericRecord> reader = new DataFileReader<>(avroFile,
new GenericDatumReader<>())) {
+ for (GenericRecord record : reader) {
+ Object value = record.get("uuidCol");
+ assertTrue(value instanceof CharSequence, "uuidCol should read back as
a string");
+ String uuid = value.toString();
+ assertEquals(UUID.fromString(uuid).toString(), uuid, "value must be a
canonical UUID string");
+ count++;
+ }
+ }
+ assertEquals(count, totalDocs);
+ }
+
/// The generator emits Pinot's stored form for BOOLEAN (int 0/1); coercion
must map it to the right Boolean and
/// leave non-boolean columns untouched.
@Test
diff --git
a/pinot-plugins/pinot-input-format/pinot-avro-base/src/main/java/org/apache/pinot/plugin/inputformat/avro/AvroSchemaUtil.java
b/pinot-plugins/pinot-input-format/pinot-avro-base/src/main/java/org/apache/pinot/plugin/inputformat/avro/AvroSchemaUtil.java
index 52ea2240bd4..0b1f8ad3ab0 100644
---
a/pinot-plugins/pinot-input-format/pinot-avro-base/src/main/java/org/apache/pinot/plugin/inputformat/avro/AvroSchemaUtil.java
+++
b/pinot-plugins/pinot-input-format/pinot-avro-base/src/main/java/org/apache/pinot/plugin/inputformat/avro/AvroSchemaUtil.java
@@ -80,8 +80,9 @@ public class AvroSchemaUtil {
/// (see `AvroWriter`). Each field is emitted as a nullable union `["null",
<type>]`.
///
/// The switch is driven by the original (logical) [DataType] rather than
the stored type, so logical types are
- /// represented faithfully instead of collapsing to their physical storage
type: BOOLEAN maps to Avro `boolean`
- /// and TIMESTAMP to a `timestamp-millis` long, not a plain `int`/`long`.
+ /// represented faithfully instead of collapsing to their physical storage
type: BOOLEAN maps to Avro `boolean`,
+ /// TIMESTAMP to a `timestamp-millis` long (not a plain `int`/`long`), and
UUID to a `uuid`-logical-type string
+ /// (not raw `bytes`).
///
/// This intentionally differs from the segment-processing converters
`AvroUtils.getAvroSchemaFromPinotSchema` and
/// `SegmentProcessorAvroUtils.convertPinotSchemaToAvroSchema`, which switch
on the stored type because they
@@ -121,9 +122,18 @@ public class AvroSchemaUtil {
case JSON:
jsonSchema.set("type", convertStringsToJsonArray("null", "string"));
return jsonSchema;
- case BYTES:
case UUID:
- // UUID is a logical type stored as fixed-width 16-byte BYTES.
+ // UUID is a logical type; represent it faithfully as an Avro string
annotated with logicalType "uuid" rather
+ // than collapsing to raw bytes, so generated sample data round-trips
as canonical UUID strings.
+ ObjectNode uuidType = JsonUtils.newObjectNode();
+ uuidType.put("type", "string");
+ uuidType.put("logicalType", "uuid");
+ ArrayNode uuidUnion = JsonUtils.newArrayNode();
+ uuidUnion.add("null");
+ uuidUnion.add(uuidType);
+ jsonSchema.set("type", uuidUnion);
+ return jsonSchema;
+ case BYTES:
jsonSchema.set("type", convertStringsToJsonArray("null", "bytes"));
return jsonSchema;
default:
diff --git
a/pinot-plugins/pinot-input-format/pinot-avro-base/src/test/java/org/apache/pinot/plugin/inputformat/avro/AvroSchemaUtilTest.java
b/pinot-plugins/pinot-input-format/pinot-avro-base/src/test/java/org/apache/pinot/plugin/inputformat/avro/AvroSchemaUtilTest.java
index 924add4bb78..09142d201f0 100644
---
a/pinot-plugins/pinot-input-format/pinot-avro-base/src/test/java/org/apache/pinot/plugin/inputformat/avro/AvroSchemaUtilTest.java
+++
b/pinot-plugins/pinot-input-format/pinot-avro-base/src/test/java/org/apache/pinot/plugin/inputformat/avro/AvroSchemaUtilTest.java
@@ -41,8 +41,6 @@ public class AvroSchemaUtilTest {
assertPrimitiveType(DataType.STRING, "string");
assertPrimitiveType(DataType.JSON, "string");
assertPrimitiveType(DataType.BYTES, "bytes");
- // UUID is a logical type stored as BYTES; it must keep mapping to Avro
bytes (not fall through to default).
- assertPrimitiveType(DataType.UUID, "bytes");
// Logical types must not collapse to their stored INT/LONG type.
assertPrimitiveType(DataType.BOOLEAN, "boolean");
@@ -60,6 +58,18 @@ public class AvroSchemaUtilTest {
() -> AvroSchemaUtil.toAvroSchemaJsonObject(new
DimensionFieldSpec("col", DataType.BIG_DECIMAL, true)));
}
+ /// A UUID column is represented faithfully as an Avro string annotated with
logicalType "uuid".
+ /// `DataGenerator#buildSpec` always marks the recommender schema FieldSpec
single-value, so this emits a scalar
+ /// union like every sibling type.
+ @Test
+ public void testToAvroSchemaJsonObjectForUuid() {
+ JsonNode type = typeOf(DataType.UUID);
+ assertEquals(type.get(0).asText(), "null");
+ JsonNode uuidBranch = type.get(1);
+ assertEquals(uuidBranch.get("type").asText(), "string");
+ assertEquals(uuidBranch.get("logicalType").asText(), "uuid");
+ }
+
private static void assertPrimitiveType(DataType dataType, String
expectedAvroType) {
JsonNode type = typeOf(dataType);
assertEquals(type.get(0).asText(), "null");
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
