This is an automated email from the ASF dual-hosted git repository.
raghavyadav01 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 068b458883d Add optional metadata map to FieldSpec (#18984)
068b458883d is described below
commit 068b458883d645ee2c686ab2a5e123ef2ffe8cff
Author: Navina Ramesh <[email protected]>
AuthorDate: Tue Jul 14 17:29:37 2026 -0400
Add optional metadata map to FieldSpec (#18984)
Add an optional, additive Map<String, String> metadata to FieldSpec for
free-form per-column metadata: @JsonInclude(NON_EMPTY) so it is omitted when
unset/empty, excluded from isBackwardCompatibleWith, and round-tripped via
the
shared appendFieldIdAndAliases helper (so TimeFieldSpec preserves it too).
The keys and their interpretation are defined by whoever populates it; the
core
schema attaches no semantics. Rolling-upgrade safe: old readers ignore the
unknown property (concrete subclasses are
@JsonIgnoreProperties(ignoreUnknown)),
new writers omit it when empty.
---
.../java/org/apache/pinot/spi/data/FieldSpec.java | 36 ++++++--
.../org/apache/pinot/spi/data/FieldSpecTest.java | 96 ++++++++++++++++++++++
2 files changed, 127 insertions(+), 5 deletions(-)
diff --git a/pinot-spi/src/main/java/org/apache/pinot/spi/data/FieldSpec.java
b/pinot-spi/src/main/java/org/apache/pinot/spi/data/FieldSpec.java
index c5c3a491f6c..5ecefcf9f85 100644
--- a/pinot-spi/src/main/java/org/apache/pinot/spi/data/FieldSpec.java
+++ b/pinot-spi/src/main/java/org/apache/pinot/spi/data/FieldSpec.java
@@ -38,6 +38,8 @@ import java.util.List;
import java.util.Map;
import java.util.Objects;
import javax.annotation.Nullable;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.collections4.MapUtils;
import org.apache.pinot.spi.utils.BooleanUtils;
import org.apache.pinot.spi.utils.ByteArray;
import org.apache.pinot.spi.utils.BytesUtils;
@@ -172,6 +174,14 @@ public abstract class FieldSpec implements
Comparable<FieldSpec>, Serializable {
@Nullable
protected List<String> _aliases;
+ // Optional, free-form per-column metadata. It is additive, excluded from
backward-compatibility
+ // checks, and omitted from serialization when unset/empty. The keys and
their interpretation are
+ // defined by whoever populates it; the core schema attaches no semantics to
it.
+ @JsonProperty("metadata")
+ @JsonInclude(JsonInclude.Include.NON_EMPTY)
+ @Nullable
+ protected Map<String, String> _metadata;
+
protected String _name;
protected DataType _dataType;
protected boolean _singleValueField = true;
@@ -249,7 +259,7 @@ public abstract class FieldSpec implements
Comparable<FieldSpec>, Serializable {
}
public void setTags(@Nullable List<String> tags) {
- _tags = tags;
+ _tags = CollectionUtils.isEmpty(tags) ? null : tags;
}
@Nullable
@@ -267,7 +277,16 @@ public abstract class FieldSpec implements
Comparable<FieldSpec>, Serializable {
}
public void setAliases(@Nullable List<String> aliases) {
- _aliases = aliases == null || aliases.isEmpty() ? null : aliases;
+ _aliases = CollectionUtils.isEmpty(aliases) ? null : aliases;
+ }
+
+ @Nullable
+ public Map<String, String> getMetadata() {
+ return _metadata;
+ }
+
+ public void setMetadata(@Nullable Map<String, String> metadata) {
+ _metadata = MapUtils.isEmpty(metadata) ? null : metadata;
}
public DataType getDataType() {
@@ -624,7 +643,7 @@ public abstract class FieldSpec implements
Comparable<FieldSpec>, Serializable {
return jsonObject;
}
- /// Appends `fieldId` and `aliases` (when set) to the given JSON object.
+ /// Appends `fieldId`, `aliases`, and `metadata` (when set) to the given
JSON object.
///
/// Subclasses that build JSON without calling [FieldSpec#toJsonObject()],
such as [TimeFieldSpec], use this helper
/// to preserve these fields during schema round-trip serialization.
@@ -639,6 +658,12 @@ public abstract class FieldSpec implements
Comparable<FieldSpec>, Serializable {
}
jsonObject.set("aliases", aliasesArray);
}
+ if (MapUtils.isNotEmpty(_metadata)) {
+ ObjectNode metadataNode = jsonObject.putObject("metadata");
+ for (Map.Entry<String, String> entry : _metadata.entrySet()) {
+ metadataNode.put(entry.getKey(), entry.getValue());
+ }
+ }
}
protected void appendDefaultNullValue(ObjectNode jsonNode) {
@@ -716,14 +741,15 @@ public abstract class FieldSpec implements
Comparable<FieldSpec>, Serializable {
&& Objects.equals(_description, that._description)
&& Objects.equals(_tags, that._tags)
&& Objects.equals(_fieldId, that._fieldId)
- && Objects.equals(_aliases, that._aliases);
+ && Objects.equals(_aliases, that._aliases)
+ && Objects.equals(_metadata, that._metadata);
}
@Override
public int hashCode() {
return Objects.hash(_name, _dataType, _singleValueField, _notNull,
_maxLength, _maxLengthExceedStrategy,
_allowTrailingZeros, _dataType.hashCode(_defaultNullValue),
_transformFunction, _virtualColumnProvider,
- _description, _tags, _fieldId, _aliases);
+ _description, _tags, _fieldId, _aliases, _metadata);
}
/**
diff --git
a/pinot-spi/src/test/java/org/apache/pinot/spi/data/FieldSpecTest.java
b/pinot-spi/src/test/java/org/apache/pinot/spi/data/FieldSpecTest.java
index b156bb84865..fe5aa979181 100644
--- a/pinot-spi/src/test/java/org/apache/pinot/spi/data/FieldSpecTest.java
+++ b/pinot-spi/src/test/java/org/apache/pinot/spi/data/FieldSpecTest.java
@@ -26,6 +26,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.LinkedList;
import java.util.List;
+import java.util.Map;
import java.util.Random;
import java.util.concurrent.TimeUnit;
import org.apache.pinot.spi.utils.ByteArray;
@@ -776,4 +777,99 @@ public class FieldSpecTest {
assertThat(newSpec.isBackwardCompatibleWith(oldSpec)).isTrue();
}
+
+ @Test
+ public void testMetadataSerdeRoundtrip()
+ throws Exception {
+ DimensionFieldSpec fieldSpec = new DimensionFieldSpec("addr", JSON, true);
+ fieldSpec.setMetadata(Map.of("2", "person"));
+
+ String json = fieldSpec.toJsonObject().toString();
+ assertThat(json).contains("\"metadata\":{\"2\":\"person\"}");
+
+ DimensionFieldSpec deserialized = JsonUtils.stringToObject(json,
DimensionFieldSpec.class);
+ assertThat(deserialized.getMetadata()).isEqualTo(Map.of("2", "person"));
+
+ String jacksonJson = JsonUtils.objectToString(fieldSpec);
+ DimensionFieldSpec fromJackson = JsonUtils.stringToObject(jacksonJson,
DimensionFieldSpec.class);
+ assertThat(fromJackson.getMetadata()).isEqualTo(Map.of("2", "person"));
+ }
+
+ @Test
+ public void testMetadataOmittedWhenNotSet()
+ throws Exception {
+ DimensionFieldSpec fieldSpec = new DimensionFieldSpec("col1", INT, true);
+
+ String json = fieldSpec.toJsonObject().toString();
+ assertThat(json).as("metadata should be absent when
null").doesNotContain("metadata");
+
+ String jacksonJson = JsonUtils.objectToString(fieldSpec);
+ assertThat(jacksonJson).as("metadata should be absent when
null").doesNotContain("metadata");
+ }
+
+ @Test
+ public void testEmptyMetadataOmittedFromJson()
+ throws Exception {
+ DimensionFieldSpec fieldSpec = new DimensionFieldSpec("col1", INT, true);
+ fieldSpec.setMetadata(Map.of());
+ assertThat(fieldSpec.getMetadata()).as("empty metadata normalizes to
null").isNull();
+
+ String json = fieldSpec.toJsonObject().toString();
+ assertThat(json).as("empty metadata should be
absent").doesNotContain("metadata");
+ assertThat(JsonUtils.stringToObject(json,
DimensionFieldSpec.class)).isEqualTo(fieldSpec);
+
+ String jacksonJson = JsonUtils.objectToString(fieldSpec);
+ assertThat(jacksonJson).as("empty metadata absent
(Jackson)").doesNotContain("metadata");
+ assertThat(JsonUtils.stringToObject(jacksonJson,
DimensionFieldSpec.class)).isEqualTo(fieldSpec);
+ }
+
+ @Test
+ public void testTimeFieldSpecRoundTripsMetadataThroughSchema()
+ throws Exception {
+ // TimeFieldSpec.toJsonObject() builds its JSON via the shared
appendFieldIdAndAliases helper;
+ // ensure metadata round-trips through schema (de)serialization for legacy
TIME columns too.
+ TimeFieldSpec timeFieldSpec = new TimeFieldSpec(new
TimeGranularitySpec(LONG, TimeUnit.DAYS, "ts"));
+ timeFieldSpec.setMetadata(Map.of("7", "event_ts"));
+
+ Schema schema = new Schema();
+ schema.setSchemaName("ts_schema");
+ schema.addField(timeFieldSpec);
+
+ String json = schema.toSingleLineJsonString();
+ assertThat(json).contains("\"metadata\":{\"7\":\"event_ts\"}");
+
+ FieldSpec deserialized = Schema.fromString(json).getFieldSpecFor("ts");
+ assertThat(deserialized.getMetadata()).isEqualTo(Map.of("7", "event_ts"));
+ }
+
+ @Test
+ public void testOldJsonWithoutMetadataDeserializesCleanly()
+ throws Exception {
+ String oldJson = "{\"name\":\"col1\",\"dataType\":\"STRING\"}";
+ DimensionFieldSpec fieldSpec = JsonUtils.stringToObject(oldJson,
DimensionFieldSpec.class);
+ assertThat(fieldSpec.getMetadata()).isNull();
+ }
+
+ @Test
+ public void testMetadataInEqualsAndHashCode() {
+ DimensionFieldSpec spec1 = new DimensionFieldSpec("col1", JSON, true);
+ DimensionFieldSpec spec2 = new DimensionFieldSpec("col1", JSON, true);
+ spec2.setMetadata(Map.of("2", "person"));
+
+ assertThat(spec1).isNotEqualTo(spec2);
+ assertThat(spec1.hashCode()).isNotEqualTo(spec2.hashCode());
+
+ spec1.setMetadata(Map.of("2", "person"));
+ assertThat(spec1).isEqualTo(spec2);
+ assertThat(spec1.hashCode()).isEqualTo(spec2.hashCode());
+ }
+
+ @Test
+ public void testMetadataNotInBackwardCompatibility() {
+ DimensionFieldSpec oldSpec = new DimensionFieldSpec("col1", JSON, true);
+ DimensionFieldSpec newSpec = new DimensionFieldSpec("col1", JSON, true);
+ newSpec.setMetadata(Map.of("2", "person"));
+
+ assertThat(newSpec.isBackwardCompatibleWith(oldSpec)).isTrue();
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
