[GitHub] [pinot] Lvnszn opened a new issue #7818: 【BUG】Field not in json will be persist.

[GitBox]

Lvnszn opened a new issue #7818:
URL: https://github.com/apache/pinot/issues/7818


   ## TableConfig
   ```
   {
     "REALTIME": {
       "tableName": "tracker_analyze_REALTIME",
       "tableType": "REALTIME",
       "segmentsConfig": {
         "schemaName": "tracker_analyze",
         "timeColumnName": "event_time",
         "timeType": "MILLISECONDS",
         "replication": "1",
         "replicasPerPartition": "1"
       },
       "tenants": {
         "broker": "DefaultTenant",
         "server": "DefaultTenant"
       },
       "tableIndexConfig": {
         "invertedIndexColumns": [
           "_app_id"
         ],
         "autoGeneratedInvertedIndex": false,
         "createInvertedIndexDuringSegmentGeneration": false,
         "loadMode": "MMAP",
         "streamConfigs": {
           "streamType": "kafka",
           "stream.kafka.topic.name": "analytics-event",
           "stream.kafka.broker.list": "127.0.0.1:9092",
           "stream.kafka.consumer.type": "lowlevel",
           "stream.kafka.consumer.prop.auto.offset.reset": "smallest",
           "stream.kafka.consumer.factory.class.name": 
"org.apache.pinot.plugin.stream.kafka20.KafkaConsumerFactory",
           "stream.kafka.decoder.class.name": 
"org.apache.pinot.plugin.stream.kafka.KafkaJSONMessageDecoder",
           "realtime.segment.flush.threshold.time": "24h",
           "realtime.segment.flush.segment.size": "200M"
         },
         "enableDefaultStarTree": false,
         "enableDynamicStarTreeCreation": false,
         "aggregateMetrics": false,
         "nullHandlingEnabled": false
       },
       "metadata": {},
       "ingestionConfig": {
         "transformConfigs": [
           {
             "columnName": "value",
             "transformFunction": "JSONPATHSTRING(custom_props, \"$.value\", 
\"\")"
           },
           {
             "columnName": "sku",
             "transformFunction": "JSONPATHSTRING(custom_props, \"$.sku\", 
\"\")"
           }
         ]
       },
       "isDimTable": false
     }
   }
   ```
   
   ## Schema
   ```
   {
     "schemaName": "tracker_analyze",
     "dimensionFieldSpecs": [
       {
         "name": "event_name",
         "dataType": "STRING"
       },
       {
         "name": "value",
         "dataType": "STRING"
       },
       {
         "name": "sku",
         "dataType": "STRING"
       },
       {
         "name": "_screen_width",
         "dataType": "INT",
         "defaultNullValue": -1
       },
       {
         "name": "_screen_height",
         "dataType": "INT",
         "defaultNullValue": -1
       },
       {
         "name": "_timezone",
         "dataType": "STRING"
       },
       {
         "name": "keyword",
         "dataType": "STRING"
       }
     ],
     "dateTimeFieldSpecs": [
       {
         "name": "event_time",
         "dataType": "LONG",
         "format": "1:MILLISECONDS:EPOCH",
         "granularity": "1:MILLISECONDS"
       }
     ]
   }
   ```
   
   ## Data
   ```
   {
        "_screen_width": 414,
        "_screen_height": 736,
        "_language": "zh_CN",
        "_timezone": "UTC+8",
        "event_name": "zqytest",
        "event_time": 1637662911612,
        "_user_id": "onedi",
        "sku": "1234",
        "product_name": "外面的商品",
        "custom_props": {
                "keyword": "看看吧",
                "product_name": "商品十一号",
                "value": "可以了吧"
        }
   }
   ```
   
   ## Query
   ```
   select * from tracker_analyze
   ```
   
   ## Result
   ```
   |sku|value|...|
   |1234|可以了吧|...|
   ```
   
   ## Expect
   ```
   |sku|value|...|
   ||可以了吧|...|
   ```


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org

For queries about this service, please contact Infrastructure at:
us...@infra.apache.org



---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For additional commands, e-mail: commits-h...@pinot.apache.org