This is an automated email from the ASF dual-hosted git repository. xxyu pushed a commit to branch doc5.0 in repository https://gitbox.apache.org/repos/asf/kylin.git
The following commit(s) were added to refs/heads/doc5.0 by this push: new c62dbf3fc8 Update community activity page c62dbf3fc8 is described below commit c62dbf3fc81f79381b1810c369a0c9551e6208a1 Author: XiaoxiangYu <x...@apache.org> AuthorDate: Mon Jan 16 19:44:39 2023 +0800 Update community activity page --- .../2022-12-18-Introduction_of_Metadata/index.md | 13 + .../protocol-buffer/metadata.proto | 364 +++++++++++++++------ .../diagram/ER-diagram/metadata_store.png | Bin .../diagram/ER-diagram/metadata_store.puml | 0 .../write_and_read_persistent_entity.puml | 0 .../write_and_read_persistent_entity_cn.png | Bin .../write_and_read_persistent_entity_cn.puml | 0 .../class_diagram/metastore_resource_store.png | Bin .../class_diagram/metastore_resource_store.puml | 0 .../2023_01_16-Introduction_of_Metastore/index.md} | 38 ++- website/blog/authors.yml | 5 +- website/docs/community.md | 18 +- .../development/how_to_understand_kylin_design.md | 7 +- website/sidebars.js | 6 +- 14 files changed, 313 insertions(+), 138 deletions(-) diff --git a/website/blog/2022-12-18-Introduction_of_Metadata/index.md b/website/blog/2022-12-18-Introduction_of_Metadata/index.md index 4a5ef1d012..db9703a8c7 100644 --- a/website/blog/2022-12-18-Introduction_of_Metadata/index.md +++ b/website/blog/2022-12-18-Introduction_of_Metadata/index.md @@ -7,6 +7,19 @@ hide_table_of_contents: false date: 2022-12-18T17:00 --- +:::tip Before your read +**Target Audience** +- Kylin 5.0 的开发者和用户 + +**What will you learn** +- 了解 Kylin 5.0 的元数据 Schema 设计和原理 + +💬 Kylin5 对[元数据的设计](https://github.com/apache/kylin/blob/doc5.0/website/blog/2022-12-18-Introduction_of_Metadata/protocol-buffer/metadata.proto) +做了比较大的调整,本文将针对这些调整做比较详细的介绍。 +::: + +<!--truncate--> + # Introduction of Metadata Kylin5 对元数据的组织结构做了比较大的调整,本文将针对这些调整做比较详细的介绍。相比于 Kylin4,Kylin5 的元数据的一个显著特点是项目级隔离,也即每个项目彼此独立、互不干扰。本文会从项目开始,分别展开Table、Model、IndexPlan、Segment、Job 各个部分的内容。更宽泛地说,Kylin5 的元数据还包括元数据更新审计日志(AuditLog)、事务表(Epoch)、权限(ACL)、查询历史(Query History) 等内容。所有的这些内容都很重要,但作为入门级的介绍,本文不涉及更宽泛的内容,而是尽量把篇幅控制在元数据中最为基础的那部分,以帮助更多的开发者快速地了解和参与到 Kylin5 的研发。当然,作为一篇入门级介绍性文章,非研发人员阅读也能有所收获。接下来让我们切入主题。 diff --git a/website/protocol-buffer/metadata.proto b/website/blog/2022-12-18-Introduction_of_Metadata/protocol-buffer/metadata.proto similarity index 57% rename from website/protocol-buffer/metadata.proto rename to website/blog/2022-12-18-Introduction_of_Metadata/protocol-buffer/metadata.proto index f1dc3f877d..1173f6e676 100644 --- a/website/protocol-buffer/metadata.proto +++ b/website/blog/2022-12-18-Introduction_of_Metadata/protocol-buffer/metadata.proto @@ -16,112 +16,249 @@ * limitations under the License. */ +/************************************************************************************************** + * 文档说明 + * + * Kylin 5.0 的元数据和之前的版本存在比较大的额不兼容情况, 所以这里希望梳理元数据字段的含义, 方便用户和开发者进行 + * 理解和开发. (https://kylin.apache.org/5.0/blog/introduction_of_metadata_cn) + * + * 标签说明 + * - [TODO] 这个说明待补充 + * - [Trivial] 这个配置项不太重要, 不太影响产品行为, 可以不关心 + * + * 对应版本 : 5.0.0-alpha + * 更新时间 : 2023-01-16 + ************************************************************************************************/ syntax = "proto3"; -package org.apache.kylin.protobuf; +package org.apache.kylin.metadata; -option java_multiple_files = true; -option java_package = "org.apache.kylin.protobuf"; -option java_outer_classname = "KylinDataModel"; +// ================================================================================================ +// ================================================================================================ +// ===================================== Part I =================================================== +/** + * 包含了所有元数据的公共字段 + */ +message RootPersistentEntity { + /** + * 该元数据的全局唯一标示符 + */ + string uuid = 1; + int64 createTime = 2; + int64 lastModified = 3; + + // [Trivial] 标记这个元数据是由哪个版本的 Kylin 生成的, 可以用作元数据版本校验, 当前实际未使用 + string version = 4; + /** + * 用于避免元数据写入相互覆盖的自增字段 + */ + int64 mvcc = 5; + /** + * TODO + */ + bool isBroken = 6; +} message ProjectInstance { - string name = 1; + RootPersistentEntity basicProp = 1; + string name = 2; + string owner = 3; + ProjectStatusEnum status = 4; + int64 createTimeUTC = 5; + string defaultDatabase = 6; + string description = 7; + + /** + * TODO + */ + string principal = 8; + + /** + * TODO + */ + string keytab = 9; + + + MaintainModelType maintain_model_type = 10; // [Trivial] + + /** + * 项目级别的配置(键值对), 可以覆盖全局的配置 + */ + map<string, string> override_kylin_properties = 11; + + /** + * 项目级别的 Segment 自动合并策略的配置 + */ + SegmentConfig segment_config = 12; + enum ProjectStatusEnum { + /** + * TODO:项目 Disable 有什么影响 + */ DISABLED = 0; ENABLED = 1; }; - ProjectStatusEnum status = 2; - string default_database = 3; - string description = 9; - // common properties - string uuid = 4; - string owner = 5; - int64 createTime = 6; - int64 lastModified = 7; - string version = 8; - - // Configuration settings - map<string, string> settings = 10; + /** + * TODO + */ + enum MaintainModelType { + MANUAL_MAINTAIN = 0; + } } message TableDesc { - string uuid = 1; - int64 lastModified = 2; - int64 createTime = 3; - string name = 4; + RootPersistentEntity basicProp = 1; + + string name = 2; + repeated ColumnDesc columns = 3; + + SourceTypeEnum sourceType = 4; + + /** + * 用于区分一个表是不是视图, 外部表还是内部表 + */ + CatalogTableType tableType = 5; + + bool isTop = 6; // [Trivial] for front end only + + string data_gen = 7; // [Trivial] + + /** + * TODO + */ + string increment_loading = 8; + + string last_snapshot_path = 9; + + int64 last_snapshot_size = 10; + int64 snapshot_last_modified = 11; + + int32 query_hit_count = 12; + + string partition_column = 13; + + map<string, string> snapshot_partitions = 14; + map<string, string> snapshot_partitions_info = 15; + + int64 snapshot_total_rows = 16; + string snapshot_partition_col = 17; + string selected_snapshot_partition_col = 18; + string temp_snapshot_path = 19; + bool snapshot_has_broken = 20; + + string database = 21; + + bool transactional = 22; + + bool rangePartition = 23; + PartitionDesc partition_desc = 24; - SourceTypeEnum sourceType = 5; enum SourceTypeEnum { ID_HIVE = 0; - ID_STREAMING = 1; + ID_STREAMING = 1; // UNDER DEVELOPED ID_SPARKSQL = 5; ID_EXTERNAL = 7; ID_JDBC = 8; ID_SPARK = 9; - ID_CSV = 11; - ID_FILE = 13; } - CatalogTableType tableType = 6; + /** + * 参考 org.apache.spark.sql.catalyst.catalog.CatalogTableType + */ enum CatalogTableType { EXTERNAL = 0; MANAGED = 1; VIEW = 2; } +} - string project = 10; - string database = 11; - repeated ColumnDesc columns = 9; - repeated ColumnDesc partitionColumn = 14; +// ================================================================================================ +// ================================================================================================ +// ===================================== Part II ================================================== - string lastSnapshotPath = 12; - int64 lastSnapshotSize = 13; - map<string, int64> snapshotPartitions = 15; - map<string, SnapshotPartitionInfo> snapshotPartitionInfo = 16; - int64 snapshotTotalRows = 17; - string selectedSnapshotPartitionCol = 18; - string snapshotPartitionCol = 19; - int64 snapshotLastModified = 20; - bool snapshotHasBroken = 21; -} +/** + * Kylin 5 之前的 DataModel 和 CubeDesc 的合并 + */ +message NDataModel { -message DataModel { - // common properties - string uuid = 1; - int64 createTime = 2; - int64 lastModified = 3; - string version = 4; - string alias = 5; - string owner = 6; - string description = 7; + /** + * common properties(from RootPersistentEntity) + */ + RootPersistentEntity basicProp = 1; + + // model basic properties + string alias = 11; + string owner = 12; + + // 谁上次修改了模型级别配置项 + string configLastModifier = 13; + + // 上次修改了模型级别配置项的时刻 + int64 configLastModified = 14; + string description = 15; + + // model schema + string rootFactTableName = 20; + string rootFactTableAlias = 21; + + // 描述了事实表和维度表当前是如何连接的 + repeated JoinTableDesc joinTables = 22; - string rootFactTableName = 8; - repeated JoinTableDesc joinTables = 9; - string filterCondition = 10; + // 打平表时是过滤数据的条件 + string filterCondition = 23; + PartitionDesc partitionDesc = 24; - repeated NamedColumn allNamedColumns = 13; //dimensions - repeated Measure allMeasures = 14; //measures - repeated ComputedColumnDesc computedColumnDescs = 15; - PartitionDesc partitionDesc = 11; - DataCheckDesc dataCheckDesc = 18; + // dimension and other columns + repeated NamedColumn allNamedColumns = 25; + repeated Measure allMeasures = 26; + repeated ComputedColumnDesc computedColumnDescs = 27; + + // misc + DataCheckDesc dataCheckDesc = 40; + + // [Trivial] + string managementType = 41; + + // check https://kylin.apache.org/5.0/blog/introduction_of_metadata_cn#significant-change + int32 semanticVersion = 42; + + int32 storageType = 43; + + // [Trivial] + ModelType modelType = 44; + + // [Trivial] + int32 recommendationsCount = 45; + + // [Trivial] for front end only + string canvas = 46; + + // Broken reason because of schema change + BrokenReason brokenReason = 47; + RealizationCapacity capacity = 48; // [Trivial] + + string multiPartitionDesc = 49; + string multiPartitionKeyMapping = 50; + + // [Trivial] for streaming feature + string fusionId = 51; + + // Some configuration + SegmentConfig segmentConfig = 61; - SegmentConfig segmentConfig = 17; - ModelType modelType = 26; enum ModelType { - BATCH = 0; - STREAMING = 1; - HYBRID = 2; + BATCH = 0; // Source is Hive only + STREAMING = 1; // Source is Kafka only, still under developing + HYBRID = 2; // Source is Kafka and Hive, still under developing UNKNOWN = 3; } - RealizationCapacity capacity = 12; - enum RealizationCapacity { + enum RealizationCapacity {// Useless SMALL = 0; MEDIUM = 1; LARGE = 2; } - BrokenReason brokenReason = 20; enum BrokenReason { SCHEMA = 0; NULL = 1; @@ -129,7 +266,51 @@ message DataModel { } } -message JoinTableDesc{ +/** + * IndexPlan 是 Kylin5 中用来组织 Index/Layout 的元数据,它里面包含两个最重要的信息 RuleBasedIndex 和 Indexes, + * 前者用于管理聚合组索引,后者用于管理被物化的索引。 + */ +message IndexPlan { + string description = 1; + int64 retentionRange = 8; + int32 engineType = 9; + repeated int64 autoMergeTimeRanges = 7; + + RuleBasedIndex ruleBasedIndex = 3; // agg group + repeated IndexEntity indexes = 4; + repeated DictionaryDesc dictionaries = 10; + + repeated IndexEntity toBeDeletedIndexes = 6; + int64 nextAggregationIndexId = 11; + int64 nextTableIndexId = 12; + repeated int32 aggShardByColumns = 13; + map<int64, int32> layoutBucketNumMapping = 15; + + map<string, string> overrideProps = 5; +} + +/** + * Kylin 5 之前的 CubeInstance, 保存了索引和 Segment 相关数据的描述信息和统计数据 + */ +message DataFlow { + RealizationStatusEnum status = 1; + enum RealizationStatusEnum { + OFFLINE = 0; + ONLINE = 1; + BROKEN = 2; + } + + int32 cost = 2; + int32 queryHitCount = 3; + int64 lastQueryTime = 4; + repeated DataSegment segments = 6; +} + +// ================================================================================================ +// ================================================================================================ +// ===================================== Part III ================================================= + +message JoinTableDesc { string table = 1; TableKind kind = 2; enum TableKind { @@ -184,25 +365,6 @@ message ComputedColumnDesc { string uuid = 7; } -message IndexPlan { - string description = 1; - int64 retentionRange = 8; - int32 engineType = 9; - repeated int64 autoMergeTimeRanges = 7; - - RuleBasedIndex ruleBasedIndex = 3; // agg group - repeated IndexEntity indexes = 4; - repeated DictionaryDesc dictionaries = 10; - - repeated IndexEntity toBeDeletedIndexes = 6; - int64 nextAggregationIndexId = 11; - int64 nextTableIndexId = 12; - repeated int32 aggShardByColumns = 13; - map<int64, int32> layoutBucketNumMapping = 15; - - map<string, string> overrideProps = 5; -} - message RuleBasedIndex { repeated int32 dimensions = 2; // dimension id repeated int32 measures = 3; //measure id @@ -256,20 +418,6 @@ message DataCheckDesc { int64 faultActions = 3; } -message DataFlow { - RealizationStatusEnum status = 1; - enum RealizationStatusEnum { - OFFLINE = 0; - ONLINE = 1; - BROKEN = 2; - } - - int32 cost = 2; - int32 queryHitCount = 3; - int64 lastQueryTime = 4; - repeated DataSegment segments = 6; -} - message DataSegment { string id = 1; string name = 2; @@ -405,8 +553,19 @@ message ColumnDesc { string name = 2; string datatype = 3; - string comment = 5; + string comment = 4; + + string data_gen = 5; + string index = 6; + + /** + * 如果是一个普通列,那么这里是 null, 否则表示这是一个可计算列 + */ + string computedColumnExpr = 7; string caseSensitiveName = 8; + + // [Trivial] 没有意义 + bool isPartitioned = 9; } message SnapshotPartitionInfo { @@ -460,7 +619,4 @@ message TimeRange { } message SegmentConfig { -} - - - +} \ No newline at end of file diff --git a/website/docs/development/dev_design/diagram/ER-diagram/metadata_store.png b/website/blog/2023_01_16-Introduction_of_Metastore/diagram/ER-diagram/metadata_store.png similarity index 100% rename from website/docs/development/dev_design/diagram/ER-diagram/metadata_store.png rename to website/blog/2023_01_16-Introduction_of_Metastore/diagram/ER-diagram/metadata_store.png diff --git a/website/docs/development/dev_design/diagram/ER-diagram/metadata_store.puml b/website/blog/2023_01_16-Introduction_of_Metastore/diagram/ER-diagram/metadata_store.puml similarity index 100% rename from website/docs/development/dev_design/diagram/ER-diagram/metadata_store.puml rename to website/blog/2023_01_16-Introduction_of_Metastore/diagram/ER-diagram/metadata_store.puml diff --git a/website/docs/development/dev_design/diagram/activity_diagram/write_and_read_persistent_entity.puml b/website/blog/2023_01_16-Introduction_of_Metastore/diagram/activity_diagram/write_and_read_persistent_entity.puml similarity index 100% rename from website/docs/development/dev_design/diagram/activity_diagram/write_and_read_persistent_entity.puml rename to website/blog/2023_01_16-Introduction_of_Metastore/diagram/activity_diagram/write_and_read_persistent_entity.puml diff --git a/website/docs/development/dev_design/diagram/activity_diagram/write_and_read_persistent_entity_cn.png b/website/blog/2023_01_16-Introduction_of_Metastore/diagram/activity_diagram/write_and_read_persistent_entity_cn.png similarity index 100% rename from website/docs/development/dev_design/diagram/activity_diagram/write_and_read_persistent_entity_cn.png rename to website/blog/2023_01_16-Introduction_of_Metastore/diagram/activity_diagram/write_and_read_persistent_entity_cn.png diff --git a/website/docs/development/dev_design/diagram/activity_diagram/write_and_read_persistent_entity_cn.puml b/website/blog/2023_01_16-Introduction_of_Metastore/diagram/activity_diagram/write_and_read_persistent_entity_cn.puml similarity index 100% rename from website/docs/development/dev_design/diagram/activity_diagram/write_and_read_persistent_entity_cn.puml rename to website/blog/2023_01_16-Introduction_of_Metastore/diagram/activity_diagram/write_and_read_persistent_entity_cn.puml diff --git a/website/docs/development/dev_design/diagram/class_diagram/metastore_resource_store.png b/website/blog/2023_01_16-Introduction_of_Metastore/diagram/class_diagram/metastore_resource_store.png similarity index 100% rename from website/docs/development/dev_design/diagram/class_diagram/metastore_resource_store.png rename to website/blog/2023_01_16-Introduction_of_Metastore/diagram/class_diagram/metastore_resource_store.png diff --git a/website/docs/development/dev_design/diagram/class_diagram/metastore_resource_store.puml b/website/blog/2023_01_16-Introduction_of_Metastore/diagram/class_diagram/metastore_resource_store.puml similarity index 100% rename from website/docs/development/dev_design/diagram/class_diagram/metastore_resource_store.puml rename to website/blog/2023_01_16-Introduction_of_Metastore/diagram/class_diagram/metastore_resource_store.puml diff --git a/website/docs/development/dev_design/metastore_design.md b/website/blog/2023_01_16-Introduction_of_Metastore/index.md similarity index 94% rename from website/docs/development/dev_design/metastore_design.md rename to website/blog/2023_01_16-Introduction_of_Metastore/index.md index 6259899b8c..70f2ef76db 100644 --- a/website/docs/development/dev_design/metastore_design.md +++ b/website/blog/2023_01_16-Introduction_of_Metastore/index.md @@ -1,22 +1,28 @@ --- -title: Metastore Design of Kylin 5 -language: en -sidebar_label: Metastore Design of Kylin 5 -pagination_label: Metastore Design of Kylin 5 -toc_min_heading_level: 2 -toc_max_heading_level: 6 -pagination_prev: development/how_to_understand_kylin_design -pagination_next: null -showLastUpdateAuthor: true -showLastUpdateTime: true -keywords: - - dev-design -draft: false -last_update: - date: 09/16/2022 - author: Xiaoxiang Yu +title: Introduction of Metastore(CN) +slug: introduction_of_metastore_cn +authors: xxyu +tags: [metastore, kylin5] +hide_table_of_contents: false +date: 2023-01-16T10:00 --- +:::tip Before your read +**Target Audience** +- 对 Kylin 5.0 元数据存储, 元数据缓存, 以及节点间元数据同步机制感兴趣的用户和开发者. +- 在二次开发过程中, 想了解对 Kylin 5.0 进行元数据读写操作的最佳实践和注意事项的开发者. +- 想对 Kylin 5.0 的元数据进行升级改造的开发者. + + +**What will you learn** +- 了解在 Kylin 5 如何进行读写元数据操作 + +💬 Kylin 5 的开发者需要了解元数据读写的逻辑的实现和技术细节 +::: + +<!--truncate--> + + ### Target Audience 这篇文档是为有以下需求的用户和开发者而准备的: diff --git a/website/blog/authors.yml b/website/blog/authors.yml index 02552d7a82..189d62dcec 100644 --- a/website/blog/authors.yml +++ b/website/blog/authors.yml @@ -1,10 +1,11 @@ xxyu: - name: 敏丞 + name: Xiaoxiang Yu title: PMC member and Release Manager of Apache Kylin url: https://github.com/hit-lacus image_url: https://github.com/hit-lacus.png pfzhan: - name: pfzhan + name: Pengfei Zhan + title: Dev of Kyligence Inc url: https://github.com/dethrive image_url: https://github.com/dethrive.png diff --git a/website/docs/community.md b/website/docs/community.md index f3ea10b7d6..0001adf1d0 100644 --- a/website/docs/community.md +++ b/website/docs/community.md @@ -9,15 +9,17 @@ sidebar_position: 70 - [Source code for Kylin 5.0](https://github.com/apache/kylin/tree/kylin5) - [Source code for Kylin 5.0 website](https://github.com/apache/kylin/tree/doc5.0) -### Community Statistics & JIRA Reports +### Mailing List +- [List for User](https://lists.apache.org/list.html?u...@kylin.apache.org) +- [List for Developer](https://lists.apache.org/list.html?d...@kylin.apache.org) +### Community Activity and Statistics - [Created vs Resolved Issues Report(recent year)](https://issues.apache.org/jira/secure/ConfigureReport.jspa?projectOrFilterId=project-12316121&periodName=weekly&daysprevious=365&cumulative=true&versionLabels=none&selectedProjectId=12316121&reportKey=com.atlassian.jira.jira-core-reports-plugin%3Acreatedvsresolved-report&atl_token=A5KQ-2QAV-T4JA-FDED_610a4d55e17d44d325672c3cb5df9a88c65906c8_lin&Next=Next) - [Recently Created Issues Report(recent year)](https://issues.apache.org/jira/secure/ConfigureReport.jspa?projectOrFilterId=project-12316121&periodName=weekly&daysprevious=365&selectedProjectId=12316121&reportKey=com.atlassian.jira.jira-core-reports-plugin%3Arecentlycreated-report&atl_token=A5KQ-2QAV-T4JA-FDED_610a4d55e17d44d325672c3cb5df9a88c65906c8_lin&Next=Next) -- [Monthly Github activities summary](https://github.com/apache/kylin/pulse/monthly) -- [Github contributors of default branch](https://github.com/apache/kylin/graphs/contributors) -- [Github Traffic (for committer only)](https://github.com/apache/kylin/graphs/traffic) - +- [Monthly GitHub activities summary](https://github.com/apache/kylin/pulse/monthly) +- [GitHub's contributors of default branch](https://github.com/apache/kylin/graphs/contributors) kylin5 is not default branch at the moment +- [GitHub Traffic (PV & UV) (for committer only)](https://github.com/apache/kylin/graphs/traffic) +- [Release download statistics for day/week/quarter/year(for committer only)](https://logging1-he-de.apache.org/stats/) Managed by Infra Team and documented at https://infra.apache.org/release-download-pages.html -### Mailing List -- [List for User](https://lists.apache.org/list.html?u...@kylin.apache.org) -- [List for Developer](https://lists.apache.org/list.html?d...@kylin.apache.org) \ No newline at end of file +### Credit Table for Contributor +- [Part of Kylin 5.0 Contributor Credits](https://cwiki.apache.org/confluence/display/KYLIN/Kylin+5.0+Contribution+Detail+Table) \ No newline at end of file diff --git a/website/docs/development/how_to_understand_kylin_design.md b/website/docs/development/how_to_understand_kylin_design.md index e660d1ac8d..f65fbe8915 100644 --- a/website/docs/development/how_to_understand_kylin_design.md +++ b/website/docs/development/how_to_understand_kylin_design.md @@ -6,7 +6,7 @@ pagination_label: Overall Design of Kylin 5 toc_min_heading_level: 2 toc_max_heading_level: 6 pagination_prev: development/how_to_release -pagination_next: development/dev_design/metastore_design +pagination_next: null showLastUpdateAuthor: true showLastUpdateTime: true keywords: @@ -28,9 +28,10 @@ Unless more comments, all source code analysis are based on [this code snapshot] - [x] Transaction(CRUD of Metadata) - [ ] Epoch, AuditLog, etc. 2. Metadata Format/Schema - - [ ] DataModel, IndexPlan, and Dataflow - - [ ] Index and Layout + - [x] DataModel, IndexPlan, and Dataflow + - [x] Index and Layout - [ ] Computed Column + - [ ] Schema Change 3. Query Engine - [ ] How a SQL query was executed in Kylin? - [ ] Query Cache diff --git a/website/sidebars.js b/website/sidebars.js index 0511fe27d8..c4f1120e6f 100644 --- a/website/sidebars.js +++ b/website/sidebars.js @@ -1000,11 +1000,7 @@ const sidebars = { { type: 'doc', id: 'development/how_to_understand_kylin_design' - }, - { - type: 'doc', - id: 'development/dev_design/metastore_design' - }, + } ], },