This is an automated email from the ASF dual-hosted git repository. beto pushed a commit to branch semantic-layer-ui-semantic-layer in repository https://gitbox.apache.org/repos/asf/superset.git
commit 926ed42d69f41f1a02cd9c1268c8fabb82bdaf91 Author: Beto Dealmeida <[email protected]> AuthorDate: Tue Mar 3 18:41:51 2026 -0500 feat(semantic layers): UI for semantic layers --- .../src/superset_core/semantic_layers/config.py | 73 +++ .../semantic_layers/semantic_layer.py | 2 + superset-frontend/package.json | 7 + superset-frontend/src/features/home/SubMenu.tsx | 29 +- .../features/semanticLayers/SemanticLayerModal.tsx | 620 +++++++++++++++++++++ superset-frontend/src/pages/DatabaseList/index.tsx | 467 +++++++++++++--- superset/commands/semantic_layer/create.py | 5 + superset/commands/semantic_layer/update.py | 5 + superset/semantic_layers/api.py | 263 ++++++++- 9 files changed, 1358 insertions(+), 113 deletions(-) diff --git a/superset-core/src/superset_core/semantic_layers/config.py b/superset-core/src/superset_core/semantic_layers/config.py new file mode 100644 index 00000000000..c1b92a21008 --- /dev/null +++ b/superset-core/src/superset_core/semantic_layers/config.py @@ -0,0 +1,73 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from __future__ import annotations + +from typing import Any + +from pydantic import BaseModel + + +def build_configuration_schema( + config_class: type[BaseModel], + configuration: BaseModel | None = None, +) -> dict[str, Any]: + """ + Build a JSON schema from a Pydantic configuration class. + + Handles generic boilerplate that any semantic layer with dynamic fields needs: + + - Reorders properties to match model field order (Pydantic sorts alphabetically) + - When ``configuration`` is None, sets ``enum: []`` on all ``x-dynamic`` properties + so the frontend renders them as empty dropdowns + + Semantic layer implementations call this instead of ``model_json_schema()`` directly, + then only need to add their own dynamic population logic. + """ + schema = config_class.model_json_schema() + + # Pydantic sorts properties alphabetically; restore model field order + field_order = [ + field.alias or name + for name, field in config_class.model_fields.items() + ] + schema["properties"] = { + key: schema["properties"][key] + for key in field_order + if key in schema["properties"] + } + + if configuration is None: + for prop_schema in schema["properties"].values(): + if prop_schema.get("x-dynamic"): + prop_schema["enum"] = [] + + return schema + + +def check_dependencies( + prop_schema: dict[str, Any], + configuration: BaseModel, +) -> bool: + """ + Check whether a dynamic property's dependencies are satisfied. + + Reads the ``x-dependsOn`` list from the property schema and returns ``True`` + when every referenced attribute on ``configuration`` is truthy. + """ + dependencies = prop_schema.get("x-dependsOn", []) + return all(getattr(configuration, dep, None) for dep in dependencies) diff --git a/superset-core/src/superset_core/semantic_layers/semantic_layer.py b/superset-core/src/superset_core/semantic_layers/semantic_layer.py index 1fc421cf359..fb79f1dab5a 100644 --- a/superset-core/src/superset_core/semantic_layers/semantic_layer.py +++ b/superset-core/src/superset_core/semantic_layers/semantic_layer.py @@ -32,6 +32,8 @@ class SemanticLayer(ABC, Generic[ConfigT, SemanticViewT]): Abstract base class for semantic layers. """ + configuration_class: type[BaseModel] + @classmethod @abstractmethod def from_configuration( diff --git a/superset-frontend/package.json b/superset-frontend/package.json index 48e037de5b0..d8e1c90cd72 100644 --- a/superset-frontend/package.json +++ b/superset-frontend/package.json @@ -116,7 +116,14 @@ "@luma.gl/gltf": "~9.2.5", "@luma.gl/shadertools": "~9.2.5", "@luma.gl/webgl": "~9.2.5", + "@fontsource/fira-code": "^5.2.7", + "@fontsource/inter": "^5.2.8", + "@great-expectations/jsonforms-antd-renderers": "^2.2.10", + "@jsonforms/core": "^3.7.0", + "@jsonforms/react": "^3.7.0", + "@jsonforms/vanilla-renderers": "^3.7.0", "@reduxjs/toolkit": "^1.9.3", + "@rjsf/antd": "^5.24.13", "@rjsf/core": "^5.24.13", "@rjsf/utils": "^5.24.3", "@rjsf/validator-ajv8": "^5.24.13", diff --git a/superset-frontend/src/features/home/SubMenu.tsx b/superset-frontend/src/features/home/SubMenu.tsx index 107700a2539..bb27cc95a76 100644 --- a/superset-frontend/src/features/home/SubMenu.tsx +++ b/superset-frontend/src/features/home/SubMenu.tsx @@ -145,6 +145,7 @@ export interface ButtonProps { buttonStyle: 'primary' | 'secondary' | 'dashed' | 'link' | 'tertiary'; loading?: boolean; icon?: IconType; + component?: ReactNode; } export interface SubMenuProps { @@ -295,18 +296,22 @@ const SubMenuComponent: FunctionComponent<SubMenuProps> = props => { </SubMenu> ))} </Menu> - {props.buttons?.map((btn, i) => ( - <Button - key={i} - buttonStyle={btn.buttonStyle} - icon={btn.icon} - onClick={btn.onClick} - data-test={btn['data-test']} - loading={btn.loading ?? false} - > - {btn.name} - </Button> - ))} + {props.buttons?.map((btn, i) => + btn.component ? ( + <span key={i}>{btn.component}</span> + ) : ( + <Button + key={i} + buttonStyle={btn.buttonStyle} + icon={btn.icon} + onClick={btn.onClick} + data-test={btn['data-test']} + loading={btn.loading ?? false} + > + {btn.name} + </Button> + ), + )} </div> </Row> {props.children} diff --git a/superset-frontend/src/features/semanticLayers/SemanticLayerModal.tsx b/superset-frontend/src/features/semanticLayers/SemanticLayerModal.tsx new file mode 100644 index 00000000000..37694a9e9a6 --- /dev/null +++ b/superset-frontend/src/features/semanticLayers/SemanticLayerModal.tsx @@ -0,0 +1,620 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import { useState, useEffect, useCallback, useRef } from 'react'; +import { t } from '@apache-superset/core'; +import { styled } from '@apache-superset/core/ui'; +import { SupersetClient } from '@superset-ui/core'; +import { Input, Spin } from 'antd'; +import { Select } from '@superset-ui/core/components'; +import { Icons } from '@superset-ui/core/components/Icons'; +import { JsonForms, withJsonFormsControlProps } from '@jsonforms/react'; +import type { + JsonSchema, + UISchemaElement, + ControlProps, +} from '@jsonforms/core'; +import { + rankWith, + and, + isStringControl, + formatIs, + schemaMatches, +} from '@jsonforms/core'; +import { + rendererRegistryEntries, + cellRegistryEntries, + TextControl, +} from '@great-expectations/jsonforms-antd-renderers'; +import type { ErrorObject } from 'ajv'; +import { + StandardModal, + ModalFormField, + MODAL_STANDARD_WIDTH, + MODAL_MEDIUM_WIDTH, +} from 'src/components/Modal'; + +/** + * Custom renderer that renders `Input.Password` for fields with + * `format: "password"` in the JSON Schema (e.g. Pydantic `SecretStr`). + */ +function PasswordControl(props: ControlProps) { + const uischema = { + ...props.uischema, + options: { ...props.uischema.options, type: 'password' }, + }; + return TextControl({ ...props, uischema }); +} +const PasswordRenderer = withJsonFormsControlProps(PasswordControl); +const passwordEntry = { + tester: rankWith(3, and(isStringControl, formatIs('password'))), + renderer: PasswordRenderer, +}; + +/** + * Renderer for `const` properties (e.g. Pydantic discriminator fields). + * Renders nothing visually but ensures the const value is set in form data, + * so discriminated unions resolve correctly on the backend. + */ +function ConstControl({ data, handleChange, path, schema }: ControlProps) { + const constValue = (schema as Record<string, unknown>).const; + useEffect(() => { + if (constValue !== undefined && data !== constValue) { + handleChange(path, constValue); + } + }, [constValue, data, handleChange, path]); + return null; +} +const ConstRenderer = withJsonFormsControlProps(ConstControl); +const constEntry = { + tester: rankWith(10, schemaMatches(s => s !== undefined && 'const' in s)), + renderer: ConstRenderer, +}; + +/** + * Renderer for fields marked `x-dynamic` in the JSON Schema. + * Shows a loading spinner inside the input while the schema is being + * refreshed with dynamic values from the backend. + */ +function DynamicFieldControl(props: ControlProps) { + const { refreshingSchema, formData: cfgData } = props.config ?? {}; + const deps = (props.schema as Record<string, unknown>)?.['x-dependsOn']; + const refreshing = + refreshingSchema && + Array.isArray(deps) && + areDependenciesSatisfied(deps as string[], (cfgData as Record<string, unknown>) ?? {}); + + if (!refreshing) { + return TextControl(props); + } + + const uischema = { + ...props.uischema, + options: { + ...props.uischema.options, + placeholderText: t('Loading...'), + inputProps: { suffix: <Spin size="small" /> }, + }, + }; + return TextControl({ ...props, uischema, enabled: false }); +} +const DynamicFieldRenderer = withJsonFormsControlProps(DynamicFieldControl); +const dynamicFieldEntry = { + tester: rankWith( + 3, + and( + isStringControl, + schemaMatches( + s => (s as Record<string, unknown>)?.['x-dynamic'] === true, + ), + ), + ), + renderer: DynamicFieldRenderer, +}; + +const renderers = [ + ...rendererRegistryEntries, + passwordEntry, + constEntry, + dynamicFieldEntry, +]; + +type Step = 'type' | 'config'; +type ValidationMode = 'ValidateAndHide' | 'ValidateAndShow'; + +const SCHEMA_REFRESH_DEBOUNCE_MS = 500; + +/** + * Removes empty `enum` arrays from schema properties. The JSON Schema spec + * requires `enum` to have at least one item, and AJV rejects empty arrays. + * Fields with empty enums are rendered as plain text inputs instead. + */ +function sanitizeSchema(schema: JsonSchema): JsonSchema { + if (!schema.properties) return schema; + const properties: Record<string, JsonSchema> = {}; + for (const [key, prop] of Object.entries(schema.properties)) { + if ( + typeof prop === 'object' && + prop !== null && + 'enum' in prop && + Array.isArray(prop.enum) && + prop.enum.length === 0 + ) { + const { enum: _empty, ...rest } = prop; + properties[key] = rest; + } else { + properties[key] = prop as JsonSchema; + } + } + return { ...schema, properties }; +} + +/** + * Builds a JSON Forms UI schema from a JSON Schema, using the first + * `examples` entry as placeholder text for each string property. + */ +function buildUiSchema( + schema: JsonSchema, +): UISchemaElement | undefined { + if (!schema.properties) return undefined; + + // Use explicit property order from backend if available, + // otherwise fall back to the JSON object key order + const propertyOrder: string[] = + (schema as Record<string, unknown>)['x-propertyOrder'] as string[] ?? + Object.keys(schema.properties); + + const elements = propertyOrder + .filter(key => key in (schema.properties ?? {})) + .map(key => { + const prop = schema.properties![key]; + const control: Record<string, unknown> = { + type: 'Control', + scope: `#/properties/${key}`, + }; + if (typeof prop === 'object' && prop !== null) { + const options: Record<string, unknown> = {}; + if ( + 'examples' in prop && + Array.isArray(prop.examples) && + prop.examples.length > 0 + ) { + options.placeholderText = String(prop.examples[0]); + } + if ('description' in prop && typeof prop.description === 'string') { + options.tooltip = prop.description; + } + if (Object.keys(options).length > 0) { + control.options = options; + } + } + return control; + }); + return { type: 'VerticalLayout', elements } as UISchemaElement; +} + +/** + * Extracts dynamic field dependency mappings from the schema. + * Returns a map of field name → list of dependency field names. + */ +function getDynamicDependencies( + schema: JsonSchema, +): Record<string, string[]> { + const deps: Record<string, string[]> = {}; + if (!schema.properties) return deps; + for (const [key, prop] of Object.entries(schema.properties)) { + if ( + typeof prop === 'object' && + prop !== null && + 'x-dynamic' in prop && + 'x-dependsOn' in prop && + Array.isArray((prop as Record<string, unknown>)['x-dependsOn']) + ) { + deps[key] = (prop as Record<string, unknown>)[ + 'x-dependsOn' + ] as string[]; + } + } + return deps; +} + +/** + * Checks whether all dependency values are filled (non-empty). + * Handles nested objects (like auth) by checking they have at least one key. + */ +function areDependenciesSatisfied( + dependencies: string[], + data: Record<string, unknown>, +): boolean { + return dependencies.every(dep => { + const value = data[dep]; + if (value === null || value === undefined || value === '') return false; + if (typeof value === 'object' && Object.keys(value).length === 0) + return false; + return true; + }); +} + +/** + * Serializes the dependency values for a set of fields into a stable string + * for comparison, so we only re-fetch when dependency values actually change. + */ +function serializeDependencyValues( + dynamicDeps: Record<string, string[]>, + data: Record<string, unknown>, +): string { + const allDepKeys = new Set<string>(); + for (const deps of Object.values(dynamicDeps)) { + for (const dep of deps) { + allDepKeys.add(dep); + } + } + const snapshot: Record<string, unknown> = {}; + for (const key of [...allDepKeys].sort()) { + snapshot[key] = data[key]; + } + return JSON.stringify(snapshot); +} + +const ModalContent = styled.div` + padding: ${({ theme }) => theme.sizeUnit * 4}px; +`; + +const BackLink = styled.button` + background: none; + border: none; + color: ${({ theme }) => theme.colorPrimary}; + cursor: pointer; + padding: 0; + font-size: ${({ theme }) => theme.fontSize[1]}px; + margin-bottom: ${({ theme }) => theme.sizeUnit * 2}px; + display: inline-flex; + align-items: center; + gap: ${({ theme }) => theme.sizeUnit}px; + + &:hover { + text-decoration: underline; + } +`; + +interface SemanticLayerType { + id: string; + name: string; + description: string; +} + +interface SemanticLayerModalProps { + show: boolean; + onHide: () => void; + addDangerToast: (msg: string) => void; + addSuccessToast: (msg: string) => void; + semanticLayerUuid?: string; +} + +export default function SemanticLayerModal({ + show, + onHide, + addDangerToast, + addSuccessToast, + semanticLayerUuid, +}: SemanticLayerModalProps) { + const isEditMode = !!semanticLayerUuid; + const [step, setStep] = useState<Step>('type'); + const [name, setName] = useState(''); + const [selectedType, setSelectedType] = useState<string | null>(null); + const [types, setTypes] = useState<SemanticLayerType[]>([]); + const [loading, setLoading] = useState(false); + const [configSchema, setConfigSchema] = useState<JsonSchema | null>(null); + const [uiSchema, setUiSchema] = useState<UISchemaElement | undefined>( + undefined, + ); + const [formData, setFormData] = useState<Record<string, unknown>>({}); + const [saving, setSaving] = useState(false); + const [hasErrors, setHasErrors] = useState(true); + const [refreshingSchema, setRefreshingSchema] = useState(false); + const [validationMode, setValidationMode] = + useState<ValidationMode>('ValidateAndHide'); + const errorsRef = useRef<ErrorObject[]>([]); + const debounceTimerRef = useRef<ReturnType<typeof setTimeout> | null>(null); + const lastDepSnapshotRef = useRef<string>(''); + const dynamicDepsRef = useRef<Record<string, string[]>>({}); + + const fetchTypes = useCallback(async () => { + setLoading(true); + try { + const { json } = await SupersetClient.get({ + endpoint: '/api/v1/semantic_layer/types', + }); + setTypes(json.result ?? []); + } catch { + addDangerToast( + t('An error occurred while fetching semantic layer types'), + ); + } finally { + setLoading(false); + } + }, [addDangerToast]); + + const applySchema = useCallback((rawSchema: JsonSchema) => { + const schema = sanitizeSchema(rawSchema); + setConfigSchema(schema); + setUiSchema(buildUiSchema(schema)); + dynamicDepsRef.current = getDynamicDependencies(rawSchema); + }, []); + + const fetchConfigSchema = useCallback( + async (type: string, configuration?: Record<string, unknown>) => { + const isInitialFetch = !configuration; + if (isInitialFetch) setLoading(true); + else setRefreshingSchema(true); + try { + const { json } = await SupersetClient.post({ + endpoint: '/api/v1/semantic_layer/schema/configuration', + jsonPayload: { type, configuration }, + }); + applySchema(json.result); + if (isInitialFetch) setStep('config'); + } catch { + if (isInitialFetch) { + addDangerToast( + t('An error occurred while fetching the configuration schema'), + ); + } + } finally { + if (isInitialFetch) setLoading(false); + else setRefreshingSchema(false); + } + }, + [addDangerToast, applySchema], + ); + + const fetchExistingLayer = useCallback( + async (uuid: string) => { + setLoading(true); + try { + const { json } = await SupersetClient.get({ + endpoint: `/api/v1/semantic_layer/${uuid}`, + }); + const layer = json.result; + setName(layer.name ?? ''); + setSelectedType(layer.type); + setFormData(layer.configuration ?? {}); + setHasErrors(false); + // Fetch base schema (no configuration → no Snowflake connection) to + // show the form immediately. The existing maybeRefreshSchema machinery + // will trigger an enriched fetch in the background once deps are + // satisfied, and DynamicFieldControl will show per-field spinners. + const { json: schemaJson } = await SupersetClient.post({ + endpoint: '/api/v1/semantic_layer/schema/configuration', + jsonPayload: { type: layer.type }, + }); + applySchema(schemaJson.result); + setStep('config'); + } catch { + addDangerToast( + t('An error occurred while fetching the semantic layer'), + ); + } finally { + setLoading(false); + } + }, + [addDangerToast, applySchema], + ); + + useEffect(() => { + if (show) { + if (isEditMode && semanticLayerUuid) { + fetchTypes(); + fetchExistingLayer(semanticLayerUuid); + } else { + fetchTypes(); + } + } else { + setStep('type'); + setName(''); + setSelectedType(null); + setTypes([]); + setConfigSchema(null); + setUiSchema(undefined); + setFormData({}); + setHasErrors(true); + setRefreshingSchema(false); + setValidationMode('ValidateAndHide'); + errorsRef.current = []; + lastDepSnapshotRef.current = ''; + dynamicDepsRef.current = {}; + if (debounceTimerRef.current) clearTimeout(debounceTimerRef.current); + } + }, [show, fetchTypes, isEditMode, semanticLayerUuid, fetchExistingLayer]); + + const handleStepAdvance = () => { + if (selectedType) { + fetchConfigSchema(selectedType); + } + }; + + const handleBack = () => { + setStep('type'); + setConfigSchema(null); + setUiSchema(undefined); + setFormData({}); + setValidationMode('ValidateAndHide'); + errorsRef.current = []; + lastDepSnapshotRef.current = ''; + dynamicDepsRef.current = {}; + if (debounceTimerRef.current) clearTimeout(debounceTimerRef.current); + }; + + const handleCreate = async () => { + setSaving(true); + try { + if (isEditMode && semanticLayerUuid) { + await SupersetClient.put({ + endpoint: `/api/v1/semantic_layer/${semanticLayerUuid}`, + jsonPayload: { name, configuration: formData }, + }); + addSuccessToast(t('Semantic layer updated')); + } else { + await SupersetClient.post({ + endpoint: '/api/v1/semantic_layer/', + jsonPayload: { name, type: selectedType, configuration: formData }, + }); + addSuccessToast(t('Semantic layer created')); + } + onHide(); + } catch { + addDangerToast( + isEditMode + ? t('An error occurred while updating the semantic layer') + : t('An error occurred while creating the semantic layer'), + ); + } finally { + setSaving(false); + } + }; + + const handleSave = () => { + if (step === 'type') { + handleStepAdvance(); + } else { + setValidationMode('ValidateAndShow'); + if (errorsRef.current.length === 0) { + handleCreate(); + } + } + }; + + const maybeRefreshSchema = useCallback( + (data: Record<string, unknown>) => { + if (!selectedType) return; + + const dynamicDeps = dynamicDepsRef.current; + if (Object.keys(dynamicDeps).length === 0) return; + + // Check if any dynamic field has all dependencies satisfied + const hasSatisfiedDeps = Object.values(dynamicDeps).some(deps => + areDependenciesSatisfied(deps, data), + ); + if (!hasSatisfiedDeps) return; + + // Only re-fetch if dependency values actually changed + const snapshot = serializeDependencyValues(dynamicDeps, data); + if (snapshot === lastDepSnapshotRef.current) return; + lastDepSnapshotRef.current = snapshot; + + if (debounceTimerRef.current) clearTimeout(debounceTimerRef.current); + debounceTimerRef.current = setTimeout(() => { + fetchConfigSchema(selectedType, data); + }, SCHEMA_REFRESH_DEBOUNCE_MS); + }, + [selectedType, fetchConfigSchema], + ); + + const handleFormChange = useCallback( + ({ data, errors }: { data: Record<string, unknown>; errors?: ErrorObject[] }) => { + setFormData(data); + errorsRef.current = errors ?? []; + setHasErrors(errorsRef.current.length > 0); + if ( + validationMode === 'ValidateAndShow' && + errorsRef.current.length === 0 + ) { + handleCreate(); + } + maybeRefreshSchema(data); + }, + [validationMode, handleCreate, maybeRefreshSchema], + ); + + const selectedTypeName = + types.find(type => type.id === selectedType)?.name ?? ''; + + const title = isEditMode + ? t('Edit %s', selectedTypeName || t('Semantic Layer')) + : step === 'type' + ? t('New Semantic Layer') + : t('Configure %s', selectedTypeName); + + return ( + <StandardModal + show={show} + onHide={onHide} + onSave={handleSave} + title={title} + icon={isEditMode ? <Icons.EditOutlined /> : <Icons.PlusOutlined />} + width={step === 'type' ? MODAL_STANDARD_WIDTH : MODAL_MEDIUM_WIDTH} + saveDisabled={ + step === 'type' ? !selectedType : saving || !name.trim() || hasErrors + } + saveText={step === 'type' ? undefined : isEditMode ? t('Save') : t('Create')} + saveLoading={saving} + contentLoading={loading} + > + {step === 'type' ? ( + <ModalContent> + <ModalFormField label={t('Type')}> + <Select + ariaLabel={t('Semantic layer type')} + placeholder={t('Select a semantic layer type')} + value={selectedType} + onChange={value => setSelectedType(value as string)} + options={types.map(type => ({ + value: type.id, + label: type.name, + }))} + getPopupContainer={() => document.body} + dropdownAlign={{ + points: ['tl', 'bl'], + offset: [0, 4], + overflow: { adjustX: 0, adjustY: 1 }, + }} + /> + </ModalFormField> + </ModalContent> + ) : ( + <ModalContent> + {!isEditMode && ( + <BackLink type="button" onClick={handleBack}> + <Icons.CaretLeftOutlined iconSize="s" /> + {t('Back')} + </BackLink> + )} + <ModalFormField label={t('Name')} required> + <Input + value={name} + onChange={e => setName(e.target.value)} + placeholder={t('Name of the semantic layer')} + /> + </ModalFormField> + {configSchema && ( + <JsonForms + schema={configSchema} + uischema={uiSchema} + data={formData} + renderers={renderers} + cells={cellRegistryEntries} + config={{ refreshingSchema, formData }} + validationMode={validationMode} + onChange={handleFormChange} + /> + )} + </ModalContent> + )} + </StandardModal> + ); +} diff --git a/superset-frontend/src/pages/DatabaseList/index.tsx b/superset-frontend/src/pages/DatabaseList/index.tsx index e8bf7ab29c2..99cc90a5a6d 100644 --- a/superset-frontend/src/pages/DatabaseList/index.tsx +++ b/superset-frontend/src/pages/DatabaseList/index.tsx @@ -17,8 +17,13 @@ * under the License. */ import { t } from '@apache-superset/core'; -import { getExtensionsRegistry, SupersetClient } from '@superset-ui/core'; -import { styled } from '@apache-superset/core/ui'; +import { + getExtensionsRegistry, + SupersetClient, + isFeatureEnabled, + FeatureFlag, +} from '@superset-ui/core'; +import { css, styled, useTheme } from '@apache-superset/core/ui'; import { useState, useMemo, useEffect, useCallback } from 'react'; import rison from 'rison'; import { useSelector } from 'react-redux'; @@ -33,7 +38,9 @@ import { import withToasts from 'src/components/MessageToasts/withToasts'; import SubMenu, { SubMenuProps } from 'src/features/home/SubMenu'; import { + Button, DeleteModal, + Dropdown, Tooltip, List, Loading, @@ -43,6 +50,7 @@ import { ListView, ListViewFilterOperator as FilterOperator, ListViewFilters, + type ListViewFetchDataConfig, } from 'src/components'; import { Typography } from '@superset-ui/core/components/Typography'; import { getUrlParam } from 'src/utils/urlUtils'; @@ -55,6 +63,7 @@ import { UserWithPermissionsAndRoles } from 'src/types/bootstrapTypes'; import type { MenuObjectProps } from 'src/types/bootstrapTypes'; import DatabaseModal from 'src/features/databases/DatabaseModal'; import UploadDataModal from 'src/features/databases/UploadDataModel'; +import SemanticLayerModal from 'src/features/semanticLayers/SemanticLayerModal'; import { DatabaseObject } from 'src/features/databases/types'; import { QueryObjectColumns } from 'src/views/CRUD/types'; import { WIDER_DROPDOWN_WIDTH } from 'src/components/ListView/utils'; @@ -70,6 +79,11 @@ const dbConfigExtraExtension = extensionsRegistry.get( const PAGE_SIZE = 25; +type ConnectionItem = DatabaseObject & { + source_type?: 'database' | 'semantic_layer'; + sl_type?: string; +}; + interface DatabaseDeleteObject extends DatabaseObject { charts: any; dashboards: any; @@ -108,20 +122,106 @@ function DatabaseList({ addSuccessToast, user, }: DatabaseListProps) { + const theme = useTheme(); + const showSemanticLayers = isFeatureEnabled(FeatureFlag.SemanticLayers); + + // Standard database list view resource (used when SL flag is OFF) const { state: { - loading, - resourceCount: databaseCount, - resourceCollection: databases, + loading: dbLoading, + resourceCount: dbCount, + resourceCollection: dbCollection, }, hasPerm, - fetchData, - refreshData, + fetchData: dbFetchData, + refreshData: dbRefreshData, } = useListViewResource<DatabaseObject>( 'database', t('database'), addDangerToast, ); + + // Combined endpoint state (used when SL flag is ON) + const [combinedItems, setCombinedItems] = useState<ConnectionItem[]>([user]); + const [combinedCount, setCombinedCount] = useState(0); + const [combinedLoading, setCombinedLoading] = useState(true); + const [lastFetchConfig, setLastFetchConfig] = + useState<ListViewFetchDataConfig | null>(null); + + const combinedFetchData = useCallback( + (config: ListViewFetchDataConfig) => { + setLastFetchConfig(config); + setCombinedLoading(true); + const { pageIndex, pageSize, sortBy, filters: filterValues } = config; + + const sourceTypeFilter = filterValues.find(f => f.id === 'source_type'); + const otherFilters = filterValues + .filter(f => f.id !== 'source_type') + .filter( + ({ value }) => value !== '' && value !== null && value !== undefined, + ) + .map(({ id, operator: opr, value }) => ({ + col: id, + opr, + value: + value && typeof value === 'object' && 'value' in value + ? value.value + : value, + })); + + const sourceTypeValue = + sourceTypeFilter?.value && typeof sourceTypeFilter.value === 'object' + ? (sourceTypeFilter.value as { value: string }).value + : sourceTypeFilter?.value; + if (sourceTypeValue) { + otherFilters.push({ + col: 'source_type', + opr: 'eq', + value: sourceTypeValue, + }); + } + + const queryParams = rison.encode_uri({ + order_column: sortBy[0].id, + order_direction: sortBy[0].desc ? 'desc' : 'asc', + page: pageIndex, + page_size: pageSize, + ...(otherFilters.length ? { filters: otherFilters } : {}), + }); + + return SupersetClient.get({ + endpoint: `/api/v1/semantic_layer/connections/?q=${queryParams}`, + }) + .then(({ json = {} }) => { + setCombinedItems(json.result); + setCombinedCount(json.count); + }) + .catch(() => { + addDangerToast(t('An error occurred while fetching connections')); + }) + .finally(() => { + setCombinedLoading(false); + }); + }, + [addDangerToast], + ); + + const combinedRefreshData = useCallback(() => { + if (lastFetchConfig) { + return combinedFetchData(lastFetchConfig); + } + return undefined; + }, [lastFetchConfig, combinedFetchData]); + + // Select the right data source based on feature flag + const loading = showSemanticLayers ? combinedLoading : dbLoading; + const databaseCount = showSemanticLayers ? combinedCount : dbCount; + const databases: ConnectionItem[] = showSemanticLayers + ? combinedItems + : dbCollection; + const fetchData = showSemanticLayers ? combinedFetchData : dbFetchData; + const refreshData = showSemanticLayers ? combinedRefreshData : dbRefreshData; + const fullUser = useSelector<any, UserWithPermissionsAndRoles>( state => state.user, ); @@ -148,6 +248,13 @@ function DatabaseList({ useState<boolean>(false); const [columnarUploadDataModalOpen, setColumnarUploadDataModalOpen] = useState<boolean>(false); + const [semanticLayerModalOpen, setSemanticLayerModalOpen] = + useState<boolean>(false); + const [slCurrentlyEditing, setSlCurrentlyEditing] = useState<string | null>( + null, + ); + const [slCurrentlyDeleting, setSlCurrentlyDeleting] = + useState<ConnectionItem | null>(null); const [allowUploads, setAllowUploads] = useState<boolean>(false); const isAdmin = isUserAdmin(fullUser); @@ -320,18 +427,63 @@ function DatabaseList({ }; if (canCreate) { - menuData.buttons = [ - { - 'data-test': 'btn-create-database', - icon: <Icons.PlusOutlined iconSize="m" />, - name: t('Database'), - buttonStyle: 'primary', - onClick: () => { - // Ensure modal will be opened in add mode - handleDatabaseEditModal({ modalOpen: true }); + const openDatabaseModal = () => + handleDatabaseEditModal({ modalOpen: true }); + + if (isFeatureEnabled(FeatureFlag.SemanticLayers)) { + menuData.buttons = [ + { + name: t('New'), + buttonStyle: 'primary', + component: ( + <Dropdown + menu={{ + items: [ + { + key: 'database', + label: t('Database'), + onClick: openDatabaseModal, + }, + { + key: 'semantic-layer', + label: t('Semantic Layer'), + onClick: () => { + setSemanticLayerModalOpen(true); + }, + }, + ], + }} + trigger={['click']} + > + <Button + data-test="btn-create-new" + buttonStyle="primary" + icon={<Icons.PlusOutlined iconSize="m" />} + > + {t('New')} + <Icons.DownOutlined + iconSize="s" + css={css` + margin-left: ${theme.sizeUnit * 1.5}px; + margin-right: -${theme.sizeUnit * 2}px; + `} + /> + </Button> + </Dropdown> + ), }, - }, - ]; + ]; + } else { + menuData.buttons = [ + { + 'data-test': 'btn-create-database', + icon: <Icons.PlusOutlined iconSize="m" />, + name: t('Database'), + buttonStyle: 'primary', + onClick: openDatabaseModal, + }, + ]; + } } const handleDatabaseExport = useCallback( @@ -401,6 +553,23 @@ function DatabaseList({ const initialSort = [{ id: 'changed_on_delta_humanized', desc: true }]; + function handleSemanticLayerDelete(item: ConnectionItem) { + SupersetClient.delete({ + endpoint: `/api/v1/semantic_layer/${item.uuid}`, + }).then( + () => { + refreshData(); + addSuccessToast(t('Deleted: %s', item.database_name)); + setSlCurrentlyDeleting(null); + }, + createErrorHandler(errMsg => + addDangerToast( + t('There was an issue deleting %s: %s', item.database_name, errMsg), + ), + ), + ); + } + const columns = useMemo( () => [ { @@ -413,7 +582,7 @@ function DatabaseList({ accessor: 'backend', Header: t('Backend'), size: 'xl', - disableSortBy: true, // TODO: api support for sorting by 'backend' + disableSortBy: true, id: 'backend', }, { @@ -427,13 +596,12 @@ function DatabaseList({ <span>{t('AQE')}</span> </Tooltip> ), - Cell: ({ - row: { - original: { allow_run_async: allowRunAsync }, - }, - }: { - row: { original: { allow_run_async: boolean } }; - }) => <BooleanDisplay value={allowRunAsync} />, + Cell: ({ row: { original } }: any) => + original.source_type === 'semantic_layer' ? ( + <span>–</span> + ) : ( + <BooleanDisplay value={original.allow_run_async} /> + ), size: 'sm', id: 'allow_run_async', }, @@ -448,33 +616,36 @@ function DatabaseList({ <span>{t('DML')}</span> </Tooltip> ), - Cell: ({ - row: { - original: { allow_dml: allowDML }, - }, - }: any) => <BooleanDisplay value={allowDML} />, + Cell: ({ row: { original } }: any) => + original.source_type === 'semantic_layer' ? ( + <span>–</span> + ) : ( + <BooleanDisplay value={original.allow_dml} /> + ), size: 'sm', id: 'allow_dml', }, { accessor: 'allow_file_upload', Header: t('File upload'), - Cell: ({ - row: { - original: { allow_file_upload: allowFileUpload }, - }, - }: any) => <BooleanDisplay value={allowFileUpload} />, + Cell: ({ row: { original } }: any) => + original.source_type === 'semantic_layer' ? ( + <span>–</span> + ) : ( + <BooleanDisplay value={original.allow_file_upload} /> + ), size: 'md', id: 'allow_file_upload', }, { accessor: 'expose_in_sqllab', Header: t('Expose in SQL Lab'), - Cell: ({ - row: { - original: { expose_in_sqllab: exposeInSqllab }, - }, - }: any) => <BooleanDisplay value={exposeInSqllab} />, + Cell: ({ row: { original } }: any) => + original.source_type === 'semantic_layer' ? ( + <span>–</span> + ) : ( + <BooleanDisplay value={original.expose_in_sqllab} /> + ), size: 'md', id: 'expose_in_sqllab', }, @@ -494,6 +665,49 @@ function DatabaseList({ }, { Cell: ({ row: { original } }: any) => { + const isSemanticLayer = + original.source_type === 'semantic_layer'; + + if (isSemanticLayer) { + if (!canEdit && !canDelete) return null; + return ( + <Actions className="actions"> + {canDelete && ( + <Tooltip + id="delete-action-tooltip" + title={t('Delete')} + placement="bottom" + > + <span + role="button" + tabIndex={0} + className="action-button" + onClick={() => setSlCurrentlyDeleting(original)} + > + <Icons.DeleteOutlined iconSize="l" /> + </span> + </Tooltip> + )} + {canEdit && ( + <Tooltip + id="edit-action-tooltip" + title={t('Edit')} + placement="bottom" + > + <span + role="button" + tabIndex={0} + className="action-button" + onClick={() => setSlCurrentlyEditing(original.uuid)} + > + <Icons.EditOutlined iconSize="l" /> + </span> + </Tooltip> + )} + </Actions> + ); + } + const handleEdit = () => handleDatabaseEditModal({ database: original, modalOpen: true }); const handleDelete = () => openDatabaseDeleteModal(original); @@ -579,6 +793,12 @@ function DatabaseList({ hidden: !canEdit && !canDelete, disableSortBy: true, }, + { + accessor: 'source_type', + hidden: true, + disableSortBy: true, + id: 'source_type', + }, { accessor: QueryObjectColumns.ChangedBy, hidden: true, @@ -596,8 +816,8 @@ function DatabaseList({ ], ); - const filters: ListViewFilters = useMemo( - () => [ + const filters: ListViewFilters = useMemo(() => { + const baseFilters: ListViewFilters = [ { Header: t('Name'), key: 'search', @@ -605,62 +825,83 @@ function DatabaseList({ input: 'search', operator: FilterOperator.Contains, }, - { - Header: t('Expose in SQL Lab'), - key: 'expose_in_sql_lab', - id: 'expose_in_sqllab', - input: 'select', - operator: FilterOperator.Equals, - unfilteredLabel: t('All'), - selects: [ - { label: t('Yes'), value: true }, - { label: t('No'), value: false }, - ], - }, - { - Header: ( - <Tooltip - id="allow-run-async-filter-header-tooltip" - title={t('Asynchronous query execution')} - placement="top" - > - <span>{t('AQE')}</span> - </Tooltip> - ), - key: 'allow_run_async', - id: 'allow_run_async', + ]; + + if (showSemanticLayers) { + baseFilters.push({ + Header: t('Source'), + key: 'source_type', + id: 'source_type', input: 'select', operator: FilterOperator.Equals, unfilteredLabel: t('All'), selects: [ - { label: t('Yes'), value: true }, - { label: t('No'), value: false }, + { label: t('Database'), value: 'database' }, + { label: t('Semantic Layer'), value: 'semantic_layer' }, ], - }, - { - Header: t('Modified by'), - key: 'changed_by', - id: 'changed_by', - input: 'select', - operator: FilterOperator.RelationOneMany, - unfilteredLabel: t('All'), - fetchSelects: createFetchRelated( - 'database', - 'changed_by', - createErrorHandler(errMsg => - t( - 'An error occurred while fetching dataset datasource values: %s', - errMsg, + }); + } + + if (!showSemanticLayers) { + baseFilters.push( + { + Header: t('Expose in SQL Lab'), + key: 'expose_in_sql_lab', + id: 'expose_in_sqllab', + input: 'select', + operator: FilterOperator.Equals, + unfilteredLabel: t('All'), + selects: [ + { label: t('Yes'), value: true }, + { label: t('No'), value: false }, + ], + }, + { + Header: ( + <Tooltip + id="allow-run-async-filter-header-tooltip" + title={t('Asynchronous query execution')} + placement="top" + > + <span>{t('AQE')}</span> + </Tooltip> + ), + key: 'allow_run_async', + id: 'allow_run_async', + input: 'select', + operator: FilterOperator.Equals, + unfilteredLabel: t('All'), + selects: [ + { label: t('Yes'), value: true }, + { label: t('No'), value: false }, + ], + }, + { + Header: t('Modified by'), + key: 'changed_by', + id: 'changed_by', + input: 'select', + operator: FilterOperator.RelationOneMany, + unfilteredLabel: t('All'), + fetchSelects: createFetchRelated( + 'database', + 'changed_by', + createErrorHandler(errMsg => + t( + 'An error occurred while fetching dataset datasource values: %s', + errMsg, + ), ), + user, ), - user, - ), - paginate: true, - dropdownStyle: { minWidth: WIDER_DROPDOWN_WIDTH }, - }, - ], - [user], - ); + paginate: true, + dropdownStyle: { minWidth: WIDER_DROPDOWN_WIDTH }, + }, + ); + } + + return baseFilters; + }, [showSemanticLayers]); return ( <> @@ -703,6 +944,48 @@ function DatabaseList({ allowedExtensions={COLUMNAR_EXTENSIONS} type="columnar" /> + <SemanticLayerModal + show={semanticLayerModalOpen} + onHide={() => { + setSemanticLayerModalOpen(false); + refreshData(); + }} + addDangerToast={addDangerToast} + addSuccessToast={addSuccessToast} + /> + <SemanticLayerModal + show={!!slCurrentlyEditing} + onHide={() => { + setSlCurrentlyEditing(null); + refreshData(); + }} + addDangerToast={addDangerToast} + addSuccessToast={addSuccessToast} + semanticLayerUuid={slCurrentlyEditing ?? undefined} + /> + {slCurrentlyDeleting && ( + <DeleteModal + description={ + <p> + {t('Are you sure you want to delete')}{' '} + <b>{slCurrentlyDeleting.database_name}</b>? + </p> + } + onConfirm={() => { + if (slCurrentlyDeleting) { + handleSemanticLayerDelete(slCurrentlyDeleting); + } + }} + onHide={() => setSlCurrentlyDeleting(null)} + open + title={ + <ModalTitleWithIcon + icon={<Icons.DeleteOutlined />} + title={t('Delete Semantic Layer?')} + /> + } + /> + )} {databaseCurrentlyDeleting && ( <DeleteModal description={ diff --git a/superset/commands/semantic_layer/create.py b/superset/commands/semantic_layer/create.py index 48ec0603c2a..8729f75a13b 100644 --- a/superset/commands/semantic_layer/create.py +++ b/superset/commands/semantic_layer/create.py @@ -16,6 +16,7 @@ # under the License. from __future__ import annotations +import json import logging from functools import partial from typing import Any @@ -48,6 +49,10 @@ class CreateSemanticLayerCommand(BaseCommand): ) def run(self) -> Model: self.validate() + if isinstance(self._properties.get("configuration"), dict): + self._properties["configuration"] = json.dumps( + self._properties["configuration"] + ) return SemanticLayerDAO.create(attributes=self._properties) def validate(self) -> None: diff --git a/superset/commands/semantic_layer/update.py b/superset/commands/semantic_layer/update.py index 5242406af8c..66c02d568ab 100644 --- a/superset/commands/semantic_layer/update.py +++ b/superset/commands/semantic_layer/update.py @@ -16,6 +16,7 @@ # under the License. from __future__ import annotations +import json import logging from functools import partial from typing import Any @@ -87,6 +88,10 @@ class UpdateSemanticLayerCommand(BaseCommand): def run(self) -> Model: self.validate() assert self._model + if isinstance(self._properties.get("configuration"), dict): + self._properties["configuration"] = json.dumps( + self._properties["configuration"] + ) return SemanticLayerDAO.update(self._model, attributes=self._properties) def validate(self) -> None: diff --git a/superset/semantic_layers/api.py b/superset/semantic_layers/api.py index 340fe90a6a2..f41e11e5888 100644 --- a/superset/semantic_layers/api.py +++ b/superset/semantic_layers/api.py @@ -16,15 +16,18 @@ # under the License. from __future__ import annotations +import json import logging from typing import Any -from flask import request, Response -from flask_appbuilder.api import expose, protect, safe +from flask import make_response, request, Response +from flask_appbuilder.api import expose, protect, rison, safe +from flask_appbuilder.api.schemas import get_list_schema from flask_appbuilder.models.sqla.interface import SQLAInterface from marshmallow import ValidationError +from pydantic import ValidationError as PydanticValidationError -from superset import event_logger +from superset import db, event_logger, is_feature_enabled from superset.commands.semantic_layer.create import CreateSemanticLayerCommand from superset.commands.semantic_layer.delete import DeleteSemanticLayerCommand from superset.commands.semantic_layer.exceptions import ( @@ -44,6 +47,7 @@ from superset.commands.semantic_layer.update import ( ) from superset.constants import MODEL_API_RW_METHOD_PERMISSION_MAP from superset.daos.semantic_layer import SemanticLayerDAO +from superset.models.core import Database from superset.semantic_layers.models import SemanticLayer, SemanticView from superset.semantic_layers.registry import registry from superset.semantic_layers.schemas import ( @@ -63,15 +67,93 @@ logger = logging.getLogger(__name__) def _serialize_layer(layer: SemanticLayer) -> dict[str, Any]: + config = layer.configuration + if isinstance(config, str): + config = json.loads(config) return { "uuid": str(layer.uuid), "name": layer.name, "description": layer.description, "type": layer.type, "cache_timeout": layer.cache_timeout, + "configuration": config or {}, + "changed_on_delta_humanized": layer.changed_on_delta_humanized(), } +def _infer_discriminators( + schema: dict[str, Any], + data: dict[str, Any], +) -> dict[str, Any]: + """ + Infer discriminator values for union fields when the frontend omits them. + + Walks the schema's properties looking for discriminated unions (fields with a + ``discriminator.mapping``). For each one, tries to match the submitted data + against one of the variants by checking which variant's required fields are + present, then injects the discriminator value. + """ + defs = schema.get("$defs", {}) + for prop_name, prop_schema in schema.get("properties", {}).items(): + value = data.get(prop_name) + if not isinstance(value, dict): + continue + + # Find discriminated union via discriminator mapping + mapping = ( + prop_schema.get("discriminator", {}).get("mapping") + if "discriminator" in prop_schema + else None + ) + if not mapping: + continue + + discriminator_field = prop_schema["discriminator"].get("propertyName") + if not discriminator_field or discriminator_field in value: + continue + + # Try each variant: match by required fields present in the data + for disc_value, ref in mapping.items(): + ref_name = ref.rsplit("/", 1)[-1] if "/" in ref else ref + variant_def = defs.get(ref_name, {}) + required = set(variant_def.get("required", [])) + # Exclude the discriminator itself from the check + required.discard(discriminator_field) + if required and required.issubset(value.keys()): + data = { + **data, + prop_name: {**value, discriminator_field: disc_value}, + } + break + + return data + + +def _parse_partial_config( + cls: Any, + config: dict[str, Any], +) -> Any: + """ + Parse a partial configuration, handling discriminator inference and + falling back to lenient validation when strict parsing fails. + """ + config_class = cls.configuration_class + + # Infer discriminator values the frontend may have omitted + schema = config_class.model_json_schema() + config = _infer_discriminators(schema, config) + + try: + return config_class.model_validate(config) + except (PydanticValidationError, ValueError): + pass + + try: + return config_class.model_validate(config, context={"partial": True}) + except (PydanticValidationError, ValueError): + return None + + class SemanticViewRestApi(BaseSupersetModelRestApi): datamodel = SQLAInterface(SemanticView) @@ -223,13 +305,18 @@ class SemanticLayerRestApi(BaseSupersetApi): parsed_config = None if config := body.get("configuration"): - try: - parsed_config = cls.from_configuration(config).configuration # type: ignore[attr-defined] - except Exception: # pylint: disable=broad-except - parsed_config = None + parsed_config = _parse_partial_config(cls, config) - schema = cls.get_configuration_schema(parsed_config) - return self.response(200, result=schema) + try: + schema = cls.get_configuration_schema(parsed_config) + except Exception: # pylint: disable=broad-except + # Connection or query failures during schema enrichment should not + # prevent the form from rendering — return the base schema instead. + schema = cls.get_configuration_schema(None) + + resp = make_response(json.dumps({"result": schema}, sort_keys=False), 200) + resp.headers["Content-Type"] = "application/json; charset=utf-8" + return resp @expose("/<uuid>/schema/runtime", methods=("POST",)) @protect() @@ -436,6 +523,164 @@ class SemanticLayerRestApi(BaseSupersetApi): ) return self.response_422(message=str(ex)) + @expose("/connections/", methods=("GET",)) + @protect() + @safe + @statsd_metrics + @rison(get_list_schema) + @event_logger.log_this_with_context( + action=lambda self, *args, **kwargs: f"{self.__class__.__name__}" + ".connections", + log_to_statsd=False, + ) + def connections(self, **kwargs: Any) -> FlaskResponse: + """List databases and semantic layers combined. + --- + get: + summary: List databases and semantic layers combined + parameters: + - in: query + name: q + content: + application/json: + schema: + $ref: '#/components/schemas/get_list_schema' + responses: + 200: + description: Combined list of databases and semantic layers + 401: + $ref: '#/components/responses/401' + 500: + $ref: '#/components/responses/500' + """ + args = kwargs.get("rison", {}) + page = args.get("page", 0) + page_size = args.get("page_size", 25) + order_column = args.get("order_column", "changed_on") + order_direction = args.get("order_direction", "desc") + filters = args.get("filters", []) + + source_type = "all" + name_filter = None + for f in filters: + if f.get("col") == "source_type": + source_type = f.get("value", "all") + elif f.get("col") == "database_name" and f.get("opr") == "ct": + name_filter = f.get("value") + + if not is_feature_enabled("SEMANTIC_LAYERS"): + source_type = "database" + + reverse = order_direction == "desc" + + def _sort_key_changed_on( + item: tuple[str, Database | SemanticLayer], + ) -> Any: + return item[1].changed_on or "" + + def _sort_key_name( + item: tuple[str, Database | SemanticLayer], + ) -> str: + obj = item[1] + raw = ( + obj.database_name # type: ignore[union-attr] + if item[0] == "database" + else obj.name + ) + return raw.lower() + + sort_key_map = { + "changed_on_delta_humanized": _sort_key_changed_on, + "database_name": _sort_key_name, + } + sort_key = sort_key_map.get(order_column, _sort_key_changed_on) + + # Fetch databases (lightweight: only loads ORM objects, no eager joins) + db_items: list[tuple[str, Database]] = [] + if source_type in ("all", "database"): + db_q = db.session.query(Database) + if name_filter: + db_q = db_q.filter( + Database.database_name.ilike(f"%{name_filter}%") + ) + db_items = [("database", obj) for obj in db_q.all()] + + # Fetch semantic layers + sl_items: list[tuple[str, SemanticLayer]] = [] + if source_type in ("all", "semantic_layer"): + sl_q = db.session.query(SemanticLayer) + if name_filter: + sl_q = sl_q.filter( + SemanticLayer.name.ilike(f"%{name_filter}%") + ) + sl_items = [("semantic_layer", obj) for obj in sl_q.all()] + + # Merge, sort, count, paginate + all_items: list[tuple[str, Any]] = db_items + sl_items # type: ignore + all_items.sort(key=sort_key, reverse=reverse) # type: ignore + total_count = len(all_items) + + start = page * page_size + page_items = all_items[start : start + page_size] + + # Serialize + result = [] + for item_type, obj in page_items: + if item_type == "database": + result.append(self._serialize_database(obj)) + else: + result.append(self._serialize_semantic_layer(obj)) + + return self.response(200, count=total_count, result=result) + + @staticmethod + def _serialize_database(obj: Database) -> dict[str, Any]: + changed_by = obj.changed_by + return { + "source_type": "database", + "id": obj.id, + "uuid": str(obj.uuid), + "database_name": obj.database_name, + "backend": obj.backend, + "allow_run_async": obj.allow_run_async, + "allow_dml": obj.allow_dml, + "allow_file_upload": obj.allow_file_upload, + "expose_in_sqllab": obj.expose_in_sqllab, + "changed_on_delta_humanized": obj.changed_on_delta_humanized(), + "changed_by": { + "first_name": changed_by.first_name, + "last_name": changed_by.last_name, + } + if changed_by + else None, + } + + @staticmethod + def _serialize_semantic_layer(obj: SemanticLayer) -> dict[str, Any]: + changed_by = obj.changed_by + sl_type = obj.type + cls = registry.get(sl_type) + type_name = cls.name if cls else sl_type + return { + "source_type": "semantic_layer", + "uuid": str(obj.uuid), + "database_name": obj.name, + "backend": type_name, + "sl_type": sl_type, + "description": obj.description, + "allow_run_async": None, + "allow_dml": None, + "allow_file_upload": None, + "expose_in_sqllab": None, + "changed_on_delta_humanized": obj.changed_on_delta_humanized(), + "changed_by": { + "first_name": changed_by.first_name, + "last_name": changed_by.last_name, + } + if changed_by + else None, + } + @expose("/", methods=("GET",)) @protect() @safe
