This is an automated email from the ASF dual-hosted git repository. villebro pushed a commit to branch villebro/ext-automanifest in repository https://gitbox.apache.org/repos/asf/superset.git
commit c5dce675a01b993eae8e1aed872d83d6160961df Author: Ville Brofeldt <[email protected]> AuthorDate: Fri Feb 6 08:31:07 2026 -0800 feat(extensions): autogenerate fe and be contributions --- .../extensions/contribution-types.md | 126 ++++++-- superset-core/src/superset_core/api/rest_api.py | 220 +++++++++++-- .../src/superset_core/extensions/__init__.py | 48 +++ .../src/superset_core/extensions/context.py | 197 ++++++++++++ .../src/superset_core/extensions/types.py | 93 +++++- superset-core/src/superset_core/mcp/__init__.py | 259 ++++++++++++---- .../src/superset_extensions_cli/cli.py | 157 +++++++++- .../tests/test_backend_discovery.py | 236 ++++++++++++++ superset/core/api/core_api_injection.py | 110 ++++++- superset/core/mcp/core_mcp_injection.py | 339 ++++++++++++++++----- superset/extensions/manager.py | 333 ++++++++++++++++++++ superset/initialization/__init__.py | 38 +-- 12 files changed, 1907 insertions(+), 249 deletions(-) diff --git a/docs/developer_portal/extensions/contribution-types.md b/docs/developer_portal/extensions/contribution-types.md index f802bd609cb..d88b141e5ff 100644 --- a/docs/developer_portal/extensions/contribution-types.md +++ b/docs/developer_portal/extensions/contribution-types.md @@ -24,15 +24,74 @@ under the License. # Contribution Types -To facilitate the development of extensions, we define a set of well-defined contribution types that extensions can implement. These contribution types serve as the building blocks for extensions, allowing them to interact with the host application and provide new functionality. +Extensions provide functionality through **contributions** - well-defined extension points that integrate with the host application. -## Frontend +## Why Contributions? -Frontend contribution types allow extensions to extend Superset's user interface with new views, commands, and menu items. +The contribution system provides several key benefits: + +- **Transparency**: Administrators can review exactly what functionality an extension provides before installation. The `manifest.json` documents all REST APIs, MCP tools, views, and other contributions in a single, readable location. + +- **Security**: Only contributions explicitly declared in the manifest are registered during startup. Extensions cannot expose functionality they haven't declared, preventing hidden or undocumented code from executing. + +- **Discoverability**: The manifest serves as a contract between extensions and the host application, making it easy to understand what an extension does without reading its source code. + +## How Contributions Work + +Contributions are automatically inferred from source code during build. The build tool scans your code and generates a `manifest.json` with all discovered contributions. + +For advanced use cases, contributions can be manually specified in `extension.json` (overrides auto-discovery). + +## Backend Contributions + +### REST API Endpoints + +Register REST APIs under `/api/v1/extensions/`: + +```python +from superset_core.api import RestApi, extension_api +from flask_appbuilder import expose + +@extension_api(id="my_api", name="My Extension API") +class MyExtensionAPI(RestApi): + @expose("/endpoint", methods=["GET"]) + def get_data(self): + return self.response(200, result={"message": "Hello"}) +``` + +### MCP Tools + +Register MCP tools for AI agents: + +```python +from superset_core.mcp import tool + +@tool(tags=["database"]) +def query_database(sql: str, database_id: int) -> dict: + """Execute a SQL query against a database.""" + return execute_query(sql, database_id) +``` + +### MCP Prompts + +Register MCP prompts: + +```python +from superset_core.mcp import prompt + +@prompt(tags={"analysis"}) +async def analyze_data(ctx, dataset: str) -> str: + """Generate analysis for a dataset.""" + return f"Analyze the {dataset} dataset..." +``` + +See [MCP Integration](./mcp) for more details. + +## Frontend Contributions ### Views -Extensions can add new views or panels to the host application, such as custom SQL Lab panels, dashboards, or other UI components. Each view is registered with a unique ID and can be activated or deactivated as needed. Contribution areas are uniquely identified (e.g., `sqllab.panels` for SQL Lab panels), enabling seamless integration into specific parts of the application. +Add panels or views to the UI: ```json "frontend": { @@ -53,7 +112,7 @@ Extensions can add new views or panels to the host application, such as custom S ### Commands -Extensions can define custom commands that can be executed within the host application, such as context-aware actions or menu options. Each command can specify properties like a unique command identifier, an icon, a title, and a description. These commands can be invoked by users through menus, keyboard shortcuts, or other UI elements, enabling extensions to add rich, interactive functionality to Superset. +Define executable commands: ```json "frontend": { @@ -63,7 +122,7 @@ Extensions can define custom commands that can be executed within the host appli "command": "my_extension.copy_query", "icon": "CopyOutlined", "title": "Copy Query", - "description": "Copy the current query to clipboard" + "description": "Copy the current query" } ] } @@ -72,7 +131,7 @@ Extensions can define custom commands that can be executed within the host appli ### Menus -Extensions can contribute new menu items or context menus to the host application, providing users with additional actions and options. Each menu item can specify properties such as the target view, the command to execute, its placement (primary, secondary, or context), and conditions for when it should be displayed. Menu contribution areas are uniquely identified (e.g., `sqllab.editor` for the SQL Lab editor), allowing extensions to seamlessly integrate their functionality into specific [...] +Add items to menus: ```json "frontend": { @@ -107,7 +166,7 @@ Extensions can contribute new menu items or context menus to the host applicatio ### Editors -Extensions can replace Superset's default text editors with custom implementations. This enables enhanced editing experiences using alternative editor frameworks like Monaco, CodeMirror, or custom solutions. When an extension registers an editor for a language, it replaces the default Ace editor in all locations that use that language (SQL Lab, Dashboard Properties, CSS editors, etc.). +Replace the default text editor: ```json "frontend": { @@ -116,8 +175,7 @@ Extensions can replace Superset's default text editors with custom implementatio { "id": "my_extension.monaco_sql", "name": "Monaco SQL Editor", - "languages": ["sql"], - "description": "Monaco-based SQL editor with IntelliSense" + "languages": ["sql"] } ] } @@ -126,30 +184,44 @@ Extensions can replace Superset's default text editors with custom implementatio See [Editors Extension Point](./extension-points/editors) for implementation details. -## Backend +## Configuration -Backend contribution types allow extensions to extend Superset's server-side capabilities with new API endpoints, MCP tools, and MCP prompts. - -### REST API Endpoints +### extension.json -Extensions can register custom REST API endpoints under the `/api/v1/extensions/` namespace. This dedicated namespace prevents conflicts with built-in endpoints and provides a clear separation between core and extension functionality. +Specify which files to scan for contributions: ```json -"backend": { - "entryPoints": ["my_extension.entrypoint"], - "files": ["backend/src/my_extension/**/*.py"] +{ + "id": "my_extension", + "name": "My Extension", + "version": "1.0.0", + "backend": { + "entryPoints": ["my_extension.entrypoint"], + "files": ["backend/src/**/*.py"] + }, + "frontend": { + "moduleFederation": { + "exposes": ["./index"] + } + } } ``` -The entry point module registers the API with Superset: +### Manual Contributions (Advanced) -```python -from superset_core.api.rest_api import add_extension_api -from .api import MyExtensionAPI +Override auto-discovery by specifying contributions directly: -add_extension_api(MyExtensionAPI) +```json +{ + "backend": { + "contributions": { + "mcpTools": [ + { "id": "query_db", "name": "query_db", "module": "my_ext.tools.query_db" } + ], + "restApis": [ + { "id": "my_api", "name": "My API", "module": "my_ext.api.MyAPI", "basePath": "/my_api" } + ] + } + } +} ``` - -### MCP Tools and Prompts - -Extensions can contribute Model Context Protocol (MCP) tools and prompts that AI agents can discover and use. See [MCP Integration](./mcp) for detailed documentation. diff --git a/superset-core/src/superset_core/api/rest_api.py b/superset-core/src/superset_core/api/rest_api.py index 05ead50a906..2726a845460 100644 --- a/superset-core/src/superset_core/api/rest_api.py +++ b/superset-core/src/superset_core/api/rest_api.py @@ -16,20 +16,69 @@ # under the License. """ -REST API functions for superset-core. +REST API decorators and base classes for superset-core. -Provides dependency-injected REST API utility functions that will be replaced by -host implementations during initialization. +Provides decorator stubs that will be replaced by host implementations +during initialization. Usage: - from superset_core.api.rest_api import add_api, add_extension_api - - add_api(MyCustomAPI) - add_extension_api(MyExtensionAPI) + from superset_core.api import RestApi, api, extension_api + + # For host application APIs + @api + class MyAPI(RestApi): + @expose("/endpoint", methods=["GET"]) + def get_data(self): + return self.response(200, result={}) + + # For extension APIs (auto-discovered, registered under /extensions/) + @extension_api(id="my_api", name="My Extension API") + class MyExtensionAPI(RestApi): + @expose("/endpoint", methods=["GET"]) + def get_data(self): + return self.response(200, result={}) """ +from __future__ import annotations + +from dataclasses import dataclass +from typing import Callable, TypeVar + from flask_appbuilder.api import BaseApi +T = TypeVar("T", bound=type) + + +# ============================================================================= +# Metadata dataclass - attached to decorated classes for discovery +# ============================================================================= + + +@dataclass +class RestApiMetadata: + """ + Metadata stored on classes decorated with @extension_api. + + Attached to classes as __rest_api_metadata__ for build-time discovery. + Includes auto-inferred Flask-AppBuilder configuration fields. + """ + + id: str + name: str + description: str | None = None + base_path: str = "" # Defaults to /{id} + module: str = "" # Format: "package.module.ClassName" + + # Auto-inferred Flask-AppBuilder fields + resource_name: str = "" # Used for URL generation and permissions + openapi_spec_tag: str = "" # Used for OpenAPI documentation grouping + class_permission_name: str = "" # Used for RBAC permissions + + +# ============================================================================= +# Base class +# ============================================================================= + class RestApi(BaseApi): """ @@ -42,31 +91,150 @@ class RestApi(BaseApi): allow_browser_login = True -def add_api(api: type[RestApi]) -> None: - """ - Add a REST API to the Superset API. +# ============================================================================= +# Decorator stubs - replaced by host application during initialization +# ============================================================================= - Host implementations will replace this function during initialization - with a concrete implementation providing actual functionality. - :param api: A REST API instance. - :returns: None. +def api(cls: T) -> T: """ - raise NotImplementedError("Function will be replaced during initialization") + Decorator to register a REST API with the host application. + This is a stub that raises NotImplementedError until the host application + initializes the concrete implementation via dependency injection. -def add_extension_api(api: type[RestApi]) -> None: - """ - Add an extension REST API to the Superset API. + Usage: + @api + class MyAPI(RestApi): + @expose("/endpoint", methods=["GET"]) + def get_data(self): + return self.response(200, result={}) - Host implementations will replace this function during initialization - with a concrete implementation providing actual functionality. + Args: + cls: The API class to register - :param api: An extension REST API instance. These are placed under - the /extensions resource. - :returns: None. - """ - raise NotImplementedError("Function will be replaced during initialization") + Returns: + The decorated class + Raises: + NotImplementedError: Before host implementation is initialized + """ + raise NotImplementedError( + "REST API decorator not initialized. " + "This decorator should be replaced during Superset startup." + ) + + +def extension_api( + id: str, # noqa: A002 + name: str, + description: str | None = None, + base_path: str | None = None, + resource_name: str | None = None, + openapi_spec_tag: str | None = None, + class_permission_name: str | None = None, +) -> Callable[[T], T]: + """ + Decorator to mark a class as an extension REST API. + + This is a stub that raises NotImplementedError until the host application + initializes the concrete implementation via dependency injection. + + In BUILD mode, stores metadata for discovery without registration. + + Auto-infers Flask-AppBuilder fields from decorator parameters: + - resource_name: defaults to id (lowercase) + - openapi_spec_tag: defaults to name + - class_permission_name: defaults to resource_name + - base_path: defaults to /{id} + + Extension APIs are: + - Auto-discovered at build time + - Registered under /api/v1/extensions/{id}/ + - Subject to manifest validation for security + + Usage: + @extension_api(id="my_api", name="My Extension API") + class MyExtensionAPI(RestApi): + # These are auto-set by the decorator: + # resource_name = "my_api" + # openapi_spec_tag = "My Extension API" + # class_permission_name = "my_api" + + @expose("/endpoint", methods=["GET"]) + def get_data(self): + return self.response(200, result={}) + + Args: + id: Unique identifier for this API (used in URL path and resource_name) + name: Human-readable name for the API (used for openapi_spec_tag) + description: Description of the API (defaults to class docstring) + base_path: Base URL path (defaults to /{id}) + resource_name: Override resource_name (defaults to id) + openapi_spec_tag: Override OpenAPI tag (defaults to name) + class_permission_name: Override permission name (defaults to resource_name) + + Returns: + Decorator that attaches __rest_api_metadata__ and auto-configures + Flask-AppBuilder fields + + Raises: + NotImplementedError: Before host implementation is initialized (except in + BUILD mode) + """ -__all__ = ["RestApi", "add_api", "add_extension_api"] + def decorator(cls: T) -> T: + # Auto-infer Flask-AppBuilder fields + inferred_resource_name = resource_name or id.lower() + inferred_openapi_spec_tag = openapi_spec_tag or name + inferred_class_permission_name = class_permission_name or inferred_resource_name + inferred_base_path = base_path or f"/{id}" + + # Set Flask-AppBuilder attributes on the class + cls.resource_name = inferred_resource_name # type: ignore[attr-defined] + cls.openapi_spec_tag = inferred_openapi_spec_tag # type: ignore[attr-defined] + cls.class_permission_name = inferred_class_permission_name # type: ignore[attr-defined] + + # Try to get context for BUILD mode detection + try: + from superset_core.extensions.context import get_context + + ctx = get_context() + + # In BUILD mode, store metadata for discovery + if ctx.is_build_mode: + api_description = description + if api_description is None and cls.__doc__: + api_description = cls.__doc__.strip().split("\n")[0] + + metadata = RestApiMetadata( + id=id, + name=name, + description=api_description, + base_path=inferred_base_path, + module=f"{cls.__module__}.{cls.__name__}", + resource_name=inferred_resource_name, + openapi_spec_tag=inferred_openapi_spec_tag, + class_permission_name=inferred_class_permission_name, + ) + cls.__rest_api_metadata__ = metadata # type: ignore[attr-defined] + return cls + except ImportError: + # Context not available - fall through to error + pass + + # Default behavior: raise error for host to replace + raise NotImplementedError( + "Extension REST API decorator not initialized. " + "This decorator should be replaced during Superset startup." + ) + + return decorator + + +__all__ = [ + "RestApi", + "RestApiMetadata", + "api", + "extension_api", +] diff --git a/superset-core/src/superset_core/extensions/__init__.py b/superset-core/src/superset_core/extensions/__init__.py index 13a83393a91..4e77bcb4a2b 100644 --- a/superset-core/src/superset_core/extensions/__init__.py +++ b/superset-core/src/superset_core/extensions/__init__.py @@ -14,3 +14,51 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. + +"""Extension framework types and context.""" + +from superset_core.extensions.context import ( + ContributionType, + get_context, + PendingContribution, + RegistrationContext, + RegistrationMode, +) +from superset_core.extensions.types import ( + BackendContributions, + ExtensionConfig, + ExtensionConfigBackend, + ExtensionConfigFrontend, + FrontendContributions, + Manifest, + ManifestBackend, + ManifestFrontend, + McpPromptContribution, + McpToolContribution, + ModuleFederationConfig, + RestApiContribution, +) + +__all__ = [ + # Context + "ContributionType", + "get_context", + "PendingContribution", + "RegistrationContext", + "RegistrationMode", + # Types - Config + "ExtensionConfig", + "ExtensionConfigBackend", + "ExtensionConfigFrontend", + # Types - Manifest + "Manifest", + "ManifestBackend", + "ManifestFrontend", + # Types - Contributions + "BackendContributions", + "FrontendContributions", + "McpToolContribution", + "McpPromptContribution", + "RestApiContribution", + "ModuleFederationConfig", +] diff --git a/superset-core/src/superset_core/extensions/context.py b/superset-core/src/superset_core/extensions/context.py new file mode 100644 index 00000000000..53663f8f261 --- /dev/null +++ b/superset-core/src/superset_core/extensions/context.py @@ -0,0 +1,197 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +Registration context for contribution discovery and security. + +Controls how decorators behave based on the current execution context: +- host: Register immediately (for host application components) +- extension: Store metadata only, defer to ExtensionManager (security boundary) +- build: Store metadata only, for CLI discovery + +The manifest.json serves as the security allowlist for extensions. +Only contributions declared in the manifest will be registered. +""" + +from __future__ import annotations + +from contextlib import contextmanager +from dataclasses import dataclass, field +from enum import Enum +from typing import Any, Callable, Iterator, Literal, TYPE_CHECKING + +if TYPE_CHECKING: + from superset_core.api.rest_api import RestApiMetadata + from superset_core.mcp import PromptMetadata, ToolMetadata + + +# Type alias for contribution types +ContributionType = Literal["tool", "prompt", "restApi"] + + +class RegistrationMode(Enum): + """Registration modes for decorator behavior.""" + + HOST = "host" # Register immediately (host application) + EXTENSION = "extension" # Defer registration (manifest validation) + BUILD = "build" # Metadata only (CLI discovery) + + +@dataclass +class PendingContribution: + """A contribution waiting for registration after manifest validation.""" + + func: Callable[..., Any] + metadata: ToolMetadata | PromptMetadata | RestApiMetadata + contrib_type: ContributionType + + +@dataclass +class RegistrationContext: + """ + Global context controlling decorator registration behavior. + + In host mode, decorators register immediately with MCP. + In extension mode, decorators store metadata and the ExtensionManager + validates against the manifest before completing registration. + In build mode, decorators only store metadata for discovery. + """ + + _mode: RegistrationMode = RegistrationMode.HOST + _current_extension_id: str | None = None + _pending_contributions: dict[str, list[PendingContribution]] = field( + default_factory=dict + ) + + def set_mode(self, mode: RegistrationMode) -> None: + """Set the global registration mode.""" + self._mode = mode + + @property + def mode(self) -> RegistrationMode: + """Get the current registration mode.""" + return self._mode + + @property + def is_host_mode(self) -> bool: + """True if in host mode (immediate registration).""" + return self._mode == RegistrationMode.HOST + + @property + def is_extension_mode(self) -> bool: + """True if in extension mode (deferred registration).""" + return self._mode == RegistrationMode.EXTENSION + + @property + def is_build_mode(self) -> bool: + """True if in build mode (metadata only).""" + return self._mode == RegistrationMode.BUILD + + @property + def current_extension_id(self) -> str | None: + """Get the current extension ID being loaded.""" + return self._current_extension_id + + @contextmanager + def extension_context(self, extension_id: str) -> Iterator[None]: + """ + Context manager for loading an extension. + + While in this context, decorators defer registration and store + contributions for manifest validation. + + Args: + extension_id: The extension being loaded + + Yields: + None + """ + old_mode = self._mode + old_ext = self._current_extension_id + + self._mode = RegistrationMode.EXTENSION + self._current_extension_id = extension_id + self._pending_contributions[extension_id] = [] + + try: + yield + finally: + self._mode = old_mode + self._current_extension_id = old_ext + + def add_pending_contribution( + self, + func: Callable[..., Any], + metadata: ToolMetadata | PromptMetadata | RestApiMetadata, + contrib_type: ContributionType, + ) -> None: + """ + Add a contribution pending manifest validation. + + Called by decorators in extension mode. + + Args: + func: The decorated function + metadata: The contribution metadata + contrib_type: Type of contribution ("tool", "prompt", "restApi") + """ + if self._current_extension_id is None: + raise RuntimeError( + "Cannot add pending contribution outside extension context" + ) + + self._pending_contributions[self._current_extension_id].append( + PendingContribution( + func=func, + metadata=metadata, + contrib_type=contrib_type, + ) + ) + + def get_pending_contributions(self, extension_id: str) -> list[PendingContribution]: + """ + Get pending contributions for an extension. + + Called by ExtensionManager during manifest validation. + + Args: + extension_id: The extension to get contributions for + + Returns: + List of pending contributions + """ + return self._pending_contributions.get(extension_id, []) + + def clear_pending_contributions(self, extension_id: str) -> None: + """ + Clear pending contributions after registration. + + Called by ExtensionManager after successful registration. + + Args: + extension_id: The extension to clear contributions for + """ + self._pending_contributions.pop(extension_id, None) + + +# Global singleton instance +_context = RegistrationContext() + + +def get_context() -> RegistrationContext: + """Get the global registration context.""" + return _context diff --git a/superset-core/src/superset_core/extensions/types.py b/superset-core/src/superset_core/extensions/types.py index 49f3eefe749..2615c65131f 100644 --- a/superset-core/src/superset_core/extensions/types.py +++ b/superset-core/src/superset_core/extensions/types.py @@ -87,6 +87,8 @@ class ContributionConfig(BaseModel): } } """ +class FrontendContributions(BaseModel): + """Frontend UI contributions.""" commands: list[dict[str, Any]] = Field( default_factory=list, @@ -104,6 +106,68 @@ class ContributionConfig(BaseModel): default_factory=list, description="Editor contributions", ) + editors: list[dict[str, Any]] = Field( + default_factory=list, + description="Editor contributions", + ) + + +class McpToolContribution(BaseModel): + """MCP tool contribution.""" + + id: str + name: str + description: str | None = None + module: str + tags: list[str] = Field(default_factory=list) + protect: bool = True + + +class McpPromptContribution(BaseModel): + """MCP prompt contribution.""" + + id: str + name: str + title: str | None = None + description: str | None = None + module: str + tags: list[str] = Field(default_factory=list) + protect: bool = True + + +class RestApiContribution(BaseModel): + """REST API contribution.""" + + id: str + name: str + description: str | None = None + module: str + basePath: str # noqa: N815 + + # Auto-inferred Flask-AppBuilder fields + resourceName: str = "" # noqa: N815 + openapiSpecTag: str = "" # noqa: N815 + classPermissionName: str = "" # noqa: N815 + + +class BackendContributions(BaseModel): + """Backend contributions.""" + + mcp_tools: list[McpToolContribution] = Field( + default_factory=list, + description="MCP tools", + alias="mcp.tools", + ) + mcp_prompts: list[McpPromptContribution] = Field( + default_factory=list, + description="MCP prompts", + alias="mcp.prompts", + ) + rest_apis: list[RestApiContribution] = Field( + default_factory=list, + description="REST APIs", + alias="rest.apis", + ) class BaseExtension(BaseModel): @@ -158,9 +222,10 @@ class BaseExtension(BaseModel): class ExtensionConfigFrontend(BaseModel): """Frontend section in extension.json.""" - contributions: ContributionConfig = Field( - default_factory=ContributionConfig, - description="UI contribution points", + # Optional: if provided, takes precedence over discovery + contributions: FrontendContributions | None = Field( + default=None, + description="Frontend contributions (optional, overrides discovery)", ) moduleFederation: ModuleFederationConfig = Field( # noqa: N815 default_factory=ModuleFederationConfig, @@ -179,13 +244,20 @@ class ExtensionConfigBackend(BaseModel): default_factory=list, description="Glob patterns for backend Python files", ) + # Optional: if provided, takes precedence over discovery + contributions: BackendContributions | None = Field( + default=None, + description="Backend contributions (optional, overrides discovery)", + ) class ExtensionConfig(BaseExtension): """ Schema for extension.json (source configuration). - This file is authored by developers to define extension metadata. + Authored by developers. Contributions can be: + - Auto-discovered from decorated code (default) + - Manually specified in contributions field (overrides discovery) """ frontend: ExtensionConfigFrontend | None = Field( @@ -206,9 +278,9 @@ class ExtensionConfig(BaseExtension): class ManifestFrontend(BaseModel): """Frontend section in manifest.json.""" - contributions: ContributionConfig = Field( - default_factory=ContributionConfig, - description="UI contribution points", + contributions: FrontendContributions = Field( + default_factory=FrontendContributions, + description="Frontend contributions", ) moduleFederation: ModuleFederationConfig = Field( # noqa: N815 default_factory=ModuleFederationConfig, @@ -227,13 +299,18 @@ class ManifestBackend(BaseModel): default_factory=list, description="Python module entry points to load", ) + contributions: BackendContributions = Field( + default_factory=BackendContributions, + description="Backend contributions", + ) class Manifest(BaseExtension): """ Schema for manifest.json (built output). - This file is generated by the build tool from extension.json. + Generated by the build tool. Contains all contributions + (discovered or manually specified) that will be registered at runtime. """ id: str = Field( diff --git a/superset-core/src/superset_core/mcp/__init__.py b/superset-core/src/superset_core/mcp/__init__.py index a5e241eb22a..e97aa5fed90 100644 --- a/superset-core/src/superset_core/mcp/__init__.py +++ b/superset-core/src/superset_core/mcp/__init__.py @@ -16,31 +16,78 @@ # under the License. """ -MCP (Model Context Protocol) tool registration for Superset MCP server. +MCP (Model Context Protocol) tool and prompt registration for Superset. -This module provides a decorator interface to register MCP tools with the -host application. +This module provides decorator stubs that are replaced by the host application +during initialization. Each decorator defines metadata dataclasses that are +used for build-time discovery. Usage: - from superset_core.mcp import tool + from superset_core.mcp import tool, prompt - @tool(name="my_tool", description="Custom business logic", tags=["extension"]) - def my_extension_tool(param: str) -> dict: - return {"message": f"Hello {param}!"} + @tool(tags=["database"]) + def query_database(sql: str) -> dict: + '''Execute a SQL query against a database.''' + return execute_query(sql) - # Or use function name and docstring: - @tool - def another_tool(value: int) -> str: - '''Tool description from docstring''' - return str(value * 2) + @prompt(tags={"analysis"}) + async def analyze_data(ctx, dataset: str) -> str: + '''Generate analysis for a dataset.''' + return f"Analyze {dataset}..." """ +from __future__ import annotations + +from dataclasses import dataclass, field from typing import Any, Callable, TypeVar # Type variable for decorated functions F = TypeVar("F", bound=Callable[..., Any]) +# ============================================================================= +# Metadata dataclasses - attached to decorated functions for discovery +# ============================================================================= + + +@dataclass +class ToolMetadata: + """ + Metadata stored on functions decorated with @tool. + + Attached to functions as __tool_metadata__ for build-time discovery. + """ + + id: str + name: str + description: str | None = None + tags: list[str] = field(default_factory=list) + protect: bool = True + module: str = "" # Format: "package.module.function_name" + + +@dataclass +class PromptMetadata: + """ + Metadata stored on functions decorated with @prompt. + + Attached to functions as __prompt_metadata__ for build-time discovery. + """ + + id: str + name: str + title: str | None = None + description: str | None = None + tags: set[str] = field(default_factory=set) + protect: bool = True + module: str = "" # Format: "package.module.function_name" + + +# ============================================================================= +# Decorator stubs - replaced by host application during initialization +# ============================================================================= + + def tool( func_or_name: str | Callable[..., Any] | None = None, *, @@ -48,53 +95,90 @@ def tool( description: str | None = None, tags: list[str] | None = None, protect: bool = True, -) -> Any: # Use Any to avoid mypy issues with dependency injection +) -> Any: """ Decorator to register an MCP tool with optional authentication. - This decorator combines FastMCP tool registration with optional authentication. + This is a stub that raises NotImplementedError until the host application + initializes the concrete implementation via dependency injection. + + In BUILD mode, stores metadata for discovery without registration. Can be used as: @tool def my_tool(): ... - Or: + @tool(tags=["database"]) + def query(): ... + @tool(name="custom_name", protect=False) def my_tool(): ... Args: func_or_name: When used as @tool, this will be the function. When used as @tool("name"), this will be the name. - name: Tool name (defaults to function name, prefixed with extension ID) - description: Tool description (defaults to function docstring) - tags: List of tags for categorizing the tool (defaults to empty list) + name: Tool name (defaults to function name) + description: Tool description (defaults to first line of docstring) + tags: List of tags for categorizing the tool protect: Whether to require Superset authentication (defaults to True) Returns: - Decorator function that registers and wraps the tool, or the wrapped function + Decorated function with __tool_metadata__ attribute Raises: - NotImplementedError: If called before host implementation is initialized - - Example: - @tool(name="my_tool", description="Does something useful", tags=["utility"]) - def my_custom_tool(param: str) -> dict: - return {"result": param} - - @tool # Uses function name and docstring with auth - def simple_tool(value: int) -> str: - '''Doubles the input value''' - return str(value * 2) - - @tool(protect=False) # No authentication required - def public_tool() -> str: - '''Public tool accessible without auth''' - return "Hello world" + NotImplementedError: Before host implementation is initialized (except in + BUILD mode) """ - raise NotImplementedError( - "MCP tool decorator not initialized. " - "This decorator should be replaced during Superset startup." - ) + + def decorator(func: F) -> F: + # Try to get context for BUILD mode detection + try: + from superset_core.extensions.context import get_context + + ctx = get_context() + + # In BUILD mode, store metadata for discovery + if ctx.is_build_mode: + tool_name = name or func.__name__ + tool_description = description + if tool_description is None and func.__doc__: + tool_description = func.__doc__.strip().split("\n")[0] + tool_tags = tags or [] + + metadata = ToolMetadata( + id=func.__name__, + name=tool_name, + description=tool_description, + tags=tool_tags, + protect=protect, + module=f"{func.__module__}.{func.__name__}", + ) + func.__tool_metadata__ = metadata # type: ignore[attr-defined] + return func + except ImportError: + # Context not available - fall through to error + pass + + # Default behavior: raise error for host to replace + raise NotImplementedError( + "MCP tool decorator not initialized. " + "This decorator should be replaced during Superset startup." + ) + + # Handle decorator usage patterns + if callable(func_or_name): + return decorator(func_or_name) + + # Return parameterized decorator + actual_name = func_or_name if isinstance(func_or_name, str) else name + + def parameterized_decorator(func: F) -> F: + nonlocal name + if actual_name is not None: + name = actual_name + return decorator(func) + + return parameterized_decorator def prompt( @@ -105,57 +189,98 @@ def prompt( description: str | None = None, tags: set[str] | None = None, protect: bool = True, -) -> Any: # Use Any to avoid mypy issues with dependency injection +) -> Any: """ Decorator to register an MCP prompt with optional authentication. - This decorator combines FastMCP prompt registration with optional authentication. + This is a stub that raises NotImplementedError until the host application + initializes the concrete implementation via dependency injection. + + In BUILD mode, stores metadata for discovery without registration. Can be used as: @prompt - async def my_prompt_handler(): ... + async def my_prompt(ctx): ... - Or: - @prompt("my_prompt") - async def my_prompt_handler(): ... + @prompt(tags={"analysis"}) + async def analyze(ctx): ... - Or: - @prompt("my_prompt", protected=False, title="Custom Title") - async def my_prompt_handler(): ... + @prompt("custom_name", title="Custom Title") + async def my_prompt(ctx): ... Args: func_or_name: When used as @prompt, this will be the function. When used as @prompt("name"), this will be the name. - name: Prompt name (defaults to function name if not provided) + name: Prompt name (defaults to function name) title: Prompt title (defaults to function name) - description: Prompt description (defaults to function docstring) + description: Prompt description (defaults to first line of docstring) tags: Set of tags for categorizing the prompt protect: Whether to require Superset authentication (defaults to True) Returns: - Decorator function that registers and wraps the prompt, or the wrapped function + Decorated function with __prompt_metadata__ attribute Raises: - NotImplementedError: If called before host implementation is initialized - - Example: - @prompt - async def my_prompt_handler(ctx: Context) -> str: - '''Interactive prompt for doing something.''' - return "Prompt instructions here..." - - @prompt("custom_prompt", protect=False, title="Custom Title") - async def public_prompt_handler(ctx: Context) -> str: - '''Public prompt accessible without auth''' - return "Public prompt accessible without auth" + NotImplementedError: Before host implementation is initialized (except in + BUILD mode) """ - raise NotImplementedError( - "MCP prompt decorator not initialized. " - "This decorator should be replaced during Superset startup." - ) + + def decorator(func: F) -> F: + # Try to get context for BUILD mode detection + try: + from superset_core.extensions.context import get_context + + ctx = get_context() + + # In BUILD mode, store metadata for discovery + if ctx.is_build_mode: + prompt_name = name or func.__name__ + prompt_title = title or func.__name__ + prompt_description = description + if prompt_description is None and func.__doc__: + prompt_description = func.__doc__.strip().split("\n")[0] + prompt_tags = tags or set() + + metadata = PromptMetadata( + id=func.__name__, + name=prompt_name, + title=prompt_title, + description=prompt_description, + tags=prompt_tags, + protect=protect, + module=f"{func.__module__}.{func.__name__}", + ) + func.__prompt_metadata__ = metadata # type: ignore[attr-defined] + return func + except ImportError: + # Context not available - fall through to error + pass + + # Default behavior: raise error for host to replace + raise NotImplementedError( + "MCP prompt decorator not initialized. " + "This decorator should be replaced during Superset startup." + ) + + # Handle decorator usage patterns + if callable(func_or_name): + return decorator(func_or_name) + + # Return parameterized decorator + actual_name = func_or_name if isinstance(func_or_name, str) else name + + def parameterized_decorator(func: F) -> F: + nonlocal name + if actual_name is not None: + name = actual_name + return decorator(func) + + return parameterized_decorator __all__ = [ "tool", "prompt", + "ToolMetadata", + "PromptMetadata", ] diff --git a/superset-extensions-cli/src/superset_extensions_cli/cli.py b/superset-extensions-cli/src/superset_extensions_cli/cli.py index b26a0eb87d3..27b238d5241 100644 --- a/superset-extensions-cli/src/superset_extensions_cli/cli.py +++ b/superset-extensions-cli/src/superset_extensions_cli/cli.py @@ -15,6 +15,8 @@ # specific language governing permissions and limitations # under the License. +import importlib.util +import inspect import json # noqa: TID251 import re import shutil @@ -28,11 +30,18 @@ from typing import Any, Callable import click import semver from jinja2 import Environment, FileSystemLoader -from superset_core.extensions.types import ( +from superset_core.extensions import ( + BackendContributions, ExtensionConfig, + FrontendContributions, Manifest, ManifestBackend, ManifestFrontend, + McpPromptContribution, + McpToolContribution, + RegistrationMode, + RestApiContribution, + get_context, ) from watchdog.events import FileSystemEventHandler from watchdog.observers import Observer @@ -55,6 +64,115 @@ REMOTE_ENTRY_REGEX = re.compile(r"^remoteEntry\..+\.js$") FRONTEND_DIST_REGEX = re.compile(r"/frontend/dist") +def discover_backend_contributions( + cwd: Path, files_patterns: list[str] +) -> BackendContributions: + """ + Discover backend contributions by importing modules and inspecting decorated objects. + + Sets context to BUILD mode so decorators only store metadata, no registration. + """ + contributions = BackendContributions() + + # Set build mode so decorators don't try to register + ctx = get_context() + ctx.set_mode(RegistrationMode.BUILD) + + try: + # Collect all Python files matching patterns + py_files: list[Path] = [] + for pattern in files_patterns: + py_files.extend(cwd.glob(pattern)) + + for py_file in py_files: + if not py_file.is_file() or py_file.suffix != ".py": + continue + + try: + # Import module dynamically + module = _import_module_from_path(py_file) + if module is None: + continue + + # Inspect all members for decorated objects + for name, obj in inspect.getmembers(module): + if name.startswith("_"): + continue + + # Check for @tool metadata + if hasattr(obj, "__tool_metadata__"): + meta = obj.__tool_metadata__ + contributions.mcp_tools.append( + McpToolContribution( + id=meta.id, + name=meta.name, + description=meta.description, + module=meta.module, + tags=list(meta.tags), + protect=meta.protect, + ) + ) + + # Check for @prompt metadata + if hasattr(obj, "__prompt_metadata__"): + meta = obj.__prompt_metadata__ + contributions.mcp_prompts.append( + McpPromptContribution( + id=meta.id, + name=meta.name, + title=meta.title, + description=meta.description, + module=meta.module, + tags=list(meta.tags), + protect=meta.protect, + ) + ) + + # Check for @extension_api metadata + if hasattr(obj, "__rest_api_metadata__"): + meta = obj.__rest_api_metadata__ + contributions.rest_apis.append( + RestApiContribution( + id=meta.id, + name=meta.name, + description=meta.description, + module=meta.module, + basePath=meta.base_path, + resourceName=meta.resource_name, + openapiSpecTag=meta.openapi_spec_tag, + classPermissionName=meta.class_permission_name, + ) + ) + + except Exception as e: + click.secho(f"⚠️ Failed to analyze {py_file}: {e}", fg="yellow") + + finally: + # Reset to host mode + ctx.set_mode(RegistrationMode.HOST) + + return contributions + + +def _import_module_from_path(py_file: Path) -> Any: + """Import a Python module from a file path.""" + module_name = py_file.stem + spec = importlib.util.spec_from_file_location(module_name, py_file) + if spec is None or spec.loader is None: + return None + + module = importlib.util.module_from_spec(spec) + sys.modules[module_name] = module + + try: + spec.loader.exec_module(module) + return module + except Exception: + # Clean up on failure + sys.modules.pop(module_name, None) + raise + + def validate_npm() -> None: """Abort if `npm` is not on PATH.""" if shutil.which("npm") is None: @@ -151,17 +269,48 @@ def build_manifest(cwd: Path, remote_entry: str | None) -> Manifest: # Generate composite ID from publisher and name composite_id = f"{extension.publisher}.{extension.name}" + # Build frontend manifest frontend: ManifestFrontend | None = None if extension.frontend and remote_entry: + # Use manually specified contributions if present, otherwise empty + # (frontend discovery will be added later) + frontend_contributions = ( + extension.frontend.contributions + if extension.frontend.contributions + else FrontendContributions() + ) frontend = ManifestFrontend( - contributions=extension.frontend.contributions, + contributions=frontend_contributions, moduleFederation=extension.frontend.moduleFederation, remoteEntry=remote_entry, ) + # Build backend manifest with contributions backend: ManifestBackend | None = None - if extension.backend and extension.backend.entryPoints: - backend = ManifestBackend(entryPoints=extension.backend.entryPoints) + if extension.backend: + # Use manually specified contributions if present, otherwise discover + if extension.backend.contributions: + backend_contributions = extension.backend.contributions + click.secho("📋 Using manually specified backend contributions", fg="cyan") + elif extension.backend.files: + click.secho("🔍 Discovering backend contributions...", fg="cyan") + backend_contributions = discover_backend_contributions( + cwd, extension.backend.files + ) + tool_count = len(backend_contributions.mcp_tools) + prompt_count = len(backend_contributions.mcp_prompts) + api_count = len(backend_contributions.rest_apis) + click.secho( + f" Found: {tool_count} tools, {prompt_count} prompts, {api_count} APIs", + fg="green", + ) + else: + backend_contributions = BackendContributions() + + backend = ManifestBackend( + entryPoints=extension.backend.entryPoints, + contributions=backend_contributions, + ) return Manifest( id=composite_id, diff --git a/superset-extensions-cli/tests/test_backend_discovery.py b/superset-extensions-cli/tests/test_backend_discovery.py new file mode 100644 index 00000000000..31f8b500b1d --- /dev/null +++ b/superset-extensions-cli/tests/test_backend_discovery.py @@ -0,0 +1,236 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +Tests for backend contributions discovery. +""" + +from __future__ import annotations + +import tempfile +from pathlib import Path + +import pytest +from superset_extensions_cli.cli import discover_backend_contributions + + [email protected] +def test_discover_backend_contributions_finds_tools(): + """Test that discover_backend_contributions finds @tool decorated functions.""" + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir_path = Path(tmpdir) + + # Create a Python file with decorated functions + backend_file = tmpdir_path / "backend.py" + backend_code = ''' +from superset_core.mcp import tool, prompt +from superset_core.api.rest_api import RestApi, extension_api + + +@tool(tags=["database"], description="Query the database") +def query_database(sql: str) -> dict: + """Execute a SQL query against the database.""" + return {"result": "success"} + + +@tool(name="custom_tool", protect=False) +def my_custom_tool(): + """A custom tool with specific configuration.""" + return {"custom": True} + + +@prompt(tags={"analysis"}, title="Data Analysis") +async def analyze_data(ctx, dataset: str) -> str: + """Generate analysis for a dataset.""" + return f"Analysis for {dataset}" + + +@extension_api(id="test_api", name="Test API") +class TestAPI(RestApi): + """Test API for the extension.""" + + def get_data(self): + return self.response(200, result={}) +''' + backend_file.write_text(backend_code) + + # Run discovery + contributions = discover_backend_contributions(tmpdir_path, ["*.py"]) + + # Verify tools were discovered + assert len(contributions.mcp_tools) == 2 + + # Check first tool + tool1 = contributions.mcp_tools[0] + assert tool1.id == "query_database" + assert tool1.name == "query_database" + assert tool1.description == "Execute a SQL query against the database." + assert tool1.tags == ["database"] + assert tool1.protect is True + assert tool1.module == "backend.query_database" + + # Check second tool + tool2 = contributions.mcp_tools[1] + assert tool2.id == "my_custom_tool" + assert tool2.name == "custom_tool" + assert tool2.description == "A custom tool with specific configuration." + assert tool2.tags == [] + assert tool2.protect is False + assert tool2.module == "backend.my_custom_tool" + + # Verify prompt was discovered + assert len(contributions.mcp_prompts) == 1 + + prompt1 = contributions.mcp_prompts[0] + assert prompt1.id == "analyze_data" + assert prompt1.name == "analyze_data" + assert prompt1.title == "Data Analysis" + assert prompt1.description == "Generate analysis for a dataset." + assert prompt1.tags == {"analysis"} + assert prompt1.protect is True + assert prompt1.module == "backend.analyze_data" + + # Verify REST API was discovered + assert len(contributions.rest_apis) == 1 + + api1 = contributions.rest_apis[0] + assert api1.id == "test_api" + assert api1.name == "Test API" + assert api1.description == "Test API for the extension." + assert api1.basePath == "/test_api" + assert api1.module == "backend.TestAPI" + + # Verify auto-inferred Flask-AppBuilder fields + assert api1.resourceName == "test_api" # defaults to id.lower() + assert api1.openapiSpecTag == "Test API" # defaults to name + assert api1.classPermissionName == "test_api" # defaults to resource_name + + [email protected] +def test_discover_backend_contributions_handles_empty_directory(): + """Test that discovery handles directories with no Python files.""" + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir_path = Path(tmpdir) + + # Create a non-Python file + (tmpdir_path / "readme.txt").write_text("This is not Python") + + contributions = discover_backend_contributions(tmpdir_path, ["*.py"]) + + assert len(contributions.mcp_tools) == 0 + assert len(contributions.mcp_prompts) == 0 + assert len(contributions.rest_apis) == 0 + + [email protected] +def test_discover_backend_contributions_handles_syntax_errors(): + """Test that discovery handles Python files with syntax errors gracefully.""" + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir_path = Path(tmpdir) + + # Create a Python file with syntax error + bad_file = tmpdir_path / "bad.py" + bad_file.write_text("def broken_function(\n # missing closing parenthesis") + + # Create a good file with contributions + good_file = tmpdir_path / "good.py" + good_code = ''' +from superset_core.mcp import tool + +@tool +def working_tool(): + """This tool should be discovered.""" + return {} +''' + good_file.write_text(good_code) + + contributions = discover_backend_contributions(tmpdir_path, ["*.py"]) + + # Should discover the working tool despite the syntax error in other file + assert len(contributions.mcp_tools) == 1 + assert contributions.mcp_tools[0].id == "working_tool" + + [email protected] +def test_discover_backend_contributions_skips_private_functions(): + """Test that discovery skips functions starting with underscore.""" + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir_path = Path(tmpdir) + + backend_file = tmpdir_path / "backend.py" + backend_code = ''' +from superset_core.mcp import tool + +@tool +def public_tool(): + """This should be discovered.""" + return {} + +@tool +def _private_tool(): + """This should be skipped.""" + return {} +''' + backend_file.write_text(backend_code) + + contributions = discover_backend_contributions(tmpdir_path, ["*.py"]) + + # Should only find the public tool + assert len(contributions.mcp_tools) == 1 + assert contributions.mcp_tools[0].id == "public_tool" + + [email protected] +def test_discover_backend_contributions_uses_file_patterns(): + """Test that discovery respects file patterns.""" + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir_path = Path(tmpdir) + + # Create files in different subdirectories + src_dir = tmpdir_path / "src" + src_dir.mkdir() + tests_dir = tmpdir_path / "tests" + tests_dir.mkdir() + + src_file = src_dir / "tools.py" + src_code = ''' +from superset_core.mcp import tool + +@tool +def src_tool(): + """Tool from src directory.""" + return {} +''' + src_file.write_text(src_code) + + test_file = tests_dir / "test_tools.py" + test_code = ''' +from superset_core.mcp import tool + +@tool +def test_tool(): + """Tool from tests directory.""" + return {} +''' + test_file.write_text(test_code) + + # Only search in src directory + contributions = discover_backend_contributions(tmpdir_path, ["src/**/*.py"]) + + # Should only find the src tool + assert len(contributions.mcp_tools) == 1 + assert contributions.mcp_tools[0].id == "src_tool" diff --git a/superset/core/api/core_api_injection.py b/superset/core/api/core_api_injection.py index be4ea69db4c..944f88ccdee 100644 --- a/superset/core/api/core_api_injection.py +++ b/superset/core/api/core_api_injection.py @@ -137,24 +137,112 @@ def inject_task_implementations() -> None: def inject_rest_api_implementations() -> None: """ - Replace abstract REST API functions in superset_core.api.rest_api with concrete + Replace abstract REST API decorators in superset_core.api.rest_api with concrete implementations from Superset. + + The decorators: + 1. Store metadata on classes for build-time discovery + 2. In host mode: Register immediately with Flask-AppBuilder + 3. In extension mode: Defer registration (ExtensionManager validates manifest) + 4. In build mode: Store metadata only """ + import logging + from typing import Callable, TypeVar + import superset_core.api.rest_api as core_rest_api_module + from superset_core.api.rest_api import RestApiMetadata + from superset_core.extensions.context import get_context from superset.extensions import appbuilder - def add_api(api: "type[RestApi]") -> None: - view = appbuilder.add_api(api) - appbuilder._add_permission(view, True) - - def add_extension_api(api: "type[RestApi]") -> None: - api.route_base = "/extensions/" + (api.resource_name or "") - view = appbuilder.add_api(api) + logger = logging.getLogger(__name__) + T = TypeVar("T", bound=type) + + def _register_api_with_appbuilder( + api_cls: type["RestApi"], + route_base: str | None = None, + ) -> None: + """Register an API class with Flask-AppBuilder.""" + if route_base: + api_cls.route_base = route_base + view = appbuilder.add_api(api_cls) appbuilder._add_permission(view, True) - - core_rest_api_module.add_api = add_api - core_rest_api_module.add_extension_api = add_extension_api + logger.info("Registered REST API: %s", api_cls.__name__) + + def create_api_decorator(cls: T) -> T: + """ + Decorator to register a REST API with the host application. + + In host mode: Registers immediately with Flask-AppBuilder. + In extension mode: Defers registration (should not be used for extensions). + In build mode: No-op (host APIs are not discovered). + """ + ctx = get_context() + + # Build mode: no-op for host APIs + if ctx.is_build_mode: + return cls + + # Host mode: register immediately + if ctx.is_host_mode: + _register_api_with_appbuilder(cls) + return cls + + # Extension mode: host @api decorator should not be used + logger.warning( + "Host @api decorator used in extension context. " + "Use @extension_api instead for extensions." + ) + return cls + + def create_extension_api_decorator( + id: str, # noqa: A002 + name: str, + description: str | None = None, + base_path: str | None = None, + ) -> Callable[[T], T]: + """ + Decorator to mark a class as an extension REST API. + + This decorator: + 1. Stores RestApiMetadata on the class for build-time discovery + 2. In host mode: Registers immediately under /extensions/{id}/ + 3. In extension mode: Defers registration (ExtensionManager validates manifest) + 4. In build mode: Stores metadata only + """ + + def decorator(cls: T) -> T: + # Build metadata + metadata = RestApiMetadata( + id=id, + name=name, + description=description or cls.__doc__, + base_path=base_path or f"/{id}", + module=f"{cls.__module__}.{cls.__name__}", + ) + cls.__rest_api_metadata__ = metadata # type: ignore[attr-defined] + + ctx = get_context() + + # Build mode: metadata only, no registration + if ctx.is_build_mode: + return cls + + # Extension mode: defer registration to ExtensionManager + if ctx.is_extension_mode: + ctx.add_pending_contribution(cls, metadata, "restApi") + return cls + + # Host mode: register immediately (for testing/development) + route_base = f"/extensions{metadata.base_path}" + _register_api_with_appbuilder(cls, route_base) + return cls + + return decorator + + # Replace the abstract decorators with concrete implementations + core_rest_api_module.api = create_api_decorator + core_rest_api_module.extension_api = create_extension_api_decorator def inject_model_session_implementation() -> None: diff --git a/superset/core/mcp/core_mcp_injection.py b/superset/core/mcp/core_mcp_injection.py index 037e9d29e86..a1fff64355c 100644 --- a/superset/core/mcp/core_mcp_injection.py +++ b/superset/core/mcp/core_mcp_injection.py @@ -20,6 +20,12 @@ MCP dependency injection implementation. This module provides the concrete implementation of MCP abstractions that replaces the abstract functions in superset-core during initialization. + +The decorators: +1. Store metadata on functions for build-time discovery +2. In host mode: Register immediately with FastMCP +3. In extension mode: Defer registration (manifest validation by ExtensionManager) +4. In build mode: Metadata only (CLI discovery) """ import logging @@ -31,6 +37,98 @@ F = TypeVar("F", bound=Callable[..., Any]) logger = logging.getLogger(__name__) +def _register_tool_with_mcp( + func: Callable[..., Any], + tool_name: str, + tool_description: str | None, + tool_tags: list[str], + protect: bool, +) -> Callable[..., Any]: + """ + Register a tool with FastMCP. + + Args: + func: The function to register + tool_name: Name for the tool + tool_description: Description for the tool + tool_tags: Tags for categorization + protect: Whether to wrap with authentication + + Returns: + The wrapped function (with auth if protect=True) + """ + from superset.mcp_service.app import mcp + + # Conditionally apply authentication wrapper + if protect: + from superset.mcp_service.auth import mcp_auth_hook + + wrapped_func = mcp_auth_hook(func) + else: + wrapped_func = func + + from fastmcp.tools import Tool + + tool = Tool.from_function( + wrapped_func, + name=tool_name, + description=tool_description or f"Tool: {tool_name}", + tags=tool_tags, + ) + mcp.add_tool(tool) + + protected_status = "protected" if protect else "public" + logger.info("Registered MCP tool: %s (%s)", tool_name, protected_status) + + return wrapped_func + + +def _register_prompt_with_mcp( + func: Callable[..., Any], + prompt_name: str, + prompt_title: str, + prompt_description: str | None, + prompt_tags: set[str], + protect: bool, +) -> Callable[..., Any]: + """ + Register a prompt with FastMCP. + + Args: + func: The function to register + prompt_name: Name for the prompt + prompt_title: Title for the prompt + prompt_description: Description for the prompt + prompt_tags: Tags for categorization + protect: Whether to wrap with authentication + + Returns: + The wrapped function (with auth if protect=True) + """ + from superset.mcp_service.app import mcp + + # Conditionally apply authentication wrapper + if protect: + from superset.mcp_service.auth import mcp_auth_hook + + wrapped_func = mcp_auth_hook(func) + else: + wrapped_func = func + + # Register prompt with FastMCP + mcp.prompt( + name=prompt_name, + title=prompt_title, + description=prompt_description or f"Prompt: {prompt_name}", + tags=prompt_tags, + )(wrapped_func) + + protected_status = "protected" if protect else "public" + logger.info("Registered MCP prompt: %s (%s)", prompt_name, protected_status) + + return wrapped_func + + def create_tool_decorator( func_or_name: str | Callable[..., Any] | None = None, *, @@ -42,8 +140,11 @@ def create_tool_decorator( """ Create the concrete MCP tool decorator implementation. - This combines FastMCP tool registration with optional Superset authentication, - replacing the need for separate @mcp.tool and @mcp_auth_hook decorators. + This decorator: + 1. Stores ToolMetadata on the function for build-time discovery + 2. In host mode: Registers immediately with FastMCP + 3. In extension mode: Defers registration (ExtensionManager validates manifest) + 4. In build mode: Stores metadata only Supports both @tool and @tool() syntax. @@ -56,58 +157,60 @@ def create_tool_decorator( protect: Whether to apply Superset authentication (defaults to True) Returns: - Decorator that registers and wraps the tool with optional authentication, - or the wrapped function when used without parentheses + Decorated function with __tool_metadata__ attribute """ def decorator(func: F) -> F: - try: - # Import here to avoid circular imports - from superset.mcp_service.app import mcp - - # Use provided values or extract from function - tool_name = name or func.__name__ - tool_description = description or func.__doc__ or f"Tool: {tool_name}" - tool_tags = tags or [] - - # Conditionally apply authentication wrapper - if protect: - from superset.mcp_service.auth import mcp_auth_hook - - wrapped_func = mcp_auth_hook(func) - else: - wrapped_func = func + from superset_core.extensions.context import get_context + from superset_core.mcp import ToolMetadata + + # Use provided values or extract from function + tool_name = name or func.__name__ + tool_description = description + if tool_description is None and func.__doc__: + tool_description = func.__doc__.strip().split("\n")[0] + tool_tags = tags or [] + + # Store metadata on function for discovery + metadata = ToolMetadata( + id=func.__name__, + name=tool_name, + description=tool_description, + tags=tool_tags, + protect=protect, + module=f"{func.__module__}.{func.__name__}", + ) + func.__tool_metadata__ = metadata # type: ignore[attr-defined] + + ctx = get_context() + + # Build mode: metadata only, no registration + if ctx.is_build_mode: + return func - from fastmcp.tools import Tool + # Extension mode: defer registration to ExtensionManager + if ctx.is_extension_mode: + ctx.add_pending_contribution(func, metadata, "tool") + return func - tool = Tool.from_function( - wrapped_func, - name=tool_name, - description=tool_description, - tags=tool_tags, + # Host mode: register immediately + try: + wrapped = _register_tool_with_mcp( + func, tool_name, tool_description, tool_tags, protect ) - mcp.add_tool(tool) - - protected_status = "protected" if protect else "public" - logger.info("Registered MCP tool: %s (%s)", tool_name, protected_status) - return wrapped_func - + wrapped.__tool_metadata__ = metadata # type: ignore[attr-defined] + return wrapped # type: ignore[return-value] except Exception as e: - logger.error("Failed to register MCP tool %s: %s", name or func.__name__, e) - # Return the original function so extension doesn't break + logger.error("Failed to register MCP tool %s: %s", tool_name, e) return func - # If called as @tool (without parentheses) + # Handle decorator usage patterns if callable(func_or_name): - # Type cast is safe here since we've confirmed it's callable return decorator(func_or_name) # type: ignore[arg-type] - # If called as @tool() or @tool(name="...") - # func_or_name would be the name parameter or None actual_name = func_or_name if isinstance(func_or_name, str) else name def parameterized_decorator(func: F) -> F: - # Use the actual_name if provided via func_or_name nonlocal name if actual_name is not None: name = actual_name @@ -128,8 +231,11 @@ def create_prompt_decorator( """ Create the concrete MCP prompt decorator implementation. - This combines FastMCP prompt registration with optional Superset authentication, - replacing the need for separate @mcp.prompt and @mcp_auth_hook decorators. + This decorator: + 1. Stores PromptMetadata on the function for build-time discovery + 2. In host mode: Registers immediately with FastMCP + 3. In extension mode: Defers registration (ExtensionManager validates manifest) + 4. In build mode: Stores metadata only Supports both @prompt and @prompt(...) syntax. @@ -143,59 +249,67 @@ def create_prompt_decorator( protect: Whether to apply Superset authentication (defaults to True) Returns: - Decorator that registers and wraps the prompt with optional authentication, - or the wrapped function when used without parentheses + Decorated function with __prompt_metadata__ attribute """ def decorator(func: F) -> F: - try: - # Import here to avoid circular imports - from superset.mcp_service.app import mcp - - # Use provided values or extract from function - prompt_name = name or func.__name__ - prompt_title = title or func.__name__ - prompt_description = description or func.__doc__ or f"Prompt: {prompt_name}" - prompt_tags = tags or set() - - # Conditionally apply authentication wrapper - if protect: - from superset.mcp_service.auth import mcp_auth_hook - - wrapped_func = mcp_auth_hook(func) - else: - wrapped_func = func - - # Register prompt with FastMCP using the same pattern as existing code - mcp.prompt( - name=prompt_name, - title=prompt_title, - description=prompt_description, - tags=prompt_tags, - )(wrapped_func) - - protected_status = "protected" if protect else "public" - logger.info("Registered MCP prompt: %s (%s)", prompt_name, protected_status) - return wrapped_func + from superset_core.extensions.context import get_context + from superset_core.mcp import PromptMetadata + + # Use provided values or extract from function + prompt_name = name or func.__name__ + prompt_title = title or func.__name__ + prompt_description = description + if prompt_description is None and func.__doc__: + prompt_description = func.__doc__.strip().split("\n")[0] + prompt_tags = tags or set() + + # Store metadata on function for discovery + metadata = PromptMetadata( + id=func.__name__, + name=prompt_name, + title=prompt_title, + description=prompt_description, + tags=prompt_tags, + protect=protect, + module=f"{func.__module__}.{func.__name__}", + ) + func.__prompt_metadata__ = metadata # type: ignore[attr-defined] + + ctx = get_context() + + # Build mode: metadata only, no registration + if ctx.is_build_mode: + return func - except Exception as e: - logger.error( - "Failed to register MCP prompt %s: %s", name or func.__name__, e + # Extension mode: defer registration to ExtensionManager + if ctx.is_extension_mode: + ctx.add_pending_contribution(func, metadata, "prompt") + return func + + # Host mode: register immediately + try: + wrapped = _register_prompt_with_mcp( + func, + prompt_name, + prompt_title, + prompt_description, + prompt_tags, + protect, ) - # Return the original function so extension doesn't break + wrapped.__prompt_metadata__ = metadata # type: ignore[attr-defined] + return wrapped # type: ignore[return-value] + except Exception as e: + logger.error("Failed to register MCP prompt %s: %s", prompt_name, e) return func - # If called as @prompt (without parentheses) + # Handle decorator usage patterns if callable(func_or_name): - # Type cast is safe here since we've confirmed it's callable return decorator(func_or_name) # type: ignore[arg-type] - # If called as @prompt() or @prompt(name="...") - # func_or_name would be the name parameter or None actual_name = func_or_name if isinstance(func_or_name, str) else name def parameterized_decorator(func: F) -> F: - # Use the actual_name if provided via name_or_fn nonlocal name if actual_name is not None: name = actual_name @@ -204,6 +318,69 @@ def create_prompt_decorator( return parameterized_decorator +def register_tool_from_manifest( + func: Callable[..., Any], + metadata: Any, # ToolMetadata + extension_id: str, +) -> Callable[..., Any]: + """ + Register a tool from an extension after manifest validation. + + Called by ExtensionManager after verifying the contribution + is declared in the extension's manifest. + + Args: + func: The decorated function + metadata: ToolMetadata from the function + extension_id: The extension ID (used for namespacing) + + Returns: + The registered wrapped function + """ + # Namespace the tool name with extension ID + prefixed_name = f"{extension_id}.{metadata.name}" + + return _register_tool_with_mcp( + func, + prefixed_name, + metadata.description, + metadata.tags, + metadata.protect, + ) + + +def register_prompt_from_manifest( + func: Callable[..., Any], + metadata: Any, # PromptMetadata + extension_id: str, +) -> Callable[..., Any]: + """ + Register a prompt from an extension after manifest validation. + + Called by ExtensionManager after verifying the contribution + is declared in the extension's manifest. + + Args: + func: The decorated function + metadata: PromptMetadata from the function + extension_id: The extension ID (used for namespacing) + + Returns: + The registered wrapped function + """ + # Namespace the prompt name with extension ID + prefixed_name = f"{extension_id}.{metadata.name}" + + return _register_prompt_with_mcp( + func, + prefixed_name, + metadata.title or metadata.name, + metadata.description, + metadata.tags, + metadata.protect, + ) + + def initialize_core_mcp_dependencies() -> None: """ Initialize MCP dependency injection by replacing abstract functions diff --git a/superset/extensions/manager.py b/superset/extensions/manager.py new file mode 100644 index 00000000000..56e100cec80 --- /dev/null +++ b/superset/extensions/manager.py @@ -0,0 +1,333 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +Extension manager for loading and validating extensions. + +The ExtensionManager coordinates extension loading: +1. Discovers extensions from configured paths +2. Validates contributions against manifests +3. Registers only declared contributions + +Security model: The manifest.json serves as an allowlist. +Only contributions declared in the manifest are registered. +""" + +from __future__ import annotations + +import logging +from typing import Any, Callable, TYPE_CHECKING, TypeVar + +from superset_core.extensions.context import get_context, PendingContribution +from superset_core.extensions.types import ( + Manifest, +) + +from superset.extensions.types import LoadedExtension +from superset.extensions.utils import ( + eager_import, + install_in_memory_importer, +) + +if TYPE_CHECKING: + from flask import Flask + +logger = logging.getLogger(__name__) + +T = TypeVar("T") + + +class ContributionValidationError(Exception): + """Raised when a contribution is not declared in the manifest.""" + + pass + + +class ExtensionManager: + """ + Manages extension lifecycle and contribution registration. + + Extensions are loaded and their contributions are validated against + the manifest before registration. This ensures only declared + functionality is exposed. + """ + + def __init__(self) -> None: + self._extensions: dict[str, LoadedExtension] = {} + self._contribution_registry: dict[str, dict[str, Any]] = { + "mcpTools": {}, + "mcpPrompts": {}, + "restApis": {}, + } + + def init_app(self, app: Flask) -> None: + """ + Initialize extension manager with Flask app. + + Loads extensions from configuration and registers contributions. + + Args: + app: Flask application instance + """ + from superset.extensions.utils import get_extensions + + with app.app_context(): + extensions = get_extensions() + for extension_id, extension in extensions.items(): + try: + self._load_extension(extension, app) + except Exception as e: + logger.error( + "Failed to load extension %s: %s", + extension_id, + e, + exc_info=True, + ) + + def _load_extension(self, extension: LoadedExtension, app: Flask) -> None: + """ + Load a single extension and register its contributions. + + Args: + extension: The loaded extension + app: Flask application instance + """ + extension_id = extension.id + manifest = extension.manifest + + logger.info("Loading extension: %s (%s)", extension.name, extension_id) + + # Store extension reference + self._extensions[extension_id] = extension + + # Install in-memory importer for backend code + if extension.backend: + install_in_memory_importer(extension.backend, extension.source_base_path) + + # Get registration context + ctx = get_context() + + # Load entry points within extension context + with ctx.extension_context(extension_id): + if manifest.backend and manifest.backend.entryPoints: + for entry_point in manifest.backend.entryPoints: + logger.debug( + "Loading entry point: %s for extension %s", + entry_point, + extension_id, + ) + try: + eager_import(entry_point) + except Exception as e: + logger.error( + "Failed to load entry point %s: %s", + entry_point, + e, + exc_info=True, + ) + raise + + # Validate and register pending contributions + self._validate_and_register_contributions(extension_id, manifest) + + def _validate_and_register_contributions( + self, extension_id: str, manifest: Manifest + ) -> None: + """ + Validate pending contributions against manifest and register. + + Args: + extension_id: Extension being validated + manifest: Extension manifest + + Raises: + ContributionValidationError: If undeclared contribution found + """ + ctx = get_context() + pending = ctx.get_pending_contributions(extension_id) + backend_contribs = manifest.backend.contributions if manifest.backend else None + + if not pending: + return + + # Build allowlists from manifest + allowed_tools: set[str] = set() + allowed_prompts: set[str] = set() + allowed_apis: set[str] = set() + + if backend_contribs: + allowed_tools = {t.id for t in backend_contribs.mcpTools} + allowed_prompts = {p.id for p in backend_contribs.mcpPrompts} + allowed_apis = {a.id for a in backend_contribs.restApis} + + for contrib in pending: + self._validate_single_contribution( + extension_id, + contrib, + allowed_tools, + allowed_prompts, + allowed_apis, + ) + self._register_contribution(extension_id, contrib) + + # Clear pending after successful registration + ctx.clear_pending_contributions(extension_id) + + logger.info( + "Registered %d contributions for extension %s", + len(pending), + extension_id, + ) + + def _validate_single_contribution( + self, + extension_id: str, + contrib: PendingContribution, + allowed_tools: set[str], + allowed_prompts: set[str], + allowed_apis: set[str], + ) -> None: + """ + Validate a single contribution against the allowlist. + + Args: + extension_id: Extension owning the contribution + contrib: The pending contribution + allowed_tools: Set of allowed tool IDs + allowed_prompts: Set of allowed prompt IDs + allowed_apis: Set of allowed API IDs + + Raises: + ContributionValidationError: If not in allowlist + """ + contrib_id = contrib.metadata.name + contrib_type = contrib.contrib_type + + if contrib_type == "tool": + if contrib_id not in allowed_tools: + raise ContributionValidationError( + f"Extension '{extension_id}' attempted to register undeclared " + f"MCP tool '{contrib_id}'. Add it to manifest.json to allow." + ) + elif contrib_type == "prompt": + if contrib_id not in allowed_prompts: + raise ContributionValidationError( + f"Extension '{extension_id}' attempted to register undeclared " + f"MCP prompt '{contrib_id}'. Add it to manifest.json to allow." + ) + elif contrib_type == "restApi": + if contrib_id not in allowed_apis: + raise ContributionValidationError( + f"Extension '{extension_id}' attempted to register undeclared " + f"REST API '{contrib_id}'. Add it to manifest.json to allow." + ) + + def _register_contribution( + self, extension_id: str, contrib: PendingContribution + ) -> None: + """ + Register a validated contribution. + + Args: + extension_id: Extension owning the contribution + contrib: The contribution to register + """ + from superset.core.mcp.core_mcp_injection import ( + register_prompt_from_manifest, + register_tool_from_manifest, + ) + + contrib_type = contrib.contrib_type + contrib_id = contrib.metadata.name + + if contrib_type == "tool": + register_tool_from_manifest(contrib.func, contrib.metadata, extension_id) + self._contribution_registry["mcpTools"][contrib_id] = { + "extension": extension_id, + "func": contrib.func, + "metadata": contrib.metadata, + } + elif contrib_type == "prompt": + register_prompt_from_manifest(contrib.func, contrib.metadata, extension_id) + self._contribution_registry["mcpPrompts"][contrib_id] = { + "extension": extension_id, + "func": contrib.func, + "metadata": contrib.metadata, + } + elif contrib_type == "restApi": + # REST APIs are registered through Flask-AppBuilder + # during the extension loading process + self._contribution_registry["restApis"][contrib_id] = { + "extension": extension_id, + "cls": contrib.func, + "metadata": contrib.metadata, + } + + def get_extension(self, extension_id: str) -> LoadedExtension | None: + """ + Get a loaded extension by ID. + + Args: + extension_id: Extension identifier + + Returns: + LoadedExtension or None if not found + """ + return self._extensions.get(extension_id) + + def get_contribution( + self, contrib_type: str, contrib_id: str + ) -> Callable[..., Any] | type | None: + """ + Get a registered contribution by type and ID. + + Args: + contrib_type: Contribution type (mcpTools, mcpPrompts, restApis) + contrib_id: Contribution identifier + + Returns: + The contribution function/class or None if not found + """ + registry = self._contribution_registry.get(contrib_type, {}) + if entry := registry.get(contrib_id): + return entry.get("func") or entry.get("cls") + return None + + def list_contributions(self, contrib_type: str) -> list[str]: + """ + List all registered contribution IDs of a type. + + Args: + contrib_type: Contribution type + + Returns: + List of contribution IDs + """ + return list(self._contribution_registry.get(contrib_type, {}).keys()) + + def list_extensions(self) -> list[str]: + """ + List all loaded extension IDs. + + Returns: + List of extension IDs + """ + return list(self._extensions.keys()) + + +# Global extension manager instance +extension_manager = ExtensionManager() diff --git a/superset/initialization/__init__.py b/superset/initialization/__init__.py index c3dacc7d6ea..583292f8200 100644 --- a/superset/initialization/__init__.py +++ b/superset/initialization/__init__.py @@ -564,36 +564,24 @@ class SupersetAppInitializer: # pylint: disable=too-many-public-methods self.init_extensions() def init_extensions(self) -> None: - from superset.extensions.utils import ( - eager_import, - get_extensions, - install_in_memory_importer, - ) + """ + Initialize extensions using the ExtensionManager. + + The ExtensionManager: + 1. Discovers extensions from configured paths + 2. Loads extension backend code + 3. Validates contributions against manifests + 4. Registers only declared contributions + """ + from superset.extensions.manager import extension_manager try: - extensions = get_extensions() - except Exception: # pylint: disable=broad-except # noqa: S110 + extension_manager.init_app(self.superset_app) + except Exception as ex: # pylint: disable=broad-except # If the db hasn't been initialized yet, an exception will be raised. # It's fine to ignore this, as in this case there are no extensions # present yet. - return - - for extension in extensions.values(): - if backend_files := extension.backend: - install_in_memory_importer( - backend_files, - source_base_path=extension.source_base_path, - ) - - backend = extension.manifest.backend - - if backend and (entrypoints := backend.entryPoints): - for entrypoint in entrypoints: - try: - eager_import(entrypoint) - except Exception as ex: # pylint: disable=broad-except # noqa: S110 - # Surface exceptions during initialization of extensions - print(ex) + logger.warning("Failed to initialize extensions: %s", ex) def init_app_in_ctx(self) -> None: """
