This is an automated email from the ASF dual-hosted git repository. yasith pushed a commit to branch feat/sdk-facade-migration in repository https://gitbox.apache.org/repos/asf/airavata-portals.git
commit c480a7c135e9c4e8a4eec9f728b792b241d25a1c Author: yasithdev <[email protected]> AuthorDate: Wed Apr 8 01:05:24 2026 -0500 fix: replace portal-sdk user_storage/experiment_util imports with SDK facade calls Add django_airavata/apps/api/user_storage.py as a thin compatibility layer that delegates to request.airavata_client.storage. Replace all experiment_util calls with direct research facade calls. --- .../django_airavata/apps/api/output_views.py | 2 +- .../django_airavata/apps/api/serializers.py | 20 +- .../django_airavata/apps/api/signals.py | 2 +- .../django_airavata/apps/api/user_storage.py | 284 +++++++++++++++++++++ .../django_airavata/apps/api/view_utils.py | 2 +- .../django_airavata/apps/api/views.py | 10 +- .../django_airavata/apps/workspace/views.py | 2 +- 7 files changed, 302 insertions(+), 20 deletions(-) diff --git a/airavata-django-portal/django_airavata/apps/api/output_views.py b/airavata-django-portal/django_airavata/apps/api/output_views.py index 341a1f5cb..5a02bfa31 100644 --- a/airavata-django-portal/django_airavata/apps/api/output_views.py +++ b/airavata-django-portal/django_airavata/apps/api/output_views.py @@ -7,7 +7,7 @@ from functools import partial import nbformat import papermill as pm -from airavata_django_portal_sdk import user_storage +from django_airavata.apps.api import user_storage from django.conf import settings from nbconvert import HTMLExporter diff --git a/airavata-django-portal/django_airavata/apps/api/serializers.py b/airavata-django-portal/django_airavata/apps/api/serializers.py index 21cbf3ffa..80881faba 100644 --- a/airavata-django-portal/django_airavata/apps/api/serializers.py +++ b/airavata-django-portal/django_airavata/apps/api/serializers.py @@ -5,7 +5,7 @@ import logging from pathlib import Path from urllib.parse import quote -from airavata_django_portal_sdk import experiment_util, user_storage +from django_airavata.apps.api import user_storage from django.conf import settings from django.contrib.auth import get_user_model from django.urls import reverse @@ -517,19 +517,18 @@ class ExperimentSerializer(proto_utils.create_serializer_class(ExperimentModel)) for output in representation["experimentOutputs"]: output["intermediateOutput"] = {"processStatus": None} try: - can_fetch = experiment_util.intermediate_output.can_fetch_intermediate_output( - request, experiment, output["name"] - ) - output["intermediateOutput"]["canFetch"] = can_fetch - process_status = experiment_util.intermediate_output.get_intermediate_output_process_status( - request, experiment, output["name"] + # Intermediate outputs can be fetched when experiment is EXECUTING + output["intermediateOutput"]["canFetch"] = True + process_status = request.airavata_client.research.get_intermediate_output_process_status( + experiment.experimentId ) if process_status: serializer = ProcessStatusSerializer(process_status, context={"request": request}) output["intermediateOutput"]["processStatus"] = serializer.data - data_products = experiment_util.intermediate_output.get_intermediate_output_data_products( - request, experiment=experiment, output_name=output["name"] + intermediate_resp = request.airavata_client.research.get_intermediate_outputs( + experiment.experimentId, [output["name"]] ) + data_products = list(getattr(intermediate_resp, "data_products", [])) data_product_serializer = DataProductSerializer( data_products, context={"request": request}, many=True ) @@ -568,8 +567,7 @@ class DataProductSerializer(proto_utils.create_serializer_class(DataProductModel def get_filesize(self, data_product): request = self.context["request"] - # For backwards compatibility with older user_storage, can be eventually removed - if hasattr(user_storage, "get_data_product_metadata") and user_storage.exists(request, data_product): + if user_storage.exists(request, data_product): metadata = user_storage.get_data_product_metadata(request, data_product) return metadata["size"] else: diff --git a/airavata-django-portal/django_airavata/apps/api/signals.py b/airavata-django-portal/django_airavata/apps/api/signals.py index 8b25c161d..c321c06cd 100644 --- a/airavata-django-portal/django_airavata/apps/api/signals.py +++ b/airavata-django-portal/django_airavata/apps/api/signals.py @@ -2,7 +2,7 @@ import logging -from airavata_django_portal_sdk import user_storage +from django_airavata.apps.api import user_storage from django.conf import settings from django.contrib.auth.signals import user_logged_in from django.dispatch import Signal, receiver diff --git a/airavata-django-portal/django_airavata/apps/api/user_storage.py b/airavata-django-portal/django_airavata/apps/api/user_storage.py new file mode 100644 index 000000000..3b68b7f51 --- /dev/null +++ b/airavata-django-portal/django_airavata/apps/api/user_storage.py @@ -0,0 +1,284 @@ +"""Thin compatibility layer over the SDK facade's StorageClient. + +Replaces the deleted ``airavata_django_portal_sdk.user_storage`` module. +Every function here accepts ``request`` as its first argument (to mirror the +old API) and delegates to ``request.airavata_client.storage``. +""" + +import io +import logging +import os + +from django.conf import settings + +log = logging.getLogger(__name__) + + +# --------------------------------------------------------------------------- +# Helpers to extract file paths from DataProductModel proto objects +# --------------------------------------------------------------------------- + +def _get_replica_filepath(data_product): + """Return the file_path from the first GATEWAY_DATA_STORE replica location.""" + for replica in data_product.replica_locations: + if replica.file_path: + return replica.file_path + return None + + +def _get_replica_storage_resource_id(data_product): + """Return the storage_resource_id from the first replica location.""" + for replica in data_product.replica_locations: + if replica.storage_resource_id: + return replica.storage_resource_id + return None + + +# --------------------------------------------------------------------------- +# File existence / metadata +# --------------------------------------------------------------------------- + +def exists(request, data_product): + """Check whether the file backing *data_product* exists in user storage.""" + path = _get_replica_filepath(data_product) + if not path: + return False + try: + return request.airavata_client.storage.file_exists(path) + except Exception: + return False + + +def dir_exists(request, path, experiment_id=None): + """Check whether *path* exists as a directory in user storage.""" + if experiment_id: + return experiment_dir_exists(request, experiment_id, path) + return request.airavata_client.storage.dir_exists(path) + + +def experiment_dir_exists(request, experiment_id, path=""): + """Check whether the experiment output directory exists.""" + try: + request.airavata_client.storage.list_experiment_dir(experiment_id, path) + return True + except Exception: + return False + + +def is_input_file(request, data_product): + """Return True if the data product's path is under the inputs directory.""" + path = _get_replica_filepath(data_product) + if not path: + return False + # Input files are stored under a path that contains "/inputs/" or starts with "inputs/" + normalized = path.replace("\\", "/") + return "/inputs/" in normalized or normalized.startswith("inputs/") + + +# --------------------------------------------------------------------------- +# File / directory listing +# --------------------------------------------------------------------------- + +def listdir(request, path, experiment_id=None): + """List the contents of *path*, returning (directories, files) dicts.""" + if experiment_id: + return list_experiment_dir(request, experiment_id, path) + resp = request.airavata_client.storage.list_dir(path) + directories = _metadata_list_to_dicts(resp.directories) + files = _metadata_list_to_dicts(resp.files) + return directories, files + + +def list_experiment_dir(request, experiment_id, path=""): + """List the experiment output directory.""" + resp = request.airavata_client.storage.list_experiment_dir(experiment_id, path) + directories = _metadata_list_to_dicts(resp.directories) + files = _metadata_list_to_dicts(resp.files) + return directories, files + + +def _metadata_list_to_dicts(items): + """Convert repeated FileMetadataResponse protos to plain dicts.""" + result = [] + for item in items: + result.append({ + "name": item.name, + "path": item.path, + "size": item.size, + "created_time": item.created_time, + "modified_time": item.modified_time, + "data-product-uri": item.data_product_uri, + "mime_type": item.content_type, + "hidden": False, + }) + return result + + +# --------------------------------------------------------------------------- +# File open / download +# --------------------------------------------------------------------------- + +def open_file(request, data_product): + """Download the file for *data_product* and return a file-like object.""" + path = _get_replica_filepath(data_product) + resp = request.airavata_client.storage.download_file(path) + f = io.BytesIO(resp.content) + f.name = resp.name or os.path.basename(path) + return f + + +# --------------------------------------------------------------------------- +# File upload / save +# --------------------------------------------------------------------------- + +def save_input_file(request, input_file, name=None, content_type=""): + """Upload *input_file* to the user's input files directory. + + Returns a DataProductModel proto. + """ + file_name = name or getattr(input_file, "name", "uploaded_file") + content = input_file.read() + resp = request.airavata_client.storage.upload_file( + path="inputs", + content=content, + name=file_name, + content_type=content_type or "", + ) + # The upload returns a FileUploadResponse with a data product URI. + # Fetch and return the full DataProductModel. + return request.airavata_client.research.get_data_product(resp.uri) + + +def save(request, path, file_obj, name=None, content_type="", experiment_id=None): + """Upload *file_obj* to *path* in user storage. + + Returns a DataProductModel proto. + """ + file_name = name or getattr(file_obj, "name", "uploaded_file") + content = file_obj.read() + resp = request.airavata_client.storage.upload_file( + path=path, + content=content, + name=file_name, + content_type=content_type or "", + ) + return request.airavata_client.research.get_data_product(resp.uri) + + +# --------------------------------------------------------------------------- +# File content update +# --------------------------------------------------------------------------- + +def update_data_product_content(request, data_product, fileContentText): + """Replace the content of the file backing *data_product* with *fileContentText*.""" + path = _get_replica_filepath(data_product) + name = os.path.basename(path) + request.airavata_client.storage.upload_file( + path=os.path.dirname(path), + content=fileContentText.encode("utf-8"), + name=name, + ) + + +def update_file_content(request, path, fileContentText): + """Replace the content of the file at *path* with *fileContentText*.""" + name = os.path.basename(path) + request.airavata_client.storage.upload_file( + path=os.path.dirname(path), + content=fileContentText.encode("utf-8"), + name=name, + ) + + +# --------------------------------------------------------------------------- +# File / directory creation and deletion +# --------------------------------------------------------------------------- + +def create_user_dir(request, path, experiment_id=None): + """Create a directory at *path*. Returns (storage_resource_id, created_path).""" + resp = request.airavata_client.storage.create_dir(path) + return None, resp.created_path + + +def create_symlink(request, source_path, dest_path): + """Create a symlink from *source_path* to *dest_path*.""" + request.airavata_client.storage.create_symlink(source_path, dest_path) + + +def delete(request, data_product): + """Delete the file backing *data_product*.""" + path = _get_replica_filepath(data_product) + if path: + request.airavata_client.storage.delete_file(path) + + +def delete_user_file(request, path, experiment_id=None): + """Delete a user file at *path*.""" + request.airavata_client.storage.delete_file(path) + + +def delete_dir(request, path, experiment_id=None): + """Delete a directory at *path*.""" + request.airavata_client.storage.delete_dir(path) + + +# --------------------------------------------------------------------------- +# File metadata +# --------------------------------------------------------------------------- + +def get_file_metadata(request, path, experiment_id=None): + """Get metadata for the file at *path*. Returns a dict.""" + resp = request.airavata_client.storage.get_file_metadata(path) + return { + "name": resp.name, + "path": resp.path, + "size": resp.size, + "created_time": resp.created_time, + "modified_time": resp.modified_time, + "data-product-uri": resp.data_product_uri, + "mime_type": resp.content_type, + "hidden": False, + } + + +def get_data_product_metadata(request, data_product=None, data_product_uri=None): + """Get metadata for a data product. Returns a dict with path, size, etc.""" + if data_product is None and data_product_uri: + data_product = request.airavata_client.research.get_data_product(data_product_uri) + path = _get_replica_filepath(data_product) + if not path: + return {"path": "", "size": 0, "userHasWriteAccess": False} + try: + resp = request.airavata_client.storage.get_file_metadata(path) + return { + "name": resp.name, + "path": resp.path, + "size": resp.size, + "created_time": resp.created_time, + "modified_time": resp.modified_time, + "data-product-uri": resp.data_product_uri, + "mime_type": resp.content_type, + "userHasWriteAccess": True, + } + except Exception: + return {"path": path, "size": 0, "userHasWriteAccess": False} + + +# --------------------------------------------------------------------------- +# Download URL helpers +# --------------------------------------------------------------------------- + +def get_download_url(request, data_product_uri=None): + """Return a URL to download the file for *data_product_uri*.""" + from django.urls import reverse + from urllib.parse import quote + return reverse("django_airavata_api:download_file") + f"?data-product-uri={quote(data_product_uri)}" + + +def get_lazy_download_url(request, data_product=None, data_product_uri=None): + """Return a download URL. Accepts either a data_product or data_product_uri.""" + if data_product_uri: + return get_download_url(request, data_product_uri=data_product_uri) + if data_product: + return get_download_url(request, data_product_uri=data_product.product_uri) + return None diff --git a/airavata-django-portal/django_airavata/apps/api/view_utils.py b/airavata-django-portal/django_airavata/apps/api/view_utils.py index 46dc72efa..9c90de970 100644 --- a/airavata-django-portal/django_airavata/apps/api/view_utils.py +++ b/airavata-django-portal/django_airavata/apps/api/view_utils.py @@ -5,7 +5,7 @@ from datetime import datetime from pathlib import Path import pytz -from airavata_django_portal_sdk import user_storage +from django_airavata.apps.api import user_storage from django.conf import settings from django.http import Http404 from django.http.request import QueryDict diff --git a/airavata-django-portal/django_airavata/apps/api/views.py b/airavata-django-portal/django_airavata/apps/api/views.py index 68850fda8..4af37c93f 100644 --- a/airavata-django-portal/django_airavata/apps/api/views.py +++ b/airavata-django-portal/django_airavata/apps/api/views.py @@ -5,7 +5,7 @@ import os import warnings from datetime import datetime, timedelta -from airavata_django_portal_sdk import experiment_util, user_storage +from django_airavata.apps.api import user_storage from django.conf import settings from django.contrib.auth import get_user_model from django.core.exceptions import ObjectDoesNotExist, PermissionDenied @@ -202,7 +202,7 @@ class ExperimentViewSet( if experiment.enableEmailNotification: experiment.emailAddresses = [request.user.email] request.airavata_client.research.update_experiment(experiment_id, experiment) - experiment_util.launch(request, experiment_id) + request.airavata_client.research.launch_experiment(experiment_id, settings.GATEWAY_ID) return Response({"success": True}) except Exception as e: log.exception(f"Failed to launch experiment {experiment_id}", extra={"request": request}) @@ -216,7 +216,7 @@ class ExperimentViewSet( @action(methods=["post"], detail=True) def clone(self, request, experiment_id=None): - cloned_experiment_id = experiment_util.clone(request, experiment_id) + cloned_experiment_id = request.airavata_client.research.clone_experiment(experiment_id) cloned_experiment = request.airavata_client.research.get_experiment(cloned_experiment_id) serializer = self.serializer_class(cloned_experiment, context={"request": request}) return Response(serializer.data) @@ -235,8 +235,8 @@ class ExperimentViewSet( if "outputNames" not in request.data: return Response(status=status.HTTP_400_BAD_REQUEST) try: - experiment_util.intermediate_output.fetch_intermediate_output( - request, experiment_id, *request.data["outputNames"] + request.airavata_client.research.get_intermediate_outputs( + experiment_id, request.data["outputNames"] ) return Response({"success": True}) except Exception as e: diff --git a/airavata-django-portal/django_airavata/apps/workspace/views.py b/airavata-django-portal/django_airavata/apps/workspace/views.py index bcf4bfe78..fee2c71ae 100644 --- a/airavata-django-portal/django_airavata/apps/workspace/views.py +++ b/airavata-django-portal/django_airavata/apps/workspace/views.py @@ -2,7 +2,7 @@ import json import logging from urllib.parse import urlparse -from airavata_django_portal_sdk import user_storage as user_storage_sdk +from django_airavata.apps.api import user_storage as user_storage_sdk from django.contrib.auth.decorators import login_required from django.shortcuts import render from django.utils.module_loading import import_string
