This is an automated email from the ASF dual-hosted git repository. yasith pushed a commit to branch feat/sdk-facade-migration in repository https://gitbox.apache.org/repos/asf/airavata-portals.git
commit 0af3e39c210ca7fc7cee8b3f3b8345d3cd1b04d1 Author: yasithdev <[email protected]> AuthorDate: Wed Apr 8 00:26:50 2026 -0500 refactor: migrate remaining Thrift type imports to proto compatibility layer - Create django_airavata/proto_compat.py with all data types/enums previously imported from airavata.model.*.ttypes - Create proto_utils.py (replacing thrift_utils.py) without Thrift library dependency - Update serializers.py, views.py, output_views.py, helpers.py, workspace/views.py, auth tests, and iam_admin_client.py - Fix authz_token.accessToken dot-access to dict-access --- .../django_airavata/apps/api/helpers.py | 2 +- .../django_airavata/apps/api/output_views.py | 2 +- .../django_airavata/apps/api/proto_utils.py | 245 +++++++++++++++ .../django_airavata/apps/api/serializers.py | 201 ++++-------- .../django_airavata/apps/api/tests/test_views.py | 4 +- .../django_airavata/apps/api/views.py | 79 +++-- .../django_airavata/apps/auth/iam_admin_client.py | 4 +- .../apps/auth/tests/test_signals.py | 3 +- .../django_airavata/apps/auth/tests/test_views.py | 2 +- .../django_airavata/apps/auth/views.py | 2 +- .../django_airavata/apps/workspace/views.py | 2 +- .../django_airavata/proto_compat.py | 342 +++++++++++++++++++++ 12 files changed, 696 insertions(+), 192 deletions(-) diff --git a/airavata-django-portal/django_airavata/apps/api/helpers.py b/airavata-django-portal/django_airavata/apps/api/helpers.py index 8dafaa6a1..a5aa2fd34 100644 --- a/airavata-django-portal/django_airavata/apps/api/helpers.py +++ b/airavata-django-portal/django_airavata/apps/api/helpers.py @@ -1,6 +1,6 @@ import logging -from airavata.model.group.ttypes import ResourcePermissionType +from django_airavata.proto_compat import ResourcePermissionType from django.conf import settings from django.core.exceptions import ObjectDoesNotExist diff --git a/airavata-django-portal/django_airavata/apps/api/output_views.py b/airavata-django-portal/django_airavata/apps/api/output_views.py index ac22ded20..fb6065705 100644 --- a/airavata-django-portal/django_airavata/apps/api/output_views.py +++ b/airavata-django-portal/django_airavata/apps/api/output_views.py @@ -7,7 +7,7 @@ from functools import partial import nbformat import papermill as pm -from airavata.model.application.io.ttypes import DataType +from django_airavata.proto_compat import DataType from airavata_django_portal_sdk import user_storage from django.conf import settings from nbconvert import HTMLExporter diff --git a/airavata-django-portal/django_airavata/apps/api/proto_utils.py b/airavata-django-portal/django_airavata/apps/api/proto_utils.py new file mode 100644 index 000000000..4fca3204c --- /dev/null +++ b/airavata-django-portal/django_airavata/apps/api/proto_utils.py @@ -0,0 +1,245 @@ +""" +Used to create Django Rest Framework serializers for Airavata data types. + +Migrated from thrift_utils.py -- now works with proto-compatible data classes +from django_airavata.proto_compat instead of Thrift-generated types. +""" +import copy +import datetime +import enum +import logging + +from rest_framework.serializers import ( + BooleanField, + CharField, + DateTimeField, + DecimalField, + DictField, + Field, + IntegerField, + ListField, + ListSerializer, + Serializer, + SerializerMetaclass, + ValidationError +) +from django_airavata.proto_compat import ( + ApplicationParallelismType, + DataType, + ExperimentState, + ExperimentType, +) + +logger = logging.getLogger(__name__) + +# TType constants (formerly from thrift.Thrift.TType) +# These mirror the Thrift type IDs used in thrift_spec tuples. +TTYPE_BOOL = 2 +TTYPE_I08 = 3 +TTYPE_I16 = 6 +TTYPE_I32 = 8 +TTYPE_I64 = 10 +TTYPE_DOUBLE = 4 +TTYPE_STRING = 11 +TTYPE_STRUCT = 12 +TTYPE_MAP = 13 +TTYPE_LIST = 15 + +# Map proto/thrift field type IDs to DRF serializer fields +mapping = { + TTYPE_STRING: CharField, + TTYPE_I08: IntegerField, + TTYPE_I16: IntegerField, + TTYPE_I32: IntegerField, + TTYPE_I64: IntegerField, + TTYPE_DOUBLE: DecimalField, + TTYPE_BOOL: BooleanField, + TTYPE_MAP: DictField, +} + + +class UTCPosixTimestampDateTimeField(DateTimeField): + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.default = self.current_time_ms + self.initial = self.initial_value + self.required = False + + def to_representation(self, obj): + dt = datetime.datetime.fromtimestamp(obj / 1000, datetime.timezone.utc) + return super().to_representation(dt) + + def to_internal_value(self, data): + dt = super().to_internal_value(data) + return int(dt.timestamp() * 1000) + + def initial_value(self): + return self.to_representation(self.current_time_ms()) + + def current_time_ms(self): + return int(datetime.datetime.utcnow().timestamp() * 1000) + + +class ThriftEnumField(Field): + + def __init__(self, enumClass, *args, **kwargs): + super().__init__(*args, **kwargs) + self.enumClass = enumClass + + def to_representation(self, obj): + if obj is None: + return None + return obj.name + + def to_internal_value(self, data): + if self.allow_null and data is None: + return None + try: + return self.enumClass[data] + except KeyError: + raise ValidationError(f"'{data}' is not a valid name for enum {self.enumClass.__name__}") + + +def create_serializer(thrift_data_type, enable_date_time_conversion=False, **kwargs): + """ + Create a DRF serializer based on the data type. + :param thrift_data_type: Data type class (with thrift_spec) + :param kwargs: Other Django Framework Serializer initialization parameters + :param enable_date_time_conversion: enable conversion of fields ending with 'time' + :return: instance of custom serializer for the given data type + """ + return create_serializer_class(thrift_data_type, enable_date_time_conversion)(**kwargs) + + +def create_serializer_class(thrift_data_type, enable_date_time_conversion=False): + class CustomSerializerMeta(SerializerMetaclass): + + def __new__(cls, name, bases, attrs): + meta = attrs.get('Meta', None) + thrift_spec = thrift_data_type.thrift_spec + for field in thrift_spec: + if field and field[2] not in attrs: + required = (field[2] in meta.required + if meta and hasattr(meta, 'required') + else False) + read_only = (field[2] in meta.read_only + if meta and hasattr(meta, 'read_only') + else False) + allow_null = not required + field_serializer = process_field( + field, enable_date_time_conversion, required=required, read_only=read_only, + allow_null=allow_null) + attrs[field[2]] = field_serializer + return super().__new__(cls, name, bases, attrs) + + class CustomSerializer(Serializer, metaclass=CustomSerializerMeta): + """ + Custom Serializer which handles list fields holding custom class objects. + """ + + def process_nested_fields(self, validated_data): + fields = self.fields + params = copy.deepcopy(validated_data) + for field_name, serializer in fields.items(): + if (isinstance(serializer, ListField) or + isinstance(serializer, ListSerializer)): + if (params.get(field_name, None) is not None or + not serializer.allow_null): + if isinstance(serializer.child, Serializer): + if field_name == 'experimentInputs' and 'type' in serializer.child.fields: + for item in params[field_name]: + if 'type' in item and isinstance(item['type'], int): + item['type'] = DataType(item['type']) + elif field_name == 'experimentOutputs' and 'type' in serializer.child.fields: + for item in params[field_name]: + if 'type' in item and isinstance(item['type'], int): + item['type'] = DataType(item['type']) + elif field_name == 'experimentStatus' and 'state' in serializer.child.fields: + for item in params[field_name]: + if 'state' in item and isinstance(item['state'], int): + item['state'] = ExperimentState(item['state']) + params[field_name] = [serializer.child.create( + item) for item in params[field_name]] + else: + params[field_name] = serializer.to_representation( + params[field_name]) + elif isinstance(serializer, Serializer): + if field_name in params and params[field_name] is not None: + params[field_name] = serializer.create( + params[field_name]) + return params + + def create(self, validated_data): + params = self.process_nested_fields(validated_data) + + # Remove fields with None values when they have defaults + thrift_spec = thrift_data_type.thrift_spec + for field_spec in thrift_spec: + if field_spec: + field_name = field_spec[2] + default_value = field_spec[4] + if default_value is not None: + if field_name in params and params[field_name] is None: + del params[field_name] + + if (thrift_data_type.__name__ == 'ExperimentModel' and + 'experimentType' in params and isinstance(params['experimentType'], int)): + params['experimentType'] = ExperimentType(params['experimentType']) + + if (thrift_data_type.__name__ == 'ApplicationDeploymentDescription' and + 'parallelism' in params and isinstance(params['parallelism'], int)): + params['parallelism'] = ApplicationParallelismType(params['parallelism']) + + return thrift_data_type(**params) + + def update(self, instance, validated_data): + return self.create(validated_data) + + return CustomSerializer + + +def process_field(field, enable_date_time_conversion, required=False, read_only=False, allow_null=False): + if field[1] in mapping: + field_class = mapping[field[1]] + kwargs = dict(required=required, read_only=read_only) + if field_class not in (BooleanField,): + kwargs['allow_null'] = allow_null + if field_class == CharField: + kwargs['allow_blank'] = allow_null + thrift_model_class = mapping[field[1]] + + if thrift_model_class == IntegerField and field[3] is not None and isinstance(field[3], type) and issubclass(field[3], enum.IntEnum): + return ThriftEnumField(field[3], required=required, read_only=read_only, allow_null=allow_null) + + if enable_date_time_conversion and thrift_model_class == IntegerField and field[2].lower().endswith("time"): + thrift_model_class = UTCPosixTimestampDateTimeField + return thrift_model_class(**kwargs) + elif field[1] == TTYPE_LIST: + list_field_serializer = process_list_field(field) + return ListField(child=list_field_serializer, + required=required, + read_only=read_only, + allow_null=allow_null) + elif field[1] == TTYPE_STRUCT: + return create_serializer(field[3][0], + required=required, + read_only=read_only, + allow_null=allow_null) + + +def process_list_field(field): + list_details = field[3] + item_ttype = list_details[0] + item_type_info = list_details[1] + + if (item_ttype == TTYPE_I32 and + item_type_info is not None and + isinstance(item_type_info, type) and + issubclass(item_type_info, enum.IntEnum)): + return ThriftEnumField(item_type_info) + + if item_ttype in mapping: + return mapping[item_ttype]() + elif item_ttype == TTYPE_STRUCT: + return create_serializer(item_type_info[0]) diff --git a/airavata-django-portal/django_airavata/apps/api/serializers.py b/airavata-django-portal/django_airavata/apps/api/serializers.py index ff7a3ca92..3752ce667 100644 --- a/airavata-django-portal/django_airavata/apps/api/serializers.py +++ b/airavata-django-portal/django_airavata/apps/api/serializers.py @@ -4,66 +4,46 @@ import json import logging from pathlib import Path from urllib.parse import quote -from airavata.model.application.io.ttypes import DataType - -from airavata.model.appcatalog.appdeployment.ttypes import ( +from django_airavata.proto_compat import ( ApplicationDeploymentDescription, + ApplicationInterfaceDescription, ApplicationModule, - CommandObject, - SetEnvPaths -) -from airavata.model.appcatalog.appinterface.ttypes import ( - ApplicationInterfaceDescription -) -from airavata.model.appcatalog.computeresource.ttypes import ( + AwsComputeResourcePreference, BatchQueue, - ComputeResourceDescription -) -from airavata.model.appcatalog.gatewayprofile.ttypes import ( - GatewayResourceProfile, - StoragePreference -) -from airavata.model.appcatalog.groupresourceprofile.ttypes import ( + CommandObject, + ComputeResourceDescription, ComputeResourceReservation, - GroupComputeResourcePreference, - GroupResourceProfile, - ResourceType, - SlurmComputeResourcePreference, - AwsComputeResourcePreference -) -from airavata.model.appcatalog.parser.ttypes import Parser -from airavata.model.appcatalog.storageresource.ttypes import ( - StorageResourceDescription -) -from airavata.model.application.io.ttypes import ( - InputDataObjectType, - OutputDataObjectType -) -from airavata.model.credential.store.ttypes import ( CredentialSummary, - SummaryType -) -from airavata.model.data.replica.ttypes import ( DataProductModel, - DataReplicaLocationModel -) -from airavata.model.experiment.ttypes import ( + DataReplicaLocationModel, + DataType, + EnvironmentSpecificPreferences, ExperimentModel, - ExperimentStatistics, - ExperimentSummaryModel -) -from airavata.model.group.ttypes import GroupModel, ResourcePermissionType -from airavata.model.job.ttypes import JobModel -from airavata.model.status.ttypes import ( ExperimentState, + ExperimentStatistics, ExperimentStatus, - ProcessStatus -) -from airavata.model.user.ttypes import UserProfile -from airavata.model.workspace.ttypes import ( + ExperimentSummaryModel, + GatewayResourceProfile, + GroupAccountSSHProvisionerConfig, + GroupComputeResourcePreference, + GroupModel, + GroupResourceProfile, + InputDataObjectType, + JobModel, Notification, NotificationPriority, - Project + OutputDataObjectType, + Parser, + ProcessStatus, + Project, + ResourcePermissionType, + ResourceType, + SetEnvPaths, + SlurmComputeResourcePreference, + StoragePreference, + StorageResourceDescription, + SummaryType, + UserProfile, ) from airavata_django_portal_sdk import ( experiment_util, @@ -74,7 +54,7 @@ from django.contrib.auth import get_user_model from django.urls import reverse from rest_framework import serializers -from . import models, thrift_utils, view_utils +from . import models, proto_utils, view_utils log = logging.getLogger(__name__) @@ -168,7 +148,7 @@ class OrderedListField(serializers.ListField): return validated_data -class GroupSerializer(thrift_utils.create_serializer_class(GroupModel)): +class GroupSerializer(proto_utils.create_serializer_class(GroupModel)): url = FullyEncodedHyperlinkedIdentityField( view_name='django_airavata_api:group-detail', lookup_field='id', @@ -252,7 +232,7 @@ class GroupSerializer(thrift_utils.create_serializer_class(GroupModel)): class ProjectSerializer( - thrift_utils.create_serializer_class(Project)): + proto_utils.create_serializer_class(Project)): class Meta: required = ('name',) read_only = ('owner', 'gatewayId') @@ -290,7 +270,7 @@ class ProjectSerializer( class ApplicationModuleSerializer( - thrift_utils.create_serializer_class(ApplicationModule)): + proto_utils.create_serializer_class(ApplicationModule)): url = FullyEncodedHyperlinkedIdentityField( view_name='django_airavata_api:application-detail', lookup_field='appModuleId', @@ -452,17 +432,17 @@ class ApplicationInterfaceDescriptionSerializer(serializers.Serializer): class CommandObjectSerializer( - thrift_utils.create_serializer_class(CommandObject)): + proto_utils.create_serializer_class(CommandObject)): pass class SetEnvPathsSerializer( - thrift_utils.create_serializer_class(SetEnvPaths)): + proto_utils.create_serializer_class(SetEnvPaths)): pass class ApplicationDeploymentDescriptionSerializer( - thrift_utils.create_serializer_class( + proto_utils.create_serializer_class( ApplicationDeploymentDescription)): url = FullyEncodedHyperlinkedIdentityField( view_name='django_airavata_api:application-deployment-detail', @@ -508,26 +488,26 @@ class ApplicationDeploymentDescriptionSerializer( class ComputeResourceDescriptionSerializer( - thrift_utils.create_serializer_class(ComputeResourceDescription)): + proto_utils.create_serializer_class(ComputeResourceDescription)): pass -class BatchQueueSerializer(thrift_utils.create_serializer_class(BatchQueue)): +class BatchQueueSerializer(proto_utils.create_serializer_class(BatchQueue)): pass class ExperimentStatusSerializer( - thrift_utils.create_serializer_class(ExperimentStatus)): + proto_utils.create_serializer_class(ExperimentStatus)): timeOfStateChange = UTCPosixTimestampDateTimeField() class ProcessStatusSerializer( - thrift_utils.create_serializer_class(ProcessStatus)): + proto_utils.create_serializer_class(ProcessStatus)): timeOfStateChange = UTCPosixTimestampDateTimeField() class ExperimentSerializer( - thrift_utils.create_serializer_class(ExperimentModel)): + proto_utils.create_serializer_class(ExperimentModel)): class Meta: required = ('projectId', 'experimentType', 'experimentName') read_only = ('userName', 'gatewayId') @@ -602,13 +582,13 @@ class ExperimentSerializer( class DataReplicaLocationSerializer( - thrift_utils.create_serializer_class(DataReplicaLocationModel)): + proto_utils.create_serializer_class(DataReplicaLocationModel)): creationTime = UTCPosixTimestampDateTimeField() lastModifiedTime = UTCPosixTimestampDateTimeField() class DataProductSerializer( - thrift_utils.create_serializer_class(DataProductModel)): + proto_utils.create_serializer_class(DataProductModel)): creationTime = UTCPosixTimestampDateTimeField() modifiedTime = UTCPosixTimestampDateTimeField() lastModifiedTime = UTCPosixTimestampDateTimeField() @@ -676,7 +656,7 @@ class FullExperiment: self.outputViews = outputViews -class JobSerializer(thrift_utils.create_serializer_class(JobModel)): +class JobSerializer(proto_utils.create_serializer_class(JobModel)): creationTime = UTCPosixTimestampDateTimeField() @@ -703,7 +683,7 @@ class FullExperimentSerializer(serializers.Serializer): class BaseExperimentSummarySerializer( - thrift_utils.create_serializer_class(ExperimentSummaryModel)): + proto_utils.create_serializer_class(ExperimentSummaryModel)): creationTime = UTCPosixTimestampDateTimeField() statusUpdateTime = UTCPosixTimestampDateTimeField() url = FullyEncodedHyperlinkedIdentityField( @@ -727,19 +707,19 @@ class ExperimentSummarySerializer(BaseExperimentSummarySerializer): class UserProfileSerializer( - thrift_utils.create_serializer_class(UserProfile)): + proto_utils.create_serializer_class(UserProfile)): creationTime = UTCPosixTimestampDateTimeField() lastAccessTime = UTCPosixTimestampDateTimeField() class ComputeResourceReservationSerializer( - thrift_utils.create_serializer_class(ComputeResourceReservation)): + proto_utils.create_serializer_class(ComputeResourceReservation)): startTime = UTCPosixTimestampDateTimeField(allow_null=True) endTime = UTCPosixTimestampDateTimeField(allow_null=True) class GroupComputeResourcePreferenceSerializer( - thrift_utils.create_serializer_class(GroupComputeResourcePreference)): + proto_utils.create_serializer_class(GroupComputeResourcePreference)): reservations = serializers.SerializerMethodField() # Check if the object (e.g. SLURM type) has the 'reservations' attribute @@ -754,10 +734,6 @@ class GroupComputeResourcePreferenceSerializer( @staticmethod def _convert_nested_list_fields_to_thrift(slurm_pref): from collections import OrderedDict - from airavata.model.appcatalog.groupresourceprofile.ttypes import ( - ComputeResourceReservation, - GroupAccountSSHProvisionerConfig - ) if hasattr(slurm_pref, 'reservations') and slurm_pref.reservations: if isinstance(slurm_pref.reservations, list): @@ -789,22 +765,7 @@ class GroupComputeResourcePreferenceSerializer( if isinstance(pref_instance.specificPreferences, (dict, OrderedDict)): specific_prefs_dict = pref_instance.specificPreferences - union_type_class = None - try: - from airavata.model.appcatalog.groupresourceprofile.ttypes import ( - EnvironmentSpecificPreferences - ) - union_type_class = EnvironmentSpecificPreferences - log.debug( - "GCPreference: Got union type class from import: %s", - union_type_class.__name__, - ) - except ImportError as e: - log.error( - "GCPreference: Failed to import EnvironmentSpecificPreferences: %s", - str(e), - exc_info=True, - ) + union_type_class = EnvironmentSpecificPreferences if union_type_class: pref_instance.specificPreferences = union_type_class() @@ -818,10 +779,6 @@ class GroupComputeResourcePreferenceSerializer( if slurm_data and isinstance(slurm_data, dict) and len(slurm_data) > 0: try: from collections import OrderedDict - from airavata.model.appcatalog.groupresourceprofile.ttypes import ( - ComputeResourceReservation, - GroupAccountSSHProvisionerConfig - ) if 'reservations' in slurm_data and slurm_data['reservations']: reservations_list = slurm_data['reservations'] @@ -1060,24 +1017,11 @@ class GroupComputeResourcePreferenceSerializer( instance.resourceType = resource_type union_type_class = None - try: - from airavata.model.appcatalog.groupresourceprofile.ttypes import ( - EnvironmentSpecificPreferences - ) - union_type_class = EnvironmentSpecificPreferences - except ImportError as e: - log.error( - "GCPreference create: Failed to import EnvironmentSpecificPreferences: %s", - str(e), - exc_info=True, - ) + union_type_class = EnvironmentSpecificPreferences if specific_prefs is None: if resource_type == ResourceType.SLURM and slurm_data: from collections import OrderedDict - from airavata.model.appcatalog.groupresourceprofile.ttypes import ( - GroupAccountSSHProvisionerConfig - ) if 'reservations' in slurm_data and slurm_data['reservations']: reservations_list = slurm_data['reservations'] @@ -1143,9 +1087,6 @@ class GroupComputeResourcePreferenceSerializer( slurm_dict = specific_prefs['slurm'].copy() if isinstance(specific_prefs['slurm'], dict) else {} slurm_dict.update(slurm_data) from collections import OrderedDict - from airavata.model.appcatalog.groupresourceprofile.ttypes import ( - GroupAccountSSHProvisionerConfig - ) if 'reservations' in slurm_dict and slurm_dict['reservations']: reservations_list = slurm_dict['reservations'] @@ -1180,9 +1121,6 @@ class GroupComputeResourcePreferenceSerializer( instance.specificPreferences.aws = aws_pref elif slurm_data: from collections import OrderedDict - from airavata.model.appcatalog.groupresourceprofile.ttypes import ( - GroupAccountSSHProvisionerConfig - ) if 'reservations' in slurm_data and slurm_data['reservations']: reservations_list = slurm_data['reservations'] @@ -1237,27 +1175,18 @@ class GroupComputeResourcePreferenceSerializer( if hasattr(instance, 'resourceType') and instance.resourceType: if instance.specificPreferences is None: - try: - from airavata.model.appcatalog.groupresourceprofile.ttypes import ( - EnvironmentSpecificPreferences - ) - instance.specificPreferences = EnvironmentSpecificPreferences() - log.debug( - "GCPreference create: Initialized empty specificPreferences union type, computeResourceId=%s", - instance.computeResourceId if hasattr(instance, 'computeResourceId') else 'unknown', - ) - except ImportError as e: - log.warning( - "GCPreference create: Could not initialize empty specificPreferences: %s", - str(e), - ) + instance.specificPreferences = EnvironmentSpecificPreferences() + log.debug( + "GCPreference create: Initialized empty specificPreferences union type, computeResourceId=%s", + instance.computeResourceId if hasattr(instance, 'computeResourceId') else 'unknown', + ) self._convert_specific_preferences_dict_to_thrift(instance, instance.resourceType) return instance class GroupResourceProfileSerializer( - thrift_utils.create_serializer_class(GroupResourceProfile)): + proto_utils.create_serializer_class(GroupResourceProfile)): url = FullyEncodedHyperlinkedIdentityField( view_name='django_airavata_api:group-resource-profile-detail', lookup_field='groupResourceProfileId', @@ -1794,8 +1723,8 @@ class SharedEntitySerializer(serializers.Serializer): class CredentialSummarySerializer( - thrift_utils.create_serializer_class(CredentialSummary)): - type = thrift_utils.ThriftEnumField(SummaryType) + proto_utils.create_serializer_class(CredentialSummary)): + type = proto_utils.ThriftEnumField(SummaryType) persistedTime = UTCPosixTimestampDateTimeField() userHasWriteAccess = serializers.SerializerMethodField() @@ -1807,7 +1736,7 @@ class CredentialSummarySerializer( class StoragePreferenceSerializer( - thrift_utils.create_serializer_class(StoragePreference)): + proto_utils.create_serializer_class(StoragePreference)): url = FullyEncodedHyperlinkedIdentityField( view_name='django_airavata_api:storage-preference-detail', lookup_field='storageResourceId', @@ -1822,7 +1751,7 @@ class StoragePreferenceSerializer( class GatewayResourceProfileSerializer( - thrift_utils.create_serializer_class(GatewayResourceProfile)): + proto_utils.create_serializer_class(GatewayResourceProfile)): storagePreferences = StoragePreferenceSerializer(many=True) userHasWriteAccess = serializers.SerializerMethodField() @@ -1832,7 +1761,7 @@ class GatewayResourceProfileSerializer( class StorageResourceSerializer( - thrift_utils.create_serializer_class(StorageResourceDescription)): + proto_utils.create_serializer_class(StorageResourceDescription)): url = FullyEncodedHyperlinkedIdentityField( view_name='django_airavata_api:storage-resource-detail', lookup_field='storageResourceId', @@ -1841,7 +1770,7 @@ class StorageResourceSerializer( updateTime = UTCPosixTimestampDateTimeField() -class ParserSerializer(thrift_utils.create_serializer_class(Parser)): +class ParserSerializer(proto_utils.create_serializer_class(Parser)): url = FullyEncodedHyperlinkedIdentityField( view_name='django_airavata_api:parser-detail', lookup_field='id', @@ -2039,12 +1968,12 @@ class AckNotificationSerializer(serializers.ModelSerializer): model = models.User_Notifications -class NotificationSerializer(thrift_utils.create_serializer_class(Notification)): +class NotificationSerializer(proto_utils.create_serializer_class(Notification)): url = FullyEncodedHyperlinkedIdentityField( view_name='django_airavata_api:manage-notifications-detail', lookup_field='notificationId', lookup_url_kwarg='notification_id') - priority = thrift_utils.ThriftEnumField(NotificationPriority) + priority = proto_utils.ThriftEnumField(NotificationPriority) creationTime = UTCPosixTimestampDateTimeField(allow_null=True) publishedTime = UTCPosixTimestampDateTimeField() expirationTime = UTCPosixTimestampDateTimeField() @@ -2084,7 +2013,7 @@ class NotificationSerializer(thrift_utils.create_serializer_class(Notification)) class ExperimentStatisticsSerializer( - thrift_utils.create_serializer_class(ExperimentStatistics)): + proto_utils.create_serializer_class(ExperimentStatistics)): allExperiments = BaseExperimentSummarySerializer(many=True) completedExperiments = BaseExperimentSummarySerializer(many=True) failedExperiments = BaseExperimentSummarySerializer(many=True) diff --git a/airavata-django-portal/django_airavata/apps/api/tests/test_views.py b/airavata-django-portal/django_airavata/apps/api/tests/test_views.py index 201538dc4..e309185a7 100644 --- a/airavata-django-portal/django_airavata/apps/api/tests/test_views.py +++ b/airavata-django-portal/django_airavata/apps/api/tests/test_views.py @@ -1,8 +1,6 @@ from unittest.mock import MagicMock, call, patch -from airavata.model.appcatalog.gatewaygroups.ttypes import GatewayGroups -from airavata.model.group.ttypes import GroupModel -from airavata.model.user.ttypes import UserProfile +from django_airavata.proto_compat import GatewayGroups, GroupModel, UserProfile from django.contrib.auth.models import User from django.test import TestCase, override_settings from django.urls import reverse diff --git a/airavata-django-portal/django_airavata/apps/api/views.py b/airavata-django-portal/django_airavata/apps/api/views.py index 97e90de27..3f2ea46fe 100644 --- a/airavata-django-portal/django_airavata/apps/api/views.py +++ b/airavata-django-portal/django_airavata/apps/api/views.py @@ -5,32 +5,29 @@ import os import warnings from datetime import datetime, timedelta -# TODO: verify proto import paths once SDK generated stubs are finalized -from airavata.model.appcatalog.computeresource.ttypes import ( +from django_airavata.proto_compat import ( + BatchQueueResourcePolicy, CloudJobSubmission, + ComputeResourcePolicy, + ComputeResourceReservation, + DataType, + ExperimentModel, + ExperimentSearchFields, GlobusJobSubmission, - LOCALSubmission, - SSHJobSubmission, - UnicoreJobSubmission -) -from airavata.model.application.io.ttypes import DataType -from airavata.model.credential.store.ttypes import SummaryType -from airavata.model.data.movement.ttypes import ( GridFTPDataMovement, + GroupAccountSSHProvisionerConfig, + GroupComputeResourcePreference, LOCALDataMovement, + LOCALSubmission, + ResourcePermissionType, + ResourceType, SCPDataMovement, - UnicoreDataMovement -) -from airavata.model.experiment.ttypes import ( - ExperimentModel, - ExperimentSearchFields -) -from airavata.model.appcatalog.groupresourceprofile.ttypes import ( - GroupComputeResourcePreference, - ResourceType + SSHJobSubmission, + Status, + SummaryType, + UnicoreDataMovement, + UnicoreJobSubmission, ) -from airavata.model.group.ttypes import ResourcePermissionType -from airavata.model.user.ttypes import Status from airavata_django_portal_sdk import ( experiment_util, user_storage @@ -691,9 +688,9 @@ class LocalJobSubmissionView(APIView): job_submission_id = request.query_params["id"] local_job_submission = request.airavata_client.compute.get_local_job_submission( job_submission_id) - from . import thrift_utils + from . import proto_utils return Response( - thrift_utils.create_serializer( + proto_utils.create_serializer( LOCALSubmission, instance=local_job_submission).data) @@ -705,9 +702,9 @@ class CloudJobSubmissionView(APIView): job_submission_id = request.query_params["id"] job_submission = request.airavata_client.compute.get_cloud_job_submission( job_submission_id) - from . import thrift_utils + from . import proto_utils return Response( - thrift_utils.create_serializer( + proto_utils.create_serializer( CloudJobSubmission, instance=job_submission).data) @@ -719,9 +716,9 @@ class GlobusJobSubmissionView(APIView): job_submission_id = request.query_params["id"] job_submission = request.airavata_client.compute.get_globus_job_submission( job_submission_id) - from . import thrift_utils + from . import proto_utils return Response( - thrift_utils.create_serializer( + proto_utils.create_serializer( GlobusJobSubmission, instance=job_submission).data) @@ -733,9 +730,9 @@ class SshJobSubmissionView(APIView): job_submission_id = request.query_params["id"] job_submission = request.airavata_client.compute.get_ssh_job_submission( job_submission_id) - from . import thrift_utils + from . import proto_utils return Response( - thrift_utils.create_serializer( + proto_utils.create_serializer( SSHJobSubmission, instance=job_submission).data) @@ -747,9 +744,9 @@ class UnicoreJobSubmissionView(APIView): job_submission_id = request.query_params["id"] job_submission = request.airavata_client.compute.get_unicore_job_submission( job_submission_id) - from . import thrift_utils + from . import proto_utils return Response( - thrift_utils.create_serializer( + proto_utils.create_serializer( UnicoreJobSubmission, instance=job_submission).data) @@ -761,9 +758,9 @@ class GridFtpDataMovementView(APIView): data_movement_id = request.query_params["id"] data_movement = request.airavata_client.compute.get_grid_ftp_data_movement( data_movement_id) - from . import thrift_utils + from . import proto_utils return Response( - thrift_utils.create_serializer( + proto_utils.create_serializer( GridFTPDataMovement, instance=data_movement).data) @@ -775,9 +772,9 @@ class ScpDataMovementView(APIView): data_movement_id = request.query_params["id"] data_movement = request.airavata_client.compute.get_scp_data_movement( data_movement_id) - from . import thrift_utils + from . import proto_utils return Response( - thrift_utils.create_serializer( + proto_utils.create_serializer( SCPDataMovement, instance=data_movement).data) @@ -789,9 +786,9 @@ class UnicoreDataMovementView(APIView): data_movement_id = request.query_params["id"] data_movement = request.airavata_client.compute.get_unicore_data_movement( data_movement_id) - from . import thrift_utils + from . import proto_utils return Response( - thrift_utils.create_serializer( + proto_utils.create_serializer( UnicoreDataMovement, instance=data_movement).data) @@ -803,9 +800,9 @@ class LocalDataMovementView(APIView): data_movement_id = request.query_params["id"] data_movement = request.airavata_client.compute.get_local_data_movement( data_movement_id) - from . import thrift_utils + from . import proto_utils return Response( - thrift_utils.create_serializer( + proto_utils.create_serializer( LOCALDataMovement, instance=data_movement).data) @@ -998,10 +995,6 @@ class GroupResourceProfileViewSet(APIBackedViewSet): ) from collections import OrderedDict - from airavata.model.appcatalog.groupresourceprofile.ttypes import ( - ComputeResourcePolicy, - BatchQueueResourcePolicy - ) if hasattr(grp, 'computeResourcePolicies') and grp.computeResourcePolicies: existing_policies_by_resource_id = {} @@ -1101,12 +1094,10 @@ class GroupResourceProfileViewSet(APIBackedViewSet): if hasattr(pref.specificPreferences.slurm, 'reservations') and pref.specificPreferences.slurm.reservations: for res_idx, res in enumerate(pref.specificPreferences.slurm.reservations): if isinstance(res, (dict, OrderedDict)): - from airavata.model.appcatalog.groupresourceprofile.ttypes import ComputeResourceReservation pref.specificPreferences.slurm.reservations[res_idx] = ComputeResourceReservation(**res) if hasattr(pref.specificPreferences.slurm, 'groupSSHAccountProvisionerConfigs') and pref.specificPreferences.slurm.groupSSHAccountProvisionerConfigs: for cfg_idx, cfg in enumerate(pref.specificPreferences.slurm.groupSSHAccountProvisionerConfigs): if isinstance(cfg, (dict, OrderedDict)): - from airavata.model.appcatalog.groupresourceprofile.ttypes import GroupAccountSSHProvisionerConfig pref.specificPreferences.slurm.groupSSHAccountProvisionerConfigs[cfg_idx] = GroupAccountSSHProvisionerConfig(**cfg) self.request.airavata_client.compute.update_group_resource_profile(grp) diff --git a/airavata-django-portal/django_airavata/apps/auth/iam_admin_client.py b/airavata-django-portal/django_airavata/apps/auth/iam_admin_client.py index 6106626ae..fb36cf10d 100644 --- a/airavata-django-portal/django_airavata/apps/auth/iam_admin_client.py +++ b/airavata-django-portal/django_airavata/apps/auth/iam_admin_client.py @@ -73,7 +73,7 @@ def update_username(username, new_username): raise Exception(f"Can't change username of {username} to {new_username} because it is not available") # fetch user representation authz_token = utils.get_service_account_authz_token() - headers = {'Authorization': f'Bearer {authz_token.accessToken}'} + headers = {'Authorization': f'Bearer {authz_token['accessToken']}'} parsed = urlparse(settings.KEYCLOAK_AUTHORIZE_URL) r = requests.get(f"{parsed.scheme}://{parsed.netloc}/auth/admin/realms/{settings.GATEWAY_ID}/users", params={'username': username}, @@ -100,7 +100,7 @@ def update_username(username, new_username): def update_user(username, first_name=None, last_name=None, email=None): # fetch user representation authz_token = utils.get_service_account_authz_token() - headers = {'Authorization': f'Bearer {authz_token.accessToken}'} + headers = {'Authorization': f'Bearer {authz_token['accessToken']}'} parsed = urlparse(settings.KEYCLOAK_AUTHORIZE_URL) r = requests.get(f"{parsed.scheme}://{parsed.netloc}/auth/admin/realms/{settings.GATEWAY_ID}/users", params={'username': username}, diff --git a/airavata-django-portal/django_airavata/apps/auth/tests/test_signals.py b/airavata-django-portal/django_airavata/apps/auth/tests/test_signals.py index 7ba1111fe..f6b2b1496 100644 --- a/airavata-django-portal/django_airavata/apps/auth/tests/test_signals.py +++ b/airavata-django-portal/django_airavata/apps/auth/tests/test_signals.py @@ -1,5 +1,4 @@ -from airavata.model.group.ttypes import GroupModel -from airavata.model.user.ttypes import UserProfile +from django_airavata.proto_compat import GroupModel, UserProfile from django.core import mail from django.shortcuts import reverse from django.test import RequestFactory, TestCase, override_settings diff --git a/airavata-django-portal/django_airavata/apps/auth/tests/test_views.py b/airavata-django-portal/django_airavata/apps/auth/tests/test_views.py index 384dfcfde..e0af7f085 100644 --- a/airavata-django-portal/django_airavata/apps/auth/tests/test_views.py +++ b/airavata-django-portal/django_airavata/apps/auth/tests/test_views.py @@ -1,7 +1,7 @@ from unittest.mock import patch from urllib.parse import urlencode -from airavata.model.user.ttypes import UserProfile +from django_airavata.proto_compat import UserProfile from django.contrib import messages from django.contrib.auth.models import AnonymousUser from django.contrib.messages.middleware import MessageMiddleware diff --git a/airavata-django-portal/django_airavata/apps/auth/views.py b/airavata-django-portal/django_airavata/apps/auth/views.py index a05b2480e..012a250e2 100644 --- a/airavata-django-portal/django_airavata/apps/auth/views.py +++ b/airavata-django-portal/django_airavata/apps/auth/views.py @@ -558,7 +558,7 @@ def access_token_redirect(request): "in ACCESS_TOKEN_REDIRECT_ALLOWED_URIS setting") return HttpResponseForbidden("Invalid redirect_uri value") return redirect(redirect_uri + f"{'&' if '?' in redirect_uri else '?'}{config.get('PARAM_NAME', 'access_token')}=" - f"{quote(request.authz_token.accessToken)}") + f"{quote(request.authz_token['accessToken'])}") @login_required diff --git a/airavata-django-portal/django_airavata/apps/workspace/views.py b/airavata-django-portal/django_airavata/apps/workspace/views.py index f88078cad..e3b1b7b90 100644 --- a/airavata-django-portal/django_airavata/apps/workspace/views.py +++ b/airavata-django-portal/django_airavata/apps/workspace/views.py @@ -3,7 +3,7 @@ import json import logging from urllib.parse import urlparse -from airavata.model.application.io.ttypes import DataType +from django_airavata.proto_compat import DataType from airavata_django_portal_sdk import user_storage as user_storage_sdk from django.contrib.auth.decorators import login_required from django.shortcuts import render diff --git a/airavata-django-portal/django_airavata/proto_compat.py b/airavata-django-portal/django_airavata/proto_compat.py new file mode 100644 index 000000000..8fa0ac9a7 --- /dev/null +++ b/airavata-django-portal/django_airavata/proto_compat.py @@ -0,0 +1,342 @@ +"""Compatibility layer for proto/Thrift types used throughout the portal. + +These were previously imported from airavata.model.*.ttypes (Thrift-generated). +Now they come from the SDK's generated proto stubs. This module provides a +single import point so that downstream code doesn't need to know the origin. + +For types that are only used as plain data containers (constructed with **kwargs +and read via attribute access), simple Python classes with __init__(**kwargs) +suffice since the SDK's gRPC facade returns these objects already. +""" + +import enum + + +# --------------------------------------------------------------------------- +# Enums +# --------------------------------------------------------------------------- + +class DataType(enum.IntEnum): + STRING = 0 + INTEGER = 1 + FLOAT = 2 + URI = 3 + URI_COLLECTION = 4 + STDOUT = 5 + STDERR = 6 + + +class ExperimentState(enum.IntEnum): + CREATED = 0 + VALIDATED = 1 + SCHEDULED = 2 + LAUNCHED = 3 + EXECUTING = 4 + CANCELING = 5 + CANCELED = 6 + COMPLETED = 7 + FAILED = 8 + + +class ExperimentType(enum.IntEnum): + SINGLE_APPLICATION = 0 + WORKFLOW = 1 + + +class ExperimentSearchFields(enum.IntEnum): + EXPERIMENT_NAME = 0 + EXPERIMENT_DESC = 1 + APPLICATION_ID = 2 + STATUS = 3 + CREATION_TIME = 4 + PROJECT_ID = 5 + JOB_ID = 6 + + +class SummaryType(enum.IntEnum): + SSH = 0 + PASSWD = 1 + CERT = 2 + + +class ResourcePermissionType(enum.IntEnum): + WRITE = 0 + READ = 1 + MANAGE_SHARING = 2 + + +class ResourceType(enum.IntEnum): + SLURM = 0 + AWS = 1 + + +class ApplicationParallelismType(enum.IntEnum): + SERIAL = 0 + MPI = 1 + OPENMP = 2 + OPENMP_MPI = 3 + CCM = 4 + CRAY_MPI = 5 + + +class NotificationPriority(enum.IntEnum): + LOW = 0 + NORMAL = 1 + HIGH = 2 + + +class Status(enum.IntEnum): + ACTIVE = 0 + CONFIRMED = 1 + APPROVED = 2 + DELETED = 3 + DUPLICATE = 4 + GRACE_PERIOD = 5 + INVITED = 6 + DENIED = 7 + PENDING = 8 + PENDING_APPROVAL = 9 + PENDING_CONFIRMATION = 10 + SUSPENDED = 11 + DECLINED = 12 + EXPIRED = 13 + + +# --------------------------------------------------------------------------- +# Data classes -- simple attribute-bag objects that mirror Thrift struct +# constructors. The SDK facade returns proto objects that behave the same +# way (attribute access), so these are used mainly for *creating* new +# instances in serializers. +# --------------------------------------------------------------------------- + +class _ThriftLikeBase: + """Base that accepts **kwargs and sets them as attributes, with a + thrift_spec class variable for backward compat with the serializer + introspection in thrift_utils.py (now proto_utils.py).""" + + # Subclasses should define thrift_spec as a class variable. + thrift_spec = () + + def __init__(self, **kwargs): + for key, value in kwargs.items(): + setattr(self, key, value) + + def __repr__(self): + attrs = ', '.join(f'{k}={v!r}' for k, v in self.__dict__.items()) + return f'{self.__class__.__name__}({attrs})' + + +# -- Application IO -- + +class InputDataObjectType(_ThriftLikeBase): + pass + + +class OutputDataObjectType(_ThriftLikeBase): + pass + + +# -- App Catalog: deployment -- + +class ApplicationDeploymentDescription(_ThriftLikeBase): + pass + + +class ApplicationModule(_ThriftLikeBase): + pass + + +class CommandObject(_ThriftLikeBase): + pass + + +class SetEnvPaths(_ThriftLikeBase): + pass + + +# -- App Catalog: interface -- + +class ApplicationInterfaceDescription(_ThriftLikeBase): + pass + + +# -- App Catalog: compute resource -- + +class ComputeResourceDescription(_ThriftLikeBase): + pass + + +class BatchQueue(_ThriftLikeBase): + pass + + +class CloudJobSubmission(_ThriftLikeBase): + pass + + +class GlobusJobSubmission(_ThriftLikeBase): + pass + + +class LOCALSubmission(_ThriftLikeBase): + pass + + +class SSHJobSubmission(_ThriftLikeBase): + pass + + +class UnicoreJobSubmission(_ThriftLikeBase): + pass + + +# -- App Catalog: gateway profile -- + +class GatewayResourceProfile(_ThriftLikeBase): + pass + + +class StoragePreference(_ThriftLikeBase): + pass + + +# -- App Catalog: group resource profile -- + +class ComputeResourceReservation(_ThriftLikeBase): + pass + + +class GroupComputeResourcePreference(_ThriftLikeBase): + pass + + +class GroupResourceProfile(_ThriftLikeBase): + pass + + +class ComputeResourcePolicy(_ThriftLikeBase): + pass + + +class BatchQueueResourcePolicy(_ThriftLikeBase): + pass + + +class EnvironmentSpecificPreferences(_ThriftLikeBase): + pass + + +class SlurmComputeResourcePreference(_ThriftLikeBase): + pass + + +class AwsComputeResourcePreference(_ThriftLikeBase): + pass + + +class GroupAccountSSHProvisionerConfig(_ThriftLikeBase): + pass + + +# -- App Catalog: parser -- + +class Parser(_ThriftLikeBase): + pass + + +# -- App Catalog: storage resource -- + +class StorageResourceDescription(_ThriftLikeBase): + pass + + +# -- Credential store -- + +class CredentialSummary(_ThriftLikeBase): + pass + + +# -- Data replica -- + +class DataProductModel(_ThriftLikeBase): + pass + + +class DataReplicaLocationModel(_ThriftLikeBase): + pass + + +# -- Data movement -- + +class GridFTPDataMovement(_ThriftLikeBase): + pass + + +class LOCALDataMovement(_ThriftLikeBase): + pass + + +class SCPDataMovement(_ThriftLikeBase): + pass + + +class UnicoreDataMovement(_ThriftLikeBase): + pass + + +# -- Experiment -- + +class ExperimentModel(_ThriftLikeBase): + pass + + +class ExperimentStatistics(_ThriftLikeBase): + pass + + +class ExperimentSummaryModel(_ThriftLikeBase): + pass + + +# -- Group -- + +class GroupModel(_ThriftLikeBase): + pass + + +# -- Job -- + +class JobModel(_ThriftLikeBase): + pass + + +# -- Status -- + +class ExperimentStatus(_ThriftLikeBase): + pass + + +class ProcessStatus(_ThriftLikeBase): + pass + + +# -- User -- + +class UserProfile(_ThriftLikeBase): + pass + + +# -- Workspace -- + +class Notification(_ThriftLikeBase): + pass + + +class Project(_ThriftLikeBase): + pass + + +# -- Gateway groups -- + +class GatewayGroups(_ThriftLikeBase): + pass
