This is an automated email from the ASF dual-hosted git repository.
yasithdev pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airavata-portals.git
The following commit(s) were added to refs/heads/main by this push:
new 77396b168 refactor(portal): make application-deployment serializers
proto-native (Track D) (#201)
77396b168 is described below
commit 77396b168d75209929acb7af9ae4dca95ff8e6f0
Author: Yasith Jayawardana <[email protected]>
AuthorDate: Tue Jun 9 04:15:48 2026 -0400
refactor(portal): make application-deployment serializers proto-native
(Track D) (#201)
Rewrite ApplicationDeploymentDescriptionSerializer + the nested
CommandObject /
SetEnvPaths serializers to read the gRPC ApplicationDeploymentDescription
protobuf directly, emitting the same Thrift-named JSON keys.
- parallelism renders as the Thrift int via the prefix-aligned parallelism
field
helper; the OrderedListField command/env-path lists still sort by
commandOrder/envPathOrder; the queue-default cluster (defaultQueueName/
NodeCount/CPUCount/Walltime) maps proto-0/'' -> null as before.
- Repoint ApplicationDeploymentViewSet (list/instance/create/update/destroy/
queues) + ApplicationModuleViewSet.application_deployments to pass
protobuf
through directly (the queues action now mutates proto batch-queue fields),
dropping
grpc_adapters.{application_deployment,_command_object,_set_env_paths}
+ grpc_requests equivalents and the Thrift
ApplicationDeploymentDescription/CommandObject/SetEnvPaths/ParallelismType
imports.
Validated byte-for-byte (deployment full incl. sorted command/env lists +
parallelism int + queue defaults, and minimal) vs the old adapter+serializer
path; write path (incl. nested lists + parallelism reverse-bridge) produces
an
equivalent proto. manage.py check green; api test failures unchanged.
---
.../django_airavata/apps/api/grpc_adapters.py | 43 -----
.../django_airavata/apps/api/grpc_requests.py | 44 -----
.../django_airavata/apps/api/serializers.py | 185 ++++++++++++++++-----
.../django_airavata/apps/api/views.py | 39 +++--
4 files changed, 161 insertions(+), 150 deletions(-)
diff --git a/airavata-django-portal/django_airavata/apps/api/grpc_adapters.py
b/airavata-django-portal/django_airavata/apps/api/grpc_adapters.py
index e2780b9ba..c5a01bff1 100644
--- a/airavata-django-portal/django_airavata/apps/api/grpc_adapters.py
+++ b/airavata-django-portal/django_airavata/apps/api/grpc_adapters.py
@@ -19,9 +19,6 @@ from airavata.model.appcatalog.computeresource.ttypes import (
from airavata.model.appcatalog.groupresourceprofile.ttypes import (
ResourceType as _ThriftResourceType,
)
-from airavata.model.appcatalog.parallelism.ttypes import (
- ApplicationParallelismType as _ThriftParallelismType,
-)
from airavata.model.application.io.ttypes import DataType as _ThriftDataType
from airavata.model.data.replica.ttypes import (
DataProductType as _ThriftDataProductType,
@@ -169,46 +166,6 @@ def application_interface(pb):
)
-def _command_object(pb):
- """gRPC ``CommandObject`` -> ``CommandObjectSerializer`` shape."""
- return SimpleNamespace(command=pb.command, commandOrder=pb.command_order)
-
-
-def _set_env_paths(pb):
- """gRPC ``SetEnvPaths`` -> ``SetEnvPathsSerializer`` shape."""
- return SimpleNamespace(
- name=pb.name, value=pb.value, envPathOrder=pb.env_path_order)
-
-
-def application_deployment(pb):
- """gRPC ``ApplicationDeploymentDescription`` ->
``ApplicationDeploymentDescriptionSerializer`` shape.
-
- Recursively adapts the nested command/env-path lists. ``parallelism`` is an
- enum with the proto/Thrift integer mismatch (proto SERIAL=1 vs Thrift
- SERIAL=0) -> bridged by name. The queue-default cluster maps the proto-zero
- "unset" sentinels back to None so the serializer renders null as Thrift
did.
- """
- return SimpleNamespace(
- appDeploymentId=pb.app_deployment_id,
- appModuleId=pb.app_module_id,
- computeHostId=pb.compute_host_id,
- executablePath=pb.executable_path,
- parallelism=_thrift_enum(pb, 'parallelism', _ThriftParallelismType),
- appDeploymentDescription=pb.app_deployment_description,
- moduleLoadCmds=[_command_object(c) for c in pb.module_load_cmds],
- libPrependPaths=[_set_env_paths(p) for p in pb.lib_prepend_paths],
- libAppendPaths=[_set_env_paths(p) for p in pb.lib_append_paths],
- setEnvironment=[_set_env_paths(p) for p in pb.set_environment],
- preJobCommands=[_command_object(c) for c in pb.pre_job_commands],
- postJobCommands=[_command_object(c) for c in pb.post_job_commands],
- defaultQueueName=pb.default_queue_name or None,
- defaultNodeCount=pb.default_node_count or None,
- defaultCPUCount=pb.default_cpu_count or None,
- defaultWalltime=pb.default_walltime or None,
- editableByUser=pb.editable_by_user,
- )
-
-
# proto ResourceType member name -> Thrift ResourceType value (names align,
# ints differ: proto SLURM=1 vs Thrift SLURM=0).
_RESOURCE_TYPE = {
diff --git a/airavata-django-portal/django_airavata/apps/api/grpc_requests.py
b/airavata-django-portal/django_airavata/apps/api/grpc_requests.py
index 5af94b865..7cb80e006 100644
--- a/airavata-django-portal/django_airavata/apps/api/grpc_requests.py
+++ b/airavata-django-portal/django_airavata/apps/api/grpc_requests.py
@@ -17,9 +17,6 @@ import importlib
from airavata.model.appcatalog.computeresource.ttypes import (
JobSubmissionProtocol as _ThriftJobSubmissionProtocol,
)
-from airavata.model.appcatalog.parallelism.ttypes import (
- ApplicationParallelismType as _ThriftParallelismType,
-)
from airavata.model.appcatalog.groupresourceprofile.ttypes import (
ResourceType as _ThriftResourceType,
)
@@ -125,47 +122,6 @@ def application_interface(t):
)
-def _command_object(t):
- return _pb2("appcatalog.appdeployment.app_deployment_pb2").CommandObject(
- command=t.command or '',
- command_order=t.commandOrder or 0,
- )
-
-
-def _set_env_paths(t):
- return _pb2("appcatalog.appdeployment.app_deployment_pb2").SetEnvPaths(
- name=t.name or '',
- value=t.value or '',
- env_path_order=t.envPathOrder or 0,
- )
-
-
-def application_deployment(t):
- """Thrift ``ApplicationDeploymentDescription`` -> proto message."""
- dep = _pb2("appcatalog.appdeployment.app_deployment_pb2")
- return dep.ApplicationDeploymentDescription(
- app_deployment_id=t.appDeploymentId or '',
- app_module_id=t.appModuleId or '',
- compute_host_id=t.computeHostId or '',
- executable_path=t.executablePath or '',
- parallelism=_proto_enum(
- _pb2("parallelism.parallelism_pb2").ApplicationParallelismType,
- _ThriftParallelismType, t.parallelism),
- app_deployment_description=t.appDeploymentDescription or '',
- module_load_cmds=[_command_object(c) for c in (t.moduleLoadCmds or
[])],
- lib_prepend_paths=[_set_env_paths(p) for p in (t.libPrependPaths or
[])],
- lib_append_paths=[_set_env_paths(p) for p in (t.libAppendPaths or [])],
- set_environment=[_set_env_paths(p) for p in (t.setEnvironment or [])],
- pre_job_commands=[_command_object(c) for c in (t.preJobCommands or
[])],
- post_job_commands=[_command_object(c) for c in (t.postJobCommands or
[])],
- default_queue_name=t.defaultQueueName or '',
- default_node_count=t.defaultNodeCount or 0,
- default_cpu_count=t.defaultCPUCount or 0,
- default_walltime=t.defaultWalltime or 0,
- editable_by_user=bool(t.editableByUser),
- )
-
-
# proto-name -> Thrift protocol value maps (mirror grpc_adapters); inverted
# below to go Thrift value -> proto value, preserving the divergent name pairs
# (Thrift CLOUD <-> proto JSP_CLOUD; Thrift LOCAL <-> proto
diff --git a/airavata-django-portal/django_airavata/apps/api/serializers.py
b/airavata-django-portal/django_airavata/apps/api/serializers.py
index ab1e225c3..d65db1301 100644
--- a/airavata-django-portal/django_airavata/apps/api/serializers.py
+++ b/airavata-django-portal/django_airavata/apps/api/serializers.py
@@ -7,11 +7,6 @@ from pathlib import Path
from urllib.parse import quote
from airavata.model.application.io.ttypes import DataType
-from airavata.model.appcatalog.appdeployment.ttypes import (
- ApplicationDeploymentDescription,
- CommandObject,
- SetEnvPaths
-)
from airavata.model.appcatalog.appinterface.ttypes import (
ApplicationInterfaceDescription
)
@@ -761,58 +756,162 @@ class
ApplicationInterfaceDescriptionSerializer(serializers.Serializer):
return request.is_gateway_admin
-class CommandObjectSerializer(
- thrift_utils.create_serializer_class(CommandObject)):
- pass
+def _app_deployment_pb2():
+ from
airavata_sdk.generated.org.apache.airavata.model.appcatalog.appdeployment
import ( # noqa: E501
+ app_deployment_pb2,
+ )
+ return app_deployment_pb2
+
+def _parallelism_field(**kwargs):
+ from airavata.model.appcatalog.parallelism.ttypes import (
+ ApplicationParallelismType as _ThriftParallelismType,
+ )
+ from airavata_sdk.generated.org.apache.airavata.model.parallelism import (
+ parallelism_pb2,
+ )
+ return proto_enum_int_field(
+ parallelism_pb2.ApplicationParallelismType.DESCRIPTOR,
+ _ThriftParallelismType, proto_prefix='APPLICATION_PARALLELISM_TYPE_',
+ **kwargs)
-class SetEnvPathsSerializer(
- thrift_utils.create_serializer_class(SetEnvPaths)):
- pass
+class CommandObjectSerializer(serializers.Serializer):
+ """Proto-native serializer for the gRPC ``CommandObject`` message."""
-class ApplicationDeploymentDescriptionSerializer(
- thrift_utils.create_serializer_class(
- ApplicationDeploymentDescription)):
+ command = serializers.CharField(
+ allow_blank=True, allow_null=True, required=False)
+ commandOrder = serializers.IntegerField(
+ source='command_order', allow_null=True, required=False)
+
+ def create(self, validated_data):
+ return _app_deployment_pb2().CommandObject(
+ command=validated_data.get('command', '') or '',
+ command_order=validated_data.get('command_order', 0) or 0,
+ )
+
+
+class SetEnvPathsSerializer(serializers.Serializer):
+ """Proto-native serializer for the gRPC ``SetEnvPaths`` message."""
+
+ name = serializers.CharField(
+ allow_blank=True, allow_null=True, required=False)
+ value = serializers.CharField(
+ allow_blank=True, allow_null=True, required=False)
+ envPathOrder = serializers.IntegerField(
+ source='env_path_order', allow_null=True, required=False)
+
+ def create(self, validated_data):
+ return _app_deployment_pb2().SetEnvPaths(
+ name=validated_data.get('name', '') or '',
+ value=validated_data.get('value', '') or '',
+ env_path_order=validated_data.get('env_path_order', 0) or 0,
+ )
+
+
+class ApplicationDeploymentDescriptionSerializer(serializers.Serializer):
+ """Proto-native serializer for the gRPC
``ApplicationDeploymentDescription``."""
+
+ appDeploymentId = serializers.CharField(
+ source='app_deployment_id', allow_blank=True, allow_null=True,
+ required=False)
+ appModuleId = serializers.CharField(
+ source='app_module_id', allow_blank=True, allow_null=True,
+ required=False)
+ computeHostId = serializers.CharField(
+ source='compute_host_id', allow_blank=True, allow_null=True,
+ required=False)
+ executablePath = serializers.CharField(
+ source='executable_path', allow_blank=True, allow_null=True,
+ required=False)
+ parallelism = _parallelism_field(required=False, allow_null=True)
+ appDeploymentDescription = serializers.CharField(
+ source='app_deployment_description', allow_blank=True, allow_null=True,
+ required=False)
+ moduleLoadCmds = OrderedListField(
+ source='module_load_cmds', order_by='commandOrder',
+ child=CommandObjectSerializer(), allow_null=True, required=False)
+ libPrependPaths = OrderedListField(
+ source='lib_prepend_paths', order_by='envPathOrder',
+ child=SetEnvPathsSerializer(), allow_null=True, required=False)
+ libAppendPaths = OrderedListField(
+ source='lib_append_paths', order_by='envPathOrder',
+ child=SetEnvPathsSerializer(), allow_null=True, required=False)
+ setEnvironment = OrderedListField(
+ source='set_environment', order_by='envPathOrder',
+ child=SetEnvPathsSerializer(), allow_null=True, required=False)
+ preJobCommands = OrderedListField(
+ source='pre_job_commands', order_by='commandOrder',
+ child=CommandObjectSerializer(), allow_null=True, required=False)
+ postJobCommands = OrderedListField(
+ source='post_job_commands', order_by='commandOrder',
+ child=CommandObjectSerializer(), allow_null=True, required=False)
+ defaultQueueName = serializers.CharField(
+ source='default_queue_name', allow_blank=True, allow_null=True,
+ required=False)
+ defaultNodeCount = ProtoIntOrNoneField(source='default_node_count')
+ defaultCPUCount = ProtoIntOrNoneField(source='default_cpu_count')
+ defaultWalltime = ProtoIntOrNoneField(source='default_walltime')
+ editableByUser = serializers.BooleanField(
+ source='editable_by_user', required=False, default=False)
url = FullyEncodedHyperlinkedIdentityField(
view_name='django_airavata_api:application-deployment-detail',
- lookup_field='appDeploymentId',
+ lookup_field='app_deployment_id',
lookup_url_kwarg='app_deployment_id')
- # Default values returned in these results have been overridden with app
- # deployment defaults for any that exist
queues = FullyEncodedHyperlinkedIdentityField(
view_name='django_airavata_api:application-deployment-queues',
- lookup_field='appDeploymentId',
+ lookup_field='app_deployment_id',
lookup_url_kwarg='app_deployment_id')
userHasWriteAccess = serializers.SerializerMethodField()
- moduleLoadCmds = OrderedListField(
- order_by='commandOrder',
- child=CommandObjectSerializer(),
- allow_null=True)
- preJobCommands = OrderedListField(
- order_by='commandOrder',
- child=CommandObjectSerializer(),
- allow_null=True)
- postJobCommands = OrderedListField(
- order_by='commandOrder',
- child=CommandObjectSerializer(),
- allow_null=True)
- libPrependPaths = OrderedListField(
- order_by='envPathOrder',
- child=SetEnvPathsSerializer(),
- allow_null=True)
- libAppendPaths = OrderedListField(
- order_by='envPathOrder',
- child=SetEnvPathsSerializer(),
- allow_null=True)
- setEnvironment = OrderedListField(
- order_by='envPathOrder',
- child=SetEnvPathsSerializer(),
- allow_null=True)
+
+ def to_representation(self, instance):
+ ret = super().to_representation(instance)
+ # The proto string default '' must render as null for the optional
+ # default-queue-name field (the old adapter mapped pb.<f> or None).
+ if ret.get('defaultQueueName') == '':
+ ret['defaultQueueName'] = None
+ return ret
def get_userHasWriteAccess(self, appDeployment):
return user_has_access(
- self.context['request'], appDeployment.appDeploymentId)
+ self.context['request'], appDeployment.app_deployment_id)
+
+ def create(self, validated_data):
+ a = _app_deployment_pb2()
+ return a.ApplicationDeploymentDescription(
+ app_module_id=validated_data.get('app_module_id', '') or '',
+ compute_host_id=validated_data.get('compute_host_id', '') or '',
+ executable_path=validated_data.get('executable_path', '') or '',
+ parallelism=validated_data.get('parallelism', 0) or 0,
+ app_deployment_description=validated_data.get(
+ 'app_deployment_description', '') or '',
+ module_load_cmds=[
+ CommandObjectSerializer().create(c)
+ for c in validated_data.get('module_load_cmds', []) or []],
+ lib_prepend_paths=[
+ SetEnvPathsSerializer().create(p)
+ for p in validated_data.get('lib_prepend_paths', []) or []],
+ lib_append_paths=[
+ SetEnvPathsSerializer().create(p)
+ for p in validated_data.get('lib_append_paths', []) or []],
+ set_environment=[
+ SetEnvPathsSerializer().create(p)
+ for p in validated_data.get('set_environment', []) or []],
+ pre_job_commands=[
+ CommandObjectSerializer().create(c)
+ for c in validated_data.get('pre_job_commands', []) or []],
+ post_job_commands=[
+ CommandObjectSerializer().create(c)
+ for c in validated_data.get('post_job_commands', []) or []],
+ default_queue_name=validated_data.get('default_queue_name', '') or
'',
+ default_node_count=validated_data.get('default_node_count', 0) or
0,
+ default_cpu_count=validated_data.get('default_cpu_count', 0) or 0,
+ default_walltime=validated_data.get('default_walltime', 0) or 0,
+ editable_by_user=bool(validated_data.get('editable_by_user',
False)),
+ )
+
+ def update(self, instance, validated_data):
+ return self.create(validated_data)
class BatchQueueSerializer(serializers.Serializer):
diff --git a/airavata-django-portal/django_airavata/apps/api/views.py
b/airavata-django-portal/django_airavata/apps/api/views.py
index 4382fd180..327caad6d 100644
--- a/airavata-django-portal/django_airavata/apps/api/views.py
+++ b/airavata-django-portal/django_airavata/apps/api/views.py
@@ -522,12 +522,11 @@ class ApplicationModuleViewSet(APIBackedViewSet):
@action(detail=True)
def application_deployments(self, request, app_module_id):
- all_deployments = [
- grpc_adapters.application_deployment(d)
- for d in self.request.airavata.research
- .get_accessible_application_deployments(self.gateway_id)]
+ all_deployments = (
+ self.request.airavata.research
+ .get_accessible_application_deployments(self.gateway_id))
app_deployments = [
- dep for dep in all_deployments if dep.appModuleId == app_module_id]
+ dep for dep in all_deployments if dep.app_module_id ==
app_module_id]
serializer = serializers.ApplicationDeploymentDescriptionSerializer(
app_deployments, many=True, context={'request': request})
return Response(serializer.data)
@@ -662,48 +661,48 @@ class ApplicationDeploymentViewSet(APIBackedViewSet):
else:
deployments =
self.request.airavata.research.get_accessible_application_deployments(
self.gateway_id)
- return [grpc_adapters.application_deployment(d) for d in deployments]
+ return list(deployments)
def get_instance(self, lookup_value):
- return grpc_adapters.application_deployment(
-
self.request.airavata.research.get_application_deployment(lookup_value))
+ return self.request.airavata.research.get_application_deployment(
+ lookup_value)
def perform_create(self, serializer):
application_deployment = serializer.save()
app_deployment_id =
self.request.airavata.research.register_application_deployment(
- self.gateway_id,
grpc_requests.application_deployment(application_deployment))
- application_deployment.appDeploymentId = app_deployment_id
+ self.gateway_id, application_deployment)
+ application_deployment.app_deployment_id = app_deployment_id
def perform_update(self, serializer):
application_deployment = serializer.save()
self.request.airavata.research.update_application_deployment(
- application_deployment.appDeploymentId,
- grpc_requests.application_deployment(application_deployment))
+ application_deployment.app_deployment_id,
+ application_deployment)
def perform_destroy(self, instance):
self.request.airavata.research.delete_application_deployment(
- instance.appDeploymentId)
+ instance.app_deployment_id)
@action(detail=True)
def queues(self, request, app_deployment_id):
"""Return queues for this deployment with defaults overridden by
deployment defaults if they exist"""
- app_deployment = grpc_adapters.application_deployment(
+ app_deployment = (
self.request.airavata.research.get_application_deployment(
app_deployment_id))
compute_resource = request.airavata.compute.get_compute_resource(
- app_deployment.computeHostId)
+ app_deployment.compute_host_id)
# Override defaults with app deployment default queue, if defined
batch_queues = []
for batch_queue in compute_resource.batch_queues:
- if app_deployment.defaultQueueName:
- if app_deployment.defaultQueueName == batch_queue.queue_name:
+ if app_deployment.default_queue_name:
+ if app_deployment.default_queue_name == batch_queue.queue_name:
batch_queue.is_default_queue = True
batch_queue.default_node_count = (
- app_deployment.defaultNodeCount or 0)
+ app_deployment.default_node_count or 0)
batch_queue.default_cpu_count = (
- app_deployment.defaultCPUCount or 0)
+ app_deployment.default_cpu_count or 0)
batch_queue.default_walltime = (
- app_deployment.defaultWalltime or 0)
+ app_deployment.default_walltime or 0)
else:
batch_queue.is_default_queue = False
batch_queues.append(batch_queue)
