This is an automated email from the ASF dual-hosted git repository.
yasithdev pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airavata.git
The following commit(s) were added to refs/heads/master by this push:
new e23618c9fc feat(storage-service): surface per-file data-product URI in
listings (#653)
e23618c9fc is described below
commit e23618c9fc67bc18b9e18d5b65601ec08187dded
Author: Yasith Jayawardana <[email protected]>
AuthorDate: Tue Jun 9 02:12:19 2026 -0400
feat(storage-service): surface per-file data-product URI in listings (#653)
File listings (list_dir / get_file_metadata / list_experiment_dir) now
carry a
stable data product URI per file, so clients can reference each stored file
by
data product (downloads, deletes, selection) without a separate registry
table.
- DataProductInterface/DataProductRepository:
getDataProductByReplicaFilePath
(JPQL join on the replica file path + gateway) finds the data product
whose
gateway-data-store replica is a given path.
- StorageProvider/StorageProviderImpl: getOrCreateDataProductByPath returns
that
product's URI, registering a new FILE data product (single
GATEWAY_DATA_STORE
replica) if none exists. This replaces the legacy
file-path-to-data-product
mapping that the portal kept in its own ORM table.
- UserStorageGrpcService populates FileMetadataResponse.data_product_uri
for each
file (best-effort: directories and lookup failures yield an empty URI,
never a
listing failure).
Validated live: an uploaded file's get_file_metadata/list_dir return a
data_product_uri that is stable across calls (found, not re-registered) and
resolves via the data-product service to the file's replica.
storage-service +
research-service tests pass.
---
.../research/repository/DataProductRepository.java | 11 +++++
.../java/org/apache/airavata/db/DBConstants.java | 1 +
.../org/apache/airavata/db/QueryConstants.java | 4 ++
.../airavata/interfaces/DataProductInterface.java | 7 +++
.../airavata/interfaces/StorageProvider.java | 18 ++++++++
.../airavata/storage/StorageProviderImpl.java | 28 ++++++++++++
.../storage/grpc/UserStorageGrpcService.java | 50 +++++++++++++++++++---
7 files changed, 114 insertions(+), 5 deletions(-)
diff --git
a/airavata-api/research-service/src/main/java/org/apache/airavata/research/repository/DataProductRepository.java
b/airavata-api/research-service/src/main/java/org/apache/airavata/research/repository/DataProductRepository.java
index 514fb47b41..4663c7b4ec 100644
---
a/airavata-api/research-service/src/main/java/org/apache/airavata/research/repository/DataProductRepository.java
+++
b/airavata-api/research-service/src/main/java/org/apache/airavata/research/repository/DataProductRepository.java
@@ -195,6 +195,17 @@ public class DataProductRepository extends
AbstractRepository<DataProductModel,
return dataProductModelList;
}
+ @Override
+ public DataProductModel getDataProductByReplicaFilePath(String gatewayId,
String filePath)
+ throws ReplicaCatalogException {
+ Map<String, Object> queryParameters = new HashMap<>();
+ queryParameters.put(DBConstants.DataProduct.GATEWAY_ID, gatewayId);
+ queryParameters.put(DBConstants.DataProduct.FILE_PATH, filePath);
+ List<DataProductModel> matches =
+ select(QueryConstants.FIND_DATA_PRODUCT_BY_REPLICA_FILE_PATH,
1, 0, queryParameters);
+ return matches.isEmpty() ? null : matches.get(0);
+ }
+
@Override
public boolean isDataProductExists(String productUri) throws
ReplicaCatalogException {
return isExists(productUri);
diff --git a/airavata-api/src/main/java/org/apache/airavata/db/DBConstants.java
b/airavata-api/src/main/java/org/apache/airavata/db/DBConstants.java
index d6cccb4b60..606c76cd82 100644
--- a/airavata-api/src/main/java/org/apache/airavata/db/DBConstants.java
+++ b/airavata-api/src/main/java/org/apache/airavata/db/DBConstants.java
@@ -169,6 +169,7 @@ public class DBConstants {
public static final String OWNER_NAME = "ownerName";
public static final String PRODUCT_NAME = "productName";
public static final String PARENT_PRODUCT_URI = "parentProductUri";
+ public static final String FILE_PATH = "filePath";
}
public static class Workflow {
diff --git
a/airavata-api/src/main/java/org/apache/airavata/db/QueryConstants.java
b/airavata-api/src/main/java/org/apache/airavata/db/QueryConstants.java
index 52d1d8e405..8feff209d7 100644
--- a/airavata-api/src/main/java/org/apache/airavata/db/QueryConstants.java
+++ b/airavata-api/src/main/java/org/apache/airavata/db/QueryConstants.java
@@ -208,6 +208,10 @@ public interface QueryConstants {
+ DBConstants.DataProduct.GATEWAY_ID + " AND DP.ownerName LIKE :"
+ DBConstants.DataProduct.OWNER_NAME
+ " AND DP.productName LIKE :" +
DBConstants.DataProduct.PRODUCT_NAME;
+ String FIND_DATA_PRODUCT_BY_REPLICA_FILE_PATH = "SELECT DP FROM " +
"DataProductEntity" + " DP "
+ + "JOIN DP.replicaLocations R WHERE DP.gatewayId = :"
+ + DBConstants.DataProduct.GATEWAY_ID + " AND R.filePath = :" +
DBConstants.DataProduct.FILE_PATH;
+
String GET_WORKFLOW_FOR_EXPERIMENT_ID = "SELECT W FROM " +
AIRAVATA_WORKFLOW_ENTITY + " W "
+ "WHERE W.experimentId LIKE :" +
DBConstants.Workflow.EXPERIMENT_ID;
diff --git
a/airavata-api/src/main/java/org/apache/airavata/interfaces/DataProductInterface.java
b/airavata-api/src/main/java/org/apache/airavata/interfaces/DataProductInterface.java
index 8c0b62d61c..ef63de1f9a 100644
---
a/airavata-api/src/main/java/org/apache/airavata/interfaces/DataProductInterface.java
+++
b/airavata-api/src/main/java/org/apache/airavata/interfaces/DataProductInterface.java
@@ -39,6 +39,13 @@ public interface DataProductInterface {
List<DataProductModel> searchDataProductsByName(
String gatewayId, String userId, String productName, int limit,
int offset) throws ReplicaCatalogException;
+ /**
+ * Return the data product in the gateway that has a replica at the given
file path, or
+ * {@code null} if none exists.
+ */
+ DataProductModel getDataProductByReplicaFilePath(String gatewayId, String
filePath)
+ throws ReplicaCatalogException;
+
boolean isDataProductExists(String productUri) throws
ReplicaCatalogException;
boolean removeDataProduct(String productUri) throws
ReplicaCatalogException;
diff --git
a/airavata-api/src/main/java/org/apache/airavata/interfaces/StorageProvider.java
b/airavata-api/src/main/java/org/apache/airavata/interfaces/StorageProvider.java
index 51b43fa9f7..9484aede7f 100644
---
a/airavata-api/src/main/java/org/apache/airavata/interfaces/StorageProvider.java
+++
b/airavata-api/src/main/java/org/apache/airavata/interfaces/StorageProvider.java
@@ -203,4 +203,22 @@ public interface StorageProvider {
* @throws Exception if a data access error occurs
*/
boolean removeReplicaLocation(String replicaId) throws Exception;
+
+ /**
+ * Return the product URI of the data product whose gateway-data-store
replica is the given
+ * file path, registering a new {@code FILE} data product for it if none
exists. This is the
+ * server-side equivalent of the legacy per-file file-path-to-data-product
mapping, so callers
+ * (e.g. file listings) can surface a stable data product URI for each
stored file.
+ *
+ * @param gatewayId the gateway id
+ * @param ownerName the file owner's user id
+ * @param fileName the file name (used as the product name when creating)
+ * @param filePath the absolute file path on the storage resource
+ * @param storageResourceId the storage resource id holding the file
+ * @return the existing or newly registered data product URI
+ * @throws Exception if a data access error occurs
+ */
+ String getOrCreateDataProductByPath(
+ String gatewayId, String ownerName, String fileName, String
filePath, String storageResourceId)
+ throws Exception;
}
diff --git
a/airavata-api/storage-service/src/main/java/org/apache/airavata/storage/StorageProviderImpl.java
b/airavata-api/storage-service/src/main/java/org/apache/airavata/storage/StorageProviderImpl.java
index ac9b14d13d..e133574079 100644
---
a/airavata-api/storage-service/src/main/java/org/apache/airavata/storage/StorageProviderImpl.java
+++
b/airavata-api/storage-service/src/main/java/org/apache/airavata/storage/StorageProviderImpl.java
@@ -27,7 +27,10 @@ import org.apache.airavata.interfaces.StorageProvider;
import
org.apache.airavata.model.appcatalog.storageresource.proto.StorageResourceDescription;
import org.apache.airavata.model.data.movement.proto.DataMovementInterface;
import org.apache.airavata.model.data.replica.proto.DataProductModel;
+import org.apache.airavata.model.data.replica.proto.DataProductType;
import org.apache.airavata.model.data.replica.proto.DataReplicaLocationModel;
+import org.apache.airavata.model.data.replica.proto.ReplicaLocationCategory;
+import org.apache.airavata.model.data.replica.proto.ReplicaPersistentType;
import org.apache.airavata.storage.repository.StorageResourceRepository;
import org.springframework.stereotype.Service;
@@ -135,4 +138,29 @@ public class StorageProviderImpl implements
StorageProvider {
public boolean removeReplicaLocation(String replicaId) throws Exception {
return dataReplicaLocationRepository.removeReplicaLocation(replicaId);
}
+
+ @Override
+ public String getOrCreateDataProductByPath(
+ String gatewayId, String ownerName, String fileName, String
filePath, String storageResourceId)
+ throws Exception {
+ DataProductModel existing =
dataProductRepository.getDataProductByReplicaFilePath(gatewayId, filePath);
+ if (existing != null) {
+ return existing.getProductUri();
+ }
+ DataReplicaLocationModel replica =
DataReplicaLocationModel.newBuilder()
+ .setReplicaName(fileName + " gateway data store copy")
+
.setReplicaLocationCategory(ReplicaLocationCategory.GATEWAY_DATA_STORE)
+ .setReplicaPersistentType(ReplicaPersistentType.TRANSIENT)
+ .setStorageResourceId(storageResourceId != null ?
storageResourceId : "")
+ .setFilePath(filePath)
+ .build();
+ DataProductModel product = DataProductModel.newBuilder()
+ .setGatewayId(gatewayId)
+ .setOwnerName(ownerName != null ? ownerName : "")
+ .setProductName(fileName)
+ .setDataProductType(DataProductType.FILE)
+ .addReplicaLocations(replica)
+ .build();
+ return dataProductRepository.registerDataProduct(product);
+ }
}
diff --git
a/airavata-api/storage-service/src/main/java/org/apache/airavata/storage/grpc/UserStorageGrpcService.java
b/airavata-api/storage-service/src/main/java/org/apache/airavata/storage/grpc/UserStorageGrpcService.java
index 79d121e9c0..d2d06486c5 100644
---
a/airavata-api/storage-service/src/main/java/org/apache/airavata/storage/grpc/UserStorageGrpcService.java
+++
b/airavata-api/storage-service/src/main/java/org/apache/airavata/storage/grpc/UserStorageGrpcService.java
@@ -33,6 +33,7 @@ import org.apache.airavata.grpc.GrpcStatusMapper;
import org.apache.airavata.interfaces.ExperimentRegistry;
import org.apache.airavata.interfaces.FileMetadata;
import org.apache.airavata.interfaces.GatewayStoragePreferenceProvider;
+import org.apache.airavata.interfaces.StorageProvider;
import org.apache.airavata.interfaces.StorageResourceAdaptor;
import
org.apache.airavata.model.appcatalog.gatewayprofile.proto.StoragePreference;
import org.apache.airavata.model.data.movement.proto.DataMovementProtocol;
@@ -52,14 +53,37 @@ public class UserStorageGrpcService extends
UserStorageServiceGrpc.UserStorageSe
private final AdaptorSupport adaptorSupport;
private final ExperimentRegistry experimentRegistry;
private final GatewayStoragePreferenceProvider
gatewayStoragePreferenceProvider;
+ private final StorageProvider storageProvider;
public UserStorageGrpcService(
AdaptorSupport adaptorSupport,
ExperimentRegistry experimentRegistry,
- GatewayStoragePreferenceProvider gatewayStoragePreferenceProvider)
{
+ GatewayStoragePreferenceProvider gatewayStoragePreferenceProvider,
+ StorageProvider storageProvider) {
this.adaptorSupport = adaptorSupport;
this.experimentRegistry = experimentRegistry;
this.gatewayStoragePreferenceProvider =
gatewayStoragePreferenceProvider;
+ this.storageProvider = storageProvider;
+ }
+
+ /**
+ * Resolve (registering if necessary) the data product URI for a stored
file, so listings and
+ * metadata can expose a stable URI per file. Returns "" for directories
or on any failure
+ * (best-effort: a missing data product URI must not fail the listing).
+ */
+ private String resolveDataProductUri(FileMetadata meta, String path,
String storageResourceId) {
+ if (meta.isDirectory()) {
+ return "";
+ }
+ try {
+ RequestContext ctx = GrpcRequestContext.current();
+ String uri = storageProvider.getOrCreateDataProductByPath(
+ ctx.getGatewayId(), ctx.getUserId(), meta.getName(), path,
storageResourceId);
+ return uri != null ? uri : "";
+ } catch (Exception e) {
+ logger.warn("Could not resolve data product URI for {}", path, e);
+ return "";
+ }
}
private StoragePreference resolveStoragePreference(String
storageResourceId) throws Exception {
@@ -79,6 +103,15 @@ public class UserStorageGrpcService extends
UserStorageServiceGrpc.UserStorageSe
return prefs.get(0);
}
+ /** Resolve the effective storage resource id: the request's, else the
gateway default. */
+ private String resolveStorageResourceId(String storageResourceId) throws
Exception {
+ if (storageResourceId != null && !storageResourceId.isEmpty()) {
+ return storageResourceId;
+ }
+ StoragePreference pref = resolveStoragePreference(storageResourceId);
+ return pref != null ? pref.getStorageResourceId() : "";
+ }
+
private StorageResourceAdaptor getStorageAdaptor(String storageResourceId)
throws Exception {
RequestContext ctx = GrpcRequestContext.current();
String resolvedId = storageResourceId;
@@ -214,6 +247,7 @@ public class UserStorageGrpcService extends
UserStorageServiceGrpc.UserStorageSe
try {
StorageResourceAdaptor adaptor =
getStorageAdaptor(request.getStorageResourceId());
String path = resolvePath(request.getPath(),
request.getStorageResourceId());
+ String storageResourceId =
resolveStorageResourceId(request.getStorageResourceId());
List<String> entries = adaptor.listDirectory(path);
ListDirResponse.Builder responseBuilder =
ListDirResponse.newBuilder();
@@ -221,7 +255,8 @@ public class UserStorageGrpcService extends
UserStorageServiceGrpc.UserStorageSe
for (String entry : entries) {
String fullPath = path.endsWith("/") ? path + entry : path +
"/" + entry;
FileMetadata meta = adaptor.getFileMetadata(fullPath);
- FileMetadataResponse fileMeta = toFileMetadataResponse(meta,
fullPath);
+ FileMetadataResponse fileMeta = toFileMetadataResponse(
+ meta, fullPath, resolveDataProductUri(meta, fullPath,
storageResourceId));
if (meta.isDirectory()) {
responseBuilder.addDirectories(fileMeta);
} else {
@@ -316,8 +351,10 @@ public class UserStorageGrpcService extends
UserStorageServiceGrpc.UserStorageSe
try {
StorageResourceAdaptor adaptor =
getStorageAdaptor(request.getStorageResourceId());
String path = resolvePath(request.getPath(),
request.getStorageResourceId());
+ String storageResourceId =
resolveStorageResourceId(request.getStorageResourceId());
FileMetadata meta = adaptor.getFileMetadata(path);
- observer.onNext(toFileMetadataResponse(meta, path));
+ observer.onNext(toFileMetadataResponse(
+ meta, path, resolveDataProductUri(meta, path,
storageResourceId)));
observer.onCompleted();
} catch (Exception e) {
observer.onError(GrpcStatusMapper.toStatusException(e));
@@ -375,7 +412,8 @@ public class UserStorageGrpcService extends
UserStorageServiceGrpc.UserStorageSe
String fullPath =
experimentDataDir.endsWith("/") ? experimentDataDir +
entry : experimentDataDir + "/" + entry;
FileMetadata meta = adaptor.getFileMetadata(fullPath);
- FileMetadataResponse fileMeta = toFileMetadataResponse(meta,
fullPath);
+ FileMetadataResponse fileMeta = toFileMetadataResponse(
+ meta, fullPath, resolveDataProductUri(meta, fullPath,
storageResourceId));
if (meta.isDirectory()) {
responseBuilder.addDirectories(fileMeta);
} else {
@@ -416,7 +454,8 @@ public class UserStorageGrpcService extends
UserStorageServiceGrpc.UserStorageSe
}
}
- private static FileMetadataResponse toFileMetadataResponse(FileMetadata
meta, String path) {
+ private static FileMetadataResponse toFileMetadataResponse(
+ FileMetadata meta, String path, String dataProductUri) {
return FileMetadataResponse.newBuilder()
.setName(meta.getName() != null ? meta.getName() : "")
.setPath(path)
@@ -424,6 +463,7 @@ public class UserStorageGrpcService extends
UserStorageServiceGrpc.UserStorageSe
.setIsDirectory(meta.isDirectory())
.setModifiedTime(meta.getModifiedTime())
.setContentType(meta.getContentType() != null ?
meta.getContentType() : "")
+ .setDataProductUri(dataProductUri != null ? dataProductUri :
"")
.build();
}
}
