(atlas) branch master updated: ATLAS-4866: Export/Import: Skip adding processed guids in audit while importing huge zip

[pinal]

This is an automated email from the ASF dual-hosted git repository.

pinal pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/atlas.git


The following commit(s) were added to refs/heads/master by this push:
     new c28930cf4 ATLAS-4866: Export/Import: Skip adding processed guids in 
audit while importing huge zip
c28930cf4 is described below

commit c28930cf4ae152e7cd93533d8d4fca2794ad5727
Author: Pinal Shah <pinal.s...@freestoneinfotech.com>
AuthorDate: Tue Jun 25 10:44:00 2024 +0530

    ATLAS-4866: Export/Import: Skip adding processed guids in audit while 
importing huge zip
    
    Signed-off-by: Pinal Shah <pinal.s...@freestoneinfotech.com>
---
 .../impexp/ExportImportAuditService.java           | 28 +++++++++++++++++++++-
 1 file changed, 27 insertions(+), 1 deletion(-)

diff --git 
a/repository/src/main/java/org/apache/atlas/repository/impexp/ExportImportAuditService.java
 
b/repository/src/main/java/org/apache/atlas/repository/impexp/ExportImportAuditService.java
index 3afa17301..67340f8a8 100644
--- 
a/repository/src/main/java/org/apache/atlas/repository/impexp/ExportImportAuditService.java
+++ 
b/repository/src/main/java/org/apache/atlas/repository/impexp/ExportImportAuditService.java
@@ -24,16 +24,20 @@ import org.apache.atlas.discovery.AtlasDiscoveryService;
 import org.apache.atlas.exception.AtlasBaseException;
 import org.apache.atlas.model.discovery.AtlasSearchResult;
 import org.apache.atlas.model.discovery.SearchParameters;
+import org.apache.atlas.model.impexp.AtlasImportResult;
 import org.apache.atlas.model.impexp.ExportImportAuditEntry;
 import org.apache.atlas.model.instance.AtlasEntityHeader;
+import org.apache.atlas.repository.audit.EntityAuditRepository;
 import org.apache.atlas.repository.ogm.DataAccess;
 import org.apache.atlas.repository.ogm.ExportImportAuditEntryDTO;
+import org.apache.atlas.type.AtlasType;
 import org.apache.commons.lang.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.util.CollectionUtils;
 
 import javax.inject.Inject;
+import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Set;
@@ -44,12 +48,14 @@ public class ExportImportAuditService {
     private static final String ENTITY_TYPE_NAME = "__ExportImportAuditEntry";
 
     private final DataAccess dataAccess;
+    private final EntityAuditRepository  auditRepository;
     private AtlasDiscoveryService discoveryService;
 
     @Inject
-    public ExportImportAuditService(DataAccess dataAccess, 
AtlasDiscoveryService discoveryService) {
+    public ExportImportAuditService(DataAccess dataAccess, 
AtlasDiscoveryService discoveryService, EntityAuditRepository auditRepository) {
         this.dataAccess = dataAccess;
         this.discoveryService = discoveryService;
+        this.auditRepository = auditRepository;
     }
 
     @GraphTransaction
@@ -164,6 +170,26 @@ public class ExportImportAuditService {
         entry.setStartTime(startTime);
         entry.setEndTime(endTime);
 
+        if (operation == ExportImportAuditEntry.OPERATION_IMPORT) {
+
+            String auditString = AtlasType.toJson(entry);
+            byte[] auditBytes  = auditString.getBytes(StandardCharsets.UTF_8);
+            long auditSize     = auditBytes != null ? auditBytes.length : 0;
+            long auditMaxSize  = auditRepository.repositoryMaxSize();
+
+            if (auditMaxSize >= 0 && auditSize > auditMaxSize) {
+
+                AtlasImportResult importResult = AtlasType.fromJson(result, 
AtlasImportResult.class);
+                if (importResult != null && 
!CollectionUtils.isEmpty(importResult.getProcessedEntities())) {
+                    LOG.warn("audit record too long: user={}, operation={}, 
size={}, maxSize={}, processedEntityCount={}, processed entities guids in 
resultSummary not stored in audit",
+                            entry.getUserName(), entry.getOperation(), 
auditSize, auditMaxSize, importResult.getProcessedEntities().size());
+
+                    importResult.getProcessedEntities().clear();
+                    entry.setResultSummary(AtlasType.toJson(importResult));
+                }
+            }
+        }
+
         save(entry);
         LOG.info("addAuditEntry: user: {}, source: {}, target: {}, operation: 
{}", entry.getUserName(),
                 entry.getSourceServerName(), entry.getTargetServerName(), 
entry.getOperation());