ATLAS-1724, ATLAS-1722: fix export to report error while exporting a non-existing entity

repository/pom.xml

@@ -33,6 +33,12 @@
 
     <dependencies>
         <dependency>
+            <groupId>org.powermock</groupId>
+            <artifactId>powermock-reflect</artifactId>
+            <version>1.6.1</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
             <groupId>org.apache.atlas</groupId>
             <artifactId>atlas-intg</artifactId>
         </dependency>

repository/src/main/java/org/apache/atlas/repository/impexp/ExportService.java

@@ -92,12 +92,41 @@ public class ExportService {
         try {
             LOG.info("==> export(user={}, from={})", userName, requestingIP);
 
-            for (AtlasObjectId item : request.getItemsToExport()) {
-                processObjectId(item, context);
+            AtlasExportResult.OperationStatus[] statuses = processItems(request, context);
+
+            processTypesDef(context);
+
+            updateSinkWithOperationMetrics(context, statuses, getOperationDuration(startTime));
+        } catch(Exception ex) {
+            LOG.error("Operation failed: ", ex);
+        } finally {
+            atlasGraph.releaseGremlinScriptEngine(context.scriptEngine);
+            LOG.info("<== export(user={}, from={}): status {}", userName, requestingIP, context.result.getOperationStatus());
+            context.clear();
+            result.clear();
         }
 
-            long endTime = System.currentTimeMillis();
+        return context.result;
+    }
+
+    private void updateSinkWithOperationMetrics(ExportContext context, AtlasExportResult.OperationStatus[] statuses, int duration) throws AtlasBaseException {
+        context.sink.setExportOrder(context.result.getData().getEntityCreationOrder());
+        context.sink.setTypesDef(context.result.getData().getTypesDef());
+        clearContextData(context);
+        context.result.setOperationStatus(getOverallOperationStatus(statuses));
+        context.result.incrementMeticsCounter("duration", duration);
+        context.sink.setResult(context.result);
+    }
+
+    private void clearContextData(ExportContext context) {
+        context.result.setData(null);
+    }
+
+    private int getOperationDuration(long startTime) {
+        return (int) (System.currentTimeMillis() - startTime);
+    }
 
+    private void processTypesDef(ExportContext context) {
         AtlasTypesDef typesDef = context.result.getData().getTypesDef();
 
         for (String entityType : context.entityTypes) {
@@ -123,33 +152,41 @@ public class ExportService {
 
             typesDef.getEnumDefs().add(enumDef);
         }
+    }
 
-            context.sink.setExportOrder(context.result.getData().getEntityCreationOrder());
-            context.sink.setTypesDef(context.result.getData().getTypesDef());
-            context.result.setData(null);
-            context.result.setOperationStatus(AtlasExportResult.OperationStatus.SUCCESS);
-            context.result.incrementMeticsCounter("duration", (int) (endTime - startTime));
+    private AtlasExportResult.OperationStatus[] processItems(AtlasExportRequest request, ExportContext context) throws AtlasServiceException, AtlasException, AtlasBaseException {
+        AtlasExportResult.OperationStatus statuses[] = new AtlasExportResult.OperationStatus[request.getItemsToExport().size()];
+        List<AtlasObjectId> itemsToExport = request.getItemsToExport();
+        for (int i = 0; i < itemsToExport.size(); i++) {
+            AtlasObjectId item = itemsToExport.get(i);
+            statuses[i] = processObjectId(item, context);
+        }
+        return statuses;
+    }
 
-            context.sink.setResult(context.result);
-        } catch(Exception ex) {
-            LOG.error("Operation failed: ", ex);
-        } finally {
-            atlasGraph.releaseGremlinScriptEngine(context.scriptEngine);
-            LOG.info("<== export(user={}, from={}): status {}", userName, requestingIP, context.result.getOperationStatus());
-            context.clear();
-            result.clear();
+    private AtlasExportResult.OperationStatus getOverallOperationStatus(AtlasExportResult.OperationStatus... statuses) {
+        AtlasExportResult.OperationStatus overall = (statuses.length == 0) ?
+                AtlasExportResult.OperationStatus.FAIL : statuses[0];
+
+        for (AtlasExportResult.OperationStatus s : statuses) {
+            if (overall != s) {
+                overall = AtlasExportResult.OperationStatus.PARTIAL_SUCCESS;
+            }
         }
 
-        return context.result;
+        return overall;
     }
 
-    private void processObjectId(AtlasObjectId item, ExportContext context) throws AtlasServiceException, AtlasException, AtlasBaseException {
+    private AtlasExportResult.OperationStatus processObjectId(AtlasObjectId item, ExportContext context) throws AtlasServiceException, AtlasException, AtlasBaseException {
         if (LOG.isDebugEnabled()) {
             LOG.debug("==> processObjectId({})", item);
         }
 
         try {
             List<AtlasEntityWithExtInfo> entities = getStartingEntity(item, context);
+            if(entities.size() == 0) {
+                return AtlasExportResult.OperationStatus.FAIL;
+            }
 
             for (AtlasEntityWithExtInfo entityWithExtInfo : entities) {
                 processEntity(entityWithExtInfo.getEntity().getGuid(), context);
@@ -167,14 +204,15 @@ public class ExportService {
                 }
             }
         } catch (AtlasBaseException excp) {
-            context.result.setOperationStatus(AtlasExportResult.OperationStatus.PARTIAL_SUCCESS);
-
             LOG.error("Fetching entity failed for: {}", item, excp);
+            return AtlasExportResult.OperationStatus.FAIL;
         }
 
         if (LOG.isDebugEnabled()) {
             LOG.debug("<== processObjectId({})", item);
         }
+
+        return AtlasExportResult.OperationStatus.SUCCESS;
     }
 
     private List<AtlasEntityWithExtInfo> getStartingEntity(AtlasObjectId item, ExportContext context) throws AtlasBaseException {

repository/src/test/java/org/apache/atlas/repository/impexp/ExportServiceTest.java

@@ -35,7 +35,8 @@ import org.apache.atlas.repository.store.graph.v1.DeleteHandlerV1;
 import org.apache.atlas.repository.store.graph.v1.SoftDeleteHandlerV1;
 import org.apache.atlas.store.AtlasTypeDefStore;
 import org.apache.atlas.type.AtlasTypeRegistry;
-import org.junit.Assert;
+import org.testng.Assert;
+import org.powermock.reflect.Whitebox;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.testng.annotations.BeforeClass;
@@ -53,6 +54,7 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
+import static org.testng.Assert.*;
 import static org.mockito.Mockito.mock;
 
 @Guice(modules = RepositoryMetadataModule.class)
@@ -158,7 +160,6 @@ public class ExportServiceTest {
         ByteArrayOutputStream baos = new ByteArrayOutputStream();
         ZipSink zipSink = new ZipSink(baos);
         AtlasExportResult result = exportService.run(zipSink, request, userName, hostName, requestingIP);
-        Assert.assertEquals(result.getOperationStatus(), AtlasExportResult.OperationStatus.SUCCESS);
 
         zipSink.close();
 
@@ -177,10 +178,10 @@ public class ExportServiceTest {
         ZipSink zipSink = new ZipSink(baos);
         AtlasExportResult result = exportService.run(zipSink, request, "admin", hostName, requestingIP);
 
-        Assert.assertNotNull(exportService);
-        Assert.assertEquals(result.getHostName(), hostName);
-        Assert.assertEquals(result.getClientIpAddress(), requestingIP);
-        Assert.assertEquals(request, result.getRequest());
+        assertNotNull(exportService);
+        assertEquals(result.getHostName(), hostName);
+        assertEquals(result.getClientIpAddress(), requestingIP);
+        assertEquals(request, result.getRequest());
     }
 
     @Test
@@ -198,8 +199,8 @@ public class ExportServiceTest {
         ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray());
         ZipSource zipSource = new ZipSource(bais);
 
-        Assert.assertNotNull(exportService);
-        Assert.assertNotNull(zipSource.getCreationOrder());
+        assertNotNull(exportService);
+        assertNotNull(zipSource.getCreationOrder());
         Assert.assertFalse(zipSource.hasNext());
     }
 
@@ -244,54 +245,109 @@ public class ExportServiceTest {
         verifyExportForEmployeeData(zipSource);
     }
 
+    @Test
+    public void verifyOverallStatus() throws Exception {
+
+        ExportService service = new ExportService(typeRegistry);
+        assertEquals(AtlasExportResult.OperationStatus.FAIL, Whitebox.invokeMethod(service,
+                "getOverallOperationStatus"));
+
+        assertEquals(AtlasExportResult.OperationStatus.SUCCESS, Whitebox.invokeMethod(service,
+                "getOverallOperationStatus",
+                AtlasExportResult.OperationStatus.SUCCESS));
+
+        assertEquals(AtlasExportResult.OperationStatus.SUCCESS, Whitebox.invokeMethod(service,
+                "getOverallOperationStatus",
+                                AtlasExportResult.OperationStatus.SUCCESS,
+                                AtlasExportResult.OperationStatus.SUCCESS,
+                                AtlasExportResult.OperationStatus.SUCCESS));
+
+        assertEquals(AtlasExportResult.OperationStatus.PARTIAL_SUCCESS, Whitebox.invokeMethod(service,
+                "getOverallOperationStatus",
+                AtlasExportResult.OperationStatus.FAIL,
+                AtlasExportResult.OperationStatus.PARTIAL_SUCCESS,
+                AtlasExportResult.OperationStatus.SUCCESS));
+
+        assertEquals(AtlasExportResult.OperationStatus.PARTIAL_SUCCESS, Whitebox.invokeMethod(service,
+                "getOverallOperationStatus",
+                AtlasExportResult.OperationStatus.FAIL,
+                AtlasExportResult.OperationStatus.FAIL,
+                AtlasExportResult.OperationStatus.PARTIAL_SUCCESS));
+
+        assertEquals(AtlasExportResult.OperationStatus.FAIL, Whitebox.invokeMethod(service,
+                "getOverallOperationStatus",
+                AtlasExportResult.OperationStatus.FAIL,
+                AtlasExportResult.OperationStatus.FAIL,
+                AtlasExportResult.OperationStatus.FAIL));
+
+
+    }
+
+    @Test
+    public void requestingExportOfNonExistentEntity_ReturnsFailure() throws Exception {
+        AtlasExportRequest request = getRequestForEmployee();
+        tamperEmployeeRequest(request);
+        ZipSource zipSource = runExportWithParameters(request);
+
+        assertNotNull(zipSource.getCreationOrder());
+        assertEquals(zipSource.getCreationOrder().size(), 0);
+        assertEquals(AtlasExportResult.OperationStatus.FAIL, zipSource.getExportResult().getOperationStatus());
+    }
+
+    private void tamperEmployeeRequest(AtlasExportRequest request) {
+        AtlasObjectId objectId = request.getItemsToExport().get(0);
+        objectId.getUniqueAttributes().remove("name");
+        objectId.getUniqueAttributes().put("qualifiedName", "XXX@121");
+    }
+
     private void verifyExportForEmployeeData(ZipSource zipSource) throws AtlasBaseException {
         final List<String> expectedEntityTypes = Arrays.asList(new String[]{"Manager", "Employee", "Department"});
 
-        Assert.assertNotNull(zipSource.getCreationOrder());
-        Assert.assertEquals(zipSource.getCreationOrder().size(), 2);
-        Assert.assertTrue(zipSource.hasNext());
+        assertNotNull(zipSource.getCreationOrder());
+        assertEquals(zipSource.getCreationOrder().size(), 2);
+        assertTrue(zipSource.hasNext());
 
         while (zipSource.hasNext()) {
             AtlasEntity entity = zipSource.next();
 
-            Assert.assertNotNull(entity);
-            Assert.assertEquals(entity.getStatus(), AtlasEntity.Status.ACTIVE);
-            Assert.assertTrue(expectedEntityTypes.contains(entity.getTypeName()));
+            assertNotNull(entity);
+            assertEquals(AtlasEntity.Status.ACTIVE, entity.getStatus());
+            assertTrue(expectedEntityTypes.contains(entity.getTypeName()));
         }
 
         verifyTypeDefs(zipSource);
     }
 
     private void verifyExportForHrData(ZipSource zipSource) throws IOException, AtlasBaseException {
-        Assert.assertNotNull(zipSource.getCreationOrder());
-        Assert.assertTrue(zipSource.getCreationOrder().size() == 1);
-        Assert.assertTrue(zipSource.hasNext());
+        assertNotNull(zipSource.getCreationOrder());
+        assertTrue(zipSource.getCreationOrder().size() == 1);
+        assertTrue(zipSource.hasNext());
 
         AtlasEntity entity = zipSource.next();
 
-        Assert.assertNotNull(entity);
-        Assert.assertTrue(entity.getTypeName().equals("Department"));
-        Assert.assertEquals(entity.getStatus(), AtlasEntity.Status.ACTIVE);
+        assertNotNull(entity);
+        assertTrue(entity.getTypeName().equals("Department"));
+        assertEquals(entity.getStatus(), AtlasEntity.Status.ACTIVE);
         verifyTypeDefs(zipSource);
     }
 
     private void verifyExportForHrDataForConnected(ZipSource zipSource) throws IOException, AtlasBaseException {
-        Assert.assertNotNull(zipSource.getCreationOrder());
-        Assert.assertTrue(zipSource.getCreationOrder().size() == 2);
-        Assert.assertTrue(zipSource.hasNext());
+        assertNotNull(zipSource.getCreationOrder());
+        assertTrue(zipSource.getCreationOrder().size() == 2);
+        assertTrue(zipSource.hasNext());
 
         AtlasEntity entity = zipSource.next();
 
-        Assert.assertNotNull(entity);
-        Assert.assertTrue(entity.getTypeName().equals("Department"));
-        Assert.assertEquals(entity.getStatus(), AtlasEntity.Status.ACTIVE);
+        assertNotNull(entity);
+        assertTrue(entity.getTypeName().equals("Department"));
+        assertEquals(entity.getStatus(), AtlasEntity.Status.ACTIVE);
         verifyTypeDefs(zipSource);
     }
 
     private void verifyTypeDefs(ZipSource zipSource) throws AtlasBaseException {
-        Assert.assertEquals(zipSource.getTypesDef().getEnumDefs().size(), 1);
-        Assert.assertEquals(zipSource.getTypesDef().getClassificationDefs().size(), 0);
-        Assert.assertEquals(zipSource.getTypesDef().getStructDefs().size(), 1);
-        Assert.assertEquals(zipSource.getTypesDef().getEntityDefs().size(), 4);
+        assertEquals(zipSource.getTypesDef().getEnumDefs().size(), 1);
+        assertEquals(zipSource.getTypesDef().getClassificationDefs().size(), 0);
+        assertEquals(zipSource.getTypesDef().getStructDefs().size(), 1);
+        assertEquals(zipSource.getTypesDef().getEntityDefs().size(), 4);
     }
 }