ATLAS-385 Support for Lineage for entities with SuperType as DataSet (anilsg via sumasai)

dashboard/public/css/lineage.css

@@ -86,4 +86,5 @@ div.lineage {
 .alignLineage{
     text-align: center;
     margin-top: 100px;
+    font-size: 24px;
 }

dashboard/public/modules/details/detailsController.js

@@ -22,14 +22,14 @@ angular.module('dgc.details').controller('DetailsController', ['$window', '$scop
 
         $scope.tableName = false;
         $scope.isTable = false;
-
+        $scope.isLineage = false;
+        
         DetailsResource.get({
             id: $stateParams.id
 
         }, function(data) {
             $scope.details = data;
             $scope.tableName = data.values.name;
-            $scope.isTable = (typeof data.typeName !== 'undefined' && data.typeName.toLowerCase().indexOf('table') !== -1) ? true : false;
             $scope.onActivate('io');
             $scope.isTags = (typeof data.traits !== 'undefined' && typeof data.traits === 'object') ? true : false;
 
@@ -56,6 +56,9 @@ angular.module('dgc.details').controller('DetailsController', ['$window', '$scop
             }
         });
 
+        $scope.$on('show_lineage', function() {
+            $scope.isLineage = true;
+        });
 
         $scope.isNumber = angular.isNumber;
         $scope.isObject = angular.isObject;
@@ -64,7 +67,8 @@ angular.module('dgc.details').controller('DetailsController', ['$window', '$scop
         $scope.onActivate = function tabActivate(tabname) {
             $scope.$broadcast('render-lineage', {
                 type: tabname,
-                tableName: $scope.tableName
+                tableName: $scope.tableName,
+                guid : $stateParams.id
             });
         };
 

dashboard/public/modules/details/views/details.html

@@ -28,7 +28,7 @@
             <h4 ng-if="details.values && details.values.name && details.values.name != ''">
             <b>Name:</b> <span class="black">{{details.values.name}}</span></h2>
             <h4 ng-if="details.values && details.values.description && details.values.description != ''"><b>Description:</b> <span class="black">{{details.values.description}}</span></h4>
-            <h4 data-ng-show="isTable" data-disable="!tableName" data-select="onActivate('io')">
+            <h4 data-disable="!tableName" data-select="onActivate('io')" id="lineageGraph" class="hide">
                 <span class="lineage">Lineage</span> 
                 <ng-include data-table-type="io" src="'/modules/lineage/views/lineage_io.html'"/>
             </h4>

dashboard/public/modules/lineage/lineage_ioController.js

@@ -39,51 +39,64 @@ angular.module('dgc.lineage').controller('Lineage_ioController', ['$element', '$
             LineageResource.get({
                 tableName: tableData.tableName,
                 type: 'outputs'
-            }, function lineageSuccess(response1) {
-
-                LineageResource.get({
-                    tableName: tableData.tableName,
-                    type: 'inputs'
-                }, function lineageSuccess(response) {
-                    response.results.values.edges = inVertObj(response.results.values.edges);
-
-                    angular.forEach(response.results.values.edges, function(value, key) {
-                        angular.forEach(response1.results.values.edges, function(value1, key1) {
-                            if (key === key1) {
-                                var array1 = value;
-                                angular.forEach(value1, function(value2) {
-                                    array1.push(value2);
+            }).$promise.then(
+                function lineageSuccess(response1) {
+                  //  $scope.$emit('show_lineage');
+                  $('#lineageGraph').removeClass('hide');
+                    LineageResource.get({
+                        tableName: tableData.tableName,
+                        type: 'inputs'
+                    }).$promise.then(
+                        //success
+                        function lineageSuccess(response) {
+                            if (response && response.results) {
+                                response.results.values.edges = inVertObj(response.results.values.edges);
+
+                                angular.forEach(response.results.values.edges, function(value, key) {
+                                    angular.forEach(response1.results.values.edges, function(value1, key1) {
+                                        if (key === key1) {
+                                            var array1 = value;
+                                            angular.forEach(value1, function(value2) {
+                                                array1.push(value2);
+                                            });
+                                            response.results.values.edges[key] = array1;
+                                            response1.results.values.edges[key] = array1;
+                                        }
+                                    });
                                 });
-                                response.results.values.edges[key] = array1;
-                                response1.results.values.edges[key] = array1;
-                            }
-                        });
-                    });
 
-                    angular.extend(response.results.values.edges, response1.results.values.edges);
-                    angular.extend(response.results.values.vertices, response1.results.values.vertices);
+                                angular.extend(response.results.values.edges, response1.results.values.edges);
+                                angular.extend(response.results.values.vertices, response1.results.values.vertices);
 
-                    if (!_.isEmpty(response.results.values.vertices)) {
-                        loadProcess(response.results.values.edges, response.results.values.vertices)
-                            .then(function(res) {
-                                guidsList = res;
+                                if (!_.isEmpty(response.results.values.vertices)) {
+                                    loadProcess(response.results.values.edges, response.results.values.vertices)
+                                        .then(function(res) {
+                                            guidsList = res;
 
-                                $scope.lineageData = transformData(response.results);
+                                            $scope.lineageData = transformData(response.results);
 
-                                if (callRender) {
-                                    render();
+                                            if (callRender) {
+                                                render();
+                                            }
+                                        });
+                                } else {
+                                    $scope.requested = false;
                                 }
-                            });
-                    } else {
-                        $scope.requested = false;
-                    }
-                });
-
-            });
-
+                            } else {
+                                $scope.requested = false;
+                            }
+                        },
+                        function() {
+                            $scope.requested = false;
+                        }
+                    );
+                },
+                function() {
+                    $scope.requested = false;
+                }
+            );
         }
 
-
         function loadProcess(edges, vertices) {
 
             var urlCalls = [];
@@ -141,6 +154,7 @@ angular.module('dgc.lineage').controller('Lineage_ioController', ['$element', '$
                     render();
                 }
             }
+            $scope.guid = lineageData.guid;
         });
 
         function transformData(metaData) {
@@ -157,11 +171,11 @@ angular.module('dgc.lineage').controller('Lineage_ioController', ['$element', '$
                     var loadProcess = getLoadProcessTypes(guid);
                     if (typeof loadProcess !== "undefined") {
                         name = loadProcess.name;
-                        type = loadProcess.typeName;
+                        type = 'edges';
                         tip = loadProcess.tip;
                     } else {
                         name = 'Load Process';
-                        type = 'Load Process';
+                        type = 'edges';
                     }
                 }
                 var vertex = {
@@ -433,29 +447,6 @@ angular.module('dgc.lineage').controller('Lineage_ioController', ['$element', '$
                 zoomListener.scale(scale);
                 zoomListener.translate([x, y]);
             }
-
-            // Toggle children function
-
-            // function toggleChildren(d) {
-            //     if (d.children) {
-            //         d._children = d.children;
-            //         d.children = null;
-            //     } else if (d._children) {
-            //         d.children = d._children;
-            //         d._children = null;
-            //     }
-            //     return d;
-            // }
-
-            // Toggle children on click.
-
-            // function click(d) {
-            //     if (d3.event.defaultPrevented) return; // click suppressed
-            //     d = toggleChildren(d);
-            //     update(d);
-            //     //centerNode(d);
-            // }
-
             //arrow
             baseSvg.append("svg:defs")
                 .append("svg:marker")
@@ -568,10 +559,10 @@ angular.module('dgc.lineage').controller('Lineage_ioController', ['$element', '$
                 nodeEnter.append("image")
                     .attr("class", "nodeImage")
                     .attr("xlink:href", function(d) {
-                        return d.type === 'Table' ? '../img/tableicon.png' : '../img/process.png';
+                        return (d.type && d.type !== '' && d.type.toLowerCase().indexOf('edges') !== -1) ? '../img/process.png' : '../img/tableicon.png';
                     })
                     .on('mouseover', function(d) {
-                        if (d.type === 'LoadProcess' || 'Table') {
+                        if (d.type === 'edges' || 'Table') {
                             tooltip.show(d);
                         }
                     })
@@ -623,18 +614,6 @@ angular.module('dgc.lineage').controller('Lineage_ioController', ['$element', '$
                         return nameDis;
                     });
 
-                // Change the circle fill depending on whether it has children and is collapsed
-                // Change the circle fill depending on whether it has children and is collapsed
-                node.select("image.nodeImage")
-                    .attr("r", 4.5)
-                    .attr("xlink:href", function(d) {
-                        if (d._children) {
-                            return d.type === 'Table' ? '../img/tableicon1.png' : '../img/process1.png';
-                        }
-                        return d.type === 'Table' ? '../img/tableicon.png' : '../img/process.png';
-                    });
-
-
                 // Transition nodes to their new position.
                 var nodeUpdate = node.transition()
                     .duration(duration)

release-log.txt

@@ -14,6 +14,7 @@ ATLAS-54 Rename configs in hive hook (shwethags)
 ATLAS-3 Mixed Index creation fails with Date types (sumasai via shwethags)
 
 ALL CHANGES:
+ATLAS-385 Support for Lineage for entities with SuperType as DataSet (anilsg via sumasai)
 ATLAS-342 Atlas is sending an ENTITY_CREATE event to the ATLAS_ENTITIES topic even if the entity exists already (shwethags)
 ATLAS-386 Handle hive rename Table (shwethags)
 ATLAS-374 Doc: Create a wiki for documenting fault tolerance and HA options for Atlas data (yhemanth via sumasai)

repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryDeleteEntitiesTest.java

+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.repository.graph;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import javax.inject.Inject;
+
+import org.apache.atlas.RepositoryMetadataModule;
+import org.apache.atlas.TestUtils;
+import org.apache.atlas.discovery.graph.GraphBackedDiscoveryService;
+import org.apache.atlas.repository.Constants;
+import org.apache.atlas.repository.RepositoryException;
+import org.apache.atlas.typesystem.ITypedReferenceableInstance;
+import org.apache.atlas.typesystem.Referenceable;
+import org.apache.atlas.typesystem.exception.EntityNotFoundException;
+import org.apache.atlas.typesystem.types.ClassType;
+import org.apache.atlas.typesystem.types.Multiplicity;
+import org.apache.atlas.typesystem.types.TypeSystem;
+import org.testng.Assert;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Guice;
+import org.testng.annotations.Test;
+
+import com.thinkaurelius.titan.core.TitanGraph;
+import com.thinkaurelius.titan.core.util.TitanCleanup;
+import com.tinkerpop.blueprints.Vertex;
+
+/**
+ * Test for GraphBackedMetadataRepository.deleteEntities
+ *
+ * Guice loads the dependencies and injects the necessary objects
+ *
+ */
+@Guice(modules = RepositoryMetadataModule.class)
+public class GraphBackedMetadataRepositoryDeleteEntitiesTest {
+
+    @Inject
+    private GraphProvider<TitanGraph> graphProvider;
+
+    @Inject
+    private GraphBackedMetadataRepository repositoryService;
+
+    @Inject
+    private GraphBackedDiscoveryService discoveryService;
+
+    private TypeSystem typeSystem;
+
+    @BeforeClass
+    public void setUp() throws Exception {
+        typeSystem = TypeSystem.getInstance();
+        typeSystem.reset();
+
+        new GraphBackedSearchIndexer(graphProvider);
+
+        TestUtils.defineDeptEmployeeTypes(typeSystem);
+        TestUtils.createHiveTypes(typeSystem);
+    }
+
+
+    @AfterClass
+    public void tearDown() throws Exception {
+        TypeSystem.getInstance().reset();
+        try {
+            graphProvider.get().shutdown();
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+        try {
+            TitanCleanup.clear(graphProvider.get());
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+    }
+
+
+
+    @Test
+    public void testDeleteEntities() throws Exception {
+        String hrDeptGuid = createHrDeptGraph();
+
+        ITypedReferenceableInstance hrDept = repositoryService.getEntityDefinition(hrDeptGuid);
+        Object refValue = hrDept.get("employees");
+        Assert.assertTrue(refValue instanceof List);
+        List<Object> employees = (List<Object>)refValue;
+        Assert.assertEquals(employees.size(), 4);
+        List<String> employeeGuids = new ArrayList<String>(4);
+        for (Object listValue : employees) {
+            Assert.assertTrue(listValue instanceof ITypedReferenceableInstance);
+            ITypedReferenceableInstance employee = (ITypedReferenceableInstance) listValue;
+            employeeGuids.add(employee.getId()._getId());
+        }
+        
+        // There should be 4 vertices for Address structs (one for each Person.address attribute value).
+        int vertexCount = countVertices(Constants.ENTITY_TYPE_PROPERTY_KEY, "Address");
+        Assert.assertEquals(vertexCount, 4);
+        vertexCount = countVertices(Constants.ENTITY_TYPE_PROPERTY_KEY, "SecurityClearance");
+        Assert.assertEquals(vertexCount, 1);
+        
+        List<String> deletedEntities = repositoryService.deleteEntities(hrDeptGuid);
+        Assert.assertTrue(deletedEntities.contains(hrDeptGuid));
+        
+        // Verify Department entity and its contained Person entities were deleted.
+        verifyEntityDoesNotExist(hrDeptGuid);
+        for (String employeeGuid : employeeGuids) {
+            verifyEntityDoesNotExist(employeeGuid);
+        }
+        // Verify all Person.address struct vertices were removed.
+        vertexCount = countVertices(Constants.ENTITY_TYPE_PROPERTY_KEY, "Address");
+        Assert.assertEquals(vertexCount, 0);
+        
+        // Verify all SecurityClearance trait vertices were removed.
+        vertexCount = countVertices(Constants.ENTITY_TYPE_PROPERTY_KEY, "SecurityClearance");
+        Assert.assertEquals(vertexCount, 0);
+    }
+
+    @Test(dependsOnMethods = "testDeleteEntities")
+    public void testDeleteContainedEntity() throws Exception {
+        String hrDeptGuid = createHrDeptGraph();
+        ITypedReferenceableInstance hrDept = repositoryService.getEntityDefinition(hrDeptGuid);
+        Object refValue = hrDept.get("employees");
+        Assert.assertTrue(refValue instanceof List);
+        List<Object> employees = (List<Object>)refValue;
+        Assert.assertEquals(employees.size(), 4);
+        Object listValue = employees.get(2);
+        Assert.assertTrue(listValue instanceof ITypedReferenceableInstance);
+        ITypedReferenceableInstance employee = (ITypedReferenceableInstance) listValue;
+        String employeeGuid = employee.getId()._getId();
+        
+        List<String> deletedEntities = repositoryService.deleteEntities(employeeGuid);
+        Assert.assertTrue(deletedEntities.contains(employeeGuid));
+        verifyEntityDoesNotExist(employeeGuid);
+        
+    }
+
+    private String createHrDeptGraph() throws Exception {
+        Referenceable deptEg1 = TestUtils.createDeptEg1(typeSystem);
+        ClassType deptType = typeSystem.getDataType(ClassType.class, "Department");
+        ITypedReferenceableInstance hrDept2 = deptType.convert(deptEg1, Multiplicity.REQUIRED);
+
+        List<String> guids = repositoryService.createEntities(hrDept2);
+        Assert.assertNotNull(guids);
+        Assert.assertEquals(guids.size(), 5);
+        List<String> entityList = repositoryService.getEntityList("Department");
+        Assert.assertNotNull(entityList);
+        Assert.assertEquals(entityList.size(), 1);
+        String hrDeptGuid = entityList.get(0);
+        return hrDeptGuid;
+    }
+    
+    private int countVertices(String propertyName, Object value) {
+        Iterable<Vertex> vertices = graphProvider.get().getVertices(propertyName, value);
+        int vertexCount = 0;
+        for (Vertex vertex : vertices) {
+            vertexCount++;
+        }
+        return vertexCount;
+    }
+    
+    private void verifyEntityDoesNotExist(String hrDeptGuid) throws RepositoryException {
+
+        try {
+            repositoryService.getEntityDefinition(hrDeptGuid);
+            Assert.fail("EntityNotFoundException was expected but none thrown");
+        }
+        catch(EntityNotFoundException e) {
+            // good
+        }
+    }
+
+}