Skip to content

A
atlas
Commit ae1d636f by Suma Shivaprasad
Options

ATLAS-385 Support for Lineage for entities with SuperType as DataSet (anilsg via sumasai)

parent 8cbb2c13
Showing with 254 additions and 80 deletions
dashboard/public/css/lineage.css
......@@ -86,4 +86,5 @@ div.lineage {
.alignLineage{
text-align: center;
margin-top: 100px;
font-size: 24px;
}
dashboard/public/modules/details/detailsController.js
......@@ -22,14 +22,14 @@ angular.module('dgc.details').controller('DetailsController', ['$window', '$scop
$scope.tableName = false;
$scope.isTable = false;
$scope.isLineage = false;
DetailsResource.get({
id: $stateParams.id
}, function(data) {
$scope.details = data;
$scope.tableName = data.values.name;
$scope.isTable = (typeof data.typeName !== 'undefined' && data.typeName.toLowerCase().indexOf('table') !== -1) ? true : false;
$scope.onActivate('io');
$scope.isTags = (typeof data.traits !== 'undefined' && typeof data.traits === 'object') ? true : false;
......@@ -56,6 +56,9 @@ angular.module('dgc.details').controller('DetailsController', ['$window', '$scop
}
});
$scope.$on('show_lineage', function() {
$scope.isLineage = true;
});
$scope.isNumber = angular.isNumber;
$scope.isObject = angular.isObject;
......@@ -64,7 +67,8 @@ angular.module('dgc.details').controller('DetailsController', ['$window', '$scop
$scope.onActivate = function tabActivate(tabname) {
$scope.$broadcast('render-lineage', {
type: tabname,
tableName: $scope.tableName
tableName: $scope.tableName,
guid : $stateParams.id
});
};
......
dashboard/public/modules/details/views/details.html
......@@ -28,7 +28,7 @@
<h4 ng-if="details.values && details.values.name && details.values.name != ''">
<b>Name:</b> <span class="black">{{details.values.name}}</span></h2>
<h4 ng-if="details.values && details.values.description && details.values.description != ''"><b>Description:</b> <span class="black">{{details.values.description}}</span></h4>
<h4 data-ng-show="isTable" data-disable="!tableName" data-select="onActivate('io')">
<h4 data-disable="!tableName" data-select="onActivate('io')" id="lineageGraph" class="hide">
<span class="lineage">Lineage</span>
<ng-include data-table-type="io" src="'/modules/lineage/views/lineage_io.html'"/>
</h4>
......
dashboard/public/modules/lineage/lineage_ioController.js
......@@ -39,51 +39,64 @@ angular.module('dgc.lineage').controller('Lineage_ioController', ['$element', '$
LineageResource.get({
tableName: tableData.tableName,
type: 'outputs'
}, function lineageSuccess(response1) {
LineageResource.get({
tableName: tableData.tableName,
type: 'inputs'
}, function lineageSuccess(response) {
response.results.values.edges = inVertObj(response.results.values.edges);
angular.forEach(response.results.values.edges, function(value, key) {
angular.forEach(response1.results.values.edges, function(value1, key1) {
if (key === key1) {
var array1 = value;
angular.forEach(value1, function(value2) {
array1.push(value2);
}).$promise.then(
function lineageSuccess(response1) {
// $scope.$emit('show_lineage');
$('#lineageGraph').removeClass('hide');
LineageResource.get({
tableName: tableData.tableName,
type: 'inputs'
}).$promise.then(
//success
function lineageSuccess(response) {
if (response && response.results) {
response.results.values.edges = inVertObj(response.results.values.edges);
angular.forEach(response.results.values.edges, function(value, key) {
angular.forEach(response1.results.values.edges, function(value1, key1) {
if (key === key1) {
var array1 = value;
angular.forEach(value1, function(value2) {
array1.push(value2);
});
response.results.values.edges[key] = array1;
response1.results.values.edges[key] = array1;
}
});
});
response.results.values.edges[key] = array1;
response1.results.values.edges[key] = array1;
}
});
});
angular.extend(response.results.values.edges, response1.results.values.edges);
angular.extend(response.results.values.vertices, response1.results.values.vertices);
angular.extend(response.results.values.edges, response1.results.values.edges);
angular.extend(response.results.values.vertices, response1.results.values.vertices);
if (!_.isEmpty(response.results.values.vertices)) {
loadProcess(response.results.values.edges, response.results.values.vertices)
.then(function(res) {
guidsList = res;
if (!_.isEmpty(response.results.values.vertices)) {
loadProcess(response.results.values.edges, response.results.values.vertices)
.then(function(res) {
guidsList = res;
$scope.lineageData = transformData(response.results);
$scope.lineageData = transformData(response.results);
if (callRender) {
render();
if (callRender) {
render();
}
});
} else {
$scope.requested = false;
}
});
} else {
$scope.requested = false;
}
});
});
} else {
$scope.requested = false;
}
},
function() {
$scope.requested = false;
}
);
},
function() {
$scope.requested = false;
}
);
}
function loadProcess(edges, vertices) {
var urlCalls = [];
......@@ -141,6 +154,7 @@ angular.module('dgc.lineage').controller('Lineage_ioController', ['$element', '$
render();
}
}
$scope.guid = lineageData.guid;
});
function transformData(metaData) {
......@@ -157,11 +171,11 @@ angular.module('dgc.lineage').controller('Lineage_ioController', ['$element', '$
var loadProcess = getLoadProcessTypes(guid);
if (typeof loadProcess !== "undefined") {
name = loadProcess.name;
type = loadProcess.typeName;
type = 'edges';
tip = loadProcess.tip;
} else {
name = 'Load Process';
type = 'Load Process';
type = 'edges';
}
}
var vertex = {
......@@ -433,29 +447,6 @@ angular.module('dgc.lineage').controller('Lineage_ioController', ['$element', '$
zoomListener.scale(scale);
zoomListener.translate([x, y]);
}
// Toggle children function
// function toggleChildren(d) {
// if (d.children) {
// d._children = d.children;
// d.children = null;
// } else if (d._children) {
// d.children = d._children;
// d._children = null;
// }
// return d;
// }
// Toggle children on click.
// function click(d) {
// if (d3.event.defaultPrevented) return; // click suppressed
// d = toggleChildren(d);
// update(d);
// //centerNode(d);
// }
//arrow
baseSvg.append("svg:defs")
.append("svg:marker")
......@@ -568,10 +559,10 @@ angular.module('dgc.lineage').controller('Lineage_ioController', ['$element', '$
nodeEnter.append("image")
.attr("class", "nodeImage")
.attr("xlink:href", function(d) {
return d.type === 'Table' ? '../img/tableicon.png' : '../img/process.png';
return (d.type && d.type !== '' && d.type.toLowerCase().indexOf('edges') !== -1) ? '../img/process.png' : '../img/tableicon.png';
})
.on('mouseover', function(d) {
if (d.type === 'LoadProcess' || 'Table') {
if (d.type === 'edges' || 'Table') {
tooltip.show(d);
}
})
......@@ -623,18 +614,6 @@ angular.module('dgc.lineage').controller('Lineage_ioController', ['$element', '$
return nameDis;
});
// Change the circle fill depending on whether it has children and is collapsed
// Change the circle fill depending on whether it has children and is collapsed
node.select("image.nodeImage")
.attr("r", 4.5)
.attr("xlink:href", function(d) {
if (d._children) {
return d.type === 'Table' ? '../img/tableicon1.png' : '../img/process1.png';
}
return d.type === 'Table' ? '../img/tableicon.png' : '../img/process.png';
});
// Transition nodes to their new position.
var nodeUpdate = node.transition()
.duration(duration)
......
release-log.txt
......@@ -14,6 +14,7 @@ ATLAS-54 Rename configs in hive hook (shwethags)
ATLAS-3 Mixed Index creation fails with Date types (sumasai via shwethags)
ALL CHANGES:
ATLAS-385 Support for Lineage for entities with SuperType as DataSet (anilsg via sumasai)
ATLAS-342 Atlas is sending an ENTITY_CREATE event to the ATLAS_ENTITIES topic even if the entity exists already (shwethags)
ATLAS-386 Handle hive rename Table (shwethags)
ATLAS-374 Doc: Create a wiki for documenting fault tolerance and HA options for Atlas data (yhemanth via sumasai)
......
repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryDeleteEntitiesTest.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.repository.graph;
import java.util.ArrayList;
import java.util.List;
import javax.inject.Inject;
import org.apache.atlas.RepositoryMetadataModule;
import org.apache.atlas.TestUtils;
import org.apache.atlas.discovery.graph.GraphBackedDiscoveryService;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.RepositoryException;
import org.apache.atlas.typesystem.ITypedReferenceableInstance;
import org.apache.atlas.typesystem.Referenceable;
import org.apache.atlas.typesystem.exception.EntityNotFoundException;
import org.apache.atlas.typesystem.types.ClassType;
import org.apache.atlas.typesystem.types.Multiplicity;
import org.apache.atlas.typesystem.types.TypeSystem;
import org.testng.Assert;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Guice;
import org.testng.annotations.Test;
import com.thinkaurelius.titan.core.TitanGraph;
import com.thinkaurelius.titan.core.util.TitanCleanup;
import com.tinkerpop.blueprints.Vertex;
/**
* Test for GraphBackedMetadataRepository.deleteEntities
*
* Guice loads the dependencies and injects the necessary objects
*
*/
@Guice(modules = RepositoryMetadataModule.class)
public class GraphBackedMetadataRepositoryDeleteEntitiesTest {
@Inject
private GraphProvider<TitanGraph> graphProvider;
@Inject
private GraphBackedMetadataRepository repositoryService;
@Inject
private GraphBackedDiscoveryService discoveryService;
private TypeSystem typeSystem;
@BeforeClass
public void setUp() throws Exception {
typeSystem = TypeSystem.getInstance();
typeSystem.reset();
new GraphBackedSearchIndexer(graphProvider);
TestUtils.defineDeptEmployeeTypes(typeSystem);
TestUtils.createHiveTypes(typeSystem);
}
@AfterClass
public void tearDown() throws Exception {
TypeSystem.getInstance().reset();
try {
graphProvider.get().shutdown();
} catch (Exception e) {
e.printStackTrace();
}
try {
TitanCleanup.clear(graphProvider.get());
} catch (Exception e) {
e.printStackTrace();
}
}
@Test
public void testDeleteEntities() throws Exception {
String hrDeptGuid = createHrDeptGraph();
ITypedReferenceableInstance hrDept = repositoryService.getEntityDefinition(hrDeptGuid);
Object refValue = hrDept.get("employees");
Assert.assertTrue(refValue instanceof List);
List<Object> employees = (List<Object>)refValue;
Assert.assertEquals(employees.size(), 4);
List<String> employeeGuids = new ArrayList<String>(4);
for (Object listValue : employees) {
Assert.assertTrue(listValue instanceof ITypedReferenceableInstance);
ITypedReferenceableInstance employee = (ITypedReferenceableInstance) listValue;
employeeGuids.add(employee.getId()._getId());
}
// There should be 4 vertices for Address structs (one for each Person.address attribute value).
int vertexCount = countVertices(Constants.ENTITY_TYPE_PROPERTY_KEY, "Address");
Assert.assertEquals(vertexCount, 4);
vertexCount = countVertices(Constants.ENTITY_TYPE_PROPERTY_KEY, "SecurityClearance");
Assert.assertEquals(vertexCount, 1);
List<String> deletedEntities = repositoryService.deleteEntities(hrDeptGuid);
Assert.assertTrue(deletedEntities.contains(hrDeptGuid));
// Verify Department entity and its contained Person entities were deleted.
verifyEntityDoesNotExist(hrDeptGuid);
for (String employeeGuid : employeeGuids) {
verifyEntityDoesNotExist(employeeGuid);
}
// Verify all Person.address struct vertices were removed.
vertexCount = countVertices(Constants.ENTITY_TYPE_PROPERTY_KEY, "Address");
Assert.assertEquals(vertexCount, 0);
// Verify all SecurityClearance trait vertices were removed.
vertexCount = countVertices(Constants.ENTITY_TYPE_PROPERTY_KEY, "SecurityClearance");
Assert.assertEquals(vertexCount, 0);
}
@Test(dependsOnMethods = "testDeleteEntities")
public void testDeleteContainedEntity() throws Exception {
String hrDeptGuid = createHrDeptGraph();
ITypedReferenceableInstance hrDept = repositoryService.getEntityDefinition(hrDeptGuid);
Object refValue = hrDept.get("employees");
Assert.assertTrue(refValue instanceof List);
List<Object> employees = (List<Object>)refValue;
Assert.assertEquals(employees.size(), 4);
Object listValue = employees.get(2);
Assert.assertTrue(listValue instanceof ITypedReferenceableInstance);
ITypedReferenceableInstance employee = (ITypedReferenceableInstance) listValue;
String employeeGuid = employee.getId()._getId();
List<String> deletedEntities = repositoryService.deleteEntities(employeeGuid);
Assert.assertTrue(deletedEntities.contains(employeeGuid));
verifyEntityDoesNotExist(employeeGuid);
}
private String createHrDeptGraph() throws Exception {
Referenceable deptEg1 = TestUtils.createDeptEg1(typeSystem);
ClassType deptType = typeSystem.getDataType(ClassType.class, "Department");
ITypedReferenceableInstance hrDept2 = deptType.convert(deptEg1, Multiplicity.REQUIRED);
List<String> guids = repositoryService.createEntities(hrDept2);
Assert.assertNotNull(guids);
Assert.assertEquals(guids.size(), 5);
List<String> entityList = repositoryService.getEntityList("Department");
Assert.assertNotNull(entityList);
Assert.assertEquals(entityList.size(), 1);
String hrDeptGuid = entityList.get(0);
return hrDeptGuid;
}
private int countVertices(String propertyName, Object value) {
Iterable<Vertex> vertices = graphProvider.get().getVertices(propertyName, value);
int vertexCount = 0;
for (Vertex vertex : vertices) {
vertexCount++;
}
return vertexCount;
}
private void verifyEntityDoesNotExist(String hrDeptGuid) throws RepositoryException {
try {
repositoryService.getEntityDefinition(hrDeptGuid);
Assert.fail("EntityNotFoundException was expected but none thrown");
}
catch(EntityNotFoundException e) {
// good
}
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment