Skip to content
Projects
Groups
Snippets
Help
This project
Loading...
Sign in / Register
Toggle navigation
A
atlas
Project
Overview
Details
Activity
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
dataplatform
atlas
Commits
ae1d636f
Commit
ae1d636f
authored
9 years ago
by
Suma Shivaprasad
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
ATLAS-385 Support for Lineage for entities with SuperType as DataSet (anilsg via sumasai)
parent
8cbb2c13
Show whitespace changes
Inline
Side-by-side
Showing
6 changed files
with
224 additions
and
50 deletions
+224
-50
lineage.css
dashboard/public/css/lineage.css
+1
-0
detailsController.js
dashboard/public/modules/details/detailsController.js
+6
-2
details.html
dashboard/public/modules/details/views/details.html
+1
-1
lineage_ioController.js
dashboard/public/modules/lineage/lineage_ioController.js
+26
-47
release-log.txt
release-log.txt
+1
-0
GraphBackedMetadataRepositoryDeleteEntitiesTest.java
...raph/GraphBackedMetadataRepositoryDeleteEntitiesTest.java
+189
-0
No files found.
dashboard/public/css/lineage.css
View file @
ae1d636f
...
...
@@ -86,4 +86,5 @@ div.lineage {
.alignLineage
{
text-align
:
center
;
margin-top
:
100px
;
font-size
:
24px
;
}
This diff is collapsed.
Click to expand it.
dashboard/public/modules/details/detailsController.js
View file @
ae1d636f
...
...
@@ -22,6 +22,7 @@ angular.module('dgc.details').controller('DetailsController', ['$window', '$scop
$scope
.
tableName
=
false
;
$scope
.
isTable
=
false
;
$scope
.
isLineage
=
false
;
DetailsResource
.
get
({
id
:
$stateParams
.
id
...
...
@@ -29,7 +30,6 @@ angular.module('dgc.details').controller('DetailsController', ['$window', '$scop
},
function
(
data
)
{
$scope
.
details
=
data
;
$scope
.
tableName
=
data
.
values
.
name
;
$scope
.
isTable
=
(
typeof
data
.
typeName
!==
'undefined'
&&
data
.
typeName
.
toLowerCase
().
indexOf
(
'table'
)
!==
-
1
)
?
true
:
false
;
$scope
.
onActivate
(
'io'
);
$scope
.
isTags
=
(
typeof
data
.
traits
!==
'undefined'
&&
typeof
data
.
traits
===
'object'
)
?
true
:
false
;
...
...
@@ -56,6 +56,9 @@ angular.module('dgc.details').controller('DetailsController', ['$window', '$scop
}
});
$scope
.
$on
(
'show_lineage'
,
function
()
{
$scope
.
isLineage
=
true
;
});
$scope
.
isNumber
=
angular
.
isNumber
;
$scope
.
isObject
=
angular
.
isObject
;
...
...
@@ -64,7 +67,8 @@ angular.module('dgc.details').controller('DetailsController', ['$window', '$scop
$scope
.
onActivate
=
function
tabActivate
(
tabname
)
{
$scope
.
$broadcast
(
'render-lineage'
,
{
type
:
tabname
,
tableName
:
$scope
.
tableName
tableName
:
$scope
.
tableName
,
guid
:
$stateParams
.
id
});
};
...
...
This diff is collapsed.
Click to expand it.
dashboard/public/modules/details/views/details.html
View file @
ae1d636f
...
...
@@ -28,7 +28,7 @@
<h4
ng-if=
"details.values && details.values.name && details.values.name != ''"
>
<b>
Name:
</b>
<span
class=
"black"
>
{{details.values.name}}
</span></h2>
<h4
ng-if=
"details.values && details.values.description && details.values.description != ''"
><b>
Description:
</b>
<span
class=
"black"
>
{{details.values.description}}
</span></h4>
<h4
data-
ng-show=
"isTable"
data-disable=
"!tableName"
data-select=
"onActivate('io')
"
>
<h4
data-
disable=
"!tableName"
data-select=
"onActivate('io')"
id=
"lineageGraph"
class=
"hide
"
>
<span
class=
"lineage"
>
Lineage
</span>
<ng-include
data-table-type=
"io"
src=
"'/modules/lineage/views/lineage_io.html'"
/>
</h4>
...
...
This diff is collapsed.
Click to expand it.
dashboard/public/modules/lineage/lineage_ioController.js
View file @
ae1d636f
...
...
@@ -39,12 +39,17 @@ angular.module('dgc.lineage').controller('Lineage_ioController', ['$element', '$
LineageResource
.
get
({
tableName
:
tableData
.
tableName
,
type
:
'outputs'
},
function
lineageSuccess
(
response1
)
{
}).
$promise
.
then
(
function
lineageSuccess
(
response1
)
{
// $scope.$emit('show_lineage');
$
(
'#lineageGraph'
).
removeClass
(
'hide'
);
LineageResource
.
get
({
tableName
:
tableData
.
tableName
,
type
:
'inputs'
},
function
lineageSuccess
(
response
)
{
}).
$promise
.
then
(
//success
function
lineageSuccess
(
response
)
{
if
(
response
&&
response
.
results
)
{
response
.
results
.
values
.
edges
=
inVertObj
(
response
.
results
.
values
.
edges
);
angular
.
forEach
(
response
.
results
.
values
.
edges
,
function
(
value
,
key
)
{
...
...
@@ -77,12 +82,20 @@ angular.module('dgc.lineage').controller('Lineage_ioController', ['$element', '$
}
else
{
$scope
.
requested
=
false
;
}
});
});
}
else
{
$scope
.
requested
=
false
;
}
},
function
()
{
$scope
.
requested
=
false
;
}
);
},
function
()
{
$scope
.
requested
=
false
;
}
);
}
function
loadProcess
(
edges
,
vertices
)
{
...
...
@@ -141,6 +154,7 @@ angular.module('dgc.lineage').controller('Lineage_ioController', ['$element', '$
render
();
}
}
$scope
.
guid
=
lineageData
.
guid
;
});
function
transformData
(
metaData
)
{
...
...
@@ -157,11 +171,11 @@ angular.module('dgc.lineage').controller('Lineage_ioController', ['$element', '$
var
loadProcess
=
getLoadProcessTypes
(
guid
);
if
(
typeof
loadProcess
!==
"undefined"
)
{
name
=
loadProcess
.
name
;
type
=
loadProcess
.
typeName
;
type
=
'edges'
;
tip
=
loadProcess
.
tip
;
}
else
{
name
=
'Load Process'
;
type
=
'
Load Proces
s'
;
type
=
'
edge
s'
;
}
}
var
vertex
=
{
...
...
@@ -433,29 +447,6 @@ angular.module('dgc.lineage').controller('Lineage_ioController', ['$element', '$
zoomListener
.
scale
(
scale
);
zoomListener
.
translate
([
x
,
y
]);
}
// Toggle children function
// function toggleChildren(d) {
// if (d.children) {
// d._children = d.children;
// d.children = null;
// } else if (d._children) {
// d.children = d._children;
// d._children = null;
// }
// return d;
// }
// Toggle children on click.
// function click(d) {
// if (d3.event.defaultPrevented) return; // click suppressed
// d = toggleChildren(d);
// update(d);
// //centerNode(d);
// }
//arrow
baseSvg
.
append
(
"svg:defs"
)
.
append
(
"svg:marker"
)
...
...
@@ -568,10 +559,10 @@ angular.module('dgc.lineage').controller('Lineage_ioController', ['$element', '$
nodeEnter
.
append
(
"image"
)
.
attr
(
"class"
,
"nodeImage"
)
.
attr
(
"xlink:href"
,
function
(
d
)
{
return
d
.
type
===
'Table'
?
'../img/tableicon.png'
:
'../img/process
.png'
;
return
(
d
.
type
&&
d
.
type
!==
''
&&
d
.
type
.
toLowerCase
().
indexOf
(
'edges'
)
!==
-
1
)
?
'../img/process.png'
:
'../img/tableicon
.png'
;
})
.
on
(
'mouseover'
,
function
(
d
)
{
if
(
d
.
type
===
'
LoadProces
s'
||
'Table'
)
{
if
(
d
.
type
===
'
edge
s'
||
'Table'
)
{
tooltip
.
show
(
d
);
}
})
...
...
@@ -623,18 +614,6 @@ angular.module('dgc.lineage').controller('Lineage_ioController', ['$element', '$
return
nameDis
;
});
// Change the circle fill depending on whether it has children and is collapsed
// Change the circle fill depending on whether it has children and is collapsed
node
.
select
(
"image.nodeImage"
)
.
attr
(
"r"
,
4.5
)
.
attr
(
"xlink:href"
,
function
(
d
)
{
if
(
d
.
_children
)
{
return
d
.
type
===
'Table'
?
'../img/tableicon1.png'
:
'../img/process1.png'
;
}
return
d
.
type
===
'Table'
?
'../img/tableicon.png'
:
'../img/process.png'
;
});
// Transition nodes to their new position.
var
nodeUpdate
=
node
.
transition
()
.
duration
(
duration
)
...
...
This diff is collapsed.
Click to expand it.
release-log.txt
View file @
ae1d636f
...
...
@@ -14,6 +14,7 @@ ATLAS-54 Rename configs in hive hook (shwethags)
ATLAS-3 Mixed Index creation fails with Date types (sumasai via shwethags)
ALL CHANGES:
ATLAS-385 Support for Lineage for entities with SuperType as DataSet (anilsg via sumasai)
ATLAS-342 Atlas is sending an ENTITY_CREATE event to the ATLAS_ENTITIES topic even if the entity exists already (shwethags)
ATLAS-386 Handle hive rename Table (shwethags)
ATLAS-374 Doc: Create a wiki for documenting fault tolerance and HA options for Atlas data (yhemanth via sumasai)
...
...
This diff is collapsed.
Click to expand it.
repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryDeleteEntitiesTest.java
0 → 100644
View file @
ae1d636f
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package
org
.
apache
.
atlas
.
repository
.
graph
;
import
java.util.ArrayList
;
import
java.util.List
;
import
javax.inject.Inject
;
import
org.apache.atlas.RepositoryMetadataModule
;
import
org.apache.atlas.TestUtils
;
import
org.apache.atlas.discovery.graph.GraphBackedDiscoveryService
;
import
org.apache.atlas.repository.Constants
;
import
org.apache.atlas.repository.RepositoryException
;
import
org.apache.atlas.typesystem.ITypedReferenceableInstance
;
import
org.apache.atlas.typesystem.Referenceable
;
import
org.apache.atlas.typesystem.exception.EntityNotFoundException
;
import
org.apache.atlas.typesystem.types.ClassType
;
import
org.apache.atlas.typesystem.types.Multiplicity
;
import
org.apache.atlas.typesystem.types.TypeSystem
;
import
org.testng.Assert
;
import
org.testng.annotations.AfterClass
;
import
org.testng.annotations.BeforeClass
;
import
org.testng.annotations.Guice
;
import
org.testng.annotations.Test
;
import
com.thinkaurelius.titan.core.TitanGraph
;
import
com.thinkaurelius.titan.core.util.TitanCleanup
;
import
com.tinkerpop.blueprints.Vertex
;
/**
* Test for GraphBackedMetadataRepository.deleteEntities
*
* Guice loads the dependencies and injects the necessary objects
*
*/
@Guice
(
modules
=
RepositoryMetadataModule
.
class
)
public
class
GraphBackedMetadataRepositoryDeleteEntitiesTest
{
@Inject
private
GraphProvider
<
TitanGraph
>
graphProvider
;
@Inject
private
GraphBackedMetadataRepository
repositoryService
;
@Inject
private
GraphBackedDiscoveryService
discoveryService
;
private
TypeSystem
typeSystem
;
@BeforeClass
public
void
setUp
()
throws
Exception
{
typeSystem
=
TypeSystem
.
getInstance
();
typeSystem
.
reset
();
new
GraphBackedSearchIndexer
(
graphProvider
);
TestUtils
.
defineDeptEmployeeTypes
(
typeSystem
);
TestUtils
.
createHiveTypes
(
typeSystem
);
}
@AfterClass
public
void
tearDown
()
throws
Exception
{
TypeSystem
.
getInstance
().
reset
();
try
{
graphProvider
.
get
().
shutdown
();
}
catch
(
Exception
e
)
{
e
.
printStackTrace
();
}
try
{
TitanCleanup
.
clear
(
graphProvider
.
get
());
}
catch
(
Exception
e
)
{
e
.
printStackTrace
();
}
}
@Test
public
void
testDeleteEntities
()
throws
Exception
{
String
hrDeptGuid
=
createHrDeptGraph
();
ITypedReferenceableInstance
hrDept
=
repositoryService
.
getEntityDefinition
(
hrDeptGuid
);
Object
refValue
=
hrDept
.
get
(
"employees"
);
Assert
.
assertTrue
(
refValue
instanceof
List
);
List
<
Object
>
employees
=
(
List
<
Object
>)
refValue
;
Assert
.
assertEquals
(
employees
.
size
(),
4
);
List
<
String
>
employeeGuids
=
new
ArrayList
<
String
>(
4
);
for
(
Object
listValue
:
employees
)
{
Assert
.
assertTrue
(
listValue
instanceof
ITypedReferenceableInstance
);
ITypedReferenceableInstance
employee
=
(
ITypedReferenceableInstance
)
listValue
;
employeeGuids
.
add
(
employee
.
getId
().
_getId
());
}
// There should be 4 vertices for Address structs (one for each Person.address attribute value).
int
vertexCount
=
countVertices
(
Constants
.
ENTITY_TYPE_PROPERTY_KEY
,
"Address"
);
Assert
.
assertEquals
(
vertexCount
,
4
);
vertexCount
=
countVertices
(
Constants
.
ENTITY_TYPE_PROPERTY_KEY
,
"SecurityClearance"
);
Assert
.
assertEquals
(
vertexCount
,
1
);
List
<
String
>
deletedEntities
=
repositoryService
.
deleteEntities
(
hrDeptGuid
);
Assert
.
assertTrue
(
deletedEntities
.
contains
(
hrDeptGuid
));
// Verify Department entity and its contained Person entities were deleted.
verifyEntityDoesNotExist
(
hrDeptGuid
);
for
(
String
employeeGuid
:
employeeGuids
)
{
verifyEntityDoesNotExist
(
employeeGuid
);
}
// Verify all Person.address struct vertices were removed.
vertexCount
=
countVertices
(
Constants
.
ENTITY_TYPE_PROPERTY_KEY
,
"Address"
);
Assert
.
assertEquals
(
vertexCount
,
0
);
// Verify all SecurityClearance trait vertices were removed.
vertexCount
=
countVertices
(
Constants
.
ENTITY_TYPE_PROPERTY_KEY
,
"SecurityClearance"
);
Assert
.
assertEquals
(
vertexCount
,
0
);
}
@Test
(
dependsOnMethods
=
"testDeleteEntities"
)
public
void
testDeleteContainedEntity
()
throws
Exception
{
String
hrDeptGuid
=
createHrDeptGraph
();
ITypedReferenceableInstance
hrDept
=
repositoryService
.
getEntityDefinition
(
hrDeptGuid
);
Object
refValue
=
hrDept
.
get
(
"employees"
);
Assert
.
assertTrue
(
refValue
instanceof
List
);
List
<
Object
>
employees
=
(
List
<
Object
>)
refValue
;
Assert
.
assertEquals
(
employees
.
size
(),
4
);
Object
listValue
=
employees
.
get
(
2
);
Assert
.
assertTrue
(
listValue
instanceof
ITypedReferenceableInstance
);
ITypedReferenceableInstance
employee
=
(
ITypedReferenceableInstance
)
listValue
;
String
employeeGuid
=
employee
.
getId
().
_getId
();
List
<
String
>
deletedEntities
=
repositoryService
.
deleteEntities
(
employeeGuid
);
Assert
.
assertTrue
(
deletedEntities
.
contains
(
employeeGuid
));
verifyEntityDoesNotExist
(
employeeGuid
);
}
private
String
createHrDeptGraph
()
throws
Exception
{
Referenceable
deptEg1
=
TestUtils
.
createDeptEg1
(
typeSystem
);
ClassType
deptType
=
typeSystem
.
getDataType
(
ClassType
.
class
,
"Department"
);
ITypedReferenceableInstance
hrDept2
=
deptType
.
convert
(
deptEg1
,
Multiplicity
.
REQUIRED
);
List
<
String
>
guids
=
repositoryService
.
createEntities
(
hrDept2
);
Assert
.
assertNotNull
(
guids
);
Assert
.
assertEquals
(
guids
.
size
(),
5
);
List
<
String
>
entityList
=
repositoryService
.
getEntityList
(
"Department"
);
Assert
.
assertNotNull
(
entityList
);
Assert
.
assertEquals
(
entityList
.
size
(),
1
);
String
hrDeptGuid
=
entityList
.
get
(
0
);
return
hrDeptGuid
;
}
private
int
countVertices
(
String
propertyName
,
Object
value
)
{
Iterable
<
Vertex
>
vertices
=
graphProvider
.
get
().
getVertices
(
propertyName
,
value
);
int
vertexCount
=
0
;
for
(
Vertex
vertex
:
vertices
)
{
vertexCount
++;
}
return
vertexCount
;
}
private
void
verifyEntityDoesNotExist
(
String
hrDeptGuid
)
throws
RepositoryException
{
try
{
repositoryService
.
getEntityDefinition
(
hrDeptGuid
);
Assert
.
fail
(
"EntityNotFoundException was expected but none thrown"
);
}
catch
(
EntityNotFoundException
e
)
{
// good
}
}
}
This diff is collapsed.
Click to expand it.
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment