Skip to content
Projects
Groups
Snippets
Help
This project
Loading...
Sign in / Register
Toggle navigation
A
atlas
Project
Overview
Details
Activity
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
dataplatform
atlas
Commits
cb2421bc
Commit
cb2421bc
authored
6 years ago
by
Ashutosh Mestry
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
ATLAS-3020: Audit APIs for classification updates.
parent
96bdc802
master
No related merge requests found
Expand all
Hide whitespace changes
Inline
Side-by-side
Showing
24 changed files
with
532 additions
and
6 deletions
+532
-6
AtlasClientV2.java
...ient-v2/src/main/java/org/apache/atlas/AtlasClientV2.java
+20
-1
AtlasEntityHeaders.java
...a/org/apache/atlas/model/instance/AtlasEntityHeaders.java
+56
-0
CassandraBasedAuditRepository.java
...atlas/repository/audit/CassandraBasedAuditRepository.java
+7
-0
EntityAuditRepository.java
.../apache/atlas/repository/audit/EntityAuditRepository.java
+9
-0
HBaseBasedAuditRepository.java
...che/atlas/repository/audit/HBaseBasedAuditRepository.java
+51
-1
InMemoryEntityAuditRepository.java
...atlas/repository/audit/InMemoryEntityAuditRepository.java
+17
-0
NoopEntityAuditRepository.java
...che/atlas/repository/audit/NoopEntityAuditRepository.java
+7
-0
AtlasEntityStore.java
...apache/atlas/repository/store/graph/AtlasEntityStore.java
+3
-0
AtlasEntityStoreV2.java
...e/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java
+7
-0
ClassificationAssociator.java
...s/repository/store/graph/v2/ClassificationAssociator.java
+0
-0
EntityGraphRetriever.java
...atlas/repository/store/graph/v2/EntityGraphRetriever.java
+1
-1
ClassificationAssociatorTest.java
...pository/store/graph/v2/ClassificationAssociatorTest.java
+0
-0
col-entity-None.json
...rces/json/classification-association/col-entity-None.json
+10
-0
col-entity-PII-FIN_PII.json
...on/classification-association/col-entity-PII-FIN_PII.json
+32
-0
col-entity-PII.json
...urces/json/classification-association/col-entity-PII.json
+22
-0
col-entity-T1-prop-Tn.json
...son/classification-association/col-entity-T1-prop-Tn.json
+34
-0
header-FIN_PII.json
...urces/json/classification-association/header-FIN_PII.json
+32
-0
header-None.json
...esources/json/classification-association/header-None.json
+21
-0
header-PII-VENDOR_PII.json
...son/classification-association/header-PII-VENDOR_PII.json
+42
-0
header-PII.json
...resources/json/classification-association/header-PII.json
+32
-0
header-Tx-prop-T1.json
...es/json/classification-association/header-Tx-prop-T1.json
+42
-0
header-Tx.json
.../resources/json/classification-association/header-Tx.json
+27
-0
header-empty.json
...sources/json/classification-association/header-empty.json
+4
-0
EntityREST.java
...p/src/main/java/org/apache/atlas/web/rest/EntityREST.java
+56
-3
No files found.
client/client-v2/src/main/java/org/apache/atlas/AtlasClientV2.java
View file @
cb2421bc
...
...
@@ -27,7 +27,9 @@ import org.apache.atlas.model.instance.AtlasClassification;
import
org.apache.atlas.model.instance.AtlasClassification.AtlasClassifications
;
import
org.apache.atlas.model.instance.AtlasEntity.AtlasEntitiesWithExtInfo
;
import
org.apache.atlas.model.instance.AtlasEntity.AtlasEntityWithExtInfo
;
import
org.apache.atlas.model.instance.AtlasEntityHeader
;
import
org.apache.atlas.model.instance.AtlasRelationship
;
import
org.apache.atlas.model.instance.AtlasEntityHeaders
;
import
org.apache.atlas.model.instance.EntityMutationResponse
;
import
org.apache.atlas.model.lineage.AtlasLineageInfo
;
import
org.apache.atlas.model.lineage.AtlasLineageInfo.LineageDirection
;
...
...
@@ -72,6 +74,8 @@ public class AtlasClientV2 extends AtlasBaseClient {
// Relationships APIs
private
static
final
String
RELATIONSHIPS_URI
=
BASE_URI
+
"v2/relationship/"
;
private
static
final
String
BULK_HEADERS
=
"bulk/headers"
;
private
static
final
String
BULK_SET_CLASSIFICATIONS
=
"bulk/setClassifications"
;
public
AtlasClientV2
(
String
[]
baseUrl
,
String
[]
basicAuthUserNamePassword
)
{
super
(
baseUrl
,
basicAuthUserNamePassword
);
...
...
@@ -326,13 +330,26 @@ public class AtlasClientV2 extends AtlasBaseClient {
}
public
void
deleteClassifications
(
String
guid
,
List
<
AtlasClassification
>
classifications
)
throws
AtlasServiceException
{
callAPI
(
formatPathParameters
(
API_V2
.
GET_CLASSIFICATIONS
,
guid
),
AtlasClassifications
.
class
,
classifications
);
for
(
AtlasClassification
c
:
classifications
)
{
callAPI
(
formatPathParameters
(
API_V2
.
DELETE_CLASSIFICATION
,
guid
,
c
.
getTypeName
()),
AtlasClassifications
.
class
,
classifications
);
}
}
public
void
deleteClassification
(
String
guid
,
String
classificationName
)
throws
AtlasServiceException
{
callAPI
(
formatPathParameters
(
API_V2
.
DELETE_CLASSIFICATION
,
guid
,
classificationName
),
null
,
null
);
}
public
AtlasEntityHeaders
getEntityHeaders
(
long
tagUpdateStartTime
)
throws
AtlasServiceException
{
MultivaluedMap
<
String
,
String
>
queryParams
=
new
MultivaluedMapImpl
();
queryParams
.
add
(
"tagUpdateStartTime"
,
Long
.
toString
(
tagUpdateStartTime
));
return
callAPI
(
API_V2
.
GET_BULK_HEADERS
,
AtlasEntityHeaders
.
class
,
queryParams
);
}
public
String
setClassifications
(
AtlasEntityHeaders
entityHeaders
)
throws
AtlasServiceException
{
return
callAPI
(
API_V2
.
UPDATE_BULK_SET_CLASSIFICATIONS
,
String
.
class
,
entityHeaders
);
}
/* Discovery calls */
public
AtlasSearchResult
dslSearch
(
final
String
query
)
throws
AtlasServiceException
{
MultivaluedMap
<
String
,
String
>
queryParams
=
new
MultivaluedMapImpl
();
...
...
@@ -480,6 +497,8 @@ public class AtlasClientV2 extends AtlasBaseClient {
public
static
final
API_V2
DELETE_RELATIONSHIP_BY_GUID
=
new
API_V2
(
RELATIONSHIPS_URI
+
"guid/"
,
HttpMethod
.
DELETE
,
Response
.
Status
.
NO_CONTENT
);
public
static
final
API_V2
CREATE_RELATIONSHIP
=
new
API_V2
(
RELATIONSHIPS_URI
,
HttpMethod
.
POST
,
Response
.
Status
.
OK
);
public
static
final
API_V2
UPDATE_RELATIONSHIP
=
new
API_V2
(
RELATIONSHIPS_URI
,
HttpMethod
.
PUT
,
Response
.
Status
.
OK
);
public
static
final
API_V2
GET_BULK_HEADERS
=
new
API_V2
(
ENTITY_API
+
BULK_HEADERS
,
HttpMethod
.
GET
,
Response
.
Status
.
OK
);
public
static
final
API_V2
UPDATE_BULK_SET_CLASSIFICATIONS
=
new
API_V2
(
ENTITY_API
+
AtlasClientV2
.
BULK_SET_CLASSIFICATIONS
,
HttpMethod
.
POST
,
Response
.
Status
.
OK
);
private
API_V2
(
String
path
,
String
method
,
Response
.
Status
status
)
{
super
(
path
,
method
,
status
);
...
...
This diff is collapsed.
Click to expand it.
intg/src/main/java/org/apache/atlas/model/instance/AtlasEntityHeaders.java
0 → 100644
View file @
cb2421bc
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package
org
.
apache
.
atlas
.
model
.
instance
;
import
com.fasterxml.jackson.annotation.JsonAutoDetect
;
import
com.fasterxml.jackson.annotation.JsonIgnoreProperties
;
import
com.fasterxml.jackson.databind.annotation.JsonSerialize
;
import
javax.xml.bind.annotation.XmlAccessType
;
import
javax.xml.bind.annotation.XmlAccessorType
;
import
javax.xml.bind.annotation.XmlRootElement
;
import
java.util.Map
;
import
static
com
.
fasterxml
.
jackson
.
annotation
.
JsonAutoDetect
.
Visibility
.
NONE
;
import
static
com
.
fasterxml
.
jackson
.
annotation
.
JsonAutoDetect
.
Visibility
.
PUBLIC_ONLY
;
@JsonAutoDetect
(
getterVisibility
=
PUBLIC_ONLY
,
setterVisibility
=
PUBLIC_ONLY
,
fieldVisibility
=
NONE
)
@JsonSerialize
(
include
=
JsonSerialize
.
Inclusion
.
NON_NULL
)
@JsonIgnoreProperties
(
ignoreUnknown
=
true
)
@XmlRootElement
@XmlAccessorType
(
XmlAccessType
.
PROPERTY
)
public
class
AtlasEntityHeaders
{
Map
<
String
,
AtlasEntityHeader
>
guidHeaderMap
;
public
AtlasEntityHeaders
()
{
}
public
AtlasEntityHeaders
(
Map
<
String
,
AtlasEntityHeader
>
guidEntityHeaderMap
)
{
guidHeaderMap
=
guidEntityHeaderMap
;
}
public
void
setGuidHeaderMap
(
Map
<
String
,
AtlasEntityHeader
>
guidHeaderMap
)
{
this
.
guidHeaderMap
=
guidHeaderMap
;
}
public
Map
<
String
,
AtlasEntityHeader
>
getGuidHeaderMap
()
{
return
guidHeaderMap
;
}
}
This diff is collapsed.
Click to expand it.
repository/src/main/java/org/apache/atlas/repository/audit/CassandraBasedAuditRepository.java
View file @
cb2421bc
...
...
@@ -30,6 +30,7 @@ import org.apache.atlas.EntityAuditEvent;
import
org.apache.atlas.annotation.ConditionalOnAtlasProperty
;
import
org.apache.atlas.exception.AtlasBaseException
;
import
org.apache.atlas.model.audit.EntityAuditEventV2
;
import
org.apache.commons.lang.NotImplementedException
;
import
org.apache.commons.lang.StringUtils
;
import
org.slf4j.Logger
;
import
org.slf4j.LoggerFactory
;
...
...
@@ -39,6 +40,7 @@ import java.util.ArrayList;
import
java.util.HashMap
;
import
java.util.List
;
import
java.util.Map
;
import
java.util.Set
;
import
javax.inject.Singleton
;
...
...
@@ -188,6 +190,11 @@ public class CassandraBasedAuditRepository extends AbstractStorageBasedAuditRepo
}
@Override
public
Set
<
String
>
getEntitiesWithTagChanges
(
long
fromTimestamp
,
long
toTimestamp
)
throws
AtlasBaseException
{
throw
new
NotImplementedException
();
}
@Override
public
void
start
()
throws
AtlasException
{
initApplicationProperties
();
initializeSettings
();
...
...
This diff is collapsed.
Click to expand it.
repository/src/main/java/org/apache/atlas/repository/audit/EntityAuditRepository.java
View file @
cb2421bc
...
...
@@ -24,6 +24,7 @@ import org.apache.atlas.model.audit.EntityAuditEventV2;
import
org.apache.atlas.exception.AtlasBaseException
;
import
java.util.List
;
import
java.util.Set
;
/**
* Interface for repository for storing entity audit events
...
...
@@ -77,6 +78,14 @@ public interface EntityAuditRepository {
*/
List
<
EntityAuditEventV2
>
listEventsV2
(
String
entityId
,
String
startKey
,
short
n
)
throws
AtlasBaseException
;
/***
* List events for given time range where classifications have been added, deleted or updated.
* @param fromTimestamp from timestamp
* @param toTimestamp to timestamp
* @return events that match the range
* @throws AtlasBaseException
*/
Set
<
String
>
getEntitiesWithTagChanges
(
long
fromTimestamp
,
long
toTimestamp
)
throws
AtlasBaseException
;
/**
* List events for the given entity id in decreasing order of timestamp, from the given timestamp. Returns n results
...
...
This diff is collapsed.
Click to expand it.
repository/src/main/java/org/apache/atlas/repository/audit/HBaseBasedAuditRepository.java
View file @
cb2421bc
...
...
@@ -50,7 +50,10 @@ import org.apache.hadoop.hbase.client.Result;
import
org.apache.hadoop.hbase.client.ResultScanner
;
import
org.apache.hadoop.hbase.client.Scan
;
import
org.apache.hadoop.hbase.client.Table
;
import
org.apache.hadoop.hbase.filter.BinaryPrefixComparator
;
import
org.apache.hadoop.hbase.filter.CompareFilter
;
import
org.apache.hadoop.hbase.filter.PageFilter
;
import
org.apache.hadoop.hbase.filter.SingleColumnValueFilter
;
import
org.apache.hadoop.hbase.io.compress.Compression
;
import
org.apache.hadoop.hbase.io.encoding.DataBlockEncoding
;
import
org.apache.hadoop.hbase.regionserver.BloomType
;
...
...
@@ -64,9 +67,10 @@ import javax.inject.Singleton;
import
java.io.Closeable
;
import
java.io.IOException
;
import
java.util.ArrayList
;
import
java.util.
Iterator
;
import
java.util.
HashSet
;
import
java.util.List
;
import
java.util.Properties
;
import
java.util.Set
;
import
static
org
.
apache
.
atlas
.
EntityAuditEvent
.
EntityAuditAction
.
TAG_ADD
;
import
static
org
.
apache
.
atlas
.
EntityAuditEvent
.
EntityAuditAction
.
TAG_DELETE
;
...
...
@@ -546,6 +550,52 @@ public class HBaseBasedAuditRepository extends AbstractStorageBasedAuditReposito
}
@Override
public
Set
<
String
>
getEntitiesWithTagChanges
(
long
fromTimestamp
,
long
toTimestamp
)
throws
AtlasBaseException
{
final
String
classificationUpdatesAction
=
"CLASSIFICATION_"
;
if
(
LOG
.
isDebugEnabled
())
{
LOG
.
debug
(
"Listing events for fromTimestamp {}, toTimestamp {}, action {}"
,
fromTimestamp
,
toTimestamp
);
}
Table
table
=
null
;
ResultScanner
scanner
=
null
;
try
{
Set
<
String
>
guids
=
new
HashSet
<>();
table
=
connection
.
getTable
(
tableName
);
byte
[]
filterValue
=
Bytes
.
toBytes
(
classificationUpdatesAction
);
BinaryPrefixComparator
binaryPrefixComparator
=
new
BinaryPrefixComparator
(
filterValue
);
SingleColumnValueFilter
filter
=
new
SingleColumnValueFilter
(
COLUMN_FAMILY
,
COLUMN_ACTION
,
CompareFilter
.
CompareOp
.
EQUAL
,
binaryPrefixComparator
);
Scan
scan
=
new
Scan
().
setFilter
(
filter
).
setTimeRange
(
fromTimestamp
,
toTimestamp
);
Result
result
;
scanner
=
table
.
getScanner
(
scan
);
while
((
result
=
scanner
.
next
())
!=
null
)
{
EntityAuditEvent
event
=
fromKey
(
result
.
getRow
());
if
(
event
==
null
)
{
continue
;
}
guids
.
add
(
event
.
getEntityId
());
}
return
guids
;
}
catch
(
IOException
e
)
{
throw
new
AtlasBaseException
(
e
);
}
finally
{
try
{
close
(
scanner
);
close
(
table
);
}
catch
(
AtlasException
e
)
{
throw
new
AtlasBaseException
(
e
);
}
}
}
@Override
public
void
start
()
throws
AtlasException
{
Configuration
configuration
=
ApplicationProperties
.
get
();
startInternal
(
configuration
,
getHBaseConfiguration
(
configuration
));
...
...
This diff is collapsed.
Click to expand it.
repository/src/main/java/org/apache/atlas/repository/audit/InMemoryEntityAuditRepository.java
View file @
cb2421bc
...
...
@@ -21,6 +21,7 @@ package org.apache.atlas.repository.audit;
import
org.apache.atlas.AtlasException
;
import
org.apache.atlas.EntityAuditEvent
;
import
org.apache.atlas.annotation.ConditionalOnAtlasProperty
;
import
org.apache.atlas.exception.AtlasBaseException
;
import
org.apache.atlas.model.audit.EntityAuditEventV2
;
import
org.apache.commons.collections.CollectionUtils
;
import
org.springframework.stereotype.Component
;
...
...
@@ -28,7 +29,9 @@ import org.springframework.stereotype.Component;
import
javax.inject.Singleton
;
import
java.util.ArrayList
;
import
java.util.Arrays
;
import
java.util.HashSet
;
import
java.util.List
;
import
java.util.Set
;
import
java.util.SortedMap
;
import
java.util.TreeMap
;
...
...
@@ -119,6 +122,20 @@ public class InMemoryEntityAuditRepository implements EntityAuditRepository {
}
@Override
public
Set
<
String
>
getEntitiesWithTagChanges
(
long
fromTimestamp
,
long
toTimestamp
)
throws
AtlasBaseException
{
Set
<
String
>
events
=
new
HashSet
<>();
for
(
EntityAuditEventV2
event
:
auditEventsV2
.
values
())
{
long
timestamp
=
event
.
getTimestamp
();
if
(
timestamp
>
fromTimestamp
&&
timestamp
<=
toTimestamp
)
{
events
.
add
(
event
.
getEntityId
());
}
}
return
events
;
}
@Override
public
List
<
Object
>
listEvents
(
String
entityId
,
String
startKey
,
short
maxResults
)
{
List
events
=
listEventsV2
(
entityId
,
startKey
,
maxResults
);
...
...
This diff is collapsed.
Click to expand it.
repository/src/main/java/org/apache/atlas/repository/audit/NoopEntityAuditRepository.java
View file @
cb2421bc
...
...
@@ -20,12 +20,14 @@ package org.apache.atlas.repository.audit;
import
org.apache.atlas.EntityAuditEvent
;
import
org.apache.atlas.annotation.ConditionalOnAtlasProperty
;
import
org.apache.atlas.exception.AtlasBaseException
;
import
org.apache.atlas.model.audit.EntityAuditEventV2
;
import
org.springframework.stereotype.Component
;
import
javax.inject.Singleton
;
import
java.util.Collections
;
import
java.util.List
;
import
java.util.Set
;
/**
* Implementation that completely disables the audit repository.
...
...
@@ -66,6 +68,11 @@ public class NoopEntityAuditRepository implements EntityAuditRepository {
}
@Override
public
Set
<
String
>
getEntitiesWithTagChanges
(
long
fromTimestamp
,
long
toTimestamp
)
throws
AtlasBaseException
{
return
Collections
.
emptySet
();
}
@Override
public
List
<
Object
>
listEvents
(
String
entityId
,
String
startKey
,
short
n
)
{
return
Collections
.
emptyList
();
}
...
...
This diff is collapsed.
Click to expand it.
repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasEntityStore.java
View file @
cb2421bc
...
...
@@ -24,6 +24,7 @@ import org.apache.atlas.model.instance.AtlasClassification;
import
org.apache.atlas.model.instance.AtlasEntity.AtlasEntitiesWithExtInfo
;
import
org.apache.atlas.model.instance.AtlasEntity.AtlasEntityWithExtInfo
;
import
org.apache.atlas.model.instance.AtlasEntityHeader
;
import
org.apache.atlas.model.instance.AtlasEntityHeaders
;
import
org.apache.atlas.model.instance.AtlasObjectId
;
import
org.apache.atlas.model.instance.EntityMutationResponse
;
import
org.apache.atlas.repository.store.graph.v2.EntityStream
;
...
...
@@ -216,4 +217,6 @@ public interface AtlasEntityStore {
List
<
AtlasClassification
>
getClassifications
(
String
guid
)
throws
AtlasBaseException
;
AtlasClassification
getClassification
(
String
guid
,
String
classificationName
)
throws
AtlasBaseException
;
String
setClassifications
(
AtlasEntityHeaders
entityHeaders
);
}
This diff is collapsed.
Click to expand it.
repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java
View file @
cb2421bc
...
...
@@ -657,6 +657,13 @@ public class AtlasEntityStoreV2 implements AtlasEntityStore {
return
ret
;
}
@Override
@GraphTransaction
public
String
setClassifications
(
AtlasEntityHeaders
entityHeaders
)
{
ClassificationAssociator
.
Updater
associator
=
new
ClassificationAssociator
.
Updater
(
typeRegistry
,
this
);
return
associator
.
setClassifications
(
entityHeaders
.
getGuidHeaderMap
());
}
private
EntityMutationResponse
createOrUpdate
(
EntityStream
entityStream
,
boolean
isPartialUpdate
,
boolean
replaceClassifications
)
throws
AtlasBaseException
{
if
(
LOG
.
isDebugEnabled
())
{
LOG
.
debug
(
"==> createOrUpdate()"
);
...
...
This diff is collapsed.
Click to expand it.
repository/src/main/java/org/apache/atlas/repository/store/graph/v2/ClassificationAssociator.java
0 → 100644
View file @
cb2421bc
This diff is collapsed.
Click to expand it.
repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java
View file @
cb2421bc
...
...
@@ -95,7 +95,7 @@ import static org.apache.atlas.type.AtlasStructType.AtlasAttribute.AtlasRelation
import
static
org
.
apache
.
atlas
.
type
.
AtlasStructType
.
AtlasAttribute
.
AtlasRelationshipEdgeDirection
.
OUT
;
@Component
public
final
class
EntityGraphRetriever
{
public
class
EntityGraphRetriever
{
private
static
final
Logger
LOG
=
LoggerFactory
.
getLogger
(
EntityGraphRetriever
.
class
);
private
static
final
String
TERM_RELATION_NAME
=
"AtlasGlossarySemanticAssignment"
;
...
...
This diff is collapsed.
Click to expand it.
repository/src/test/java/org/apache/atlas/repository/store/graph/v2/ClassificationAssociatorTest.java
0 → 100644
View file @
cb2421bc
This diff is collapsed.
Click to expand it.
repository/src/test/resources/json/classification-association/col-entity-None.json
0 → 100644
View file @
cb2421bc
{
"typeName"
:
"hive_column"
,
"attributes"
:
{
"owner"
:
"hive"
,
"qualifiedName"
:
"hortoniabank.us_customers.nationalid@cl1"
,
"name"
:
"nationalid"
},
"guid"
:
"0ce68113-77fe-4ed1-9585-69371202bd74"
,
"status"
:
"ACTIVE"
}
This diff is collapsed.
Click to expand it.
repository/src/test/resources/json/classification-association/col-entity-PII-FIN_PII.json
0 → 100644
View file @
cb2421bc
{
"typeName"
:
"hive_column"
,
"attributes"
:
{
"owner"
:
"hive"
,
"qualifiedName"
:
"hortoniabank.us_customers.nationalid@cl1"
,
"name"
:
"nationalid"
},
"guid"
:
"0ce68113-77fe-4ed1-9585-69371202bd74"
,
"status"
:
"ACTIVE"
,
"classifications"
:
[
{
"typeName"
:
"PII"
,
"attributes"
:
{
"type"
:
"ssn"
},
"entityGuid"
:
"0ce68113-77fe-4ed1-9585-69371202bd74"
,
"entityStatus"
:
"ACTIVE"
,
"propagate"
:
true
,
"removePropagationsOnEntityDelete"
:
false
},
{
"typeName"
:
"FIN_PII"
,
"attributes"
:
{
"type"
:
"ssn"
},
"entityGuid"
:
"0ce68113-77fe-4ed1-9585-69371202bd74"
,
"entityStatus"
:
"ACTIVE"
,
"propagate"
:
true
,
"removePropagationsOnEntityDelete"
:
false
}
]
}
This diff is collapsed.
Click to expand it.
repository/src/test/resources/json/classification-association/col-entity-PII.json
0 → 100644
View file @
cb2421bc
{
"typeName"
:
"hive_column"
,
"attributes"
:
{
"owner"
:
"hive"
,
"qualifiedName"
:
"hortoniabank.us_customers.nationalid@cl1"
,
"name"
:
"nationalid"
},
"guid"
:
"0ce68113-77fe-4ed1-9585-69371202bd74"
,
"status"
:
"ACTIVE"
,
"classifications"
:
[
{
"typeName"
:
"PII"
,
"attributes"
:
{
"type"
:
"ssn"
},
"entityGuid"
:
"0ce68113-77fe-4ed1-9585-69371202bd74"
,
"entityStatus"
:
"ACTIVE"
,
"propagate"
:
true
,
"removePropagationsOnEntityDelete"
:
false
}
]
}
This diff is collapsed.
Click to expand it.
repository/src/test/resources/json/classification-association/col-entity-T1-prop-Tn.json
0 → 100644
View file @
cb2421bc
{
"typeName"
:
"hive_column"
,
"attributes"
:
{
"owner"
:
"hive"
,
"createTime"
:
1547071410000
,
"qualifiedName"
:
"stocks.daily@cl1"
,
"name"
:
"daily"
},
"guid"
:
"df122fc3-5555-40f8-a30f-3090b8a622f8"
,
"status"
:
"ACTIVE"
,
"displayText"
:
"daily"
,
"classifications"
:
[
{
"typeName"
:
"T1"
,
"attributes"
:
{},
"entityGuid"
:
"df122fc3-5555-40f8-a30f-3090b8a622f8"
,
"entityStatus"
:
"ACTIVE"
,
"propagate"
:
false
,
"validityPeriods"
:
[],
"removePropagationsOnEntityDelete"
:
false
},
{
"typeName"
:
"Tn"
,
"attributes"
:
{},
"entityGuid"
:
"22222222-5555-40f8-a30f-3090b8a622f8"
,
"entityStatus"
:
"ACTIVE"
,
"propagate"
:
false
,
"validityPeriods"
:
[],
"removePropagationsOnEntityDelete"
:
false
}
],
"meaningNames"
:
[],
"meanings"
:
[]
}
This diff is collapsed.
Click to expand it.
repository/src/test/resources/json/classification-association/header-FIN_PII.json
0 → 100644
View file @
cb2421bc
{
"guidHeaderMap"
:
{
"0ce68113-77fe-4ed1-9585-69371202bd74"
:
{
"typeName"
:
"hive_column"
,
"attributes"
:
{
"owner"
:
"hive"
,
"qualifiedName"
:
"hortoniabank.us_customers.nationalid@cl1"
,
"name"
:
"nationalid"
},
"guid"
:
"0ce68113-77fe-4ed1-9585-69371202bd74"
,
"status"
:
"ACTIVE"
,
"displayText"
:
"nationalid"
,
"classificationNames"
:
[
"FIN_PII"
],
"classifications"
:
[
{
"typeName"
:
"FIN_PII"
,
"attributes"
:
{
"type"
:
"ssn"
},
"entityGuid"
:
"0ce68113-77fe-4ed1-9585-69371202bd74"
,
"entityStatus"
:
"ACTIVE"
,
"propagate"
:
true
,
"removePropagationsOnEntityDelete"
:
false
}
],
"meaningNames"
:
[],
"meanings"
:
[]
}
}
}
This diff is collapsed.
Click to expand it.
repository/src/test/resources/json/classification-association/header-None.json
0 → 100644
View file @
cb2421bc
{
"guidHeaderMap"
:
{
"0ce68113-77fe-4ed1-9585-69371202bd74"
:
{
"typeName"
:
"hive_column"
,
"attributes"
:
{
"owner"
:
"hive"
,
"qualifiedName"
:
"hortoniabank.us_customers.nationalid@cl1"
,
"name"
:
"nationalid"
},
"guid"
:
"0ce68113-77fe-4ed1-9585-69371202bd74"
,
"status"
:
"ACTIVE"
,
"displayText"
:
"nationalid"
,
"classificationNames"
:
[
],
"classifications"
:
[
],
"meaningNames"
:
[],
"meanings"
:
[]
}
}
}
This diff is collapsed.
Click to expand it.
repository/src/test/resources/json/classification-association/header-PII-VENDOR_PII.json
0 → 100644
View file @
cb2421bc
{
"guidHeaderMap"
:
{
"0ce68113-77fe-4ed1-9585-69371202bd74"
:
{
"typeName"
:
"hive_column"
,
"attributes"
:
{
"owner"
:
"hive"
,
"qualifiedName"
:
"hortoniabank.us_customers.nationalid@cl1"
,
"name"
:
"nationalid"
},
"guid"
:
"0ce68113-77fe-4ed1-9585-69371202bd74"
,
"status"
:
"ACTIVE"
,
"displayText"
:
"nationalid"
,
"classificationNames"
:
[
"PII"
],
"classifications"
:
[
{
"typeName"
:
"PII"
,
"attributes"
:
{
"type"
:
"ssn"
},
"entityGuid"
:
"0ce68113-77fe-4ed1-9585-69371202bd74"
,
"entityStatus"
:
"ACTIVE"
,
"propagate"
:
true
,
"removePropagationsOnEntityDelete"
:
false
},
{
"typeName"
:
"VENDOR_PII"
,
"attributes"
:
{
"type"
:
"ssn"
},
"entityGuid"
:
"0ce68113-77fe-4ed1-9585-69371202bd74"
,
"entityStatus"
:
"ACTIVE"
,
"propagate"
:
true
,
"removePropagationsOnEntityDelete"
:
false
}
],
"meaningNames"
:
[],
"meanings"
:
[]
}
}
}
This diff is collapsed.
Click to expand it.
repository/src/test/resources/json/classification-association/header-PII.json
0 → 100644
View file @
cb2421bc
{
"guidHeaderMap"
:
{
"0ce68113-77fe-4ed1-9585-69371202bd74"
:
{
"typeName"
:
"hive_column"
,
"attributes"
:
{
"owner"
:
"hive"
,
"qualifiedName"
:
"hortoniabank.us_customers.nationalid@cl1"
,
"name"
:
"nationalid"
},
"guid"
:
"0ce68113-77fe-4ed1-9585-69371202bd74"
,
"status"
:
"ACTIVE"
,
"displayText"
:
"nationalid"
,
"classificationNames"
:
[
"PII"
],
"classifications"
:
[
{
"typeName"
:
"PII"
,
"attributes"
:
{
"type"
:
"ssn"
},
"entityGuid"
:
"0ce68113-77fe-4ed1-9585-69371202bd74"
,
"entityStatus"
:
"ACTIVE"
,
"propagate"
:
true
,
"removePropagationsOnEntityDelete"
:
false
}
],
"meaningNames"
:
[],
"meanings"
:
[]
}
}
}
This diff is collapsed.
Click to expand it.
repository/src/test/resources/json/classification-association/header-Tx-prop-T1.json
0 → 100644
View file @
cb2421bc
{
"guidHeaderMap"
:
{
"0ce68113-77fe-4ed1-9585-69371202bd74"
:
{
"typeName"
:
"hive_column"
,
"attributes"
:
{
"owner"
:
"hive"
,
"qualifiedName"
:
"hortoniabank.us_customers.nationalid@cl1"
,
"name"
:
"nationalid"
},
"guid"
:
"0ce68113-77fe-4ed1-9585-69371202bd74"
,
"status"
:
"ACTIVE"
,
"displayText"
:
"nationalid"
,
"classificationNames"
:
[
"T1"
,
"Tx"
],
"classifications"
:
[
{
"typeName"
:
"Tx"
,
"attributes"
:
{
"type"
:
"ssn"
},
"entityGuid"
:
"0ce68113-77fe-4ed1-9585-69371202bd74"
,
"entityStatus"
:
"ACTIVE"
,
"propagate"
:
true
,
"removePropagationsOnEntityDelete"
:
false
},
{
"typeName"
:
"T1"
,
"attributes"
:
{
"type"
:
"ssn"
},
"entityGuid"
:
"22222222-77fe-4ed1-9585-69371202bd74"
,
"entityStatus"
:
"ACTIVE"
,
"propagate"
:
true
,
"removePropagationsOnEntityDelete"
:
false
}
],
"meaningNames"
:
[],
"meanings"
:
[]
}
}
}
This diff is collapsed.
Click to expand it.
repository/src/test/resources/json/classification-association/header-Tx.json
0 → 100644
View file @
cb2421bc
{
"typeName"
:
"hive_table"
,
"attributes"
:
{
"owner"
:
"hive"
,
"createTime"
:
1547071410000
,
"qualifiedName"
:
"stocks.daily@cl1"
,
"name"
:
"daily"
},
"guid"
:
"df122fc3-5555-40f8-a30f-3090b8a622f8"
,
"status"
:
"ACTIVE"
,
"displayText"
:
"daily"
,
"classificationNames"
:
[],
"classifications"
:
[
{
"typeName"
:
"Tx"
,
"attributes"
:
{},
"entityGuid"
:
"df122fc3-5555-40f8-a30f-3090b8a622f8"
,
"entityStatus"
:
"ACTIVE"
,
"propagate"
:
false
,
"validityPeriods"
:
[],
"removePropagationsOnEntityDelete"
:
false
}
],
"meaningNames"
:
[],
"meanings"
:
[]
}
\ No newline at end of file
This diff is collapsed.
Click to expand it.
repository/src/test/resources/json/classification-association/header-empty.json
0 → 100644
View file @
cb2421bc
{
"guidHeaderMap"
:
{}
}
\ No newline at end of file
This diff is collapsed.
Click to expand it.
webapp/src/main/java/org/apache/atlas/web/rest/EntityREST.java
View file @
cb2421bc
...
...
@@ -26,10 +26,12 @@ import org.apache.atlas.model.instance.AtlasClassification;
import
org.apache.atlas.model.instance.AtlasEntity.AtlasEntitiesWithExtInfo
;
import
org.apache.atlas.model.instance.AtlasEntity.AtlasEntityWithExtInfo
;
import
org.apache.atlas.model.instance.AtlasEntityHeader
;
import
org.apache.atlas.model.instance.AtlasEntityHeaders
;
import
org.apache.atlas.model.instance.ClassificationAssociateRequest
;
import
org.apache.atlas.model.instance.EntityMutationResponse
;
import
org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef
;
import
org.apache.atlas.repository.audit.EntityAuditRepository
;
import
org.apache.atlas.repository.store.graph.v2.ClassificationAssociator
;
import
org.apache.atlas.repository.converters.AtlasInstanceConverter
;
import
org.apache.atlas.repository.store.graph.AtlasEntityStore
;
import
org.apache.atlas.repository.store.graph.v2.AtlasEntityStream
;
...
...
@@ -49,7 +51,16 @@ import org.springframework.stereotype.Service;
import
javax.inject.Inject
;
import
javax.inject.Singleton
;
import
javax.servlet.http.HttpServletRequest
;
import
javax.ws.rs.*
;
import
javax.ws.rs.Consumes
;
import
javax.ws.rs.DELETE
;
import
javax.ws.rs.DefaultValue
;
import
javax.ws.rs.GET
;
import
javax.ws.rs.POST
;
import
javax.ws.rs.PUT
;
import
javax.ws.rs.Path
;
import
javax.ws.rs.PathParam
;
import
javax.ws.rs.Produces
;
import
javax.ws.rs.QueryParam
;
import
javax.ws.rs.core.Context
;
import
javax.ws.rs.core.MediaType
;
import
java.util.ArrayList
;
...
...
@@ -77,7 +88,6 @@ public class EntityREST {
private
final
EntityAuditRepository
auditRepository
;
private
final
AtlasInstanceConverter
instanceConverter
;
@Inject
public
EntityREST
(
AtlasTypeRegistry
typeRegistry
,
AtlasEntityStore
entitiesStore
,
EntityAuditRepository
auditRepository
,
AtlasInstanceConverter
instanceConverter
)
{
...
...
@@ -601,7 +611,7 @@ public class EntityREST {
}
/**
* Bulk API to create new entities or update existing entities in Atlas.
* Bulk API to create new entities or update
s
existing entities in Atlas.
* Existing entity is matched using its unique guid if supplied or by its unique attributes eg: qualifiedName
*/
@POST
...
...
@@ -708,6 +718,49 @@ public class EntityREST {
}
}
@GET
@Path
(
"bulk/headers"
)
@Produces
(
Servlets
.
JSON_MEDIA_TYPE
)
public
AtlasEntityHeaders
getEntityHeaders
(
@QueryParam
(
"tagUpdateStartTime"
)
long
tagUpdateStartTime
)
throws
AtlasBaseException
{
AtlasPerfTracer
perf
=
null
;
try
{
long
tagUpdateEndTime
=
System
.
currentTimeMillis
();
if
(
tagUpdateStartTime
>
tagUpdateEndTime
)
{
throw
new
AtlasBaseException
(
AtlasErrorCode
.
BAD_REQUEST
,
"fromTimestamp should be less than toTimestamp"
);
}
if
(
AtlasPerfTracer
.
isPerfTraceEnabled
(
PERF_LOG
))
{
perf
=
AtlasPerfTracer
.
getPerfTracer
(
PERF_LOG
,
"EntityREST.getEntityHeaders("
+
tagUpdateStartTime
+
", "
+
tagUpdateEndTime
+
")"
);
}
ClassificationAssociator
.
Retriever
associator
=
new
ClassificationAssociator
.
Retriever
(
typeRegistry
,
auditRepository
);
return
associator
.
get
(
tagUpdateStartTime
,
tagUpdateEndTime
);
}
finally
{
AtlasPerfTracer
.
log
(
perf
);
}
}
@POST
@Path
(
"bulk/setClassifications"
)
@Produces
(
Servlets
.
JSON_MEDIA_TYPE
)
@Consumes
(
Servlets
.
JSON_MEDIA_TYPE
)
public
String
setClassifications
(
AtlasEntityHeaders
entityHeaders
)
throws
AtlasBaseException
{
AtlasPerfTracer
perf
=
null
;
try
{
if
(
AtlasPerfTracer
.
isPerfTraceEnabled
(
PERF_LOG
))
{
perf
=
AtlasPerfTracer
.
getPerfTracer
(
PERF_LOG
,
"EntityREST.setClassifications()"
);
}
ClassificationAssociator
.
Updater
associator
=
new
ClassificationAssociator
.
Updater
(
typeRegistry
,
entitiesStore
);
return
associator
.
setClassifications
(
entityHeaders
.
getGuidHeaderMap
());
}
finally
{
AtlasPerfTracer
.
log
(
perf
);
}
}
private
AtlasEntityType
ensureEntityType
(
String
typeName
)
throws
AtlasBaseException
{
AtlasEntityType
ret
=
typeRegistry
.
getEntityTypeByName
(
typeName
);
...
...
This diff is collapsed.
Click to expand it.
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment