Skip to content
Projects
Groups
Snippets
Help
This project
Loading...
Sign in / Register
Toggle navigation
A
atlas
Project
Overview
Details
Activity
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
dataplatform
atlas
Commits
48c10133
Commit
48c10133
authored
Feb 28, 2017
by
apoorvnaik
Committed by
Madhan Neethiraj
Feb 28, 2017
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
ATLAS-1606: introduced query provider to handle Gremlin version specific queries
Signed-off-by:
Madhan Neethiraj
<
madhan@apache.org
>
parent
b1717f2e
Hide whitespace changes
Inline
Side-by-side
Showing
7 changed files
with
236 additions
and
84 deletions
+236
-84
EntityLineageService.java
...java/org/apache/atlas/discovery/EntityLineageService.java
+18
-29
MetricsService.java
...c/main/java/org/apache/atlas/services/MetricsService.java
+48
-41
AtlasGremlin2QueryProvider.java
...ava/org/apache/atlas/util/AtlasGremlin2QueryProvider.java
+68
-0
AtlasGremlin3QueryProvider.java
...ava/org/apache/atlas/util/AtlasGremlin3QueryProvider.java
+27
-0
AtlasGremlinQueryProvider.java
...java/org/apache/atlas/util/AtlasGremlinQueryProvider.java
+57
-0
MetricsServiceTest.java
...st/java/org/apache/atlas/services/MetricsServiceTest.java
+1
-1
ExportService.java
...in/java/org/apache/atlas/web/resources/ExportService.java
+17
-13
No files found.
repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java
View file @
48c10133
...
...
@@ -31,6 +31,8 @@ import org.apache.atlas.repository.Constants;
import
org.apache.atlas.repository.graph.AtlasGraphProvider
;
import
org.apache.atlas.repository.graphdb.AtlasGraph
;
import
org.apache.atlas.repository.graphdb.AtlasVertex
;
import
org.apache.atlas.util.AtlasGremlinQueryProvider
;
import
org.apache.atlas.util.AtlasGremlinQueryProvider.AtlasGremlinQuery
;
import
org.apache.commons.collections.CollectionUtils
;
import
javax.inject.Inject
;
...
...
@@ -47,29 +49,12 @@ public class EntityLineageService implements AtlasLineageService {
private
static
final
String
OUTPUT_PROCESS_EDGE
=
"__Process.outputs"
;
private
final
AtlasGraph
graph
;
/**
* Gremlin query to retrieve input/output lineage for specified depth on a DataSet entity.
* return list of Atlas vertices paths.
*/
private
static
final
String
PARTIAL_LINEAGE_QUERY
=
"g.V('__guid', '%s').as('src').in('%s').out('%s')."
+
"loop('src', {it.loops <= %s}, {((it.object.'__superTypeNames') ? "
+
"(it.object.'__superTypeNames'.contains('DataSet')) : false)})."
+
"path().toList()"
;
/**
* Gremlin query to retrieve all (no fixed depth) input/output lineage for a DataSet entity.
* return list of Atlas vertices paths.
*/
private
static
final
String
FULL_LINEAGE_QUERY
=
"g.V('__guid', '%s').as('src').in('%s').out('%s')."
+
"loop('src', {((it.path.contains(it.object)) ? false : true)}, "
+
"{((it.object.'__superTypeNames') ? "
+
"(it.object.'__superTypeNames'.contains('DataSet')) : false)})."
+
"path().toList()"
;
private
final
AtlasGremlinQueryProvider
gremlinQueryProvider
;
@Inject
EntityLineageService
()
throws
DiscoveryException
{
this
.
graph
=
AtlasGraphProvider
.
getGraphInstance
();
this
.
gremlinQueryProvider
=
AtlasGremlinQueryProvider
.
INSTANCE
;
}
@Override
...
...
@@ -157,23 +142,27 @@ public class EntityLineageService implements AtlasLineageService {
String
lineageQuery
=
null
;
if
(
direction
.
equals
(
LineageDirection
.
INPUT
))
{
if
(
depth
<
1
)
{
lineageQuery
=
String
.
format
(
FULL_LINEAGE_QUERY
,
entityGuid
,
OUTPUT_PROCESS_EDGE
,
INPUT_PROCESS_EDGE
);
}
else
{
lineageQuery
=
String
.
format
(
PARTIAL_LINEAGE_QUERY
,
entityGuid
,
OUTPUT_PROCESS_EDGE
,
INPUT_PROCESS_EDGE
,
depth
);
}
lineageQuery
=
generateLineageQuery
(
entityGuid
,
depth
,
OUTPUT_PROCESS_EDGE
,
INPUT_PROCESS_EDGE
);
}
else
if
(
direction
.
equals
(
LineageDirection
.
OUTPUT
))
{
if
(
depth
<
1
)
{
lineageQuery
=
String
.
format
(
FULL_LINEAGE_QUERY
,
entityGuid
,
INPUT_PROCESS_EDGE
,
OUTPUT_PROCESS_EDGE
);
}
else
{
lineageQuery
=
String
.
format
(
PARTIAL_LINEAGE_QUERY
,
entityGuid
,
INPUT_PROCESS_EDGE
,
OUTPUT_PROCESS_EDGE
,
depth
);
}
lineageQuery
=
generateLineageQuery
(
entityGuid
,
depth
,
INPUT_PROCESS_EDGE
,
OUTPUT_PROCESS_EDGE
);
}
return
lineageQuery
;
}
private
String
generateLineageQuery
(
String
entityGuid
,
int
depth
,
String
incomingFrom
,
String
outgoingTo
)
{
String
lineageQuery
;
if
(
depth
<
1
)
{
String
query
=
gremlinQueryProvider
.
getQuery
(
AtlasGremlinQuery
.
FULL_LINEAGE
);
lineageQuery
=
String
.
format
(
query
,
entityGuid
,
incomingFrom
,
outgoingTo
);
}
else
{
String
query
=
gremlinQueryProvider
.
getQuery
(
AtlasGremlinQuery
.
PARTIAL_LINEAGE
);
lineageQuery
=
String
.
format
(
query
,
entityGuid
,
incomingFrom
,
outgoingTo
,
depth
);
}
return
lineageQuery
;
}
private
AtlasEntityHeader
toAtlasEntityHeader
(
Object
vertexObj
)
{
AtlasEntityHeader
ret
=
new
AtlasEntityHeader
();
...
...
repository/src/main/java/org/apache/atlas/services/MetricsService.java
View file @
48c10133
...
...
@@ -24,14 +24,14 @@ import org.apache.atlas.AtlasException;
import
org.apache.atlas.model.metrics.AtlasMetrics
;
import
org.apache.atlas.repository.graph.AtlasGraphProvider
;
import
org.apache.atlas.repository.graphdb.AtlasGraph
;
import
org.apache.atlas.type.AtlasTypeRegistry
;
import
org.apache.atlas.util.AtlasGremlinQueryProvider
;
import
org.apache.atlas.util.AtlasGremlinQueryProvider.AtlasGremlinQuery
;
import
org.apache.commons.configuration.Configuration
;
import
org.slf4j.Logger
;
import
org.slf4j.LoggerFactory
;
import
javax.inject.Inject
;
import
javax.script.ScriptException
;
import
java.util.Collection
;
import
java.util.List
;
import
java.util.Map
;
...
...
@@ -39,51 +39,54 @@ import java.util.Map;
public
class
MetricsService
{
private
static
final
Logger
LOG
=
LoggerFactory
.
getLogger
(
MetricsService
.
class
);
public
static
final
String
METRIC_QUERY_PREFIX
=
"atlas.metric.query."
;
public
static
final
String
METRIC_QUERY_CACHE_TTL
=
"atlas.metric.query.cache.ttlInSecs"
;
public
static
final
int
DEFAULT_CACHE_TTL_IN_SECS
=
900
;
// Query Category constants
public
static
final
String
TYPE
=
"type"
;
public
static
final
String
ENTITY
=
"entity"
;
public
static
final
String
TAG
=
"tag"
;
public
static
final
String
GENERAL
=
"general"
;
public
static
final
String
METRIC_TYPE_COUNT
=
TYPE
+
"Count"
;
public
static
final
String
METRIC_TYPE_UNUSED_COUNT
=
TYPE
+
"UnusedCount"
;
public
static
final
String
METRIC_TYPE_ENTITIES
=
TYPE
+
"Entities"
;
// Query names
protected
static
final
String
METRIC_TYPE_COUNT
=
TYPE
+
"Count"
;
protected
static
final
String
METRIC_TYPE_UNUSED_COUNT
=
TYPE
+
"UnusedCount"
;
protected
static
final
String
METRIC_TYPE_ENTITIES
=
TYPE
+
"Entities"
;
p
ublic
static
final
String
METRIC_ENTITY_COUNT
=
ENTITY
+
"Count"
;
p
ublic
static
final
String
METRIC_ENTITY_DELETED
=
ENTITY
+
"Deleted"
;
p
ublic
static
final
String
METRIC_TAGGED_ENTITIES
=
ENTITY
+
"Tagged"
;
p
ublic
static
final
String
METRIC_TAGS_PER_ENTITY
=
ENTITY
+
"Tags"
;
p
rotected
static
final
String
METRIC_ENTITY_COUNT
=
ENTITY
+
"Count"
;
p
rotected
static
final
String
METRIC_ENTITY_DELETED
=
ENTITY
+
"Deleted"
;
p
rotected
static
final
String
METRIC_TAGGED_ENTITIES
=
ENTITY
+
"Tagged"
;
p
rotected
static
final
String
METRIC_TAGS_PER_ENTITY
=
ENTITY
+
"Tags"
;
public
static
final
String
METRIC_TAG_COUNT
=
TAG
+
"Count"
;
public
static
final
String
METRIC_ENTITIES_PER_TAG
=
TAG
+
"Entities"
;
protected
static
final
String
METRIC_TAG_COUNT
=
TAG
+
"Count"
;
protected
static
final
String
METRIC_ENTITIES_PER_TAG
=
TAG
+
"Entities"
;
public
static
final
String
METRIC_QUERY_PREFIX
=
"atlas.metric.query."
;
public
static
final
String
METRIC_QUERY_CACHE_TTL
=
"atlas.metric.query.cache.ttlInSecs"
;
public
static
final
int
DEFAULT_CACHE_TTL_IN_SECS
=
900
;
public
static
final
String
METRIC_COLLECTION_TIME
=
"collectionTime"
;
private
static
Configuration
configuration
=
null
;
private
final
AtlasGraph
atlasGraph
;
private
final
AtlasTypeRegistry
atlasTypeRegistry
;
private
final
int
cacheTTLInSecs
;
private
static
Configuration
configuration
=
null
;
private
static
AtlasGremlinQueryProvider
gremlinQueryProvider
=
null
;
private
final
AtlasGraph
atlasGraph
;
private
final
int
cacheTTLInSecs
;
private
AtlasMetrics
cachedMetrics
=
null
;
private
long
cacheExpirationTime
=
0
;
@Inject
public
MetricsService
(
AtlasTypeRegistry
typeRegistry
)
throws
AtlasException
{
this
(
ApplicationProperties
.
get
(),
AtlasGraphProvider
.
getGraphInstance
()
,
typeRegistry
);
public
MetricsService
()
throws
AtlasException
{
this
(
ApplicationProperties
.
get
(),
AtlasGraphProvider
.
getGraphInstance
());
}
@VisibleForTesting
MetricsService
(
Configuration
configuration
,
AtlasGraph
graph
,
AtlasTypeRegistry
typeRegistry
)
{
MetricsService
(
Configuration
configuration
,
AtlasGraph
graph
)
{
MetricsService
.
configuration
=
configuration
;
atlasTypeRegistry
=
typeRegistry
;
atlasGraph
=
graph
;
cacheTTLInSecs
=
configuration
!=
null
?
configuration
.
getInt
(
METRIC_QUERY_CACHE_TTL
,
DEFAULT_CACHE_TTL_IN_SECS
)
:
DEFAULT_CACHE_TTL_IN_SECS
;
:
DEFAULT_CACHE_TTL_IN_SECS
;
gremlinQueryProvider
=
AtlasGremlinQueryProvider
.
INSTANCE
;
}
@SuppressWarnings
(
"unchecked"
)
...
...
@@ -96,9 +99,13 @@ public class MetricsService {
if
(
LOG
.
isDebugEnabled
())
{
LOG
.
debug
(
"Executing query: {}"
,
metricQuery
);
}
executeGremlinQuery
(
metrics
,
metricQuery
.
type
,
metricQuery
.
name
,
metricQuery
.
query
);
executeGremlinQuery
(
metrics
,
metricQuery
.
group
,
metricQuery
.
name
,
metricQuery
.
query
);
}
catch
(
ScriptException
e
)
{
LOG
.
error
(
"Gremlin execution failed for metric {}"
,
metricQuery
,
e
);
if
(
LOG
.
isDebugEnabled
())
{
LOG
.
debug
(
"Gremlin execution failed for metric {}"
,
metricQuery
,
e
);
}
else
{
LOG
.
warn
(
"Gremlin execution failed for metric {}"
,
metricQuery
);
}
}
}
...
...
@@ -145,7 +152,7 @@ public class MetricsService {
private
static
String
getQuery
(
String
type
,
String
name
,
String
defaultQuery
)
{
String
ret
=
configuration
!=
null
?
configuration
.
getString
(
METRIC_QUERY_PREFIX
+
type
+
"."
+
name
,
defaultQuery
)
:
defaultQuery
;
:
defaultQuery
;
if
(
LOG
.
isDebugEnabled
())
{
LOG
.
debug
(
"query for {}.{}: {}"
,
type
,
name
,
ret
);
...
...
@@ -161,31 +168,31 @@ public class MetricsService {
* query is not blank/empty.
*/
private
enum
MetricQuery
{
TYPE_COUNT
(
GENERAL
,
METRIC_TYPE_COUNT
,
"g.V().has('__type', 'typeSystem').filter({it.'__type.category'.name() != 'TRAIT'}).count()"
),
UNUSED_TYPE_COUNT
(
GENERAL
,
METRIC_TYPE_UNUSED_COUNT
,
"g.V('__type', 'typeSystem').filter({ it.'__type.category'.name() != 'TRAIT' && it.inE.count() == 0}).count()"
),
ENTITY_COUNT
(
GENERAL
,
METRIC_ENTITY_COUNT
,
"g.V().has('__superTypeNames', T.in, ['Referenceable']).count()"
),
TAGS_COUNT
(
GENERAL
,
METRIC_TAG_COUNT
,
"g.V().has('__type', 'typeSystem').filter({it.'__type.category'.name() == 'TRAIT'}).count()"
),
DELETED_ENTITY_COUNT
(
GENERAL
,
METRIC_ENTITY_DELETED
,
"g.V().has('__superTypeNames', T.in, ['Referenceable']).has('__status', 'DELETED').count()"
),
TYPE_COUNT
(
GENERAL
,
METRIC_TYPE_COUNT
,
AtlasGremlinQuery
.
TYPE_COUNT_METRIC
),
UNUSED_TYPE_COUNT
(
GENERAL
,
METRIC_TYPE_UNUSED_COUNT
,
AtlasGremlinQuery
.
TYPE_UNUSED_COUNT_METRIC
),
ENTITY_COUNT
(
GENERAL
,
METRIC_ENTITY_COUNT
,
AtlasGremlinQuery
.
ENTITY_COUNT_METRIC
),
TAGS_COUNT
(
GENERAL
,
METRIC_TAG_COUNT
,
AtlasGremlinQuery
.
TAG_COUNT_METRIC
),
DELETED_ENTITY_COUNT
(
GENERAL
,
METRIC_ENTITY_DELETED
,
AtlasGremlinQuery
.
ENTITY_DELETED_METRIC
),
ENTITIES_PER_TYPE
(
ENTITY
,
METRIC_TYPE_ENTITIES
,
"g.V().has('__typeName', T.in, g.V().has('__type', 'typeSystem').filter({it.'__type.category'.name() != 'TRAIT'}).'__type.name'.toSet()).groupCount{it.'__typeName'}.cap.toList()"
),
TAGGED_ENTITIES
(
ENTITY
,
METRIC_TAGGED_ENTITIES
,
"g.V().has('__superTypeNames', T.in, ['Referenceable']).has('__traitNames').count()"
),
ENTITIES_PER_TYPE
(
ENTITY
,
METRIC_TYPE_ENTITIES
,
AtlasGremlinQuery
.
ENTITIES_PER_TYPE_METRIC
),
TAGGED_ENTITIES
(
ENTITY
,
METRIC_TAGGED_ENTITIES
,
AtlasGremlinQuery
.
TAGGED_ENTITIES_METRIC
),
ENTITIES_WITH_SPECIFIC_TAG
(
TAG
,
METRIC_
TAGS_PER_ENTITY
,
"g.V().has('__typeName', T.in, g.V().has('__type', 'typeSystem').filter{it.'__type.category'.name() == 'TRAIT'}.'__type.name'.toSet()).groupCount{it.'__typeName'}.cap.toList()"
),
ENTITIES_WITH_SPECIFIC_TAG
(
TAG
,
METRIC_
ENTITIES_PER_TAG
,
AtlasGremlinQuery
.
ENTITIES_FOR_TAG_METRIC
),
;
private
final
String
type
;
private
final
String
group
;
private
final
String
name
;
private
final
String
query
;
MetricQuery
(
String
type
,
String
name
,
String
q
uery
)
{
this
.
type
=
type
;
this
.
name
=
name
;
this
.
query
=
MetricsService
.
getQuery
(
type
,
name
,
query
);
MetricQuery
(
String
group
,
String
name
,
AtlasGremlinQuery
gremlinQ
uery
)
{
this
.
group
=
group
;
this
.
name
=
name
;
this
.
query
=
MetricsService
.
getQuery
(
group
,
name
,
gremlinQueryProvider
.
getQuery
(
gremlinQuery
)
);
}
@Override
public
String
toString
()
{
return
"MetricQuery{"
+
"
type='"
+
type
+
'\''
+
return
"MetricQuery{"
+
"
group='"
+
group
+
'\''
+
", name='"
+
name
+
'\''
+
", query='"
+
query
+
'\''
+
'}'
;
...
...
repository/src/main/java/org/apache/atlas/util/AtlasGremlin2QueryProvider.java
0 → 100644
View file @
48c10133
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package
org
.
apache
.
atlas
.
util
;
public
class
AtlasGremlin2QueryProvider
extends
AtlasGremlinQueryProvider
{
@Override
public
String
getQuery
(
final
AtlasGremlinQuery
gremlinQuery
)
{
switch
(
gremlinQuery
)
{
case
TYPE_COUNT_METRIC:
return
"g.V().has('__type', 'typeSystem').filter({it.'__type.category'.name() != 'TRAIT'}).count()"
;
case
TYPE_UNUSED_COUNT_METRIC:
return
"g.V('__type', 'typeSystem').filter({ it.'__type.category'.name() != 'TRAIT' && it.inE.count() == 0}).count()"
;
case
ENTITY_COUNT_METRIC:
return
"g.V().has('__superTypeNames', T.in, ['Referenceable']).count()"
;
case
TAG_COUNT_METRIC:
return
"g.V().has('__type', 'typeSystem').filter({it.'__type.category'.name() == 'TRAIT'}).count()"
;
case
ENTITY_DELETED_METRIC:
return
"g.V().has('__superTypeNames', T.in, ['Referenceable']).has('__status', 'DELETED').count()"
;
case
ENTITIES_PER_TYPE_METRIC:
return
"g.V().has('__typeName', T.in, g.V().has('__type', 'typeSystem').filter({it.'__type.category'.name() != 'TRAIT'}).'__type.name'.toSet()).groupCount{it.'__typeName'}.cap.toList()"
;
case
TAGGED_ENTITIES_METRIC:
return
"g.V().has('__superTypeNames', T.in, ['Referenceable']).has('__traitNames').count()"
;
case
ENTITIES_FOR_TAG_METRIC:
return
"g.V().has('__typeName', T.in, g.V().has('__type', 'typeSystem').filter{it.'__type.category'.name() == 'TRAIT'}.'__type.name'.toSet()).groupCount{it.'__typeName'}.cap.toList()"
;
case
EXPORT_BY_GUID:
return
"g.V('__guid', startGuid).bothE().bothV().has('__guid').__guid.dedup().toList()"
;
case
EXPORT_TYPE_STARTS_WITH:
return
"g.V().has('__typeName','%s').filter({it.'%s'.startsWith(attrValue)}).has('__guid').__guid.toList()"
;
case
EXPORT_TYPE_ENDS_WITH:
return
"g.V().has('__typeName','%s').filter({it.'%s'.endsWith(attrValue)}).has('__guid').__guid.toList()"
;
case
EXPORT_TYPE_CONTAINS:
return
"g.V().has('__typeName','%s').filter({it.'%s'.contains(attrValue)}).has('__guid').__guid.toList()"
;
case
EXPORT_TYPE_MATCHES:
return
"g.V().has('__typeName','%s').filter({it.'%s'.matches(attrValue)}).has('__guid').__guid.toList()"
;
case
EXPORT_TYPE_DEFAULT:
return
"g.V().has('__typeName','%s').has('%s', attrValue).has('__guid').__guid.toList()"
;
case
FULL_LINEAGE:
return
"g.V('__guid', '%s').as('src').in('%s').out('%s')."
+
"loop('src', {((it.path.contains(it.object)) ? false : true)}, "
+
"{((it.object.'__superTypeNames') ? "
+
"(it.object.'__superTypeNames'.contains('DataSet')) : false)})."
+
"path().toList()"
;
case
PARTIAL_LINEAGE:
return
"g.V('__guid', '%s').as('src').in('%s').out('%s')."
+
"loop('src', {it.loops <= %s}, {((it.object.'__superTypeNames') ? "
+
"(it.object.'__superTypeNames'.contains('DataSet')) : false)})."
+
"path().toList()"
;
}
// Should never reach this point
return
null
;
}
}
repository/src/main/java/org/apache/atlas/util/AtlasGremlin3QueryProvider.java
0 → 100644
View file @
48c10133
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package
org
.
apache
.
atlas
.
util
;
public
class
AtlasGremlin3QueryProvider
extends
AtlasGremlin2QueryProvider
{
@Override
public
String
getQuery
(
final
AtlasGremlinQuery
gremlinQuery
)
{
// In case any overrides are necessary, a specific switch case can be added here to
// return Gremlin 3 specific query otherwise delegate to super.getQuery
return
super
.
getQuery
(
gremlinQuery
);
}
}
repository/src/main/java/org/apache/atlas/util/AtlasGremlinQueryProvider.java
0 → 100644
View file @
48c10133
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package
org
.
apache
.
atlas
.
util
;
import
org.apache.atlas.repository.graph.AtlasGraphProvider
;
import
org.apache.atlas.repository.graphdb.GremlinVersion
;
/**
* Generic Gremlin query provider which is agnostic of the Gremlin/TinkerPop version being used in Atlas
*/
public
abstract
class
AtlasGremlinQueryProvider
{
public
static
final
AtlasGremlinQueryProvider
INSTANCE
=
AtlasGraphProvider
.
getGraphInstance
().
getSupportedGremlinVersion
()
==
GremlinVersion
.
THREE
?
new
AtlasGremlin3QueryProvider
()
:
new
AtlasGremlin2QueryProvider
();
abstract
public
String
getQuery
(
final
AtlasGremlinQuery
gremlinQuery
);
public
enum
AtlasGremlinQuery
{
// Metrics related Queries
TYPE_COUNT_METRIC
,
TYPE_UNUSED_COUNT_METRIC
,
ENTITY_COUNT_METRIC
,
TAG_COUNT_METRIC
,
ENTITY_DELETED_METRIC
,
ENTITIES_PER_TYPE_METRIC
,
TAGGED_ENTITIES_METRIC
,
ENTITIES_FOR_TAG_METRIC
,
// Import Export related Queries
EXPORT_BY_GUID
,
EXPORT_TYPE_STARTS_WITH
,
EXPORT_TYPE_ENDS_WITH
,
EXPORT_TYPE_CONTAINS
,
EXPORT_TYPE_MATCHES
,
EXPORT_TYPE_DEFAULT
,
// Lineage Queries
FULL_LINEAGE
,
PARTIAL_LINEAGE
;
// Discovery Queries
}
}
repository/src/test/java/org/apache/atlas/services/MetricsServiceTest.java
View file @
48c10133
...
...
@@ -63,7 +63,7 @@ public class MetricsServiceTest {
when
(
mockTypeRegistry
.
getAllEntityDefNames
()).
thenReturn
(
Arrays
.
asList
(
"a"
,
"b"
,
"c"
));
setupMockGraph
();
metricsService
=
new
MetricsService
(
mockConfig
,
mockGraph
,
mockTypeRegistry
);
metricsService
=
new
MetricsService
(
mockConfig
,
mockGraph
);
}
private
void
setupMockGraph
()
throws
ScriptException
{
...
...
webapp/src/main/java/org/apache/atlas/web/resources/ExportService.java
View file @
48c10133
...
...
@@ -37,6 +37,8 @@ import org.apache.atlas.type.AtlasEntityType;
import
org.apache.atlas.type.AtlasStructType.AtlasAttribute
;
import
org.apache.atlas.type.AtlasTypeRegistry
;
import
org.apache.atlas.type.AtlasTypeUtil
;
import
org.apache.atlas.util.AtlasGremlinQueryProvider
;
import
org.apache.atlas.util.AtlasGremlinQueryProvider.AtlasGremlinQuery
;
import
org.apache.commons.collections.CollectionUtils
;
import
org.apache.commons.collections.MapUtils
;
import
org.apache.commons.lang3.StringUtils
;
...
...
@@ -47,7 +49,12 @@ import javax.script.Bindings;
import
javax.script.ScriptContext
;
import
javax.script.ScriptEngine
;
import
javax.script.ScriptException
;
import
java.util.*
;
import
java.util.ArrayList
;
import
java.util.Collections
;
import
java.util.HashSet
;
import
java.util.List
;
import
java.util.Map
;
import
java.util.Set
;
public
class
ExportService
{
...
...
@@ -62,21 +69,17 @@ public class ExportService {
private
final
AtlasTypeRegistry
typeRegistry
;
private
final
AtlasGraph
atlasGraph
;
private
final
EntityGraphRetriever
entityGraphRetriever
;
private
final
AtlasGremlinQueryProvider
gremlinQueryProvider
;
// query engine support
private
final
ScriptEngine
scriptEngine
;
private
final
Bindings
bindings
;
private
final
String
queryByGuid
=
"g.V('__guid', startGuid).bothE().bothV().has('__guid').__guid.dedup().toList()"
;
final
private
String
queryByAttrEquals
=
"g.V().has('__typeName','%s').has('%s', attrValue).has('__guid').__guid.toList()"
;
final
private
String
queryByAttrStartWith
=
"g.V().has('__typeName','%s').filter({it.'%s'.startsWith(attrValue)}).has('__guid').__guid.toList()"
;
final
private
String
queryByAttrEndsWith
=
"g.V().has('__typeName','%s').filter({it.'%s'.endsWith(attrValue)}).has('__guid').__guid.toList()"
;
final
private
String
queryByAttrContains
=
"g.V().has('__typeName','%s').filter({it.'%s'.contains(attrValue)}).has('__guid').__guid.toList()"
;
final
private
String
queryByAttrMatches
=
"g.V().has('__typeName','%s').filter({it.'%s'.matches(attrValue)}).has('__guid').__guid.toList()"
;
public
ExportService
(
final
AtlasTypeRegistry
typeRegistry
)
throws
AtlasBaseException
{
this
.
typeRegistry
=
typeRegistry
;
this
.
entityGraphRetriever
=
new
EntityGraphRetriever
(
this
.
typeRegistry
);
this
.
atlasGraph
=
AtlasGraphProvider
.
getGraphInstance
();
this
.
gremlinQueryProvider
=
AtlasGremlinQueryProvider
.
INSTANCE
;
this
.
scriptEngine
=
new
GremlinGroovyScriptEngine
();
...
...
@@ -186,15 +189,15 @@ public class ExportService {
final
String
queryTemplate
;
if
(
StringUtils
.
equalsIgnoreCase
(
matchType
,
MATCH_TYPE_STARTS_WITH
))
{
queryTemplate
=
queryByAttrStartWith
;
queryTemplate
=
gremlinQueryProvider
.
getQuery
(
AtlasGremlinQuery
.
EXPORT_TYPE_STARTS_WITH
)
;
}
else
if
(
StringUtils
.
equalsIgnoreCase
(
matchType
,
MATCH_TYPE_ENDS_WITH
))
{
queryTemplate
=
queryByAttrEndsWith
;
queryTemplate
=
gremlinQueryProvider
.
getQuery
(
AtlasGremlinQuery
.
EXPORT_TYPE_ENDS_WITH
)
;
}
else
if
(
StringUtils
.
equalsIgnoreCase
(
matchType
,
MATCH_TYPE_CONTAINS
))
{
queryTemplate
=
queryByAttrContains
;
queryTemplate
=
gremlinQueryProvider
.
getQuery
(
AtlasGremlinQuery
.
EXPORT_TYPE_CONTAINS
)
;
}
else
if
(
StringUtils
.
equalsIgnoreCase
(
matchType
,
MATCH_TYPE_MATCHES
))
{
queryTemplate
=
queryByAttrMatches
;
queryTemplate
=
gremlinQueryProvider
.
getQuery
(
AtlasGremlinQuery
.
EXPORT_TYPE_MATCHES
)
;
}
else
{
// default
queryTemplate
=
queryByAttrEquals
;
queryTemplate
=
gremlinQueryProvider
.
getQuery
(
AtlasGremlinQuery
.
EXPORT_TYPE_DEFAULT
)
;
}
for
(
Map
.
Entry
<
String
,
Object
>
e
:
item
.
getUniqueAttributes
().
entrySet
())
{
...
...
@@ -320,7 +323,8 @@ public class ExportService {
}
private
List
<
String
>
executeGremlinScriptForHive
(
String
guid
)
throws
ScriptException
{
return
executeGremlinScriptFor
(
this
.
queryByGuid
,
"startGuid"
,
guid
);
String
queryByGuid
=
gremlinQueryProvider
.
getQuery
(
AtlasGremlinQuery
.
EXPORT_BY_GUID
);
return
executeGremlinScriptFor
(
queryByGuid
,
"startGuid"
,
guid
);
}
private
List
<
String
>
executeGremlinScriptFor
(
String
query
,
String
parameterName
,
String
parameterValue
)
{
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment