Skip to content
Projects
Groups
Snippets
Help
This project
Loading...
Sign in / Register
Toggle navigation
A
atlas
Project
Overview
Details
Activity
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
dataplatform
atlas
Commits
377fe19f
Commit
377fe19f
authored
7 years ago
by
Madhan Neethiraj
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
ATLAS-1961: Basic search improvement in use of Solr index for attribute filtering
parent
71f9c065
Hide whitespace changes
Inline
Side-by-side
Showing
3 changed files
with
50 additions
and
39 deletions
+50
-39
EntitySearchProcessor.java
...ava/org/apache/atlas/discovery/EntitySearchProcessor.java
+38
-35
SearchContext.java
...c/main/java/org/apache/atlas/discovery/SearchContext.java
+1
-1
SearchProcessor.java
...main/java/org/apache/atlas/discovery/SearchProcessor.java
+11
-3
No files found.
repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java
View file @
377fe19f
...
@@ -20,7 +20,6 @@ package org.apache.atlas.discovery;
...
@@ -20,7 +20,6 @@ package org.apache.atlas.discovery;
import
org.apache.atlas.model.discovery.SearchParameters.FilterCriteria
;
import
org.apache.atlas.model.discovery.SearchParameters.FilterCriteria
;
import
org.apache.atlas.repository.Constants
;
import
org.apache.atlas.repository.Constants
;
import
org.apache.atlas.repository.graphdb.*
;
import
org.apache.atlas.repository.graphdb.*
;
import
org.apache.atlas.type.AtlasClassificationType
;
import
org.apache.atlas.type.AtlasEntityType
;
import
org.apache.atlas.type.AtlasEntityType
;
import
org.apache.atlas.utils.AtlasPerfTracer
;
import
org.apache.atlas.utils.AtlasPerfTracer
;
import
org.apache.commons.collections.CollectionUtils
;
import
org.apache.commons.collections.CollectionUtils
;
...
@@ -34,60 +33,70 @@ public class EntitySearchProcessor extends SearchProcessor {
...
@@ -34,60 +33,70 @@ public class EntitySearchProcessor extends SearchProcessor {
private
static
final
Logger
PERF_LOG
=
AtlasPerfTracer
.
getPerfLogger
(
"EntitySearchProcessor"
);
private
static
final
Logger
PERF_LOG
=
AtlasPerfTracer
.
getPerfLogger
(
"EntitySearchProcessor"
);
private
final
AtlasIndexQuery
indexQuery
;
private
final
AtlasIndexQuery
indexQuery
;
private
final
AtlasGraphQuery
partialG
raphQuery
;
private
final
AtlasGraphQuery
g
raphQuery
;
private
final
AtlasGraphQuery
all
GraphQuery
;
private
final
AtlasGraphQuery
filter
GraphQuery
;
public
EntitySearchProcessor
(
SearchContext
context
)
{
public
EntitySearchProcessor
(
SearchContext
context
)
{
super
(
context
);
super
(
context
);
AtlasEntityType
entityType
=
context
.
getEntityType
();
final
AtlasEntityType
entityType
=
context
.
getEntityType
();
AtlasClassificationType
classificationType
=
context
.
getClassificationType
();
final
FilterCriteria
filterCriteria
=
context
.
getSearchParameters
().
getEntityFilters
();
FilterCriteria
filterCriteria
=
context
.
getSearchParameters
().
getEntityFilters
();
final
Set
<
String
>
typeAndSubTypes
=
entityType
.
getTypeAndAllSubTypes
();
Set
<
String
>
typeAndSubTypes
=
entityType
.
getTypeAndAllSubTypes
();
final
Set
<
String
>
solrAttributes
=
new
HashSet
<>();
Set
<
String
>
solrAttributes
=
new
HashSet
<>();
final
Set
<
String
>
gremlinAttributes
=
new
HashSet
<>();
Set
<
String
>
gremlinAttributes
=
new
HashSet
<>();
final
Set
<
String
>
allAttributes
=
new
HashSet
<>();
Set
<
String
>
allAttributes
=
new
HashSet
<>();
processSearchAttributes
(
entityType
,
filterCriteria
,
solrAttributes
,
gremlinAttributes
,
allAttributes
);
processSearchAttributes
(
entityType
,
filterCriteria
,
solrAttributes
,
gremlinAttributes
,
allAttributes
);
boolean
useSolrSearch
=
typeAndSubTypes
.
size
()
<=
MAX_ENTITY_TYPES_IN_INDEX_QUERY
&&
canApplySolrFilter
(
entityType
,
filterCriteria
,
false
);
final
boolean
typeSearchBySolr
=
typeAndSubTypes
.
size
()
<=
MAX_ENTITY_TYPES_IN_INDEX_QUERY
;
final
boolean
attrSearchBySolr
=
canApplySolrFilter
(
entityType
,
filterCriteria
,
false
);
if
(
useSolrSearch
)
{
StringBuilder
solrQuery
=
new
StringBuilder
();
StringBuilder
solrQuery
=
new
StringBuilder
();
if
(
typeSearchBySolr
)
{
constructTypeTestQuery
(
solrQuery
,
typeAndSubTypes
);
constructTypeTestQuery
(
solrQuery
,
typeAndSubTypes
);
}
if
(
attrSearchBySolr
)
{
constructFilterQuery
(
solrQuery
,
entityType
,
filterCriteria
,
solrAttributes
);
constructFilterQuery
(
solrQuery
,
entityType
,
filterCriteria
,
solrAttributes
);
}
else
{
gremlinAttributes
.
addAll
(
solrAttributes
);
}
if
(
solrQuery
.
length
()
>
0
)
{
String
solrQueryString
=
STRAY_AND_PATTERN
.
matcher
(
solrQuery
).
replaceAll
(
")"
);
String
solrQueryString
=
STRAY_AND_PATTERN
.
matcher
(
solrQuery
).
replaceAll
(
")"
);
solrQueryString
=
STRAY_OR_PATTERN
.
matcher
(
solrQueryString
).
replaceAll
(
")"
);
solrQueryString
=
STRAY_OR_PATTERN
.
matcher
(
solrQueryString
).
replaceAll
(
")"
);
solrQueryString
=
STRAY_ELIPSIS_PATTERN
.
matcher
(
solrQueryString
).
replaceAll
(
""
);
solrQueryString
=
STRAY_ELIPSIS_PATTERN
.
matcher
(
solrQueryString
).
replaceAll
(
""
);
indexQuery
=
context
.
getGraph
().
indexQuery
(
Constants
.
VERTEX_INDEX
,
solrQueryString
);
indexQuery
=
context
.
getGraph
().
indexQuery
(
Constants
.
VERTEX_INDEX
,
solrQueryString
);
}
else
{
indexQuery
=
null
;
}
if
(
CollectionUtils
.
isNotEmpty
(
gremlinAttributes
)
||
classificationType
!=
null
)
{
if
(
CollectionUtils
.
isNotEmpty
(
gremlinAttributes
)
||
!
typeSearchBySolr
)
{
AtlasGraphQuery
query
=
context
.
getGraph
().
query
();
AtlasGraphQuery
query
=
context
.
getGraph
().
query
();
addClassificationNameConditionIfNecessary
(
query
);
if
(!
typeSearchBySolr
)
{
query
.
in
(
Constants
.
TYPE_NAME_PROPERTY_KEY
,
typeAndSubTypes
);
}
graphQuery
=
toGremlinFilterQuery
(
entityType
,
filterCriteria
,
gremlinAttributes
,
query
);
partialGraphQuery
=
toGremlinFilterQuery
(
entityType
,
filterCriteria
,
gremlinAttributes
,
query
);
if
(
context
.
getSearchParameters
().
getExcludeDeletedEntities
()
&&
indexQuery
==
null
)
{
}
else
{
graphQuery
.
has
(
Constants
.
STATE_PROPERTY_KEY
,
"ACTIVE"
);
partialGraphQuery
=
null
;
}
}
}
else
{
}
else
{
indexQuery
=
null
;
graphQuery
=
null
;
partialGraphQuery
=
null
;
}
}
AtlasGraphQuery
query
=
context
.
getGraph
().
query
().
in
(
Constants
.
TYPE_NAME_PROPERTY_KEY
,
typeAndSubTypes
);
AtlasGraphQuery
query
=
context
.
getGraph
().
query
().
in
(
Constants
.
TYPE_NAME_PROPERTY_KEY
,
typeAndSubTypes
);
addClassificationNameConditionIfNecessary
(
query
);
filterGraphQuery
=
toGremlinFilterQuery
(
entityType
,
filterCriteria
,
allAttributes
,
query
);
allGraphQuery
=
toGremlinFilterQuery
(
entityType
,
filterCriteria
,
allAttributes
,
query
);
if
(
context
.
getSearchParameters
().
getExcludeDeletedEntities
())
{
if
(
context
.
getSearchParameters
().
getExcludeDeletedEntities
())
{
all
GraphQuery
.
has
(
Constants
.
STATE_PROPERTY_KEY
,
"ACTIVE"
);
filter
GraphQuery
.
has
(
Constants
.
STATE_PROPERTY_KEY
,
"ACTIVE"
);
}
}
}
}
...
@@ -128,15 +137,15 @@ public class EntitySearchProcessor extends SearchProcessor {
...
@@ -128,15 +137,15 @@ public class EntitySearchProcessor extends SearchProcessor {
vertices
=
getVerticesFromIndexQueryResult
(
queryResult
);
vertices
=
getVerticesFromIndexQueryResult
(
queryResult
);
if
(
partialG
raphQuery
!=
null
)
{
if
(
g
raphQuery
!=
null
)
{
AtlasGraphQuery
guidQuery
=
context
.
getGraph
().
query
().
in
(
Constants
.
GUID_PROPERTY_KEY
,
getGuids
(
vertices
));
AtlasGraphQuery
guidQuery
=
context
.
getGraph
().
query
().
in
(
Constants
.
GUID_PROPERTY_KEY
,
getGuids
(
vertices
));
guidQuery
.
addConditionsFrom
(
partialG
raphQuery
);
guidQuery
.
addConditionsFrom
(
g
raphQuery
);
vertices
=
getVertices
(
guidQuery
.
vertices
().
iterator
());
vertices
=
getVertices
(
guidQuery
.
vertices
().
iterator
());
}
}
}
else
{
}
else
{
Iterator
<
AtlasVertex
>
queryResult
=
allG
raphQuery
.
vertices
(
qryOffset
,
limit
).
iterator
();
Iterator
<
AtlasVertex
>
queryResult
=
g
raphQuery
.
vertices
(
qryOffset
,
limit
).
iterator
();
if
(!
queryResult
.
hasNext
())
{
// no more results from query - end of search
if
(!
queryResult
.
hasNext
())
{
// no more results from query - end of search
break
;
break
;
...
@@ -182,7 +191,7 @@ public class EntitySearchProcessor extends SearchProcessor {
...
@@ -182,7 +191,7 @@ public class EntitySearchProcessor extends SearchProcessor {
AtlasGraphQuery
query
=
context
.
getGraph
().
query
().
in
(
Constants
.
GUID_PROPERTY_KEY
,
getGuids
(
entityVertices
));
AtlasGraphQuery
query
=
context
.
getGraph
().
query
().
in
(
Constants
.
GUID_PROPERTY_KEY
,
getGuids
(
entityVertices
));
query
.
addConditionsFrom
(
all
GraphQuery
);
query
.
addConditionsFrom
(
filter
GraphQuery
);
List
<
AtlasVertex
>
ret
=
getVertices
(
query
.
vertices
().
iterator
());
List
<
AtlasVertex
>
ret
=
getVertices
(
query
.
vertices
().
iterator
());
...
@@ -194,10 +203,4 @@ public class EntitySearchProcessor extends SearchProcessor {
...
@@ -194,10 +203,4 @@ public class EntitySearchProcessor extends SearchProcessor {
return
ret
;
return
ret
;
}
}
private
void
addClassificationNameConditionIfNecessary
(
AtlasGraphQuery
query
)
{
if
(
context
.
getClassificationType
()
!=
null
&&
!
context
.
needClassificationProcessor
())
{
query
.
in
(
Constants
.
TRAIT_NAMES_PROPERTY_KEY
,
context
.
getClassificationType
().
getTypeAndAllSubTypes
());
}
}
}
}
This diff is collapsed.
Click to expand it.
repository/src/main/java/org/apache/atlas/discovery/SearchContext.java
View file @
377fe19f
...
@@ -109,7 +109,7 @@ public class SearchContext {
...
@@ -109,7 +109,7 @@ public class SearchContext {
}
}
public
boolean
needClassificationProcessor
()
{
public
boolean
needClassificationProcessor
()
{
return
classificationType
!=
null
&&
(
hasAttributeFilter
(
searchParameters
.
getTagFilters
())
||
entityType
==
null
)
;
return
classificationType
!=
null
;
}
}
public
boolean
needEntityProcessor
()
{
public
boolean
needEntityProcessor
()
{
...
...
This diff is collapsed.
Click to expand it.
repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java
View file @
377fe19f
...
@@ -88,7 +88,7 @@ public abstract class SearchProcessor {
...
@@ -88,7 +88,7 @@ public abstract class SearchProcessor {
public
abstract
List
<
AtlasVertex
>
execute
();
public
abstract
List
<
AtlasVertex
>
execute
();
public
List
<
AtlasVertex
>
filter
(
List
<
AtlasVertex
>
entityVertices
)
{
public
List
<
AtlasVertex
>
filter
(
List
<
AtlasVertex
>
entityVertices
)
{
return
nextProcessor
==
null
?
entityVertices
:
nextProcessor
.
filter
(
entityVertices
);
return
nextProcessor
==
null
||
CollectionUtils
.
isEmpty
(
entityVertices
)
?
entityVertices
:
nextProcessor
.
filter
(
entityVertices
);
}
}
...
@@ -193,12 +193,20 @@ public abstract class SearchProcessor {
...
@@ -193,12 +193,20 @@ public abstract class SearchProcessor {
String
filterQuery
=
toSolrQuery
(
type
,
filterCriteria
,
solrAttributes
,
0
);
String
filterQuery
=
toSolrQuery
(
type
,
filterCriteria
,
solrAttributes
,
0
);
if
(
StringUtils
.
isNotEmpty
(
filterQuery
))
{
if
(
StringUtils
.
isNotEmpty
(
filterQuery
))
{
solrQuery
.
append
(
AND_STR
).
append
(
filterQuery
);
if
(
solrQuery
.
length
()
>
0
)
{
solrQuery
.
append
(
AND_STR
);
}
solrQuery
.
append
(
filterQuery
);
}
}
}
}
if
(
type
instanceof
AtlasEntityType
&&
context
.
getSearchParameters
().
getExcludeDeletedEntities
())
{
if
(
type
instanceof
AtlasEntityType
&&
context
.
getSearchParameters
().
getExcludeDeletedEntities
())
{
solrQuery
.
append
(
AND_STR
).
append
(
"v.\"__state\":"
).
append
(
"ACTIVE"
);
if
(
solrQuery
.
length
()
>
0
)
{
solrQuery
.
append
(
AND_STR
);
}
solrQuery
.
append
(
"v.\"__state\":"
).
append
(
"ACTIVE"
);
}
}
}
}
...
...
This diff is collapsed.
Click to expand it.
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment