Skip to content
Projects
Groups
Snippets
Help
This project
Loading...
Sign in / Register
Toggle navigation
A
atlas
Project
Overview
Details
Activity
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
dataplatform
atlas
Commits
cfb6b84f
Commit
cfb6b84f
authored
7 years ago
by
Madhan Neethiraj
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
ATLAS-1961: Basic search improvement in use of Solr index for attribute filtering (# 2)
parent
377fe19f
Hide whitespace changes
Inline
Side-by-side
Showing
4 changed files
with
61 additions
and
18 deletions
+61
-18
EntitySearchProcessor.java
...ava/org/apache/atlas/discovery/EntitySearchProcessor.java
+21
-9
FullTextSearchProcessor.java
...a/org/apache/atlas/discovery/FullTextSearchProcessor.java
+34
-2
SearchContext.java
...c/main/java/org/apache/atlas/discovery/SearchContext.java
+4
-5
SearchProcessor.java
...main/java/org/apache/atlas/discovery/SearchProcessor.java
+2
-2
No files found.
repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java
View file @
cfb6b84f
...
...
@@ -20,6 +20,7 @@ package org.apache.atlas.discovery;
import
org.apache.atlas.model.discovery.SearchParameters.FilterCriteria
;
import
org.apache.atlas.repository.Constants
;
import
org.apache.atlas.repository.graphdb.*
;
import
org.apache.atlas.type.AtlasClassificationType
;
import
org.apache.atlas.type.AtlasEntityType
;
import
org.apache.atlas.utils.AtlasPerfTracer
;
import
org.apache.commons.collections.CollectionUtils
;
...
...
@@ -39,18 +40,21 @@ public class EntitySearchProcessor extends SearchProcessor {
public
EntitySearchProcessor
(
SearchContext
context
)
{
super
(
context
);
final
AtlasEntityType
entityType
=
context
.
getEntityType
();
final
FilterCriteria
filterCriteria
=
context
.
getSearchParameters
().
getEntityFilters
();
final
Set
<
String
>
typeAndSubTypes
=
entityType
.
getTypeAndAllSubTypes
();
final
Set
<
String
>
solrAttributes
=
new
HashSet
<>();
final
Set
<
String
>
gremlinAttributes
=
new
HashSet
<>();
final
Set
<
String
>
allAttributes
=
new
HashSet
<>();
final
AtlasEntityType
entityType
=
context
.
getEntityType
();
final
FilterCriteria
filterCriteria
=
context
.
getSearchParameters
().
getEntityFilters
();
final
Set
<
String
>
typeAndSubTypes
=
entityType
.
getTypeAndAllSubTypes
();
final
Set
<
String
>
solrAttributes
=
new
HashSet
<>();
final
Set
<
String
>
gremlinAttributes
=
new
HashSet
<>();
final
Set
<
String
>
allAttributes
=
new
HashSet
<>();
final
AtlasClassificationType
classificationType
=
context
.
getClassificationType
();
final
boolean
filterClassification
=
classificationType
!=
null
&&
!
context
.
needClassificationProcessor
();
processSearchAttributes
(
entityType
,
filterCriteria
,
solrAttributes
,
gremlinAttributes
,
allAttributes
);
final
boolean
typeSearchBySolr
=
typeAndSubTypes
.
size
()
<=
MAX_ENTITY_TYPES_IN_INDEX_QUERY
;
final
boolean
attrSearchBySolr
=
canApplySolrFilter
(
entityType
,
filterCriteria
,
false
);
final
boolean
typeSearchBySolr
=
!
filterClassification
&&
typeAndSubTypes
.
size
()
<=
MAX_ENTITY_TYPES_IN_INDEX_QUERY
;
final
boolean
attrSearchBySolr
=
!
filterClassification
&&
CollectionUtils
.
isNotEmpty
(
solrAttributes
)
&&
canApplySolrFilter
(
entityType
,
filterCriteria
,
false
);
StringBuilder
solrQuery
=
new
StringBuilder
();
...
...
@@ -82,6 +86,10 @@ public class EntitySearchProcessor extends SearchProcessor {
query
.
in
(
Constants
.
TYPE_NAME_PROPERTY_KEY
,
typeAndSubTypes
);
}
if
(
filterClassification
)
{
query
.
in
(
Constants
.
TRAIT_NAMES_PROPERTY_KEY
,
classificationType
.
getTypeAndAllSubTypes
());
}
graphQuery
=
toGremlinFilterQuery
(
entityType
,
filterCriteria
,
gremlinAttributes
,
query
);
if
(
context
.
getSearchParameters
().
getExcludeDeletedEntities
()
&&
indexQuery
==
null
)
{
...
...
@@ -93,6 +101,10 @@ public class EntitySearchProcessor extends SearchProcessor {
AtlasGraphQuery
query
=
context
.
getGraph
().
query
().
in
(
Constants
.
TYPE_NAME_PROPERTY_KEY
,
typeAndSubTypes
);
if
(
filterClassification
)
{
query
.
in
(
Constants
.
TRAIT_NAMES_PROPERTY_KEY
,
classificationType
.
getTypeAndAllSubTypes
());
}
filterGraphQuery
=
toGremlinFilterQuery
(
entityType
,
filterCriteria
,
allAttributes
,
query
);
if
(
context
.
getSearchParameters
().
getExcludeDeletedEntities
())
{
...
...
@@ -115,7 +127,7 @@ public class EntitySearchProcessor extends SearchProcessor {
}
try
{
int
qryOffset
=
(
nextProcessor
==
null
)
?
context
.
getSearchParameters
().
getOffset
()
:
0
;
int
qryOffset
=
(
nextProcessor
==
null
&&
(
graphQuery
==
null
||
indexQuery
==
null
)
)
?
context
.
getSearchParameters
().
getOffset
()
:
0
;
int
limit
=
context
.
getSearchParameters
().
getLimit
();
int
resultIdx
=
qryOffset
;
...
...
This diff is collapsed.
Click to expand it.
repository/src/main/java/org/apache/atlas/discovery/FullTextSearchProcessor.java
View file @
cfb6b84f
...
...
@@ -22,12 +22,14 @@ import org.apache.atlas.repository.Constants;
import
org.apache.atlas.repository.graphdb.AtlasIndexQuery
;
import
org.apache.atlas.repository.graphdb.AtlasVertex
;
import
org.apache.atlas.utils.AtlasPerfTracer
;
import
org.apache.commons.lang3.StringUtils
;
import
org.slf4j.Logger
;
import
org.slf4j.LoggerFactory
;
import
java.util.ArrayList
;
import
java.util.Iterator
;
import
java.util.List
;
import
java.util.Set
;
public
class
FullTextSearchProcessor
extends
SearchProcessor
{
...
...
@@ -40,9 +42,39 @@ public class FullTextSearchProcessor extends SearchProcessor {
super
(
context
);
SearchParameters
searchParameters
=
context
.
getSearchParameters
();
String
queryString
=
String
.
format
(
"v.\"%s\":(%s)"
,
Constants
.
ENTITY_TEXT_PROPERTY_KEY
,
searchParameters
.
getQuery
()
);
String
Builder
queryString
=
new
StringBuilder
(
);
indexQuery
=
context
.
getGraph
().
indexQuery
(
Constants
.
FULLTEXT_INDEX
,
queryString
);
queryString
.
append
(
"v.\""
).
append
(
Constants
.
ENTITY_TEXT_PROPERTY_KEY
).
append
(
"\":("
).
append
(
searchParameters
.
getQuery
());
// if search includes entity-type criteria, adding a filter here can help avoid unnecessary
// processing (and rejection) by subsequent EntitySearchProcessor
if
(
context
.
getEntityType
()
!=
null
)
{
Set
<
String
>
typeAndSubTypeNames
=
context
.
getEntityType
().
getTypeAndAllSubTypes
();
if
(
typeAndSubTypeNames
.
size
()
<=
MAX_ENTITY_TYPES_IN_INDEX_QUERY
)
{
queryString
.
append
(
AND_STR
).
append
(
"("
).
append
(
StringUtils
.
join
(
typeAndSubTypeNames
,
SPACE_STRING
)).
append
(
")"
);
}
else
{
LOG
.
warn
(
"'{}' has too many subtypes ({}) to include in index-query; might cause poor performance"
,
context
.
getEntityType
().
getTypeName
(),
typeAndSubTypeNames
.
size
());
}
}
// if search includes classification criteria, adding a filter here can help avoid unnecessary
// processing (and rejection) by subsequent ClassificationSearchProcessor or EntitySearchProcessor
if
(
context
.
getClassificationType
()
!=
null
)
{
Set
<
String
>
typeAndSubTypeNames
=
context
.
getClassificationType
().
getTypeAndAllSubTypes
();
if
(
typeAndSubTypeNames
.
size
()
<=
MAX_CLASSIFICATION_TYPES_IN_INDEX_QUERY
)
{
queryString
.
append
(
AND_STR
).
append
(
"("
).
append
(
StringUtils
.
join
(
typeAndSubTypeNames
,
SPACE_STRING
)).
append
(
")"
);
}
else
{
LOG
.
warn
(
"'{}' has too many subtypes ({}) to include in index-query; might cause poor performance"
,
context
.
getEntityType
().
getTypeName
(),
typeAndSubTypeNames
.
size
());
}
}
queryString
.
append
(
")"
);
indexQuery
=
context
.
getGraph
().
indexQuery
(
Constants
.
FULLTEXT_INDEX
,
queryString
.
toString
());
}
@Override
...
...
This diff is collapsed.
Click to expand it.
repository/src/main/java/org/apache/atlas/discovery/SearchContext.java
View file @
cfb6b84f
...
...
@@ -61,7 +61,6 @@ public class SearchContext {
if
(
needEntityProcessor
())
{
addProcessor
(
new
EntitySearchProcessor
(
this
));
}
}
...
...
@@ -104,15 +103,15 @@ public class SearchContext {
return
toString
(
new
StringBuilder
()).
toString
();
}
public
boolean
needFullTextrocessor
()
{
boolean
needFullTextrocessor
()
{
return
StringUtils
.
isNotEmpty
(
searchParameters
.
getQuery
());
}
public
boolean
needClassificationProcessor
()
{
return
classificationType
!=
null
;
boolean
needClassificationProcessor
()
{
return
classificationType
!=
null
&&
(
entityType
==
null
||
hasAttributeFilter
(
searchParameters
.
getTagFilters
()))
;
}
public
boolean
needEntityProcessor
()
{
boolean
needEntityProcessor
()
{
return
entityType
!=
null
;
}
...
...
This diff is collapsed.
Click to expand it.
repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java
View file @
cfb6b84f
...
...
@@ -181,7 +181,7 @@ public abstract class SearchProcessor {
protected
void
constructTypeTestQuery
(
StringBuilder
solrQuery
,
Set
<
String
>
typeAndAllSubTypes
)
{
String
typeAndSubtypesString
=
StringUtils
.
join
(
typeAndAllSubTypes
,
SPACE_STRING
);
solrQuery
.
append
(
"v.\"
__typeName
\": ("
)
solrQuery
.
append
(
"v.\"
"
).
append
(
Constants
.
TYPE_NAME_PROPERTY_KEY
).
append
(
"
\": ("
)
.
append
(
typeAndSubtypesString
)
.
append
(
")"
);
}
...
...
@@ -206,7 +206,7 @@ public abstract class SearchProcessor {
solrQuery
.
append
(
AND_STR
);
}
solrQuery
.
append
(
"v.\"
__state\":"
).
append
(
"
ACTIVE"
);
solrQuery
.
append
(
"v.\"
"
).
append
(
Constants
.
STATE_PROPERTY_KEY
).
append
(
"\":
ACTIVE"
);
}
}
...
...
This diff is collapsed.
Click to expand it.
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment