Skip to content
Projects
Groups
Snippets
Help
This project
Loading...
Sign in / Register
Toggle navigation
A
atlas
Project
Overview
Details
Activity
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
dataplatform
atlas
Commits
e0d2cdc2
Commit
e0d2cdc2
authored
Jul 21, 2017
by
Madhan Neethiraj
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
ATLAS-1981: Cache escaped type-query string to avoid repeated computation
parent
f511f272
Hide whitespace changes
Inline
Side-by-side
Showing
10 changed files
with
131 additions
and
102 deletions
+131
-102
Constants.java
.../src/main/java/org/apache/atlas/repository/Constants.java
+2
-2
AtlasClassificationType.java
...n/java/org/apache/atlas/type/AtlasClassificationType.java
+22
-8
AtlasEntityType.java
.../src/main/java/org/apache/atlas/type/AtlasEntityType.java
+10
-1
AtlasStructType.java
.../src/main/java/org/apache/atlas/type/AtlasStructType.java
+40
-0
AtlasTypeRegistry.java
...rc/main/java/org/apache/atlas/type/AtlasTypeRegistry.java
+2
-2
ClassificationSearchProcessor.java
...apache/atlas/discovery/ClassificationSearchProcessor.java
+9
-8
EntityDiscoveryService.java
...va/org/apache/atlas/discovery/EntityDiscoveryService.java
+16
-16
EntitySearchProcessor.java
...ava/org/apache/atlas/discovery/EntitySearchProcessor.java
+10
-8
FullTextSearchProcessor.java
...a/org/apache/atlas/discovery/FullTextSearchProcessor.java
+10
-13
SearchProcessor.java
...main/java/org/apache/atlas/discovery/SearchProcessor.java
+10
-44
No files found.
common/src/main/java/org/apache/atlas/repository/Constants.java
View file @
e0d2cdc2
...
...
@@ -97,8 +97,8 @@ public final class Constants {
public
static
final
String
QUALIFIED_NAME
=
"Referenceable.qualifiedName"
;
public
static
final
String
TYPE_NAME_PROPERTY_KEY
=
INTERNAL_PROPERTY_KEY_PREFIX
+
"typeName"
;
public
static
final
String
INDEX_SEARCH_MAX_RESULT_SET_SIZE
=
"atlas.graph.index.search.max-result-set-size"
;
public
static
final
String
INDEX_SEARCH_
MAX_TYPES_COUNT
=
"atlas.graph.index.search.max-types-count
"
;
public
static
final
String
INDEX_SEARCH_
MAX_TAGS_COUNT
=
"atlas.graph.index.search.max-tags-count
"
;
public
static
final
String
INDEX_SEARCH_
TYPES_MAX_QUERY_STR_LENGTH
=
"atlas.graph.index.search.types.max-query-str-length
"
;
public
static
final
String
INDEX_SEARCH_
TAGS_MAX_QUERY_STR_LENGTH
=
"atlas.graph.index.search.tags.max-query-str-length
"
;
private
Constants
()
{
}
...
...
intg/src/main/java/org/apache/atlas/type/AtlasClassificationType.java
View file @
e0d2cdc2
...
...
@@ -45,10 +45,11 @@ public class AtlasClassificationType extends AtlasStructType {
private
final
AtlasClassificationDef
classificationDef
;
private
List
<
AtlasClassificationType
>
superTypes
=
Collections
.
emptyList
();
private
Set
<
String
>
allSuperTypes
=
Collections
.
emptySet
();
private
Set
<
String
>
allSubTypes
=
Collections
.
emptySet
();
private
Set
<
String
>
typeAndAllSubTypes
=
Collections
.
emptySet
();
private
List
<
AtlasClassificationType
>
superTypes
=
Collections
.
emptyList
();
private
Set
<
String
>
allSuperTypes
=
Collections
.
emptySet
();
private
Set
<
String
>
allSubTypes
=
Collections
.
emptySet
();
private
Set
<
String
>
typeAndAllSubTypes
=
Collections
.
emptySet
();
private
String
typeAndAllSubTypesQryStr
=
""
;
public
AtlasClassificationType
(
AtlasClassificationDef
classificationDef
)
{
super
(
classificationDef
);
...
...
@@ -108,6 +109,13 @@ public class AtlasClassificationType extends AtlasStructType {
}
}
@Override
public
void
resolveReferencesPhase3
(
AtlasTypeRegistry
typeRegistry
)
throws
AtlasBaseException
{
allSubTypes
=
Collections
.
unmodifiableSet
(
allSubTypes
);
typeAndAllSubTypes
=
Collections
.
unmodifiableSet
(
typeAndAllSubTypes
);
typeAndAllSubTypesQryStr
=
""
;
// will be computed on next access
}
private
void
addSubType
(
AtlasClassificationType
subType
)
{
allSubTypes
.
add
(
subType
.
getTypeName
());
typeAndAllSubTypes
.
add
(
subType
.
getTypeName
());
...
...
@@ -119,11 +127,17 @@ public class AtlasClassificationType extends AtlasStructType {
public
Set
<
String
>
getAllSuperTypes
()
{
return
allSuperTypes
;
}
public
Set
<
String
>
getAllSubTypes
()
{
return
Collections
.
unmodifiableSet
(
allSubTypes
);
}
public
Set
<
String
>
getAllSubTypes
()
{
return
allSubTypes
;
}
public
Set
<
String
>
getTypeAndAllSubTypes
()
{
return
Collections
.
unmodifiableSet
(
typeAndAllSubTypes
);
}
public
Set
<
String
>
getTypeAndAllSubTypes
()
{
return
typeAndAllSubTypes
;
}
public
String
getTypeAndAllSubTypesQryStr
()
{
if
(
StringUtils
.
isEmpty
(
typeAndAllSubTypesQryStr
))
{
typeAndAllSubTypesQryStr
=
AtlasAttribute
.
escapeIndexQueryValue
(
typeAndAllSubTypes
);
}
return
typeAndAllSubTypesQryStr
;
}
public
boolean
isSuperTypeOf
(
AtlasClassificationType
classificationType
)
{
return
classificationType
!=
null
&&
allSubTypes
.
contains
(
classificationType
.
getTypeName
());
...
...
intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java
View file @
e0d2cdc2
...
...
@@ -55,6 +55,7 @@ public class AtlasEntityType extends AtlasStructType {
private
Set
<
String
>
typeAndAllSuperTypes
=
Collections
.
emptySet
();
private
Map
<
String
,
AtlasAttribute
>
relationshipAttributes
=
Collections
.
emptyMap
();
private
Map
<
String
,
List
<
AtlasRelationshipType
>>
relationshipAttributesType
=
Collections
.
emptyMap
();
private
String
typeAndAllSubTypesQryStr
=
""
;
public
AtlasEntityType
(
AtlasEntityDef
entityDef
)
{
super
(
entityDef
);
...
...
@@ -150,7 +151,7 @@ public class AtlasEntityType extends AtlasStructType {
allSubTypes
=
Collections
.
unmodifiableSet
(
allSubTypes
);
typeAndAllSubTypes
=
Collections
.
unmodifiableSet
(
typeAndAllSubTypes
);
typeAndAllSu
perTypes
=
Collections
.
unmodifiableSet
(
typeAndAllSuperTypes
);
typeAndAllSu
bTypesQryStr
=
""
;
// will be computed on next access
relationshipAttributes
=
Collections
.
unmodifiableMap
(
relationshipAttributes
);
relationshipAttributesType
=
Collections
.
unmodifiableMap
(
relationshipAttributesType
);
}
...
...
@@ -218,6 +219,14 @@ public class AtlasEntityType extends AtlasStructType {
return
relationshipAttributesType
;
}
public
String
getTypeAndAllSubTypesQryStr
()
{
if
(
StringUtils
.
isEmpty
(
typeAndAllSubTypesQryStr
))
{
typeAndAllSubTypesQryStr
=
AtlasAttribute
.
escapeIndexQueryValue
(
typeAndAllSubTypes
);
}
return
typeAndAllSubTypesQryStr
;
}
public
boolean
hasRelationshipAttribute
(
String
attributeName
)
{
return
relationshipAttributes
.
containsKey
(
attributeName
);
}
...
...
intg/src/main/java/org/apache/atlas/type/AtlasStructType.java
View file @
e0d2cdc2
...
...
@@ -38,6 +38,7 @@ import java.util.Collections;
import
java.util.HashMap
;
import
java.util.List
;
import
java.util.Map
;
import
java.util.Set
;
/**
* class that implements behaviour of a struct-type.
...
...
@@ -713,6 +714,39 @@ public class AtlasStructType extends AtlasType {
return
key
;
}
public
static
String
escapeIndexQueryValue
(
Set
<
String
>
values
)
{
StringBuilder
sb
=
new
StringBuilder
();
sb
.
append
(
BRACE_OPEN_CHAR
);
for
(
String
value
:
values
)
{
sb
.
append
(
escapeIndexQueryValue
(
value
)).
append
(
SPACE_CHAR
);
}
sb
.
append
(
BRACE_CLOSE_CHAR
);
return
sb
.
toString
();
}
public
static
String
escapeIndexQueryValue
(
String
value
)
{
String
ret
=
value
;
if
(
StringUtils
.
containsAny
(
value
,
IDX_QRY_OFFENDING_CHARS
))
{
boolean
isQuoteAtStart
=
value
.
charAt
(
0
)
==
DOUBLE_QUOTE_CHAR
;
boolean
isQuoteAtEnd
=
value
.
charAt
(
value
.
length
()
-
1
)
==
DOUBLE_QUOTE_CHAR
;
if
(!
isQuoteAtStart
)
{
if
(!
isQuoteAtEnd
)
{
ret
=
DOUBLE_QUOTE_CHAR
+
value
+
DOUBLE_QUOTE_CHAR
;
}
else
{
ret
=
DOUBLE_QUOTE_CHAR
+
value
;
}
}
else
if
(!
isQuoteAtEnd
)
{
ret
=
value
+
DOUBLE_QUOTE_CHAR
;
}
}
return
ret
;
}
private
String
getRelationshipEdgeLabel
(
String
relationshipLabel
)
{
return
(
relationshipLabel
==
null
)
?
getEdgeLabel
(
vertexPropertyName
)
:
relationshipLabel
;
}
...
...
@@ -730,6 +764,12 @@ public class AtlasStructType extends AtlasType {
new
String
[]
{
"%"
,
"_p"
},
};
private
static
final
char
[]
IDX_QRY_OFFENDING_CHARS
=
{
'@'
,
'/'
,
' '
};
private
static
final
char
BRACE_OPEN_CHAR
=
'('
;
private
static
final
char
BRACE_CLOSE_CHAR
=
')'
;
private
static
final
char
DOUBLE_QUOTE_CHAR
=
'"'
;
private
static
final
char
SPACE_CHAR
=
' '
;
public
enum
AtlasRelationshipEdgeDirection
{
IN
,
OUT
}
}
}
intg/src/main/java/org/apache/atlas/type/AtlasTypeRegistry.java
View file @
e0d2cdc2
...
...
@@ -365,8 +365,8 @@ public class AtlasTypeRegistry {
type
.
resolveReferencesPhase2
(
this
);
}
for
(
Atlas
EntityType
entityType
:
registryData
.
entityDef
s
.
getAllTypes
())
{
entityT
ype
.
resolveReferencesPhase3
(
this
);
for
(
Atlas
Type
type
:
registryData
.
allType
s
.
getAllTypes
())
{
t
ype
.
resolveReferencesPhase3
(
this
);
}
}
...
...
repository/src/main/java/org/apache/atlas/discovery/ClassificationSearchProcessor.java
View file @
e0d2cdc2
...
...
@@ -42,23 +42,24 @@ public class ClassificationSearchProcessor extends SearchProcessor {
public
ClassificationSearchProcessor
(
SearchContext
context
)
{
super
(
context
);
AtlasClassificationType
classificationType
=
context
.
getClassificationType
();
FilterCriteria
filterCriteria
=
context
.
getSearchParameters
().
getTagFilters
();
Set
<
String
>
typeAndSubTypes
=
classificationType
.
getTypeAndAllSubTypes
();
Set
<
String
>
solrAttributes
=
new
HashSet
<>();
Set
<
String
>
gremlinAttributes
=
new
HashSet
<>();
Set
<
String
>
allAttributes
=
new
HashSet
<>();
final
AtlasClassificationType
classificationType
=
context
.
getClassificationType
();
final
FilterCriteria
filterCriteria
=
context
.
getSearchParameters
().
getTagFilters
();
final
Set
<
String
>
typeAndSubTypes
=
classificationType
.
getTypeAndAllSubTypes
();
final
String
typeAndSubTypesQryStr
=
classificationType
.
getTypeAndAllSubTypesQryStr
();
final
Set
<
String
>
solrAttributes
=
new
HashSet
<>();
final
Set
<
String
>
gremlinAttributes
=
new
HashSet
<>();
final
Set
<
String
>
allAttributes
=
new
HashSet
<>();
processSearchAttributes
(
classificationType
,
filterCriteria
,
solrAttributes
,
gremlinAttributes
,
allAttributes
);
// for classification search, if any attribute can't be handled by Solr - switch to all Gremlin
boolean
useSolrSearch
=
typeAndSubTypes
.
size
()
<=
MAX_CLASSIFICATION_TYPES_IN_INDEX_QUERY
&&
CollectionUtils
.
isEmpty
(
gremlinAttributes
)
&&
canApplySolrFilter
(
classificationType
,
filterCriteria
,
false
);
boolean
useSolrSearch
=
typeAndSubTypes
QryStr
.
length
()
<=
MAX_QUERY_STR_LENGTH_TAGS
&&
CollectionUtils
.
isEmpty
(
gremlinAttributes
)
&&
canApplySolrFilter
(
classificationType
,
filterCriteria
,
false
);
if
(
useSolrSearch
)
{
StringBuilder
solrQuery
=
new
StringBuilder
();
constructTypeTestQuery
(
solrQuery
,
typeAndSubTypes
);
constructTypeTestQuery
(
solrQuery
,
typeAndSubTypes
QryStr
);
constructFilterQuery
(
solrQuery
,
classificationType
,
filterCriteria
,
solrAttributes
);
String
solrQueryString
=
STRAY_AND_PATTERN
.
matcher
(
solrQuery
).
replaceAll
(
")"
);
...
...
repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java
View file @
e0d2cdc2
...
...
@@ -87,8 +87,8 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
private
final
GraphBackedSearchIndexer
indexer
;
private
final
SearchTracker
searchTracker
;
private
final
int
maxResultSetSize
;
private
final
int
maxTypes
Count
InIdxQuery
;
private
final
int
maxTags
Count
InIdxQuery
;
private
final
int
maxTypes
Length
InIdxQuery
;
private
final
int
maxTags
Length
InIdxQuery
;
@Inject
EntityDiscoveryService
(
MetadataRepository
metadataRepository
,
AtlasTypeRegistry
typeRegistry
,
...
...
@@ -101,8 +101,8 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
this
.
gremlinQueryProvider
=
AtlasGremlinQueryProvider
.
INSTANCE
;
this
.
typeRegistry
=
typeRegistry
;
this
.
maxResultSetSize
=
ApplicationProperties
.
get
().
getInt
(
Constants
.
INDEX_SEARCH_MAX_RESULT_SET_SIZE
,
150
);
this
.
maxTypes
CountInIdxQuery
=
ApplicationProperties
.
get
().
getInt
(
Constants
.
INDEX_SEARCH_MAX_TYPES_COUNT
,
10
);
this
.
maxTags
CountInIdxQuery
=
ApplicationProperties
.
get
().
getInt
(
Constants
.
INDEX_SEARCH_MAX_TAGS_COUNT
,
10
);
this
.
maxTypes
LengthInIdxQuery
=
ApplicationProperties
.
get
().
getInt
(
Constants
.
INDEX_SEARCH_TYPES_MAX_QUERY_STR_LENGTH
,
512
);
this
.
maxTags
LengthInIdxQuery
=
ApplicationProperties
.
get
().
getInt
(
Constants
.
INDEX_SEARCH_TAGS_MAX_QUERY_STR_LENGTH
,
512
);
}
@Override
...
...
@@ -490,8 +490,8 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
}
private
String
getQueryForFullTextSearch
(
String
userKeyedString
,
String
typeName
,
String
classification
)
{
String
typeFilter
=
getTypeFilter
(
typeRegistry
,
typeName
,
maxTypesCount
InIdxQuery
);
String
classificationFilter
=
getClassificationFilter
(
typeRegistry
,
classification
,
maxTags
Count
InIdxQuery
);
String
typeFilter
=
getTypeFilter
(
typeRegistry
,
typeName
,
maxTypesLength
InIdxQuery
);
String
classificationFilter
=
getClassificationFilter
(
typeRegistry
,
classification
,
maxTags
Length
InIdxQuery
);
StringBuilder
queryText
=
new
StringBuilder
();
...
...
@@ -619,23 +619,23 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
return
excludeDeletedEntities
&&
GraphHelper
.
getStatus
(
vertex
)
==
Status
.
DELETED
;
}
private
static
String
getClassificationFilter
(
AtlasTypeRegistry
typeRegistry
,
String
classificationName
,
int
maxTypes
Count
InIdxQuery
)
{
AtlasClassificationType
classification
=
typeRegistry
.
getClassificationTypeByName
(
classificationName
);
S
et
<
String
>
typeAndSubTypes
=
classification
!=
null
?
classification
.
getTypeAndAllSubTypes
()
:
null
;
private
static
String
getClassificationFilter
(
AtlasTypeRegistry
typeRegistry
,
String
classificationName
,
int
maxTypes
Length
InIdxQuery
)
{
AtlasClassificationType
type
=
typeRegistry
.
getClassificationTypeByName
(
classificationName
);
S
tring
typeAndSubTypesQryStr
=
type
!=
null
?
type
.
getTypeAndAllSubTypesQryStr
()
:
null
;
if
(
CollectionUtils
.
isNotEmpty
(
typeAndSubTypes
)
&&
typeAndSubTypes
.
size
()
<=
maxTypesCount
InIdxQuery
)
{
return
String
.
format
(
"(%s)"
,
StringUtils
.
join
(
typeAndSubTypes
,
" "
))
;
if
(
StringUtils
.
isNotEmpty
(
typeAndSubTypesQryStr
)
&&
typeAndSubTypesQryStr
.
length
()
<=
maxTypesLength
InIdxQuery
)
{
return
typeAndSubTypesQryStr
;
}
return
""
;
}
private
static
String
getTypeFilter
(
AtlasTypeRegistry
typeRegistry
,
String
typeName
,
int
maxTypes
Count
InIdxQuery
)
{
AtlasEntityType
type
=
typeRegistry
.
getEntityTypeByName
(
typeName
);
S
et
<
String
>
typeAndSubTypes
=
type
!=
null
?
type
.
getTypeAndAllSubTypes
()
:
null
;
private
static
String
getTypeFilter
(
AtlasTypeRegistry
typeRegistry
,
String
typeName
,
int
maxTypes
Length
InIdxQuery
)
{
AtlasEntityType
type
=
typeRegistry
.
getEntityTypeByName
(
typeName
);
S
tring
typeAndSubTypesQryStr
=
type
!=
null
?
type
.
getTypeAndAllSubTypesQryStr
()
:
null
;
if
(
CollectionUtils
.
isNotEmpty
(
typeAndSubTypes
)
&&
typeAndSubTypes
.
size
()
<=
maxTypesCount
InIdxQuery
)
{
return
String
.
format
(
"(%s)"
,
StringUtils
.
join
(
typeAndSubTypes
,
" "
))
;
if
(
StringUtils
.
isNotEmpty
(
typeAndSubTypesQryStr
)
&&
typeAndSubTypesQryStr
.
length
()
<=
maxTypesLength
InIdxQuery
)
{
return
typeAndSubTypesQryStr
;
}
return
""
;
...
...
repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java
View file @
e0d2cdc2
...
...
@@ -40,12 +40,13 @@ public class EntitySearchProcessor extends SearchProcessor {
public
EntitySearchProcessor
(
SearchContext
context
)
{
super
(
context
);
final
AtlasEntityType
entityType
=
context
.
getEntityType
();
final
FilterCriteria
filterCriteria
=
context
.
getSearchParameters
().
getEntityFilters
();
final
Set
<
String
>
typeAndSubTypes
=
entityType
.
getTypeAndAllSubTypes
();
final
Set
<
String
>
solrAttributes
=
new
HashSet
<>();
final
Set
<
String
>
gremlinAttributes
=
new
HashSet
<>();
final
Set
<
String
>
allAttributes
=
new
HashSet
<>();
final
AtlasEntityType
entityType
=
context
.
getEntityType
();
final
FilterCriteria
filterCriteria
=
context
.
getSearchParameters
().
getEntityFilters
();
final
Set
<
String
>
typeAndSubTypes
=
entityType
.
getTypeAndAllSubTypes
();
final
String
typeAndSubTypesQryStr
=
entityType
.
getTypeAndAllSubTypesQryStr
();
final
Set
<
String
>
solrAttributes
=
new
HashSet
<>();
final
Set
<
String
>
gremlinAttributes
=
new
HashSet
<>();
final
Set
<
String
>
allAttributes
=
new
HashSet
<>();
final
AtlasClassificationType
classificationType
=
context
.
getClassificationType
();
final
boolean
filterClassification
=
classificationType
!=
null
&&
!
context
.
needClassificationProcessor
();
...
...
@@ -53,13 +54,13 @@ public class EntitySearchProcessor extends SearchProcessor {
processSearchAttributes
(
entityType
,
filterCriteria
,
solrAttributes
,
gremlinAttributes
,
allAttributes
);
final
boolean
typeSearchBySolr
=
!
filterClassification
&&
typeAndSubTypes
.
size
()
<=
MAX_ENTITY_TYPES_IN_INDEX_QUERY
;
final
boolean
typeSearchBySolr
=
!
filterClassification
&&
typeAndSubTypes
QryStr
.
length
()
<=
MAX_QUERY_STR_LENGTH_TYPES
;
final
boolean
attrSearchBySolr
=
!
filterClassification
&&
CollectionUtils
.
isNotEmpty
(
solrAttributes
)
&&
canApplySolrFilter
(
entityType
,
filterCriteria
,
false
);
StringBuilder
solrQuery
=
new
StringBuilder
();
if
(
typeSearchBySolr
)
{
constructTypeTestQuery
(
solrQuery
,
typeAndSubTypes
);
constructTypeTestQuery
(
solrQuery
,
typeAndSubTypes
QryStr
);
}
if
(
attrSearchBySolr
)
{
...
...
@@ -169,6 +170,7 @@ public class EntitySearchProcessor extends SearchProcessor {
guidQuery
.
addConditionsFrom
(
graphQuery
);
entityVertices
.
clear
();
getVertices
(
guidQuery
.
vertices
().
iterator
(),
entityVertices
);
}
}
else
{
...
...
repository/src/main/java/org/apache/atlas/discovery/FullTextSearchProcessor.java
View file @
e0d2cdc2
...
...
@@ -30,7 +30,6 @@ import org.slf4j.LoggerFactory;
import
java.util.ArrayList
;
import
java.util.Iterator
;
import
java.util.List
;
import
java.util.Set
;
public
class
FullTextSearchProcessor
extends
SearchProcessor
{
...
...
@@ -50,28 +49,26 @@ public class FullTextSearchProcessor extends SearchProcessor {
// if search includes entity-type criteria, adding a filter here can help avoid unnecessary
// processing (and rejection) by subsequent EntitySearchProcessor
if
(
context
.
getEntityType
()
!=
null
)
{
S
et
<
String
>
typeAndSubTypeNames
=
context
.
getEntityType
().
getTypeAndAllSubTypes
();
S
tring
typeAndSubTypeNamesQryStr
=
context
.
getEntityType
().
getTypeAndAllSubTypesQryStr
();
if
(
typeAndSubTypeNames
.
size
()
<=
MAX_ENTITY_TYPES_IN_INDEX_QUERY
)
{
queryString
.
append
(
AND_STR
);
appendIndexQueryValue
(
typeAndSubTypeNames
,
queryString
);
if
(
typeAndSubTypeNamesQryStr
.
length
()
<=
MAX_QUERY_STR_LENGTH_TYPES
)
{
queryString
.
append
(
AND_STR
).
append
(
typeAndSubTypeNamesQryStr
);
}
else
{
LOG
.
warn
(
"'{}' has too many subtypes ({}) to include in index-query; might cause poor performance"
,
context
.
getEntityType
().
getTypeName
(),
typeAndSubTypeNames
.
size
());
LOG
.
warn
(
"'{}' has too many subtypes (
query-string-length=
{}) to include in index-query; might cause poor performance"
,
context
.
getEntityType
().
getTypeName
(),
typeAndSubTypeNames
QryStr
.
length
());
}
}
// if search includes classification criteria, adding a filter here can help avoid unnecessary
// processing (and rejection) by subsequent ClassificationSearchProcessor or EntitySearchProcessor
if
(
context
.
getClassificationType
()
!=
null
)
{
S
et
<
String
>
typeAndSubTypeNames
=
context
.
getClassificationType
().
getTypeAndAllSubTypes
();
S
tring
typeAndSubTypeNamesStr
=
context
.
getClassificationType
().
getTypeAndAllSubTypesQryStr
();
if
(
typeAndSubTypeNames
.
size
()
<=
MAX_CLASSIFICATION_TYPES_IN_INDEX_QUERY
)
{
queryString
.
append
(
AND_STR
);
appendIndexQueryValue
(
typeAndSubTypeNames
,
queryString
);
if
(
typeAndSubTypeNamesStr
.
length
()
<=
MAX_QUERY_STR_LENGTH_TAGS
)
{
queryString
.
append
(
AND_STR
).
append
(
typeAndSubTypeNamesStr
);
}
else
{
LOG
.
warn
(
"'{}' has too many subtypes ({}) to include in index-query; might cause poor performance"
,
context
.
getClassificationType
().
getTypeName
(),
typeAndSubTypeNames
.
size
());
LOG
.
warn
(
"'{}' has too many subtypes (
query-string-length=
{}) to include in index-query; might cause poor performance"
,
context
.
getClassificationType
().
getTypeName
(),
typeAndSubTypeNames
Str
.
length
());
}
}
...
...
repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java
View file @
e0d2cdc2
...
...
@@ -39,21 +39,19 @@ import java.util.regex.Pattern;
public
abstract
class
SearchProcessor
{
private
static
final
Logger
LOG
=
LoggerFactory
.
getLogger
(
SearchProcessor
.
class
);
public
static
final
Pattern
STRAY_AND_PATTERN
=
Pattern
.
compile
(
"(AND\\s+)+\\)"
);
public
static
final
Pattern
STRAY_OR_PATTERN
=
Pattern
.
compile
(
"(OR\\s+)+\\)"
);
public
static
final
Pattern
STRAY_ELIPSIS_PATTERN
=
Pattern
.
compile
(
"(\\(\\s*)\\)"
);
public
static
final
int
MAX_RESULT_SIZE
=
getApplicationProperty
(
Constants
.
INDEX_SEARCH_MAX_RESULT_SET_SIZE
,
150
);
public
static
final
int
MAX_
ENTITY_TYPES_IN_INDEX_QUERY
=
getApplicationProperty
(
Constants
.
INDEX_SEARCH_MAX_TYPES_COUNT
,
10
);
public
static
final
int
MAX_
CLASSIFICATION_TYPES_IN_INDEX_QUERY
=
getApplicationProperty
(
Constants
.
INDEX_SEARCH_MAX_TAGS_COUNT
,
10
);
public
static
final
Pattern
STRAY_AND_PATTERN
=
Pattern
.
compile
(
"(AND\\s+)+\\)"
);
public
static
final
Pattern
STRAY_OR_PATTERN
=
Pattern
.
compile
(
"(OR\\s+)+\\)"
);
public
static
final
Pattern
STRAY_ELIPSIS_PATTERN
=
Pattern
.
compile
(
"(\\(\\s*)\\)"
);
public
static
final
int
MAX_RESULT_SIZE
=
getApplicationProperty
(
Constants
.
INDEX_SEARCH_MAX_RESULT_SET_SIZE
,
150
);
public
static
final
int
MAX_
QUERY_STR_LENGTH_TYPES
=
getApplicationProperty
(
Constants
.
INDEX_SEARCH_TYPES_MAX_QUERY_STR_LENGTH
,
512
);
public
static
final
int
MAX_
QUERY_STR_LENGTH_TAGS
=
getApplicationProperty
(
Constants
.
INDEX_SEARCH_TAGS_MAX_QUERY_STR_LENGTH
,
512
);
public
static
final
String
AND_STR
=
" AND "
;
public
static
final
String
EMPTY_STRING
=
""
;
public
static
final
String
SPACE_STRING
=
" "
;
public
static
final
String
BRACE_OPEN_STR
=
"("
;
public
static
final
String
BRACE_CLOSE_STR
=
")"
;
public
static
final
char
DOUBLE_QUOTE
=
'"'
;
private
static
final
Map
<
SearchParameters
.
Operator
,
String
>
OPERATOR_MAP
=
new
HashMap
<>();
private
static
final
char
[]
OFFENDING_CHARS
=
{
'@'
,
'/'
,
' '
};
// This can grow as we discover corner cases
static
{
...
...
@@ -181,14 +179,13 @@ public abstract class SearchProcessor {
return
ret
;
}
protected
void
constructTypeTestQuery
(
StringBuilder
solrQuery
,
S
et
<
String
>
typeAndAllSubTypes
)
{
if
(
CollectionUtils
.
isNotEmpty
(
typeAndAllSubTypes
))
{
protected
void
constructTypeTestQuery
(
StringBuilder
solrQuery
,
S
tring
typeAndAllSubTypesQryStr
)
{
if
(
StringUtils
.
isNotEmpty
(
typeAndAllSubTypesQryStr
))
{
if
(
solrQuery
.
length
()
>
0
)
{
solrQuery
.
append
(
AND_STR
);
}
solrQuery
.
append
(
"v.\""
).
append
(
Constants
.
TYPE_NAME_PROPERTY_KEY
).
append
(
"\":"
);
appendIndexQueryValue
(
typeAndAllSubTypes
,
solrQuery
);
solrQuery
.
append
(
"v.\""
).
append
(
Constants
.
TYPE_NAME_PROPERTY_KEY
).
append
(
"\":"
).
append
(
typeAndAllSubTypesQryStr
);
}
}
...
...
@@ -255,7 +252,7 @@ public abstract class SearchProcessor {
if
(
OPERATOR_MAP
.
get
(
op
)
!=
null
)
{
String
qualifiedName
=
type
.
getQualifiedAttributeName
(
attrName
);
ret
=
String
.
format
(
OPERATOR_MAP
.
get
(
op
),
qualifiedName
,
escapeIndexQueryValue
(
attrVal
));
ret
=
String
.
format
(
OPERATOR_MAP
.
get
(
op
),
qualifiedName
,
AtlasStructType
.
AtlasAttribute
.
escapeIndexQueryValue
(
attrVal
));
}
}
catch
(
AtlasBaseException
ex
)
{
LOG
.
warn
(
ex
.
getMessage
());
...
...
@@ -389,16 +386,6 @@ public abstract class SearchProcessor {
return
ret
;
}
protected
String
appendIndexQueryValue
(
Set
<
String
>
values
,
StringBuilder
sb
)
{
sb
.
append
(
BRACE_OPEN_STR
);
for
(
String
value
:
values
)
{
sb
.
append
(
escapeIndexQueryValue
(
value
)).
append
(
SPACE_STRING
);
}
sb
.
append
(
BRACE_CLOSE_STR
);
return
sb
.
toString
();
}
private
static
int
getApplicationProperty
(
String
propertyName
,
int
defaultValue
)
{
try
{
return
ApplicationProperties
.
get
().
getInt
(
propertyName
,
defaultValue
);
...
...
@@ -408,25 +395,4 @@ public abstract class SearchProcessor {
return
defaultValue
;
}
private
String
escapeIndexQueryValue
(
String
value
)
{
String
ret
=
value
;
if
(
StringUtils
.
containsAny
(
value
,
OFFENDING_CHARS
))
{
boolean
isQuoteAtStart
=
value
.
charAt
(
0
)
==
DOUBLE_QUOTE
;
boolean
isQuoteAtEnd
=
value
.
charAt
(
value
.
length
()
-
1
)
==
DOUBLE_QUOTE
;
if
(!
isQuoteAtStart
)
{
if
(!
isQuoteAtEnd
)
{
ret
=
DOUBLE_QUOTE
+
value
+
DOUBLE_QUOTE
;
}
else
{
ret
=
DOUBLE_QUOTE
+
value
;
}
}
else
if
(!
isQuoteAtEnd
)
{
ret
=
value
+
DOUBLE_QUOTE
;
}
}
return
ret
;
}
}
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment