Skip to content

A
atlas
Commit 2223c82c by Ashutosh Mestry
Options

ATLAS-3370: Aggregation Metrics with quick search. Modifications to DSL Query…

ATLAS-3370: Aggregation Metrics with quick search. Modifications to DSL Query Processor and Basic Search.
parent 4b7b1a99
Showing with 237 additions and 93 deletions
addons/models/0000-Area0/0010-base_model.json
...@@ -40,7 +40,8 @@ ...@@ -40,7 +40,8 @@
"cardinality": "SINGLE", "cardinality": "SINGLE",
"isIndexable": true, "isIndexable": true,
"isOptional": false, "isOptional": false,
"isUnique": false "isUnique": false,
"indexType": "STRING"
}, },
{ {
"name": "description", "name": "description",
...@@ -56,7 +57,8 @@ ...@@ -56,7 +57,8 @@
"cardinality": "SINGLE", "cardinality": "SINGLE",
"isIndexable": true, "isIndexable": true,
"isOptional": true, "isOptional": true,
"isUnique": false "isUnique": false,
"indexType": "STRING"
} }
] ]
}, },
...@@ -113,7 +115,7 @@ ...@@ -113,7 +115,7 @@
}, },
{ {
"name": "Infrastructure", "name": "Infrastructure",
"description":"Infrastructure can be IT infrastructure, which contains hosts and servers. Infrastructure might not be IT orientated, such as 'Car' for IoT applications.", "description": "Infrastructure can be IT infrastructure, which contains hosts and servers. Infrastructure might not be IT orientated, such as 'Car' for IoT applications.",
"superTypes": [ "superTypes": [
"Asset" "Asset"
], ],
......
graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraphManagement.java
...@@ -155,9 +155,10 @@ public interface AtlasGraphManagement { ...@@ -155,9 +155,10 @@ public interface AtlasGraphManagement {
* *
* @param vertexIndex * @param vertexIndex
* @param propertyKey * @param propertyKey
* @param isStringField
* @return the index field name used for the given property * @return the index field name used for the given property
*/ */
String addMixedIndex(String vertexIndex, AtlasPropertyKey propertyKey); String addMixedIndex(String vertexIndex, AtlasPropertyKey propertyKey, boolean isStringField);
/** /**
* Gets the index field name for the vertex property. * Gets the index field name for the vertex property.
...@@ -165,5 +166,5 @@ public interface AtlasGraphManagement { ...@@ -165,5 +166,5 @@ public interface AtlasGraphManagement {
* @param propertyKey * @param propertyKey
* @return the encoded name for the index * @return the encoded name for the index
*/ */
String getIndexFieldName(String indexName, AtlasPropertyKey propertyKey); String getIndexFieldName(String indexName, AtlasPropertyKey propertyKey, boolean isStringField);
} }
graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraph.java
...@@ -51,6 +51,7 @@ import org.janusgraph.core.PropertyKey; ...@@ -51,6 +51,7 @@ import org.janusgraph.core.PropertyKey;
import org.janusgraph.core.SchemaViolationException; import org.janusgraph.core.SchemaViolationException;
import org.janusgraph.core.schema.JanusGraphIndex; import org.janusgraph.core.schema.JanusGraphIndex;
import org.janusgraph.core.schema.JanusGraphManagement; import org.janusgraph.core.schema.JanusGraphManagement;
import org.janusgraph.core.schema.Mapping;
import org.janusgraph.core.schema.Parameter; import org.janusgraph.core.schema.Parameter;
import org.janusgraph.diskstorage.BackendException; import org.janusgraph.diskstorage.BackendException;
import org.janusgraph.graphdb.database.StandardJanusGraph; import org.janusgraph.graphdb.database.StandardJanusGraph;
...@@ -75,8 +76,9 @@ import static org.apache.atlas.repository.graphdb.janus.AtlasJanusGraphDatabase. ...@@ -75,8 +76,9 @@ import static org.apache.atlas.repository.graphdb.janus.AtlasJanusGraphDatabase.
*/ */
public class AtlasJanusGraph implements AtlasGraph<AtlasJanusVertex, AtlasJanusEdge> { public class AtlasJanusGraph implements AtlasGraph<AtlasJanusVertex, AtlasJanusEdge> {
private static final Logger LOG = LoggerFactory.getLogger(AtlasJanusGraph.class); private static final Logger LOG = LoggerFactory.getLogger(AtlasJanusGraph.class);
private static final Parameter[] EMPTY_PARAMETER_ARRAY = new Parameter[0]; private static final Parameter[] EMPTY_PARAMETER_ARRAY = new Parameter[0];
private static Configuration APPLICATION_PROPERTIES = null; private static Configuration APPLICATION_PROPERTIES = null;
private final ConvertGremlinValueFunction GREMLIN_VALUE_CONVERSION_FUNCTION = new ConvertGremlinValueFunction(); private final ConvertGremlinValueFunction GREMLIN_VALUE_CONVERSION_FUNCTION = new ConvertGremlinValueFunction();
...@@ -418,10 +420,12 @@ public class AtlasJanusGraph implements AtlasGraph<AtlasJanusVertex, AtlasJanusE ...@@ -418,10 +420,12 @@ public class AtlasJanusGraph implements AtlasGraph<AtlasJanusVertex, AtlasJanusE
} }
String getIndexFieldName(AtlasPropertyKey propertyKey, JanusGraphIndex graphIndex) { String getIndexFieldName(AtlasPropertyKey propertyKey, JanusGraphIndex graphIndex, Parameter ... parameters) {
PropertyKey janusKey = AtlasJanusObjectFactory.createPropertyKey(propertyKey); PropertyKey janusKey = AtlasJanusObjectFactory.createPropertyKey(propertyKey);
if(parameters == null) {
return janusGraph.getIndexSerializer().getDefaultFieldName(janusKey, EMPTY_PARAMETER_ARRAY, graphIndex.getBackingIndex()); parameters = EMPTY_PARAMETER_ARRAY;
}
return janusGraph.getIndexSerializer().getDefaultFieldName(janusKey, parameters, graphIndex.getBackingIndex());
} }
......
graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphManagement.java
...@@ -45,6 +45,8 @@ import java.util.Set; ...@@ -45,6 +45,8 @@ import java.util.Set;
* Janus implementation of AtlasGraphManagement. * Janus implementation of AtlasGraphManagement.
*/ */
public class AtlasJanusGraphManagement implements AtlasGraphManagement { public class AtlasJanusGraphManagement implements AtlasGraphManagement {
private static final Parameter[] STRING_PARAMETER_ARRAY = new Parameter[]{Mapping.STRING.asParameter()};
private static final Logger LOG = LoggerFactory.getLogger(AtlasJanusGraphManagement.class); private static final Logger LOG = LoggerFactory.getLogger(AtlasJanusGraphManagement.class);
private static final char[] RESERVED_CHARS = { '{', '}', '"', '$', Token.SEPARATOR_CHAR }; private static final char[] RESERVED_CHARS = { '{', '}', '"', '$', Token.SEPARATOR_CHAR };
...@@ -145,7 +147,6 @@ public class AtlasJanusGraphManagement implements AtlasGraphManagement { ...@@ -145,7 +147,6 @@ public class AtlasJanusGraphManagement implements AtlasGraphManagement {
} }
PropertyKeyMaker propertyKeyBuilder = management.makePropertyKey(propertyName).dataType(propertyClass); PropertyKeyMaker propertyKeyBuilder = management.makePropertyKey(propertyName).dataType(propertyClass);
if (cardinality != null) { if (cardinality != null) {
Cardinality janusCardinality = AtlasJanusObjectFactory.createCardinality(cardinality); Cardinality janusCardinality = AtlasJanusObjectFactory.createCardinality(cardinality);
propertyKeyBuilder.cardinality(janusCardinality); propertyKeyBuilder.cardinality(janusCardinality);
...@@ -192,13 +193,25 @@ public class AtlasJanusGraphManagement implements AtlasGraphManagement { ...@@ -192,13 +193,25 @@ public class AtlasJanusGraphManagement implements AtlasGraphManagement {
} }
@Override @Override
public String addMixedIndex(String indexName, AtlasPropertyKey propertyKey) { public String addMixedIndex(String indexName, AtlasPropertyKey propertyKey, boolean isStringField) {
PropertyKey janusKey = AtlasJanusObjectFactory.createPropertyKey(propertyKey); PropertyKey janusKey = AtlasJanusObjectFactory.createPropertyKey(propertyKey);
JanusGraphIndex janusGraphIndex = management.getGraphIndex(indexName); JanusGraphIndex janusGraphIndex = management.getGraphIndex(indexName);
if(isStringField) {
management.addIndexKey(janusGraphIndex, janusKey, Mapping.STRING.asParameter());
LOG.debug("created a string type for {} with janueKey {}.", propertyKey.getName(), janusKey);
} else {
management.addIndexKey(janusGraphIndex, janusKey); management.addIndexKey(janusGraphIndex, janusKey);
LOG.debug("created a default type for {} with janueKey {}.", propertyKey.getName(), janusKey);
}
String encodedName = "";
if(isStringField) {
encodedName = graph.getIndexFieldName(propertyKey, janusGraphIndex, STRING_PARAMETER_ARRAY);
} else {
encodedName = graph.getIndexFieldName(propertyKey, janusGraphIndex);
}
String encodedName = graph.getIndexFieldName(propertyKey, janusGraphIndex);
LOG.info("property '{}' is encoded to '{}'.", propertyKey.getName(), encodedName); LOG.info("property '{}' is encoded to '{}'.", propertyKey.getName(), encodedName);
...@@ -206,13 +219,17 @@ public class AtlasJanusGraphManagement implements AtlasGraphManagement { ...@@ -206,13 +219,17 @@ public class AtlasJanusGraphManagement implements AtlasGraphManagement {
} }
@Override @Override
public String getIndexFieldName(String indexName, AtlasPropertyKey propertyKey) { public String getIndexFieldName(String indexName, AtlasPropertyKey propertyKey, boolean isStringField) {
JanusGraphIndex janusGraphIndex = management.getGraphIndex(indexName); JanusGraphIndex janusGraphIndex = management.getGraphIndex(indexName);
if(isStringField) {
return graph.getIndexFieldName(propertyKey, janusGraphIndex, STRING_PARAMETER_ARRAY);
} else {
return graph.getIndexFieldName(propertyKey, janusGraphIndex); return graph.getIndexFieldName(propertyKey, janusGraphIndex);
} }
@Override }
public AtlasGraphIndex getGraphIndex(String indexName) { public AtlasGraphIndex getGraphIndex(String indexName) {
JanusGraphIndex index = management.getGraphIndex(indexName); JanusGraphIndex index = management.getGraphIndex(indexName);
......
graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/AbstractGraphDatabaseTest.java
...@@ -99,7 +99,7 @@ public abstract class AbstractGraphDatabaseTest { ...@@ -99,7 +99,7 @@ public abstract class AbstractGraphDatabaseTest {
AtlasPropertyKey key = management.makePropertyKey(propertyName, propertyClass, cardinality); AtlasPropertyKey key = management.makePropertyKey(propertyName, propertyClass, cardinality);
try { try {
if (propertyClass != Integer.class) { if (propertyClass != Integer.class) {
management.addMixedIndex(BACKING_INDEX_NAME, key); management.addMixedIndex(BACKING_INDEX_NAME, key, false);
} }
} catch(Throwable t) { } catch(Throwable t) {
//ok //ok
......
intg/src/main/java/org/apache/atlas/model/typedef/AtlasStructDef.java
...@@ -262,7 +262,9 @@ public class AtlasStructDef extends AtlasBaseTypeDef implements Serializable { ...@@ -262,7 +262,9 @@ public class AtlasStructDef extends AtlasBaseTypeDef implements Serializable {
private static final long serialVersionUID = 1L; private static final long serialVersionUID = 1L;
public static final int DEFAULT_SEARCHWEIGHT = -1; public static final int DEFAULT_SEARCHWEIGHT = -1;
public static final String SEARCH_WEIGHT_ATTR_NAME = "searchWeight"; public static final String SEARCH_WEIGHT_ATTR_NAME = "searchWeight";
public static final String INDEX_TYPE_ATTR_NAME = "indexType";
public static final String ATTRDEF_OPTION_SOFT_REFERENCE = "isSoftReference"; public static final String ATTRDEF_OPTION_SOFT_REFERENCE = "isSoftReference";
private final String STRING_TRUE = "true"; private final String STRING_TRUE = "true";
...@@ -271,6 +273,8 @@ public class AtlasStructDef extends AtlasBaseTypeDef implements Serializable { ...@@ -271,6 +273,8 @@ public class AtlasStructDef extends AtlasBaseTypeDef implements Serializable {
*/ */
public enum Cardinality { SINGLE, LIST, SET } public enum Cardinality { SINGLE, LIST, SET }
public enum IndexType { DEFAULT, STRING}
public static final int COUNT_NOT_SET = -1; public static final int COUNT_NOT_SET = -1;
private String name; private String name;
...@@ -285,6 +289,7 @@ public class AtlasStructDef extends AtlasBaseTypeDef implements Serializable { ...@@ -285,6 +289,7 @@ public class AtlasStructDef extends AtlasBaseTypeDef implements Serializable {
private String defaultValue; private String defaultValue;
private String description; private String description;
private int searchWeight = DEFAULT_SEARCHWEIGHT; private int searchWeight = DEFAULT_SEARCHWEIGHT;
private IndexType indexType = null;
private List<AtlasConstraintDef> constraints; private List<AtlasConstraintDef> constraints;
private Map<String, String> options; private Map<String, String> options;
...@@ -296,32 +301,34 @@ public class AtlasStructDef extends AtlasBaseTypeDef implements Serializable { ...@@ -296,32 +301,34 @@ public class AtlasStructDef extends AtlasBaseTypeDef implements Serializable {
} }
public AtlasAttributeDef(String name, String typeName, int searchWeight) { public AtlasAttributeDef(String name, String typeName, int searchWeight) {
this(name, typeName, false, Cardinality.SINGLE, searchWeight); this(name, typeName, false, Cardinality.SINGLE, searchWeight, null);
} }
public AtlasAttributeDef(String name, String typeName, boolean isOptional, Cardinality cardinality) { public AtlasAttributeDef(String name, String typeName, int searchWeight, IndexType indexType) {
this(name, typeName, isOptional, cardinality, DEFAULT_SEARCHWEIGHT); this(name, typeName, false, Cardinality.SINGLE, searchWeight, indexType);
} }
private AtlasAttributeDef(String name, String typeName, boolean isOptional, Cardinality cardinality, int searchWeight) { public AtlasAttributeDef(String name, String typeName, boolean isOptional, Cardinality cardinality) {
this(name, typeName, isOptional, cardinality, COUNT_NOT_SET, COUNT_NOT_SET, false, false, false, null, searchWeight); this(name, typeName, isOptional, cardinality, DEFAULT_SEARCHWEIGHT, null);
} }
private AtlasAttributeDef(String name, String typeName, boolean isOptional, Cardinality cardinality, int searchWeight, IndexType indexType) {
this(name, typeName, isOptional, cardinality, COUNT_NOT_SET, COUNT_NOT_SET, false, false, false, null, searchWeight, indexType);
}
public AtlasAttributeDef(String name, String typeName, boolean isOptional, Cardinality cardinality, public AtlasAttributeDef(String name, String typeName, boolean isOptional, Cardinality cardinality,
int valuesMinCount, int valuesMaxCount, boolean isUnique, boolean isIndexable, boolean includeInNotification, List<AtlasConstraintDef> constraints) { int valuesMinCount, int valuesMaxCount, boolean isUnique, boolean isIndexable, boolean includeInNotification, List<AtlasConstraintDef> constraints) {
this(name, typeName, isOptional, cardinality, valuesMinCount, valuesMaxCount, isUnique, isIndexable, includeInNotification, constraints, DEFAULT_SEARCHWEIGHT); this(name, typeName, isOptional, cardinality, valuesMinCount, valuesMaxCount, isUnique, isIndexable, includeInNotification, constraints, DEFAULT_SEARCHWEIGHT, null);
} }
private AtlasAttributeDef(String name, String typeName, boolean isOptional, Cardinality cardinality, private AtlasAttributeDef(String name, String typeName, boolean isOptional, Cardinality cardinality,
int valuesMinCount, int valuesMaxCount, boolean isUnique, boolean isIndexable, boolean includeInNotification, List<AtlasConstraintDef> constraints, int searchWeight) { int valuesMinCount, int valuesMaxCount, boolean isUnique, boolean isIndexable, boolean includeInNotification, List<AtlasConstraintDef> constraints, int searchWeight, IndexType indexType) {
this(name, typeName, isOptional, cardinality, valuesMinCount, valuesMaxCount, isUnique, isIndexable, includeInNotification, null, constraints, null, null, searchWeight); this(name, typeName, isOptional, cardinality, valuesMinCount, valuesMaxCount, isUnique, isIndexable, includeInNotification, null, constraints, null, null, searchWeight, indexType);
} }
public AtlasAttributeDef(String name, String typeName, boolean isOptional, Cardinality cardinality, public AtlasAttributeDef(String name, String typeName, boolean isOptional, Cardinality cardinality,
int valuesMinCount, int valuesMaxCount, boolean isUnique, boolean isIndexable, boolean includeInNotification, String defaultValue, int valuesMinCount, int valuesMaxCount, boolean isUnique, boolean isIndexable, boolean includeInNotification, String defaultValue,
List<AtlasConstraintDef> constraints, Map<String,String> options, String description, int searchWeight) { List<AtlasConstraintDef> constraints, Map<String,String> options, String description, int searchWeight, IndexType indexType) {
setName(name); setName(name);
setTypeName(typeName); setTypeName(typeName);
setIsOptional(isOptional); setIsOptional(isOptional);
...@@ -336,6 +343,7 @@ public class AtlasStructDef extends AtlasBaseTypeDef implements Serializable { ...@@ -336,6 +343,7 @@ public class AtlasStructDef extends AtlasBaseTypeDef implements Serializable {
setOptions(options); setOptions(options);
setDescription(description); setDescription(description);
setSearchWeight(searchWeight); setSearchWeight(searchWeight);
setIndexType(indexType);
} }
public AtlasAttributeDef(AtlasAttributeDef other) { public AtlasAttributeDef(AtlasAttributeDef other) {
...@@ -354,6 +362,7 @@ public class AtlasStructDef extends AtlasBaseTypeDef implements Serializable { ...@@ -354,6 +362,7 @@ public class AtlasStructDef extends AtlasBaseTypeDef implements Serializable {
setOptions(other.getOptions()); setOptions(other.getOptions());
setDescription((other.getDescription())); setDescription((other.getDescription()));
setSearchWeight(other.getSearchWeight()); setSearchWeight(other.getSearchWeight());
setIndexType(other.getIndexType());
} }
} }
...@@ -365,6 +374,14 @@ public class AtlasStructDef extends AtlasBaseTypeDef implements Serializable { ...@@ -365,6 +374,14 @@ public class AtlasStructDef extends AtlasBaseTypeDef implements Serializable {
this.searchWeight = searchWeight; this.searchWeight = searchWeight;
} }
public void setIndexType(IndexType indexType) {
this.indexType = indexType;
}
public IndexType getIndexType() {
return indexType;
}
public String getName() { public String getName() {
return name; return name;
} }
...@@ -510,6 +527,7 @@ public class AtlasStructDef extends AtlasBaseTypeDef implements Serializable { ...@@ -510,6 +527,7 @@ public class AtlasStructDef extends AtlasBaseTypeDef implements Serializable {
sb.append(", defaultValue=").append(defaultValue); sb.append(", defaultValue=").append(defaultValue);
sb.append(", options='").append(options).append('\''); sb.append(", options='").append(options).append('\'');
sb.append(", searchWeight='").append(searchWeight).append('\''); sb.append(", searchWeight='").append(searchWeight).append('\'');
sb.append(", indexType='").append(indexType).append('\'');
sb.append(", constraints=["); sb.append(", constraints=[");
if (CollectionUtils.isNotEmpty(constraints)) { if (CollectionUtils.isNotEmpty(constraints)) {
int i = 0; int i = 0;
...@@ -545,12 +563,13 @@ public class AtlasStructDef extends AtlasBaseTypeDef implements Serializable { ...@@ -545,12 +563,13 @@ public class AtlasStructDef extends AtlasBaseTypeDef implements Serializable {
Objects.equals(description, that.description) && Objects.equals(description, that.description) &&
Objects.equals(constraints, that.constraints) && Objects.equals(constraints, that.constraints) &&
Objects.equals(options, that.options) && Objects.equals(options, that.options) &&
Objects.equals(searchWeight, that.searchWeight); Objects.equals(searchWeight, that.searchWeight) &&
Objects.equals(indexType, that.indexType);
} }
@Override @Override
public int hashCode() { public int hashCode() {
return Objects.hash(name, typeName, isOptional, cardinality, valuesMinCount, valuesMaxCount, isUnique, isIndexable, includeInNotification, defaultValue, constraints, options, description, searchWeight); return Objects.hash(name, typeName, isOptional, cardinality, valuesMinCount, valuesMaxCount, isUnique, isIndexable, includeInNotification, defaultValue, constraints, options, description, searchWeight, indexType);
} }
@Override @Override
......
intg/src/main/java/org/apache/atlas/type/AtlasStructType.java
...@@ -34,6 +34,7 @@ import org.slf4j.LoggerFactory; ...@@ -34,6 +34,7 @@ import org.slf4j.LoggerFactory;
import java.util.*; import java.util.*;
import static org.apache.atlas.model.TypeCategory.OBJECT_ID_TYPE; import static org.apache.atlas.model.TypeCategory.OBJECT_ID_TYPE;
import static org.apache.atlas.model.typedef.AtlasBaseTypeDef.ATLAS_TYPE_STRING;
import static org.apache.atlas.model.typedef.AtlasStructDef.AtlasConstraintDef.*; import static org.apache.atlas.model.typedef.AtlasStructDef.AtlasConstraintDef.*;
/** /**
...@@ -694,6 +695,7 @@ public class AtlasStructType extends AtlasType { ...@@ -694,6 +695,7 @@ public class AtlasStructType extends AtlasType {
} }
public static class AtlasAttribute { public static class AtlasAttribute {
public static final Object VERTEX_PROPERTY_PREFIX_STRING_INDEX_TYPE = "__s_";
private final AtlasStructType definedInType; private final AtlasStructType definedInType;
private final AtlasType attributeType; private final AtlasType attributeType;
private final AtlasAttributeDef attributeDef; private final AtlasAttributeDef attributeDef;
...@@ -718,14 +720,15 @@ public class AtlasStructType extends AtlasType { ...@@ -718,14 +720,15 @@ public class AtlasStructType extends AtlasType {
this.attributeDef = attrDef; this.attributeDef = attrDef;
this.attributeType = attributeType.getTypeForAttribute(); this.attributeType = attributeType.getTypeForAttribute();
this.qualifiedName = getQualifiedAttributeName(definedInType.getStructDef(), attributeDef.getName()); this.qualifiedName = getQualifiedAttributeName(definedInType.getStructDef(), attributeDef.getName());
this.vertexPropertyName = encodePropertyKey(this.qualifiedName); this.vertexPropertyName = generateVertexPropertyName(definedInType.getStructDef(), attributeDef, qualifiedName);
this.vertexUniquePropertyName = attrDef.getIsUnique() ? encodePropertyKey(getQualifiedAttributeName(definedInType.getStructDef(), UNIQUE_ATTRIBUTE_SHADE_PROPERTY_PREFIX + attributeDef.getName())) : null; this.vertexUniquePropertyName = attrDef.getIsUnique() ? encodePropertyKey(getQualifiedAttributeName(definedInType.getStructDef(), UNIQUE_ATTRIBUTE_SHADE_PROPERTY_PREFIX + attributeDef.getName())) : null;
this.relationshipName = relationshipName; this.relationshipName = relationshipName;
this.relationshipEdgeLabel = getRelationshipEdgeLabel(relationshipLabel); this.relationshipEdgeLabel = getRelationshipEdgeLabel(relationshipLabel);
boolean isOwnedRef = false; boolean isOwnedRef = false;
String inverseRefAttribute = null; String inverseRefAttribute = null;
LOG.debug("Attribute {} will use the vertext property name {}.", qualifiedName, vertexPropertyName);
if (CollectionUtils.isNotEmpty(attributeDef.getConstraints())) { if (CollectionUtils.isNotEmpty(attributeDef.getConstraints())) {
for (AtlasConstraintDef constraint : attributeDef.getConstraints()) { for (AtlasConstraintDef constraint : attributeDef.getConstraints()) {
if (constraint.isConstraintType(CONSTRAINT_TYPE_OWNED_REF)) { if (constraint.isConstraintType(CONSTRAINT_TYPE_OWNED_REF)) {
...@@ -829,6 +832,8 @@ public class AtlasStructType extends AtlasType { ...@@ -829,6 +832,8 @@ public class AtlasStructType extends AtlasType {
public int getSearchWeight() { return attributeDef.getSearchWeight(); } public int getSearchWeight() { return attributeDef.getSearchWeight(); }
public AtlasAttributeDef.IndexType getIndexType() { return attributeDef.getIndexType();}
public boolean getIsDynAttribute() { return isDynAttribute; } public boolean getIsDynAttribute() { return isDynAttribute; }
public void setIsDynAttribute(boolean isDynAttribute){ this.isDynAttribute = isDynAttribute; } public void setIsDynAttribute(boolean isDynAttribute){ this.isDynAttribute = isDynAttribute; }
...@@ -910,9 +915,19 @@ public class AtlasStructType extends AtlasType { ...@@ -910,9 +915,19 @@ public class AtlasStructType extends AtlasType {
return (relationshipLabel == null) ? getEdgeLabel(qualifiedName) : relationshipLabel; return (relationshipLabel == null) ? getEdgeLabel(qualifiedName) : relationshipLabel;
} }
private static String getQualifiedAttributeName(AtlasStructDef structDef, String attrName) { public static String getQualifiedAttributeName(AtlasStructDef structDef, String attrName) {
final String typeName = structDef.getName(); return attrName.contains(".") ? attrName : String.format("%s.%s", structDef.getName(), attrName);
return attrName.contains(".") ? attrName : String.format("%s.%s", typeName, attrName); }
public static String generateVertexPropertyName(AtlasStructDef structDef, AtlasAttributeDef attrDef, String qualifiedName) {
String vertexPropertyName = qualifiedName;
if(!attrDef.getName().contains(".") &&
AtlasAttributeDef.IndexType.STRING.equals(attrDef.getIndexType()) &&
ATLAS_TYPE_STRING.equalsIgnoreCase(attrDef.getTypeName())) {
vertexPropertyName = String.format("%s.%s%s", structDef.getName(), VERTEX_PROPERTY_PREFIX_STRING_INDEX_TYPE, attrDef.getName());
}
return encodePropertyKey(vertexPropertyName);
} }
// Keys copied from org.janusgraph.graphdb.types.system.SystemTypeManager.RESERVED_CHARS // Keys copied from org.janusgraph.graphdb.types.system.SystemTypeManager.RESERVED_CHARS
......
intg/src/main/java/org/apache/atlas/v1/model/typedef/AttributeDefinition.java
...@@ -21,8 +21,11 @@ package org.apache.atlas.v1.model.typedef; ...@@ -21,8 +21,11 @@ package org.apache.atlas.v1.model.typedef;
import com.fasterxml.jackson.annotation.JsonAutoDetect; import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.databind.annotation.JsonSerialize; import com.fasterxml.jackson.databind.annotation.JsonSerialize;
import org.apache.atlas.model.typedef.AtlasStructDef;
import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlAttribute;
import javax.xml.bind.annotation.XmlRootElement; import javax.xml.bind.annotation.XmlRootElement;
import java.io.Serializable; import java.io.Serializable;
...@@ -52,33 +55,34 @@ public class AttributeDefinition implements Serializable { ...@@ -52,33 +55,34 @@ public class AttributeDefinition implements Serializable {
private String description; private String description;
private Map<String, String> options; private Map<String, String> options;
private int searchWeight = DEFAULT_SEARCHWEIGHT; private int searchWeight = DEFAULT_SEARCHWEIGHT;
private AtlasStructDef.AtlasAttributeDef.IndexType indexType = null;
public AttributeDefinition() { public AttributeDefinition() {
} }
public AttributeDefinition(String name, String dataTypeName, Multiplicity multiplicity) { public AttributeDefinition(String name, String dataTypeName, Multiplicity multiplicity) {
this(name, dataTypeName, multiplicity, false, false, true, null, null, DEFAULT_SEARCHWEIGHT); this(name, dataTypeName, multiplicity, false, false, true, null, null, DEFAULT_SEARCHWEIGHT, null);
} }
public AttributeDefinition(String name, String dataTypeName, Multiplicity multiplicity, boolean isComposite, public AttributeDefinition(String name, String dataTypeName, Multiplicity multiplicity, boolean isComposite,
String reverseAttributeName) { String reverseAttributeName) {
this(name, dataTypeName, multiplicity, isComposite, reverseAttributeName, DEFAULT_SEARCHWEIGHT); this(name, dataTypeName, multiplicity, isComposite, reverseAttributeName, DEFAULT_SEARCHWEIGHT, null);
} }
public AttributeDefinition(String name, String dataTypeName, Multiplicity multiplicity, boolean isComposite, public AttributeDefinition(String name, String dataTypeName, Multiplicity multiplicity, boolean isComposite,
String reverseAttributeName, int searchWeight) { String reverseAttributeName, int searchWeight, AtlasStructDef.AtlasAttributeDef.IndexType indexType) {
this(name, dataTypeName, multiplicity, isComposite, false, false, reverseAttributeName, null, searchWeight); this(name, dataTypeName, multiplicity, isComposite, false, false, reverseAttributeName, null, searchWeight, indexType);
} }
public AttributeDefinition(String name, String dataTypeName, Multiplicity multiplicity, boolean isComposite, public AttributeDefinition(String name, String dataTypeName, Multiplicity multiplicity, boolean isComposite,
boolean isUnique, boolean isIndexable, String reverseAttributeName, boolean isUnique, boolean isIndexable, String reverseAttributeName,
Map<String, String> options) { Map<String, String> options) {
this(name, dataTypeName, multiplicity, isComposite, isUnique, isIndexable,reverseAttributeName, options, DEFAULT_SEARCHWEIGHT); this(name, dataTypeName, multiplicity, isComposite, isUnique, isIndexable,reverseAttributeName, options, DEFAULT_SEARCHWEIGHT, null);
} }
public AttributeDefinition(String name, String dataTypeName, Multiplicity multiplicity, boolean isComposite, public AttributeDefinition(String name, String dataTypeName, Multiplicity multiplicity, boolean isComposite,
boolean isUnique, boolean isIndexable, String reverseAttributeName, boolean isUnique, boolean isIndexable, String reverseAttributeName,
Map<String, String> options, int searchWeight) { Map<String, String> options, int searchWeight, AtlasStructDef.AtlasAttributeDef.IndexType indexType) {
this.name = name; this.name = name;
this.dataTypeName = dataTypeName; this.dataTypeName = dataTypeName;
this.multiplicity = multiplicity; this.multiplicity = multiplicity;
...@@ -88,6 +92,7 @@ public class AttributeDefinition implements Serializable { ...@@ -88,6 +92,7 @@ public class AttributeDefinition implements Serializable {
this.reverseAttributeName = reverseAttributeName; this.reverseAttributeName = reverseAttributeName;
this.options = options; this.options = options;
this.searchWeight = searchWeight; this.searchWeight = searchWeight;
this.indexType = indexType;
} }
...@@ -213,4 +218,12 @@ public class AttributeDefinition implements Serializable { ...@@ -213,4 +218,12 @@ public class AttributeDefinition implements Serializable {
public int getSearchWeight() { public int getSearchWeight() {
return searchWeight; return searchWeight;
} }
public void setIndexType(AtlasStructDef.AtlasAttributeDef.IndexType indexType) {
this.indexType = indexType;
}
public AtlasStructDef.AtlasAttributeDef.IndexType getIndexType() {
return this.indexType;
}
} }
repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java
...@@ -512,7 +512,10 @@ public abstract class SearchProcessor { ...@@ -512,7 +512,10 @@ public abstract class SearchProcessor {
break; break;
} }
ret = predicate.generatePredicate(attribute.getQualifiedName(), attrValue, attrClass); String vertexPropertyName = attribute.getVertexPropertyName();
ret = predicate.generatePredicate(
StringUtils.isEmpty(vertexPropertyName) ? attribute.getQualifiedName() : vertexPropertyName,
attrValue, attrClass);
} }
return ret; return ret;
......
repository/src/main/java/org/apache/atlas/query/GremlinClauseList.java
...@@ -37,14 +37,6 @@ class GremlinClauseList { ...@@ -37,14 +37,6 @@ class GremlinClauseList {
list.add(idx, g); list.add(idx, g);
} }
public void add(GremlinQueryComposer.GremlinClauseValue g, AtlasEntityType t) {
add(g);
}
public void add(int idx, GremlinQueryComposer.GremlinClauseValue g, AtlasEntityType t) {
add(idx, g);
}
public void add(GremlinClause clause, String... args) { public void add(GremlinClause clause, String... args) {
list.add(new GremlinQueryComposer.GremlinClauseValue(clause, clause.get(args))); list.add(new GremlinQueryComposer.GremlinClauseValue(clause, clause.get(args)));
} }
......
repository/src/main/java/org/apache/atlas/query/GremlinQueryComposer.java
...@@ -177,6 +177,7 @@ public class GremlinQueryComposer { ...@@ -177,6 +177,7 @@ public class GremlinQueryComposer {
introduceType(lhsI); introduceType(lhsI);
org = lhsI; org = lhsI;
lhsI = createInfo(lhs); lhsI = createInfo(lhs);
lhsI.setTypeName(org.getTypeName());
} }
if (!context.validator.isValidQualifiedName(lhsI.getQualifiedName(), lhsI.getRaw())) { if (!context.validator.isValidQualifiedName(lhsI.getQualifiedName(), lhsI.getRaw())) {
...@@ -192,11 +193,11 @@ public class GremlinQueryComposer { ...@@ -192,11 +193,11 @@ public class GremlinQueryComposer {
rhs = addQuotesIfNecessary(lhsI, rhs); rhs = addQuotesIfNecessary(lhsI, rhs);
SearchParameters.Operator op = SearchParameters.Operator.fromString(operator); SearchParameters.Operator op = SearchParameters.Operator.fromString(operator);
if (op == SearchParameters.Operator.LIKE) { if (op == SearchParameters.Operator.LIKE) {
add(GremlinClause.TEXT_CONTAINS, lhsI.getQualifiedName(), IdentifierHelper.getFixedRegEx(rhs)); add(GremlinClause.TEXT_CONTAINS, getPropertyForClause(lhsI), IdentifierHelper.getFixedRegEx(rhs));
} else if (op == SearchParameters.Operator.IN) { } else if (op == SearchParameters.Operator.IN) {
add(GremlinClause.HAS_OPERATOR, lhsI.getQualifiedName(), "within", rhs); add(GremlinClause.HAS_OPERATOR, getPropertyForClause(lhsI), "within", rhs);
} else { } else {
add(GremlinClause.HAS_OPERATOR, lhsI.getQualifiedName(), op.getSymbols()[1], rhs); add(GremlinClause.HAS_OPERATOR, getPropertyForClause(lhsI), op.getSymbols()[1], rhs);
} }
// record that the attribute has been processed so that the select clause doesn't add a attr presence check // record that the attribute has been processed so that the select clause doesn't add a attr presence check
attributesProcessed.add(lhsI.getQualifiedName()); attributesProcessed.add(lhsI.getQualifiedName());
...@@ -320,12 +321,12 @@ public class GremlinQueryComposer { ...@@ -320,12 +321,12 @@ public class GremlinQueryComposer {
IdentifierHelper.Info ia = createInfo(name); IdentifierHelper.Info ia = createInfo(name);
if (queryMetadata.hasSelect() && queryMetadata.hasGroupBy()) { if (queryMetadata.hasSelect() && queryMetadata.hasGroupBy()) {
addSelectTransformation(this.context.selectClauseComposer, getQualifiedName(ia), isDesc); addSelectTransformation(this.context.selectClauseComposer, getPropertyForClause(ia), isDesc);
} else if (queryMetadata.hasGroupBy()) { } else if (queryMetadata.hasGroupBy()) {
addOrderByClause(getQualifiedName(ia), isDesc); addOrderByClause(ia, isDesc);
moveToLast(GremlinClause.GROUP_BY); moveToLast(GremlinClause.GROUP_BY);
} else { } else {
addOrderByClause(getQualifiedName(ia), isDesc); addOrderByClause(ia, isDesc);
} }
} }
...@@ -347,9 +348,17 @@ public class GremlinQueryComposer { ...@@ -347,9 +348,17 @@ public class GremlinQueryComposer {
queryClauses.contains(GremlinClause.HAS_TYPE_WITHIN) != -1; queryClauses.contains(GremlinClause.HAS_TYPE_WITHIN) != -1;
} }
private String getQualifiedName(IdentifierHelper.Info ia) { private String getPropertyForClause(IdentifierHelper.Info ia) {
return context.validator.isValidQualifiedName(ia.getQualifiedName(), ia.getRaw()) ? String vertexPropertyName = lookup.getVertexPropertyName(ia.getTypeName(), ia.getAttributeName());
ia.getQualifiedName() : ia.getRaw(); if (StringUtils.isNotEmpty(vertexPropertyName)) {
return vertexPropertyName;
}
if (StringUtils.isNotEmpty(ia.getQualifiedName())) {
return ia.getQualifiedName();
}
return ia.getRaw();
} }
private void addSelectAttrExistsCheck(final SelectClauseComposer selectClauseComposer) { private void addSelectAttrExistsCheck(final SelectClauseComposer selectClauseComposer) {
...@@ -362,7 +371,7 @@ public class GremlinQueryComposer { ...@@ -362,7 +371,7 @@ public class GremlinQueryComposer {
IdentifierHelper.Info idMetadata = createInfo(qualifiedAttribute); IdentifierHelper.Info idMetadata = createInfo(qualifiedAttribute);
// Only primitive attributes need to be checked // Only primitive attributes need to be checked
if (idMetadata.isPrimitive() && !selectClauseComposer.isAggregatorIdx(i) && !attributesProcessed.contains(qualifiedAttribute)) { if (idMetadata.isPrimitive() && !selectClauseComposer.isAggregatorIdx(i) && !attributesProcessed.contains(qualifiedAttribute)) {
add(GremlinClause.HAS_PROPERTY, qualifiedAttribute); add(GremlinClause.HAS_PROPERTY, getPropertyForClause(idMetadata));
} }
} }
// All these checks should be done before the grouping happens (if any) // All these checks should be done before the grouping happens (if any)
...@@ -388,10 +397,10 @@ public class GremlinQueryComposer { ...@@ -388,10 +397,10 @@ public class GremlinQueryComposer {
} }
if (!scc.getItem(i).equals(scc.getLabel(i))) { if (!scc.getItem(i).equals(scc.getLabel(i))) {
context.addAlias(scc.getLabel(i), getQualifiedName(ia)); context.addAlias(scc.getLabel(i), ia.getQualifiedName());
} }
if (scc.updateAsApplicable(i, getQualifiedName(ia))) { if (scc.updateAsApplicable(i, getPropertyForClause(ia), ia.getQualifiedName())) {
continue; continue;
} }
...@@ -404,10 +413,10 @@ public class GremlinQueryComposer { ...@@ -404,10 +413,10 @@ public class GremlinQueryComposer {
scc.incrementTypesIntroduced(); scc.incrementTypesIntroduced();
scc.isSelectNoop = !ia.hasParts(); scc.isSelectNoop = !ia.hasParts();
if (ia.hasParts()) { if (ia.hasParts()) {
scc.assign(i, getQualifiedName(createInfo(ia.get())), GremlinClause.INLINE_GET_PROPERTY); scc.assign(i, getPropertyForClause(createInfo(ia.get())), GremlinClause.INLINE_GET_PROPERTY);
} }
} else { } else {
scc.assign(i, getQualifiedName(ia), GremlinClause.INLINE_GET_PROPERTY); scc.assign(i, getPropertyForClause(ia), GremlinClause.INLINE_GET_PROPERTY);
scc.incrementPrimitiveType(); scc.incrementPrimitiveType();
} }
} }
...@@ -456,16 +465,16 @@ public class GremlinQueryComposer { ...@@ -456,16 +465,16 @@ public class GremlinQueryComposer {
private void addSelectTransformation(final SelectClauseComposer selectClauseComposer, private void addSelectTransformation(final SelectClauseComposer selectClauseComposer,
final String orderByQualifiedAttrName, final String orderByQualifiedAttrName,
final boolean isDesc) { final boolean isDesc) {
GremlinClause fn; GremlinClause gremlinClause;
if (selectClauseComposer.isSelectNoop) { if (selectClauseComposer.isSelectNoop) {
fn = GremlinClause.SELECT_NOOP_FN; gremlinClause = GremlinClause.SELECT_NOOP_FN;
} else if (queryMetadata.hasGroupBy()) { } else if (queryMetadata.hasGroupBy()) {
fn = selectClauseComposer.onlyAggregators() ? GremlinClause.SELECT_ONLY_AGG_GRP_FN : GremlinClause.SELECT_MULTI_ATTR_GRP_FN; gremlinClause = selectClauseComposer.onlyAggregators() ? GremlinClause.SELECT_ONLY_AGG_GRP_FN : GremlinClause.SELECT_MULTI_ATTR_GRP_FN;
} else { } else {
fn = selectClauseComposer.onlyAggregators() ? GremlinClause.SELECT_ONLY_AGG_FN : GremlinClause.SELECT_FN; gremlinClause = selectClauseComposer.onlyAggregators() ? GremlinClause.SELECT_ONLY_AGG_FN : GremlinClause.SELECT_FN;
} }
if (StringUtils.isEmpty(orderByQualifiedAttrName)) { if (StringUtils.isEmpty(orderByQualifiedAttrName)) {
add(0, fn, add(0, gremlinClause,
selectClauseComposer.getLabelHeader(), selectClauseComposer.getLabelHeader(),
selectClauseComposer.hasAssignmentExpr() ? selectClauseComposer.getAssignmentExprString() : EMPTY_STRING, selectClauseComposer.hasAssignmentExpr() ? selectClauseComposer.getAssignmentExprString() : EMPTY_STRING,
selectClauseComposer.getItemsString(), selectClauseComposer.getItemsString(),
...@@ -477,7 +486,7 @@ public class GremlinQueryComposer { ...@@ -477,7 +486,7 @@ public class GremlinQueryComposer {
sortClause = isDesc ? GremlinClause.INLINE_TUPLE_SORT_DESC : GremlinClause.INLINE_TUPLE_SORT_ASC; sortClause = isDesc ? GremlinClause.INLINE_TUPLE_SORT_DESC : GremlinClause.INLINE_TUPLE_SORT_ASC;
} }
String idxStr = String.valueOf(itemIdx); String idxStr = String.valueOf(itemIdx);
add(0, fn, add(0, gremlinClause,
selectClauseComposer.getLabelHeader(), selectClauseComposer.getLabelHeader(),
selectClauseComposer.hasAssignmentExpr() ? selectClauseComposer.getAssignmentExprString() : EMPTY_STRING, selectClauseComposer.hasAssignmentExpr() ? selectClauseComposer.getAssignmentExprString() : EMPTY_STRING,
selectClauseComposer.getItemsString(), selectClauseComposer.getItemsString(),
...@@ -575,12 +584,11 @@ public class GremlinQueryComposer { ...@@ -575,12 +584,11 @@ public class GremlinQueryComposer {
} }
} }
private void addOrderByClause(String name, boolean descr) { private void addOrderByClause(IdentifierHelper.Info ia, boolean descr) {
if (LOG.isDebugEnabled()) { if (LOG.isDebugEnabled()) {
LOG.debug("addOrderByClause(name={})", name, descr); LOG.debug("addOrderByClause(name={})", ia.getRaw(), descr);
} }
IdentifierHelper.Info ia = createInfo(name);
add((!descr) ? GremlinClause.ORDER_BY : GremlinClause.ORDER_BY_DESC, ia); add((!descr) ? GremlinClause.ORDER_BY : GremlinClause.ORDER_BY_DESC, ia);
} }
...@@ -598,7 +606,7 @@ public class GremlinQueryComposer { ...@@ -598,7 +606,7 @@ public class GremlinQueryComposer {
return; return;
} }
add(clause, (idInfo.getQualifiedName() == null ? idInfo.get() : idInfo.getQualifiedName())); add(clause, getPropertyForClause(idInfo));
} }
private void add(GremlinClause clause, String... args) { private void add(GremlinClause clause, String... args) {
......
repository/src/main/java/org/apache/atlas/query/IdentifierHelper.java
...@@ -33,6 +33,9 @@ public class IdentifierHelper { ...@@ -33,6 +33,9 @@ public class IdentifierHelper {
public static String get(String quotedIdentifier) { public static String get(String quotedIdentifier) {
String ret; String ret;
if (StringUtils.isEmpty(quotedIdentifier)) {
return null;
}
if (quotedIdentifier.charAt(0) == '`') { if (quotedIdentifier.charAt(0) == '`') {
ret = extract(BACKTICK_QUOTED_IDENTIFIER, quotedIdentifier); ret = extract(BACKTICK_QUOTED_IDENTIFIER, quotedIdentifier);
...@@ -74,20 +77,25 @@ public class IdentifierHelper { ...@@ -74,20 +77,25 @@ public class IdentifierHelper {
public static boolean isQuoted(String val) { public static boolean isQuoted(String val) {
boolean ret = false; boolean ret = false;
if (StringUtils.isEmpty(val)) {
return ret;
}
if (val != null && val.length() > 1) {
char first = val.charAt(0); char first = val.charAt(0);
char last = val.charAt(val.length() - 1); char last = val.charAt(val.length() - 1);
if (first == last && (first == '\'' || first == '"' || first == '`')) { if (first == last && (first == '\'' || first == '"' || first == '`')) {
ret = true; ret = true;
} }
}
return ret; return ret;
} }
public static String removeQuotes(String rhs) { public static String removeQuotes(String rhs) {
if (StringUtils.isEmpty(rhs)) {
return rhs;
}
return rhs.replace("\"", "") return rhs.replace("\"", "")
.replace("'", "") .replace("'", "")
.replace("`", ""); .replace("`", "");
...@@ -294,5 +302,9 @@ public class IdentifierHelper { ...@@ -294,5 +302,9 @@ public class IdentifierHelper {
public boolean isNumeric() { public boolean isNumeric() {
return isNumeric; return isNumeric;
} }
public void setTypeName(String typeName) {
this.typeName = typeName;
}
} }
} }
repository/src/main/java/org/apache/atlas/query/Lookup.java
...@@ -43,4 +43,6 @@ public interface Lookup { ...@@ -43,4 +43,6 @@ public interface Lookup {
boolean isDate(GremlinQueryComposer.Context context, String attributeName); boolean isDate(GremlinQueryComposer.Context context, String attributeName);
boolean isNumeric(GremlinQueryComposer.Context context, String attrName); boolean isNumeric(GremlinQueryComposer.Context context, String attrName);
String getVertexPropertyName(String typeName, String attrName);
} }
repository/src/main/java/org/apache/atlas/query/RegistryBasedLookup.java
...@@ -254,6 +254,17 @@ class RegistryBasedLookup implements Lookup { ...@@ -254,6 +254,17 @@ class RegistryBasedLookup implements Lookup {
return ret; return ret;
} }
@Override
public String getVertexPropertyName(String typeName, String attrName) {
AtlasEntityType entityType = typeRegistry.getEntityTypeByName(typeName);
AtlasStructType.AtlasAttribute attribute = getAttribute(entityType, attrName);
if (attribute == null) {
return null;
}
return attribute.getVertexPropertyName();
}
private AtlasStructType.AtlasAttribute getAttribute(AtlasEntityType entityType, String attrName) { private AtlasStructType.AtlasAttribute getAttribute(AtlasEntityType entityType, String attrName) {
AtlasStructType.AtlasAttribute ret = null; AtlasStructType.AtlasAttribute ret = null;
......
repository/src/main/java/org/apache/atlas/query/SelectClauseComposer.java
...@@ -61,20 +61,16 @@ class SelectClauseComposer { ...@@ -61,20 +61,16 @@ class SelectClauseComposer {
this.items = Arrays.copyOf(items, items.length); this.items = Arrays.copyOf(items, items.length);
} }
public boolean updateAsApplicable(int currentIndex, String qualifiedName) { public boolean updateAsApplicable(int currentIndex, String propertyForClause, String qualifiedName) {
boolean ret = false; boolean ret = false;
if (currentIndex == getCountIdx()) { if (currentIndex == getCountIdx()) {
ret = assign(currentIndex, COUNT_STR, ret = assign(currentIndex, COUNT_STR, GremlinClause.INLINE_COUNT.get(), GremlinClause.INLINE_ASSIGNMENT);
GremlinClause.INLINE_COUNT.get(), GremlinClause.INLINE_ASSIGNMENT);
} else if (currentIndex == getMinIdx()) { } else if (currentIndex == getMinIdx()) {
ret = assign(currentIndex, MIN_STR, qualifiedName, ret = assign(currentIndex, MIN_STR, propertyForClause, GremlinClause.INLINE_ASSIGNMENT, GremlinClause.INLINE_MIN);
GremlinClause.INLINE_ASSIGNMENT, GremlinClause.INLINE_MIN);
} else if (currentIndex == getMaxIdx()) { } else if (currentIndex == getMaxIdx()) {
ret = assign(currentIndex, MAX_STR, qualifiedName, ret = assign(currentIndex, MAX_STR, propertyForClause, GremlinClause.INLINE_ASSIGNMENT, GremlinClause.INLINE_MAX);
GremlinClause.INLINE_ASSIGNMENT, GremlinClause.INLINE_MAX);
} else if (currentIndex == getSumIdx()) { } else if (currentIndex == getSumIdx()) {
ret = assign(currentIndex, SUM_STR, qualifiedName, ret = assign(currentIndex, SUM_STR, propertyForClause, GremlinClause.INLINE_ASSIGNMENT, GremlinClause.INLINE_SUM);
GremlinClause.INLINE_ASSIGNMENT, GremlinClause.INLINE_SUM);
} else { } else {
attributes[currentIndex] = qualifiedName; attributes[currentIndex] = qualifiedName;
} }
......
repository/src/main/java/org/apache/atlas/repository/converters/TypeConverterUtil.java
...@@ -307,7 +307,10 @@ public final class TypeConverterUtil { ...@@ -307,7 +307,10 @@ public final class TypeConverterUtil {
} }
public static AtlasAttributeDef toAtlasAttributeDef(final AttributeDefinition attrDefinition) { public static AtlasAttributeDef toAtlasAttributeDef(final AttributeDefinition attrDefinition) {
AtlasAttributeDef ret = new AtlasAttributeDef(attrDefinition.getName(), attrDefinition.getDataTypeName(), attrDefinition.getSearchWeight()); AtlasAttributeDef ret = new AtlasAttributeDef(attrDefinition.getName(),
attrDefinition.getDataTypeName(),
attrDefinition.getSearchWeight(),
attrDefinition.getIndexType());
ret.setIsIndexable(attrDefinition.getIsIndexable()); ret.setIsIndexable(attrDefinition.getIsIndexable());
ret.setIsUnique(attrDefinition.getIsUnique()); ret.setIsUnique(attrDefinition.getIsUnique());
...@@ -362,7 +365,16 @@ public final class TypeConverterUtil { ...@@ -362,7 +365,16 @@ public final class TypeConverterUtil {
AttributeDefinition oldAttrDef = AtlasStructDefStoreV2.toAttributeDefinition(attribute); AttributeDefinition oldAttrDef = AtlasStructDefStoreV2.toAttributeDefinition(attribute);
ret.add(new AttributeDefinition(oldAttrDef.getName(), oldAttrDef.getDataTypeName(), new Multiplicity(oldAttrDef.getMultiplicity()), oldAttrDef.getIsComposite(), oldAttrDef.getIsUnique(), oldAttrDef.getIsIndexable(), oldAttrDef.getReverseAttributeName(), oldAttrDef.getOptions(), oldAttrDef.getSearchWeight())); ret.add(new AttributeDefinition(oldAttrDef.getName(),
oldAttrDef.getDataTypeName(),
new Multiplicity(oldAttrDef.getMultiplicity()),
oldAttrDef.getIsComposite(),
oldAttrDef.getIsUnique(),
oldAttrDef.getIsIndexable(),
oldAttrDef.getReverseAttributeName(),
oldAttrDef.getOptions(),
oldAttrDef.getSearchWeight(),
oldAttrDef.getIndexType()));
} }
} }
......
repository/src/main/java/org/apache/atlas/repository/patches/UniqueAttributePatch.java
...@@ -138,7 +138,14 @@ public class UniqueAttributePatch extends AtlasPatchHandler { ...@@ -138,7 +138,14 @@ public class UniqueAttributePatch extends AtlasPatchHandler {
Class propertyClass = getIndexer().getPrimitiveClass(attribTypeName); Class propertyClass = getIndexer().getPrimitiveClass(attribTypeName);
AtlasCardinality cardinality = getIndexer().toAtlasCardinality(attributeDef.getCardinality()); AtlasCardinality cardinality = getIndexer().toAtlasCardinality(attributeDef.getCardinality());
getIndexer().createVertexIndex(management, uniquePropertyName, UniqueKind.PER_TYPE_UNIQUE, propertyClass, cardinality, isIndexable, true); getIndexer().createVertexIndex(management,
uniquePropertyName,
UniqueKind.PER_TYPE_UNIQUE,
propertyClass,
cardinality,
isIndexable,
true,
AtlasAttributeDef.IndexType.STRING.equals(attribute.getIndexType()));
} }
getIndexer().commit(management); getIndexer().commit(management);
......
repository/src/main/java/org/apache/atlas/repository/store/bootstrap/AtlasTypeDefStoreInitializer.java
...@@ -55,6 +55,7 @@ import org.apache.commons.collections.MapUtils; ...@@ -55,6 +55,7 @@ import org.apache.commons.collections.MapUtils;
import org.apache.commons.configuration.Configuration; import org.apache.commons.configuration.Configuration;
import org.apache.commons.lang.ObjectUtils; import org.apache.commons.lang.ObjectUtils;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.apache.jute.Index;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import org.springframework.core.annotation.Order; import org.springframework.core.annotation.Order;
...@@ -1041,6 +1042,18 @@ public class AtlasTypeDefStoreInitializer implements ActiveStateChangeHandler { ...@@ -1041,6 +1042,18 @@ public class AtlasTypeDefStoreInitializer implements ActiveStateChangeHandler {
} }
atlasAttributeDef.setSearchWeight(searchWeight); atlasAttributeDef.setSearchWeight(searchWeight);
LOG.info("Updating Model attribute {}'s property{} to {}.", atlasAttributeDef.getName(), entry.getKey(), entry.getValue()); LOG.info("Updating Model attribute {}'s property{} to {}.", atlasAttributeDef.getName(), entry.getKey(), entry.getValue());
} else if (AtlasAttributeDef.INDEX_TYPE_ATTR_NAME.equalsIgnoreCase(entry.getKey())) {
String indexTypeString = (String) entry.getValue();
if(!StringUtils.isEmpty(indexTypeString)) {
try {
AtlasAttributeDef.IndexType indexType = AtlasAttributeDef.IndexType.valueOf(indexTypeString);
atlasAttributeDef.setIndexType(indexType);
} catch (IllegalArgumentException e) {
String msg = String.format("Value %s provided for the attribute %s is not valid.", indexTypeString, AtlasAttributeDef.INDEX_TYPE_ATTR_NAME);
LOG.error(msg);
throw new RuntimeException(msg);
}
}
} else { } else {
//sanity exception //sanity exception
//more attributes can be added as needed. //more attributes can be added as needed.
......
repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasStructDefStoreV2.java
...@@ -540,6 +540,7 @@ public class AtlasStructDefStoreV2 extends AtlasAbstractDefStoreV2<AtlasStructDe ...@@ -540,6 +540,7 @@ public class AtlasStructDefStoreV2 extends AtlasAbstractDefStoreV2<AtlasStructDe
attribInfo.put("defaultValue", attributeDef.getDefaultValue()); attribInfo.put("defaultValue", attributeDef.getDefaultValue());
attribInfo.put("description", attributeDef.getDescription()); attribInfo.put("description", attributeDef.getDescription());
attribInfo.put("searchWeight", attributeDef.getSearchWeight()); attribInfo.put("searchWeight", attributeDef.getSearchWeight());
attribInfo.put("indexType", attributeDef.getIndexType());
if(attributeDef.getOptions() != null) { if(attributeDef.getOptions() != null) {
attribInfo.put("options", AtlasType.toJson(attributeDef.getOptions())); attribInfo.put("options", AtlasType.toJson(attributeDef.getOptions()));
...@@ -634,6 +635,11 @@ public class AtlasStructDefStoreV2 extends AtlasAbstractDefStoreV2<AtlasStructDe ...@@ -634,6 +635,11 @@ public class AtlasStructDefStoreV2 extends AtlasAbstractDefStoreV2<AtlasStructDe
} else { } else {
ret.setSearchWeight(-1); ret.setSearchWeight(-1);
} }
String indexType = (String) attribInfo.get("indexType");
if(!StringUtils.isEmpty(indexType)) {
ret.setIndexType(AtlasAttributeDef.IndexType.valueOf(indexType));
}
return ret; return ret;
} }
...@@ -652,7 +658,7 @@ public class AtlasStructDefStoreV2 extends AtlasAbstractDefStoreV2<AtlasStructDe ...@@ -652,7 +658,7 @@ public class AtlasStructDefStoreV2 extends AtlasAbstractDefStoreV2<AtlasStructDe
ret.setDescription(attrDef.getDescription()); ret.setDescription(attrDef.getDescription());
ret.setDefaultValue(attrDef.getDefaultValue()); ret.setDefaultValue(attrDef.getDefaultValue());
ret.setSearchWeight(attrDef.getSearchWeight()); ret.setSearchWeight(attrDef.getSearchWeight());
ret.setIndexType(attrDef.getIndexType());
return ret; return ret;
} }
} }
......
repository/src/test/java/org/apache/atlas/query/GremlinQueryComposerTest.java
...@@ -162,7 +162,7 @@ public class GremlinQueryComposerTest { ...@@ -162,7 +162,7 @@ public class GremlinQueryComposerTest {
verify("from DB where name = \"Reporting\" select name, owner", getExpected(exSel, exMain)); verify("from DB where name = \"Reporting\" select name, owner", getExpected(exSel, exMain));
verify("from DB where (name = \"Reporting\") select name, owner", getExpected(exSel, exMain)); verify("from DB where (name = \"Reporting\") select name, owner", getExpected(exSel, exMain));
verify("Table where Asset.name like \"Tab*\"", verify("Table where Asset.name like \"Tab*\"",
"g.V().has('__typeName', 'Table').has('Table.name', org.janusgraph.core.attribute.Text.textRegex(\"Tab.*\")).dedup().limit(25).toList()"); "g.V().has('__typeName', 'Table').has('Asset.__s_name', org.janusgraph.core.attribute.Text.textRegex(\"Tab.*\")).dedup().limit(25).toList()");
verify("from Table where (db.name = \"Reporting\")", verify("from Table where (db.name = \"Reporting\")",
"g.V().has('__typeName', 'Table').out('__Table.db').has('DB.name', eq(\"Reporting\")).dedup().in('__Table.db').dedup().limit(25).toList()"); "g.V().has('__typeName', 'Table').out('__Table.db').has('DB.name', eq(\"Reporting\")).dedup().in('__Table.db').dedup().limit(25).toList()");
} }
...@@ -184,8 +184,8 @@ public class GremlinQueryComposerTest { ...@@ -184,8 +184,8 @@ public class GremlinQueryComposerTest {
@Test @Test
public void subType() { public void subType() {
String exMain = "g.V().has('__typeName', within('Asset','Table')).has('Asset.name').has('Asset.owner').dedup().limit(25).toList()"; String exMain = "g.V().has('__typeName', within('Asset','Table')).has('Asset.__s_name').has('Asset.__s_owner').dedup().limit(25).toList()";
String exSel = "def f(r){ t=[['name','owner']]; r.each({t.add([it.value('Asset.name'),it.value('Asset.owner')])}); t.unique(); }"; String exSel = "def f(r){ t=[['name','owner']]; r.each({t.add([it.value('Asset.__s_name'),it.value('Asset.__s_owner')])}); t.unique(); }";
verify("Asset select name, owner", getExpected(exSel, exMain)); verify("Asset select name, owner", getExpected(exSel, exMain));
} }
...@@ -343,8 +343,8 @@ public class GremlinQueryComposerTest { ...@@ -343,8 +343,8 @@ public class GremlinQueryComposerTest {
verify("Table has db", 1); verify("Table has db", 1);
verify("Table groupby(db) select name", 1); verify("Table groupby(db) select name", 1);
verify("Table groupby(name) select name, max(db)", 1); verify("Table groupby(name) select name, max(db)", 1);
verify("Table select db, columns", 2); verify("Table select db, columns", 1);
verify("Table select db, owner, columns", 3); verify("Table select db, owner, columns", 2);
} }
private void verify(String dsl, String expectedGremlin, int expectedNumberOfErrors) { private void verify(String dsl, String expectedGremlin, int expectedNumberOfErrors) {
...@@ -530,5 +530,16 @@ public class GremlinQueryComposerTest { ...@@ -530,5 +530,16 @@ public class GremlinQueryComposerTest {
context.setNumericTypeFormatter("f"); context.setNumericTypeFormatter("f");
return attrName.equals("partitionSize"); return attrName.equals("partitionSize");
} }
@Override
public String getVertexPropertyName(String typeName, String attrName) {
if (typeName.equals("Asset")) {
if (attrName.equals("name") || attrName.equals("owner")) {
return String.format("%s.__s_%s", typeName, attrName);
}
}
return null;
}
} }
} }
webapp/src/test/resources/json/search-parameters/combination-filters.json
...@@ -139,7 +139,7 @@ ...@@ -139,7 +139,7 @@
"typeName": "hdfs_path", "typeName": "hdfs_path",
"excludeDeletedEntities": true, "excludeDeletedEntities": true,
"classification" : "fooTag", "classification" : "fooTag",
"query": "test ; \\{ \\} \\[ \\]", "query": "test",
"limit": 25, "limit": 25,
"offset": 0, "offset": 0,
"entityFilters": null, "entityFilters": null,
...@@ -169,7 +169,7 @@ ...@@ -169,7 +169,7 @@
"typeName": "hdfs_path", "typeName": "hdfs_path",
"excludeDeletedEntities": true, "excludeDeletedEntities": true,
"classification" : "fooTag", "classification" : "fooTag",
"query": "test ; \\{ \\} \\[ \\]", "query": "test",
"limit": 25, "limit": 25,
"offset": 0, "offset": 0,
"entityFilters": { "entityFilters": {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment