Skip to content

A
atlas
Commit 89f70609 by Jeff Hagelberg
Options

ATLAS-1114: Performance improvements for create/update entity (2 of 2)

parent 6cd68119
Showing with 942 additions and 299 deletions
repository/src/main/java/org/apache/atlas/repository/audit/InMemoryEntityAuditRepository.java
......@@ -18,17 +18,17 @@
package org.apache.atlas.repository.audit;
import org.apache.atlas.AtlasException;
import org.apache.atlas.EntityAuditEvent;
import com.google.inject.Singleton;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.SortedMap;
import java.util.TreeMap;
import org.apache.atlas.AtlasException;
import org.apache.atlas.EntityAuditEvent;
import com.google.inject.Singleton;
/**
* Entity audit repository where audit events are stored in-memory. Used only for integration tests
*/
......@@ -50,8 +50,10 @@ public class InMemoryEntityAuditRepository implements EntityAuditRepository {
}
}
//synchronized to avoid concurrent modification exception that occurs if events are added
//while we are iterating through the map
@Override
public List<EntityAuditEvent> listEvents(String entityId, String startKey, short maxResults)
public synchronized List<EntityAuditEvent> listEvents(String entityId, String startKey, short maxResults)
throws AtlasException {
List<EntityAuditEvent> events = new ArrayList<>();
String myStartKey = startKey;
......
repository/src/main/java/org/apache/atlas/repository/graph/DeleteHandler.java
......@@ -22,6 +22,7 @@ import static org.apache.atlas.repository.graph.GraphHelper.EDGE_LABEL_PREFIX;
import static org.apache.atlas.repository.graph.GraphHelper.string;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.Iterator;
......@@ -70,7 +71,7 @@ public abstract class DeleteHandler {
* @param instanceVertices
* @throws AtlasException
*/
public void deleteEntities(List<AtlasVertex> instanceVertices) throws AtlasException {
public void deleteEntities(Collection<AtlasVertex> instanceVertices) throws AtlasException {
RequestContext requestContext = RequestContext.get();
Set<AtlasVertex> deletionCandidateVertices = new HashSet<>();
......
repository/src/main/java/org/apache/atlas/repository/graph/FullTextMapper.java
......@@ -17,10 +17,15 @@
*/
package org.apache.atlas.repository.graph;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.atlas.AtlasException;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.typesystem.ITypedInstance;
import org.apache.atlas.typesystem.ITypedReferenceableInstance;
import org.apache.atlas.typesystem.persistence.Id;
import org.apache.atlas.typesystem.types.AttributeInfo;
import org.apache.atlas.typesystem.types.DataTypes;
import org.apache.atlas.typesystem.types.EnumValue;
......@@ -29,23 +34,22 @@ import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
public class FullTextMapper {
private static final Logger LOG = LoggerFactory.getLogger(FullTextMapper.class);
private final GraphToTypedInstanceMapper graphToTypedInstanceMapper;
private final TypedInstanceToGraphMapper typedInstanceToGraphMapper;
private static final GraphHelper graphHelper = GraphHelper.getInstance();
private static final String FULL_TEXT_DELIMITER = " ";
private final Map<String, ITypedReferenceableInstance> instanceCache;
FullTextMapper(GraphToTypedInstanceMapper graphToTypedInstanceMapper) {
FullTextMapper(TypedInstanceToGraphMapper typedInstanceToGraphMapper,
GraphToTypedInstanceMapper graphToTypedInstanceMapper) {
this.graphToTypedInstanceMapper = graphToTypedInstanceMapper;
this.typedInstanceToGraphMapper = typedInstanceToGraphMapper;
instanceCache = new HashMap<>();
}
......@@ -126,8 +130,12 @@ public class FullTextMapper {
case CLASS:
if (followReferences) {
String refGuid = ((ITypedReferenceableInstance) value).getId()._getId();
AtlasVertex refVertex = graphHelper.getVertexForGUID(refGuid);
Id refId = ((ITypedReferenceableInstance) value).getId();
String refGuid = refId._getId();
AtlasVertex refVertex = typedInstanceToGraphMapper.lookupVertex(refId);
if(refVertex == null) {
refVertex = graphHelper.getVertexForGUID(refGuid);
}
return mapRecursive(refVertex, false);
}
break;
......
repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepository.java
......@@ -19,9 +19,11 @@
package org.apache.atlas.repository.graph;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import org.apache.atlas.AtlasClient;
import org.apache.atlas.AtlasException;
......@@ -357,8 +359,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
instanceToGraphMapper.mapTypedInstanceToGraph(TypedInstanceToGraphMapper.Operation.UPDATE_FULL,
entitiesUpdated);
RequestContext requestContext = RequestContext.get();
return new AtlasClient.EntityResult(requestContext.getCreatedEntityIds(),
requestContext.getUpdatedEntityIds(), requestContext.getDeletedEntityIds());
return createEntityResultFromContext(requestContext);
} catch (AtlasException e) {
throw new RepositoryException(e);
}
......@@ -375,13 +376,14 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
TypedInstanceToGraphMapper instanceToGraphMapper = new TypedInstanceToGraphMapper(graphToInstanceMapper, deleteHandler);
instanceToGraphMapper.mapTypedInstanceToGraph(TypedInstanceToGraphMapper.Operation.UPDATE_PARTIAL, entity);
RequestContext requestContext = RequestContext.get();
return new AtlasClient.EntityResult(requestContext.getCreatedEntityIds(),
requestContext.getUpdatedEntityIds(), requestContext.getDeletedEntityIds());
return createEntityResultFromContext(requestContext);
} catch (AtlasException e) {
throw new RepositoryException(e);
}
}
@Override
@GraphTransaction
public AtlasClient.EntityResult deleteEntities(List<String> guids) throws RepositoryException {
......@@ -390,32 +392,41 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
throw new IllegalArgumentException("guids must be non-null and non-empty");
}
List<AtlasVertex> vertices = new ArrayList<>(guids.size());
for (String guid : guids) {
if (guid == null) {
LOG.warn("deleteEntities: Ignoring null guid");
continue;
}
try {
AtlasVertex instanceVertex = graphHelper.getVertexForGUID(guid);
vertices.add(instanceVertex);
} catch (EntityNotFoundException e) {
// Retrieve vertices for requested guids.
Map<String, AtlasVertex> vertices = graphHelper.getVerticesForGUIDs(guids);
Collection<AtlasVertex> deletionCandidates = vertices.values();
if(LOG.isDebugEnabled()) {
for(String guid : guids) {
if(! vertices.containsKey(guid)) {
// Entity does not exist - treat as non-error, since the caller
// wanted to delete the entity and it's already gone.
LOG.info("Deletion request ignored for non-existent entity with guid {}", guid);
LOG.debug("Deletion request ignored for non-existent entity with guid " + guid);
}
}
}
if (deletionCandidates.isEmpty()) {
LOG.info("No deletion candidate entities were found for guids %s", guids);
return new AtlasClient.EntityResult(Collections.<String>emptyList(), Collections.<String>emptyList(), Collections.<String>emptyList());
}
try {
deleteHandler.deleteEntities(vertices);
deleteHandler.deleteEntities(deletionCandidates);
}
catch (AtlasException e) {
throw new RepositoryException(e);
}
RequestContext requestContext = RequestContext.get();
return new AtlasClient.EntityResult(requestContext.getCreatedEntityIds(),
requestContext.getUpdatedEntityIds(), requestContext.getDeletedEntityIds());
return createEntityResultFromContext(requestContext);
}
private AtlasClient.EntityResult createEntityResultFromContext(RequestContext requestContext) {
return new AtlasClient.EntityResult(
requestContext.getCreatedEntityIds(),
requestContext.getUpdatedEntityIds(),
requestContext.getDeletedEntityIds());
}
public AtlasGraph getGraph() {
......
repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java
......@@ -18,9 +18,20 @@
package org.apache.atlas.repository.graph;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.BiMap;
import com.google.common.collect.HashBiMap;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.Stack;
import java.util.UUID;
import org.apache.atlas.ApplicationProperties;
import org.apache.atlas.AtlasException;
import org.apache.atlas.RequestContext;
......@@ -51,13 +62,17 @@ import org.apache.atlas.typesystem.types.Multiplicity;
import org.apache.atlas.typesystem.types.TypeSystem;
import org.apache.atlas.typesystem.types.ValueConversionException;
import org.apache.atlas.typesystem.types.utils.TypesUtil;
import org.apache.atlas.util.AttributeValueMap;
import org.apache.atlas.util.IndexedInstance;
import org.apache.atlas.utils.ParamChecker;
import org.apache.commons.lang.StringUtils;
import org.codehaus.jettison.json.JSONArray;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.*;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.BiMap;
import com.google.common.collect.HashBiMap;
/**
* Utility class for graph operations.
......@@ -516,6 +531,60 @@ public final class GraphHelper {
return findVertex(Constants.GUID_PROPERTY_KEY, guid);
}
/**
* Finds the Vertices that correspond to the given property values. Property
* values that are not found in the graph will not be in the map.
*
* @return propertyValue to AtlasVertex map with the result.
*/
private Map<String, AtlasVertex> getVerticesForPropertyValues(String property, List<String> values)
throws RepositoryException {
if(values.isEmpty()) {
return Collections.emptyMap();
}
Collection<String> nonNullValues = new HashSet<>(values.size());
for(String value : values) {
if(value != null) {
nonNullValues.add(value);
}
}
//create graph query that finds vertices with the guids
AtlasGraphQuery query = graph.query();
query.in(property, nonNullValues);
Iterable<AtlasVertex> results = query.vertices();
Map<String, AtlasVertex> result = new HashMap<>(values.size());
//Process the result, using the guidToIndexMap to figure out where
//each vertex should go in the result list.
for(AtlasVertex vertex : results) {
if(vertex.exists()) {
String propertyValue = vertex.getProperty(property, String.class);
if(LOG.isDebugEnabled()) {
LOG.debug("Found a vertex {} with {} = {}", string(vertex), property, propertyValue);
}
result.put(propertyValue, vertex);
}
}
return result;
}
/**
* Finds the Vertices that correspond to the given GUIDs. GUIDs
* that are not found in the graph will not be in the map.
*
* @return GUID to AtlasVertex map with the result.
*/
public Map<String, AtlasVertex> getVerticesForGUIDs(List<String> guids)
throws RepositoryException {
return getVerticesForPropertyValues(Constants.GUID_PROPERTY_KEY, guids);
}
public static String getQualifiedNameForMapKey(String prefix, String key) {
return prefix + "." + key;
}
......@@ -638,6 +707,112 @@ public final class GraphHelper {
}
/**
* Finds vertices that match at least one unique attribute of the instances specified. The AtlasVertex at a given index in the result corresponds
* to the IReferencableInstance at that same index that was passed in. The number of elements in the resultant list is guaranteed to match the
* number of instances that were passed in. If no vertex is found for a given instance, that entry will be null in the resultant list.
*
*
* @param classType
* @param instancesForClass
* @return
* @throws AtlasException
*/
public List<AtlasVertex> getVerticesForInstancesByUniqueAttribute(ClassType classType, List<? extends IReferenceableInstance> instancesForClass) throws AtlasException {
//For each attribute, need to figure out what values to search for and which instance(s)
//those values correspond to.
Map<String, AttributeValueMap> map = new HashMap<String, AttributeValueMap>();
for (AttributeInfo attributeInfo : classType.fieldMapping().fields.values()) {
if (attributeInfo.isUnique) {
String propertyKey = getQualifiedFieldName(classType, attributeInfo.name);
AttributeValueMap mapForAttribute = new AttributeValueMap();
for(int idx = 0; idx < instancesForClass.size(); idx++) {
IReferenceableInstance instance = instancesForClass.get(idx);
Object value = instance.get(attributeInfo.name);
mapForAttribute.put(value, instance, idx);
}
map.put(propertyKey, mapForAttribute);
}
}
AtlasVertex[] result = new AtlasVertex[instancesForClass.size()];
if(map.isEmpty()) {
//no unique attributes
return Arrays.asList(result);
}
//construct gremlin query
AtlasGraphQuery query = graph.query();
query.has(Constants.ENTITY_TYPE_PROPERTY_KEY, classType.getName());
query.has(Constants.STATE_PROPERTY_KEY,Id.EntityState.ACTIVE.name());
List<AtlasGraphQuery> orChildren = new ArrayList<AtlasGraphQuery>();
//build up an or expression to find vertices which match at least one of the unique attribute constraints
//For each unique attributes, we add a within clause to match vertices that have a value of that attribute
//that matches the value in some instance.
for(Map.Entry<String, AttributeValueMap> entry : map.entrySet()) {
AtlasGraphQuery orChild = query.createChildQuery();
String propertyName = entry.getKey();
AttributeValueMap valueMap = entry.getValue();
Set<Object> values = valueMap.getAttributeValues();
if(values.size() == 1) {
orChild.has(propertyName, values.iterator().next());
}
else if(values.size() > 1) {
orChild.in(propertyName, values);
}
orChildren.add(orChild);
}
if(orChildren.size() == 1) {
AtlasGraphQuery child = orChildren.get(0);
query.addConditionsFrom(child);
}
else if(orChildren.size() > 1) {
query.or(orChildren);
}
Iterable<AtlasVertex> queryResult = query.vertices();
for(AtlasVertex matchingVertex : queryResult) {
Collection<IndexedInstance> matches = getInstancesForVertex(map, matchingVertex);
for(IndexedInstance wrapper : matches) {
result[wrapper.getIndex()]= matchingVertex;
}
}
return Arrays.asList(result);
}
//finds the instance(s) that correspond to the given vertex
private Collection<IndexedInstance> getInstancesForVertex(Map<String, AttributeValueMap> map, AtlasVertex foundVertex) {
//loop through the unique attributes. For each attribute, check to see if the vertex property that
//corresponds to that attribute has a value from one or more of the instances that were passed in.
for(Map.Entry<String, AttributeValueMap> entry : map.entrySet()) {
String propertyName = entry.getKey();
AttributeValueMap valueMap = entry.getValue();
Object vertexValue = foundVertex.getProperty(propertyName, Object.class);
Collection<IndexedInstance> instances = valueMap.get(vertexValue);
if(instances != null && instances.size() > 0) {
//return first match. Let the underling graph determine if this is a problem
//(i.e. if the other unique attributes change be changed safely to match what
//the user requested).
return instances;
}
//try another attribute
}
return Collections.emptyList();
}
/**
* Guid and AtlasVertex combo
*/
public static class VertexInfo {
......
repository/src/main/java/org/apache/atlas/repository/graph/TypedInstanceToGraphMapper.java
......@@ -17,7 +17,19 @@
*/
package org.apache.atlas.repository.graph;
import com.google.inject.Inject;
import static org.apache.atlas.repository.graph.GraphHelper.string;
import java.security.MessageDigest;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.apache.atlas.AtlasException;
import org.apache.atlas.RequestContext;
import org.apache.atlas.repository.Constants;
......@@ -47,18 +59,9 @@ import org.apache.atlas.utils.MD5Utils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.security.MessageDigest;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import static org.apache.atlas.repository.graph.GraphHelper.string;
import com.google.common.base.Function;
import com.google.common.collect.Lists;
import com.google.inject.Inject;
public final class TypedInstanceToGraphMapper {
......@@ -86,18 +89,20 @@ public final class TypedInstanceToGraphMapper {
void mapTypedInstanceToGraph(Operation operation, ITypedReferenceableInstance... typedInstances)
throws AtlasException {
RequestContext requestContext = RequestContext.get();
Collection<IReferenceableInstance> allNewInstances = new ArrayList<>();
for (ITypedReferenceableInstance typedInstance : typedInstances) {
if (LOG.isDebugEnabled()) {
LOG.debug("Adding/updating entity {}", typedInstance);
allNewInstances.addAll(walkClassInstances(typedInstance));
}
Collection<IReferenceableInstance> newInstances = walkClassInstances(typedInstance);
TypeUtils.Pair<List<ITypedReferenceableInstance>, List<ITypedReferenceableInstance>> instancesPair =
createVerticesAndDiscoverInstances(newInstances);
createVerticesAndDiscoverInstances(allNewInstances);
List<ITypedReferenceableInstance> entitiesToCreate = instancesPair.left;
List<ITypedReferenceableInstance> entitiesToUpdate = instancesPair.right;
FullTextMapper fulltextMapper = new FullTextMapper(graphToTypedInstanceMapper);
FullTextMapper fulltextMapper = new FullTextMapper(this, graphToTypedInstanceMapper);
switch (operation) {
case CREATE:
List<String> ids = addOrUpdateAttributesAndTraits(operation, entitiesToCreate);
......@@ -119,7 +124,7 @@ public final class TypedInstanceToGraphMapper {
default:
throw new UnsupportedOperationException("Not handled - " + operation);
}
}
}
private Collection<IReferenceableInstance> walkClassInstances(ITypedReferenceableInstance typedInstance)
......@@ -257,68 +262,112 @@ public final class TypedInstanceToGraphMapper {
List<ITypedReferenceableInstance> instancesToCreate = new ArrayList<>();
List<ITypedReferenceableInstance> instancesToUpdate = new ArrayList<>();
for (IReferenceableInstance instance : instances) {
if (LOG.isDebugEnabled()) {
LOG.debug("Discovering instance to create/update for {}", instance.toShortString());
}
Map<Id,AtlasVertex> foundVertices = findExistingVertices(instances);
//cache all the ids
idToVertexMap.putAll(foundVertices);
ITypedReferenceableInstance newInstance;
Set<Id> processedIds = new HashSet<>();
for(IReferenceableInstance instance : instances) {
Id id = instance.getId();
if (!idToVertexMap.containsKey(id)) {
AtlasVertex instanceVertex;
if (id.isAssigned()) { // has a GUID
if (LOG.isDebugEnabled()) {
LOG.debug("Instance has an assigned id {}", instance.getId()._getId());
}
instanceVertex = graphHelper.getVertexForGUID(id.id);
if (!(instance instanceof ReferenceableInstance)) {
throw new IllegalStateException(
String.format("%s is not of type ITypedReferenceableInstance", instance.toShortString()));
if(processedIds.contains(id)) {
continue;
}
newInstance = (ITypedReferenceableInstance) instance;
instancesToUpdate.add(newInstance);
} else {
//Check if there is already an instance with the same unique attribute value
AtlasVertex instanceVertex = foundVertices.get(id);
ClassType classType = typeSystem.getDataType(ClassType.class, instance.getTypeName());
instanceVertex = graphHelper.getVertexForInstanceByUniqueAttribute(classType, instance);
//no entity with the given unique attribute, create new
if (instanceVertex == null) {
if (LOG.isDebugEnabled()) {
if(instanceVertex == null) {
if(LOG.isDebugEnabled()) {
LOG.debug("Creating new vertex for instance {}", instance.toShortString());
}
newInstance = classType.convert(instance, Multiplicity.REQUIRED);
ITypedReferenceableInstance newInstance = classType.convert(instance, Multiplicity.REQUIRED);
instanceVertex = graphHelper.createVertexWithIdentity(newInstance, classType.getAllSuperTypeNames());
instancesToCreate.add(newInstance);
//Map only unique attributes for cases of circular references
mapInstanceToVertex(newInstance, instanceVertex, classType.fieldMapping().fields, true, Operation.CREATE);
idToVertexMap.put(id, instanceVertex);
} else {
if (LOG.isDebugEnabled()) {
}
else {
if(LOG.isDebugEnabled()) {
LOG.debug("Re-using existing vertex {} for instance {}", string(instanceVertex), instance.toShortString());
}
if (!(instance instanceof ReferenceableInstance)) {
if (!(instance instanceof ITypedReferenceableInstance)) {
throw new IllegalStateException(
String.format("%s is not of type ITypedReferenceableInstance", instance.toShortString()));
}
newInstance = (ITypedReferenceableInstance) instance;
instancesToUpdate.add(newInstance);
ITypedReferenceableInstance existingInstance = (ITypedReferenceableInstance) instance;
instancesToUpdate.add(existingInstance);
}
processedIds.add(id);
}
return TypeUtils.Pair.of(instancesToCreate, instancesToUpdate);
}
//Set the id in the new instance
idToVertexMap.put(id, instanceVertex);
private Map<Id,AtlasVertex> findExistingVertices(Collection<IReferenceableInstance> instances) throws AtlasException {
VertexLookupContext context = new VertexLookupContext(this);
Map<Id,AtlasVertex> result = new HashMap<>();
for(IReferenceableInstance instance : instances) {
context.addInstance(instance);
}
List<Id> instancesToLoad = new ArrayList<>(context.getInstancesToLoadByGuid());
List<String> guidsToLoad = Lists.transform(instancesToLoad, new Function<Id,String>() {
@Override
public String apply(Id instance) {
Id id = getExistingId(instance);
return id.id;
}
});
Map<String, AtlasVertex> instanceVertices = graphHelper.getVerticesForGUIDs(guidsToLoad);
List<String> missingGuids = new ArrayList<>();
for(int i = 0 ; i < instancesToLoad.size(); i++) {
String guid = guidsToLoad.get(i);
AtlasVertex instanceVertex = instanceVertices.get(guid);
if(instanceVertex == null) {
missingGuids.add(guid);
continue;
}
Id instance = instancesToLoad.get(i);
if(LOG.isDebugEnabled()) {
LOG.debug("Found vertex {} for instance {}", string(instanceVertex), instance);
}
result.put(instance, instanceVertex);
}
if(missingGuids.size() > 0) {
throw new EntityNotFoundException("Could not find entities in the repository with the following GUIDs: " + missingGuids);
}
for(Map.Entry<ClassType,List<IReferenceableInstance>> entry : context.getInstancesToLoadByUniqueAttribute().entrySet()) {
ClassType type = entry.getKey();
List<IReferenceableInstance> instancesForClass = entry.getValue();
List<AtlasVertex> correspondingVertices = graphHelper.getVerticesForInstancesByUniqueAttribute(type, instancesForClass);
for(int i = 0; i < instancesForClass.size(); i++) {
IReferenceableInstance inst = instancesForClass.get(i);
AtlasVertex vertex = correspondingVertices.get(i);
result.put(getExistingId(inst), vertex);
}
return TypeUtils.Pair.of(instancesToCreate, instancesToUpdate);
}
return result;
}
private void addFullTextProperty(List<ITypedReferenceableInstance> instances, FullTextMapper fulltextMapper) throws AtlasException {
for (ITypedReferenceableInstance typedInstance : instances) { // Traverse
AtlasVertex instanceVertex = getClassVertex(typedInstance);
......@@ -582,6 +631,7 @@ public final class TypedInstanceToGraphMapper {
// add a new vertex for the struct or trait instance
AtlasVertex structInstanceVertex = graphHelper.createVertexWithoutIdentity(structInstance.getTypeName(), null,
Collections.<String>emptySet()); // no super types for struct type
if (LOG.isDebugEnabled()) {
LOG.debug("created vertex {} for struct {} value {}", string(structInstanceVertex), attributeInfo.name,
structInstance.toShortString());
......@@ -649,21 +699,24 @@ public final class TypedInstanceToGraphMapper {
return graphHelper.getOrCreateEdge(instanceVertex, toVertex, edgeLabel);
}
private AtlasVertex getClassVertex(ITypedReferenceableInstance typedReference) throws EntityNotFoundException {
AtlasVertex referenceVertex = null;
private <V,E> AtlasVertex<V,E> getClassVertex(ITypedReferenceableInstance typedReference) throws EntityNotFoundException {
AtlasVertex<V,E> referenceVertex = null;
Id id = null;
if (typedReference != null) {
id = typedReference instanceof Id ? (Id) typedReference : typedReference.getId();
if (id.isAssigned()) {
referenceVertex = graphHelper.getVertexForGUID(id.id);
} else {
id = getExistingId(typedReference);
referenceVertex = idToVertexMap.get(id);
if(referenceVertex == null && id.isAssigned()) {
referenceVertex = graphHelper.getVertexForGUID(id.id);
}
}
return referenceVertex;
}
Id getExistingId(IReferenceableInstance instance) {
return instance instanceof Id ? (Id) instance : instance.getId();
}
private Id getId(ITypedReferenceableInstance typedReference) throws EntityNotFoundException {
if (typedReference == null) {
throw new IllegalArgumentException("typedReference must be non-null");
......@@ -768,4 +821,8 @@ public final class TypedInstanceToGraphMapper {
GraphHelper.setProperty(instanceVertex, vertexPropertyName, propertyValue);
}
public AtlasVertex lookupVertex(Id refId) {
return idToVertexMap.get(refId);
}
}
repository/src/main/java/org/apache/atlas/repository/graph/VertexLookupContext.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.repository.graph;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.apache.atlas.AtlasException;
import org.apache.atlas.typesystem.IReferenceableInstance;
import org.apache.atlas.typesystem.ITypedReferenceableInstance;
import org.apache.atlas.typesystem.persistence.Id;
import org.apache.atlas.typesystem.types.AttributeInfo;
import org.apache.atlas.typesystem.types.ClassType;
import org.apache.atlas.typesystem.types.DataTypes;
import org.apache.atlas.typesystem.types.IDataType;
import org.apache.atlas.typesystem.types.Multiplicity;
import org.apache.atlas.typesystem.types.TypeSystem;
import org.apache.atlas.typesystem.types.DataTypes.TypeCategory;
/**
* Helper class for TypedInstanceGraphMapper. Determines which instances
* should be loaded by GUID and which ones should be loaded by unique attribute.
* In addition, it sorts the instances that should be loaded by unique
* attribute by class.
*
*/
public class VertexLookupContext {
private final TypedInstanceToGraphMapper mapper;
private static final TypeSystem typeSystem = TypeSystem.getInstance();
private Map<ClassType,List<IReferenceableInstance>> instancesWithoutGuids = new HashMap<>();
private Set<Id> guidsToLookup = new HashSet<>();
/**
* @param typedInstanceToGraphMapper
*/
VertexLookupContext(TypedInstanceToGraphMapper typedInstanceToGraphMapper) {
mapper = typedInstanceToGraphMapper;
}
/**
* Adds an instance to be loaded.
*
*/
public void addInstance(IReferenceableInstance instance) throws AtlasException {
ClassType classType = typeSystem.getDataType(ClassType.class, instance.getTypeName());
ITypedReferenceableInstance newInstance = classType.convert(instance, Multiplicity.REQUIRED);
findReferencedInstancesToPreLoad(newInstance);
Id id = instance.getId();
if(mapper.lookupVertex(id) == null) {
if(id.isAssigned()) {
guidsToLookup.add(id);
}
else {
addToClassMap(classType, instance);
}
}
}
/**
* Returns the instances that should be loaded by unique attribute, sorted by
* class.
*
*/
public Map<ClassType,List<IReferenceableInstance>> getInstancesToLoadByUniqueAttribute() {
return instancesWithoutGuids;
}
/**
* Returns the Ids of the instance that should be loaded by GUID
*
* @return
*/
public Set<Id> getInstancesToLoadByGuid() {
return guidsToLookup;
}
private void addToClassMap(ClassType classType, IReferenceableInstance instance) throws AtlasException {
List<IReferenceableInstance> toUpdate = instancesWithoutGuids.get(classType);
if(toUpdate == null) {
toUpdate = new ArrayList<>();
instancesWithoutGuids.put(classType, toUpdate);
}
toUpdate.add(instance);
}
private void findReferencedInstancesToPreLoad(ITypedReferenceableInstance newInstance) throws AtlasException {
//pre-load vertices for reference fields
for(AttributeInfo info : newInstance.fieldMapping().fields.values()) {
if(info.dataType().getTypeCategory() == TypeCategory.CLASS) {
ITypedReferenceableInstance newAttributeValue = (ITypedReferenceableInstance)newInstance.get(info.name);
addAdditionalInstance(newAttributeValue);
}
if(info.dataType().getTypeCategory() == TypeCategory.ARRAY) {
IDataType elementType = ((DataTypes.ArrayType) info.dataType()).getElemType();
if(elementType.getTypeCategory() == TypeCategory.CLASS) {
List<ITypedReferenceableInstance> newElements = (List) newInstance.get(info.name);
addAdditionalInstances(newElements);
}
}
if(info.dataType().getTypeCategory() == TypeCategory.MAP) {
IDataType elementType = ((DataTypes.MapType) info.dataType()).getValueType();
if(elementType.getTypeCategory() == TypeCategory.CLASS) {
Map<Object, ITypedReferenceableInstance> newAttribute =
(Map<Object, ITypedReferenceableInstance>) newInstance.get(info.name);
if(newAttribute != null) {
addAdditionalInstances(newAttribute.values());
}
}
}
}
}
private void addAdditionalInstance(ITypedReferenceableInstance instance) {
if(instance == null) {
return;
}
Id id = mapper.getExistingId(instance);
if(! id.isAssigned()) {
return;
}
guidsToLookup.add(id);
}
private void addAdditionalInstances(Collection<ITypedReferenceableInstance> newElements) {
if(newElements != null) {
for(ITypedReferenceableInstance instance: newElements) {
addAdditionalInstance(instance);
}
}
}
}
\ No newline at end of file
repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java
......@@ -367,6 +367,10 @@ public class DefaultMetadataService implements MetadataService, ActiveStateChang
private void validateUniqueAttribute(String entityType, String attributeName) throws AtlasException {
ClassType type = typeSystem.getDataType(ClassType.class, entityType);
AttributeInfo attribute = type.fieldMapping().fields.get(attributeName);
if(attribute == null) {
throw new IllegalArgumentException(
String.format("%s is not an attribute in %s", attributeName, entityType));
}
if (!attribute.isUnique) {
throw new IllegalArgumentException(
String.format("%s.%s is not a unique attribute", entityType, attributeName));
......
repository/src/main/java/org/apache/atlas/util/AttributeValueMap.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.util;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import org.apache.atlas.repository.graph.GraphHelper;
import org.apache.atlas.typesystem.IReferenceableInstance;
/**
* Map of attribute values to a collection of IndexedInstances with that attribute value.
*
* @see GraphHelper#getVerticesForInstancesByUniqueAttributes
*
*/
public class AttributeValueMap {
//need collection in case they are adding the same entity twice?
private Map<Object,Collection<IndexedInstance>> valueMap_ = new HashMap<>();
public void put(Object value, IReferenceableInstance instance, int index) {
IndexedInstance wrapper = new IndexedInstance(instance, index);
Collection<IndexedInstance> existingValues = valueMap_.get(value);
if(existingValues == null) {
//only expect 1 value
existingValues = new HashSet<>(1);
valueMap_.put(value, existingValues);
}
existingValues.add(wrapper);
}
public Collection<IndexedInstance> get(Object value) {
return valueMap_.get(value);
}
public Set<Object> getAttributeValues() {
return valueMap_.keySet();
}
}
\ No newline at end of file
repository/src/main/java/org/apache/atlas/util/IndexedInstance.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.util;
import org.apache.atlas.repository.graph.GraphHelper;
import org.apache.atlas.typesystem.IReferenceableInstance;
/**
* Data structure that stores an IReferenceableInstance and its location within
* a list.
*
* @see GraphHelper#getVerticesForInstancesByUniqueAttributes
*/
public class IndexedInstance {
private final IReferenceableInstance instance_;
private final int index_;
public IndexedInstance(IReferenceableInstance instance, int index) {
super();
this.instance_ = instance;
this.index_ = index;
}
public IReferenceableInstance getInstance() {
return instance_;
}
public int getIndex() {
return index_;
}
@Override
public int hashCode() {
return instance_.hashCode();
}
@Override
public boolean equals(Object other) {
if(!(other instanceof IndexedInstance)) {
return false;
}
IndexedInstance otherInstance = (IndexedInstance)other;
return instance_.equals(otherInstance.getInstance());
}
}
\ No newline at end of file
repository/src/test/java/org/apache/atlas/repository/graph/GraphHelperTest.java
......@@ -18,15 +18,42 @@
package org.apache.atlas.repository.graph;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertFalse;
import static org.testng.Assert.assertNotNull;
import static org.testng.Assert.assertNull;
import static org.testng.Assert.assertTrue;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import javax.inject.Inject;
import org.apache.atlas.AtlasException;
import org.apache.atlas.RepositoryMetadataModule;
import org.apache.atlas.TestUtils;
import org.apache.atlas.repository.graph.GraphHelper.VertexInfo;
import org.apache.atlas.repository.graphdb.AtlasEdge;
import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.services.MetadataService;
import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.atlas.typesystem.ITypedReferenceableInstance;
import org.apache.atlas.typesystem.Referenceable;
import org.apache.atlas.typesystem.TypesDef;
import org.apache.atlas.typesystem.exception.TypeNotFoundException;
import org.apache.atlas.typesystem.json.InstanceSerialization;
import org.apache.atlas.typesystem.json.TypesSerialization;
import org.apache.atlas.typesystem.types.ClassType;
import org.apache.atlas.typesystem.types.Multiplicity;
import org.apache.atlas.typesystem.types.TypeSystem;
import org.codehaus.jettison.json.JSONArray;
import org.testng.Assert;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
......@@ -34,20 +61,6 @@ import org.testng.annotations.DataProvider;
import org.testng.annotations.Guice;
import org.testng.annotations.Test;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
import javax.inject.Inject;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertFalse;
import static org.testng.Assert.assertNotNull;
import static org.testng.Assert.assertNull;
import static org.testng.Assert.assertTrue;
@Guice(modules = RepositoryMetadataModule.class)
public class GraphHelperTest {
......@@ -69,6 +82,9 @@ public class GraphHelperTest {
}
@Inject
private MetadataService metadataService;
@Inject
private GraphBackedMetadataRepository repositoryService;
private TypeSystem typeSystem;
......@@ -82,7 +98,12 @@ public class GraphHelperTest {
typeSystem.reset();
new GraphBackedSearchIndexer(typeRegistry);
TypesDef typesDef = TestUtils.defineHiveTypes();
try {
metadataService.getTypeDefinition(TestUtils.TABLE_TYPE);
} catch (TypeNotFoundException e) {
metadataService.createType(TypesSerialization.toJson(typesDef));
}
TestUtils.defineDeptEmployeeTypes(typeSystem);
}
......@@ -92,6 +113,43 @@ public class GraphHelperTest {
}
@Test
public void testGetInstancesByUniqueAttributes() throws Exception {
GraphHelper helper = GraphHelper.getInstance();
List<ITypedReferenceableInstance> instances = new ArrayList<>();
List<String> guids = new ArrayList<>();
TypeSystem ts = TypeSystem.getInstance();
ClassType dbType = ts.getDataType(ClassType.class, TestUtils.DATABASE_TYPE);
for(int i = 0; i < 10; i++) {
Referenceable db = TestUtils.createDBEntity();
String guid = createInstance(db);
ITypedReferenceableInstance instance = convert(db, dbType);
instances.add(instance);
guids.add(guid);
}
//lookup vertices via getVertexForInstanceByUniqueAttributes
List<AtlasVertex> vertices = helper.getVerticesForInstancesByUniqueAttribute(dbType, instances);
assertEquals(instances.size(), vertices.size());
//assert vertex matches the vertex we get through getVertexForGUID
for(int i = 0; i < instances.size(); i++) {
String guid = guids.get(i);
AtlasVertex foundVertex = vertices.get(i);
AtlasVertex expectedVertex = helper.getVertexForGUID(guid);
assertEquals(foundVertex, expectedVertex);
}
}
@Test
public void testGetVerticesForGUIDSWithDuplicates() throws Exception {
ITypedReferenceableInstance hrDept = TestUtils.createDeptEg1(TypeSystem.getInstance());
List<String> result = repositoryService.createEntities(hrDept);
String guid = result.get(0);
Map<String, AtlasVertex> verticesForGUIDs = GraphHelper.getInstance().getVerticesForGUIDs(Arrays.asList(guid, guid));
Assert.assertEquals(verticesForGUIDs.size(), 1);
Assert.assertTrue(verticesForGUIDs.containsKey(guid));
}
@Test
public void testGetCompositeGuidsAndVertices() throws Exception {
ITypedReferenceableInstance hrDept = TestUtils.createDeptEg1(typeSystem);
List<String> createdGuids = repositoryService.createEntities(hrDept);
......@@ -144,4 +202,22 @@ public class GraphHelperTest {
assertFalse(iterator.hasNext());
assertFalse(iterator.hasNext());
}
private ITypedReferenceableInstance convert(Referenceable instance, ClassType type) throws AtlasException {
return type.convert(instance, Multiplicity.REQUIRED);
}
private String createInstance(Referenceable entity) throws Exception {
TestUtils.resetRequestContext();
String entityjson = InstanceSerialization.toJson(entity, true);
JSONArray entitiesJson = new JSONArray();
entitiesJson.put(entityjson);
List<String> guids = metadataService.createEntities(entitiesJson.toString());
if (guids != null && guids.size() > 0) {
return guids.get(guids.size() - 1);
}
return null;
}
}
typesystem/src/test/resources/atlas-application.properties
......@@ -77,6 +77,7 @@ atlas.graph.index.search.solr.zookeeper-url=${solr.zk.address}
######### Hive Lineage Configs #########
## Schema
atlas.lineage.schema.query.hive_table=hive_table where __guid='%s'\, columns
atlas.lineage.schema.query.hive_table_v1=hive_table_v1 where __guid='%s'\, columns
######### Notification Configs #########
atlas.notification.embedded=true
......
webapp/src/main/java/org/apache/atlas/web/resources/EntityResource.java
......@@ -144,7 +144,7 @@ public class EntityResource {
throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.CONFLICT));
} catch (ValueConversionException ve) {
LOG.error("Unable to persist entity instance due to a deserialization error entityDef={}", entityJson, ve);
throw new WebApplicationException(Servlets.getErrorResponse(ve.getCause(), Response.Status.BAD_REQUEST));
throw new WebApplicationException(Servlets.getErrorResponse(ve.getCause() != null ? ve.getCause() : ve, Response.Status.BAD_REQUEST));
} catch (AtlasException | IllegalArgumentException e) {
LOG.error("Unable to persist entity instance entityDef={}", entityJson, e);
throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST));
......
webapp/src/test/java/org/apache/atlas/notification/EntityNotificationIT.java
......@@ -51,8 +51,6 @@ import static org.testng.Assert.assertTrue;
@Guice(modules = NotificationModule.class)
public class EntityNotificationIT extends BaseResourceIT {
private static final String ENTITIES = "api/atlas/entities";
private static final String TRAITS = "traits";
private final String DATABASE_NAME = "db" + randomString();
private final String TABLE_NAME = "table" + randomString();
@Inject
......@@ -66,7 +64,7 @@ public class EntityNotificationIT extends BaseResourceIT {
public void setUp() throws Exception {
super.setUp();
createTypeDefinitionsV1();
Referenceable HiveDBInstance = createHiveDBInstanceV1(DATABASE_NAME);
Referenceable HiveDBInstance = createHiveDBInstanceBuiltIn(DATABASE_NAME);
dbId = createInstance(HiveDBInstance);
List<NotificationConsumer<EntityNotification>> consumers =
......@@ -77,13 +75,13 @@ public class EntityNotificationIT extends BaseResourceIT {
@Test
public void testCreateEntity() throws Exception {
Referenceable tableInstance = createHiveTableInstanceV1(DATABASE_NAME, TABLE_NAME, dbId);
Referenceable tableInstance = createHiveTableInstanceBuiltIn(DATABASE_NAME, TABLE_NAME, dbId);
tableId = createInstance(tableInstance);
final String guid = tableId._getId();
waitForNotification(notificationConsumer, MAX_WAIT_TIME,
newNotificationPredicate(EntityNotification.OperationType.ENTITY_CREATE, HIVE_TABLE_TYPE, guid));
newNotificationPredicate(EntityNotification.OperationType.ENTITY_CREATE, HIVE_TABLE_TYPE_BUILTIN, guid));
}
@Test(dependsOnMethods = "testCreateEntity")
......@@ -96,29 +94,29 @@ public class EntityNotificationIT extends BaseResourceIT {
atlasClientV1.updateEntityAttribute(guid, property, newValue);
waitForNotification(notificationConsumer, MAX_WAIT_TIME,
newNotificationPredicate(EntityNotification.OperationType.ENTITY_UPDATE, HIVE_TABLE_TYPE, guid));
newNotificationPredicate(EntityNotification.OperationType.ENTITY_UPDATE, HIVE_TABLE_TYPE_BUILTIN, guid));
}
@Test
public void testDeleteEntity() throws Exception {
final String tableName = "table-" + randomString();
final String dbName = "db-" + randomString();
Referenceable HiveDBInstance = createHiveDBInstanceV1(dbName);
Referenceable HiveDBInstance = createHiveDBInstanceBuiltIn(dbName);
Id dbId = createInstance(HiveDBInstance);
Referenceable tableInstance = createHiveTableInstanceV1(dbName, tableName, dbId);
Referenceable tableInstance = createHiveTableInstanceBuiltIn(dbName, tableName, dbId);
final Id tableId = createInstance(tableInstance);
final String guid = tableId._getId();
waitForNotification(notificationConsumer, MAX_WAIT_TIME,
newNotificationPredicate(EntityNotification.OperationType.ENTITY_CREATE, HIVE_TABLE_TYPE, guid));
newNotificationPredicate(EntityNotification.OperationType.ENTITY_CREATE, HIVE_TABLE_TYPE_BUILTIN, guid));
final String name = (String) tableInstance.get(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME);
atlasClientV1.deleteEntity(HIVE_TABLE_TYPE, AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, name);
atlasClientV1.deleteEntity(HIVE_TABLE_TYPE_BUILTIN, AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, name);
waitForNotification(notificationConsumer, MAX_WAIT_TIME,
newNotificationPredicate(EntityNotification.OperationType.ENTITY_DELETE, HIVE_TABLE_TYPE, guid));
newNotificationPredicate(EntityNotification.OperationType.ENTITY_DELETE, HIVE_TABLE_TYPE_BUILTIN, guid));
}
@Test(dependsOnMethods = "testCreateEntity")
......@@ -141,7 +139,7 @@ public class EntityNotificationIT extends BaseResourceIT {
atlasClientV1.addTrait(guid, traitInstance);
EntityNotification entityNotification = waitForNotification(notificationConsumer, MAX_WAIT_TIME,
newNotificationPredicate(EntityNotification.OperationType.TRAIT_ADD, HIVE_TABLE_TYPE, guid));
newNotificationPredicate(EntityNotification.OperationType.TRAIT_ADD, HIVE_TABLE_TYPE_BUILTIN, guid));
IReferenceableInstance entity = entityNotification.getEntity();
assertTrue(entity.getTraits().contains(traitName));
......@@ -166,7 +164,7 @@ public class EntityNotificationIT extends BaseResourceIT {
atlasClientV1.addTrait(guid, traitInstance);
entityNotification = waitForNotification(notificationConsumer, MAX_WAIT_TIME,
newNotificationPredicate(EntityNotification.OperationType.TRAIT_ADD, HIVE_TABLE_TYPE, guid));
newNotificationPredicate(EntityNotification.OperationType.TRAIT_ADD, HIVE_TABLE_TYPE_BUILTIN, guid));
allTraits = entityNotification.getAllTraits();
allTraitNames = new LinkedList<>();
......@@ -187,7 +185,7 @@ public class EntityNotificationIT extends BaseResourceIT {
atlasClientV1.deleteTrait(guid, traitName);
EntityNotification entityNotification = waitForNotification(notificationConsumer, MAX_WAIT_TIME,
newNotificationPredicate(EntityNotification.OperationType.TRAIT_DELETE, HIVE_TABLE_TYPE, guid));
newNotificationPredicate(EntityNotification.OperationType.TRAIT_DELETE, HIVE_TABLE_TYPE_BUILTIN, guid));
assertFalse(entityNotification.getEntity().getTraits().contains(traitName));
}
......
webapp/src/test/java/org/apache/atlas/notification/NotificationHookConsumerIT.java
......@@ -68,7 +68,7 @@ public class NotificationHookConsumerIT extends BaseResourceIT {
new Referenceable(randomString())));
//send valid message
final Referenceable entity = new Referenceable(DATABASE_TYPE);
final Referenceable entity = new Referenceable(DATABASE_TYPE_BUILTIN);
String dbName = "db" + randomString();
entity.set(NAME, dbName);
entity.set(DESCRIPTION, randomString());
......@@ -79,7 +79,7 @@ public class NotificationHookConsumerIT extends BaseResourceIT {
waitFor(MAX_WAIT_TIME, new Predicate() {
@Override
public boolean evaluate() throws Exception {
JSONArray results = searchByDSL(String.format("%s where name='%s'", DATABASE_TYPE, entity.get(NAME)));
JSONArray results = searchByDSL(String.format("%s where name='%s'", DATABASE_TYPE_BUILTIN, entity.get(NAME)));
return results.length() == 1;
}
});
......@@ -87,7 +87,7 @@ public class NotificationHookConsumerIT extends BaseResourceIT {
@Test
public void testCreateEntity() throws Exception {
final Referenceable entity = new Referenceable(DATABASE_TYPE);
final Referenceable entity = new Referenceable(DATABASE_TYPE_BUILTIN);
String dbName = "db" + randomString();
entity.set(NAME, dbName);
entity.set(DESCRIPTION, randomString());
......@@ -99,13 +99,13 @@ public class NotificationHookConsumerIT extends BaseResourceIT {
waitFor(MAX_WAIT_TIME, new Predicate() {
@Override
public boolean evaluate() throws Exception {
JSONArray results = searchByDSL(String.format("%s where qualifiedName='%s'", DATABASE_TYPE, entity.get(QUALIFIED_NAME)));
JSONArray results = searchByDSL(String.format("%s where qualifiedName='%s'", DATABASE_TYPE_BUILTIN, entity.get(QUALIFIED_NAME)));
return results.length() == 1;
}
});
//Assert that user passed in hook message is used in audit
Referenceable instance = atlasClientV1.getEntity(DATABASE_TYPE, QUALIFIED_NAME, (String) entity.get(QUALIFIED_NAME));
Referenceable instance = atlasClientV1.getEntity(DATABASE_TYPE_BUILTIN, QUALIFIED_NAME, (String) entity.get(QUALIFIED_NAME));
List<EntityAuditEvent> events =
atlasClientV1.getEntityAuditEvents(instance.getId()._getId(), (short) 1);
assertEquals(events.size(), 1);
......@@ -114,7 +114,7 @@ public class NotificationHookConsumerIT extends BaseResourceIT {
@Test
public void testUpdateEntityPartial() throws Exception {
final Referenceable entity = new Referenceable(DATABASE_TYPE);
final Referenceable entity = new Referenceable(DATABASE_TYPE_BUILTIN);
final String dbName = "db" + randomString();
entity.set(NAME, dbName);
entity.set(DESCRIPTION, randomString());
......@@ -123,26 +123,26 @@ public class NotificationHookConsumerIT extends BaseResourceIT {
atlasClientV1.createEntity(entity);
final Referenceable newEntity = new Referenceable(DATABASE_TYPE);
final Referenceable newEntity = new Referenceable(DATABASE_TYPE_BUILTIN);
newEntity.set("owner", randomString());
sendHookMessage(
new HookNotification.EntityPartialUpdateRequest(TEST_USER, DATABASE_TYPE, QUALIFIED_NAME, dbName, newEntity));
new HookNotification.EntityPartialUpdateRequest(TEST_USER, DATABASE_TYPE_BUILTIN, QUALIFIED_NAME, dbName, newEntity));
waitFor(MAX_WAIT_TIME, new Predicate() {
@Override
public boolean evaluate() throws Exception {
Referenceable localEntity = atlasClientV1.getEntity(DATABASE_TYPE, QUALIFIED_NAME, dbName);
Referenceable localEntity = atlasClientV1.getEntity(DATABASE_TYPE_BUILTIN, QUALIFIED_NAME, dbName);
return (localEntity.get("owner") != null && localEntity.get("owner").equals(newEntity.get("owner")));
}
});
//Its partial update and un-set fields are not updated
Referenceable actualEntity = atlasClientV1.getEntity(DATABASE_TYPE, QUALIFIED_NAME, dbName);
Referenceable actualEntity = atlasClientV1.getEntity(DATABASE_TYPE_BUILTIN, QUALIFIED_NAME, dbName);
assertEquals(actualEntity.get(DESCRIPTION), entity.get(DESCRIPTION));
}
@Test
public void testUpdatePartialUpdatingQualifiedName() throws Exception {
final Referenceable entity = new Referenceable(DATABASE_TYPE);
final Referenceable entity = new Referenceable(DATABASE_TYPE_BUILTIN);
final String dbName = "db" + randomString();
entity.set(NAME, dbName);
entity.set(DESCRIPTION, randomString());
......@@ -151,29 +151,29 @@ public class NotificationHookConsumerIT extends BaseResourceIT {
atlasClientV1.createEntity(entity);
final Referenceable newEntity = new Referenceable(DATABASE_TYPE);
final Referenceable newEntity = new Referenceable(DATABASE_TYPE_BUILTIN);
final String newName = "db" + randomString();
newEntity.set(QUALIFIED_NAME, newName);
sendHookMessage(
new HookNotification.EntityPartialUpdateRequest(TEST_USER, DATABASE_TYPE, QUALIFIED_NAME, dbName, newEntity));
new HookNotification.EntityPartialUpdateRequest(TEST_USER, DATABASE_TYPE_BUILTIN, QUALIFIED_NAME, dbName, newEntity));
waitFor(MAX_WAIT_TIME, new Predicate() {
@Override
public boolean evaluate() throws Exception {
JSONArray results = searchByDSL(String.format("%s where qualifiedName='%s'", DATABASE_TYPE, newName));
JSONArray results = searchByDSL(String.format("%s where qualifiedName='%s'", DATABASE_TYPE_BUILTIN, newName));
return results.length() == 1;
}
});
//no entity with the old qualified name
JSONArray results = searchByDSL(String.format("%s where qualifiedName='%s'", DATABASE_TYPE, dbName));
JSONArray results = searchByDSL(String.format("%s where qualifiedName='%s'", DATABASE_TYPE_BUILTIN, dbName));
assertEquals(results.length(), 0);
}
@Test
public void testDeleteByQualifiedName() throws Exception {
Referenceable entity = new Referenceable(DATABASE_TYPE);
Referenceable entity = new Referenceable(DATABASE_TYPE_BUILTIN);
final String dbName = "db" + randomString();
entity.set(NAME, dbName);
entity.set(DESCRIPTION, randomString());
......@@ -183,7 +183,7 @@ public class NotificationHookConsumerIT extends BaseResourceIT {
final String dbId = atlasClientV1.createEntity(entity).get(0);
sendHookMessage(
new HookNotification.EntityDeleteRequest(TEST_USER, DATABASE_TYPE, QUALIFIED_NAME, dbName));
new HookNotification.EntityDeleteRequest(TEST_USER, DATABASE_TYPE_BUILTIN, QUALIFIED_NAME, dbName));
waitFor(MAX_WAIT_TIME, new Predicate() {
@Override
public boolean evaluate() throws Exception {
......@@ -195,7 +195,7 @@ public class NotificationHookConsumerIT extends BaseResourceIT {
@Test
public void testUpdateEntityFullUpdate() throws Exception {
Referenceable entity = new Referenceable(DATABASE_TYPE);
Referenceable entity = new Referenceable(DATABASE_TYPE_BUILTIN);
final String dbName = "db" + randomString();
entity.set(NAME, dbName);
entity.set(DESCRIPTION, randomString());
......@@ -204,7 +204,7 @@ public class NotificationHookConsumerIT extends BaseResourceIT {
atlasClientV1.createEntity(entity);
final Referenceable newEntity = new Referenceable(DATABASE_TYPE);
final Referenceable newEntity = new Referenceable(DATABASE_TYPE_BUILTIN);
newEntity.set(NAME, randomString());
newEntity.set(DESCRIPTION, randomString());
newEntity.set("owner", randomString());
......@@ -216,12 +216,12 @@ public class NotificationHookConsumerIT extends BaseResourceIT {
waitFor(MAX_WAIT_TIME, new Predicate() {
@Override
public boolean evaluate() throws Exception {
JSONArray results = searchByDSL(String.format("%s where qualifiedName='%s'", DATABASE_TYPE, newEntity.get(QUALIFIED_NAME)));
JSONArray results = searchByDSL(String.format("%s where qualifiedName='%s'", DATABASE_TYPE_BUILTIN, newEntity.get(QUALIFIED_NAME)));
return results.length() == 1;
}
});
Referenceable actualEntity = atlasClientV1.getEntity(DATABASE_TYPE, QUALIFIED_NAME, dbName);
Referenceable actualEntity = atlasClientV1.getEntity(DATABASE_TYPE_BUILTIN, QUALIFIED_NAME, dbName);
assertEquals(actualEntity.get(DESCRIPTION), newEntity.get(DESCRIPTION));
assertEquals(actualEntity.get("owner"), newEntity.get("owner"));
}
......
webapp/src/test/java/org/apache/atlas/web/resources/BaseResourceIT.java
......@@ -95,6 +95,12 @@ public abstract class BaseResourceIT {
@BeforeClass
public void setUp() throws Exception {
//set high timeouts so that tests do not fail due to read timeouts while you
//are stepping through the code in a debugger
ApplicationProperties.get().setProperty("atlas.client.readTimeoutMSecs", "100000000");
ApplicationProperties.get().setProperty("atlas.client.connectTimeoutMSecs", "100000000");
Configuration configuration = ApplicationProperties.get();
atlasUrls = configuration.getStringArray(ATLAS_REST_ADDRESS);
......@@ -221,13 +227,18 @@ public abstract class BaseResourceIT {
try {
if (!update) {
entity = entitiesClientV2.createEntity(atlasEntity);
assertNotNull(entity);
assertNotNull(entity.getEntitiesByOperation(EntityMutations.EntityOperation.CREATE));
assertTrue(entity.getEntitiesByOperation(EntityMutations.EntityOperation.CREATE).size() > 0);
return entity.getEntitiesByOperation(EntityMutations.EntityOperation.CREATE).get(0);
} else {
entity = entitiesClientV2.updateEntity(atlasEntity);
}
assertNotNull(entity);
assertNotNull(entity.getEntitiesByOperation(EntityMutations.EntityOperation.UPDATE));
assertTrue(entity.getEntitiesByOperation(EntityMutations.EntityOperation.UPDATE).size() > 0);
return entity.getEntitiesByOperation(EntityMutations.EntityOperation.UPDATE).get(0);
}
} catch (AtlasServiceException e) {
LOG.error("Entity {} failed", update ? "update" : "creation", entity);
}
......@@ -242,10 +253,20 @@ public abstract class BaseResourceIT {
return modifyEntity(atlasEntity, true);
}
protected static final String DATABASE_TYPE = "hive_db";
protected static final String HIVE_TABLE_TYPE = "hive_table";
protected static final String COLUMN_TYPE = "hive_column";
protected static final String HIVE_PROCESS_TYPE = "hive_process";
protected static final String DATABASE_TYPE_V2 = "hive_db_v2";
protected static final String HIVE_TABLE_TYPE_V2 = "hive_table_v2";
protected static final String COLUMN_TYPE_V2 = "hive_column_v2";
protected static final String HIVE_PROCESS_TYPE_V2 = "hive_process_v2";
protected static final String DATABASE_TYPE = "hive_db_v1";
protected static final String HIVE_TABLE_TYPE = "hive_table_v1";
protected static final String COLUMN_TYPE = "hive_column_v1";
protected static final String HIVE_PROCESS_TYPE = "hive_process_v1";
protected static final String DATABASE_TYPE_BUILTIN = "hive_db";
protected static final String HIVE_TABLE_TYPE_BUILTIN = "hive_table";
protected static final String COLUMN_TYPE_BUILTIN = "hive_column";
protected static final String HIVE_PROCESS_TYPE_BUILTIN = "hive_process";
protected void createTypeDefinitionsV1() throws Exception {
HierarchicalTypeDefinition<ClassType> dbClsDef = TypesUtil
......@@ -323,7 +344,7 @@ public abstract class BaseResourceIT {
protected void createTypeDefinitionsV2() throws Exception {
AtlasEntityDef dbClsTypeDef = AtlasTypeUtil.createClassTypeDef(
DATABASE_TYPE,
DATABASE_TYPE_V2,
null,
AtlasTypeUtil.createUniqueRequiredAttrDef(NAME, "string"),
AtlasTypeUtil.createRequiredAttrDef(DESCRIPTION, "string"),
......@@ -332,7 +353,7 @@ public abstract class BaseResourceIT {
AtlasTypeUtil.createOptionalAttrDef("createTime", "int"));
AtlasEntityDef columnClsDef = AtlasTypeUtil
.createClassTypeDef(COLUMN_TYPE, null,
.createClassTypeDef(COLUMN_TYPE_V2, null,
AtlasTypeUtil.createOptionalAttrDef(NAME, "string"),
AtlasTypeUtil.createOptionalAttrDef("dataType", "string"),
AtlasTypeUtil.createOptionalAttrDef("comment", "string"));
......@@ -348,20 +369,21 @@ public abstract class BaseResourceIT {
));
AtlasEntityDef tblClsDef = AtlasTypeUtil
.createClassTypeDef(HIVE_TABLE_TYPE,
.createClassTypeDef(HIVE_TABLE_TYPE_V2,
ImmutableSet.of("DataSet"),
AtlasTypeUtil.createOptionalAttrDef("owner", "string"),
AtlasTypeUtil.createOptionalAttrDef("createTime", "long"),
AtlasTypeUtil.createOptionalAttrDef("lastAccessTime", "date"),
AtlasTypeUtil.createOptionalAttrDef("temporary", "boolean"),
AtlasTypeUtil.createRequiredAttrDef("db", DATABASE_TYPE),
AtlasTypeUtil.createRequiredAttrDef("columns", DataTypes.arrayTypeName(COLUMN_TYPE)),
AtlasTypeUtil.createRequiredAttrDef("db", DATABASE_TYPE_V2),
//some tests don't set the columns field or set it to null...
AtlasTypeUtil.createOptionalAttrDef("columns", DataTypes.arrayTypeName(COLUMN_TYPE_V2)),
AtlasTypeUtil.createOptionalAttrDef("tableType", "tableType"),
AtlasTypeUtil.createOptionalAttrDef("serde1", "serdeType"),
AtlasTypeUtil.createOptionalAttrDef("serde2", "serdeType"));
AtlasEntityDef loadProcessClsDef = AtlasTypeUtil
.createClassTypeDef(HIVE_PROCESS_TYPE,
.createClassTypeDef(HIVE_PROCESS_TYPE_V2,
ImmutableSet.of("Process"),
AtlasTypeUtil.createOptionalAttrDef("userName", "string"),
AtlasTypeUtil.createOptionalAttrDef("startTime", "int"),
......@@ -415,7 +437,7 @@ public abstract class BaseResourceIT {
return RandomStringUtils.randomAlphabetic(1) + RandomStringUtils.randomAlphanumeric(9);
}
protected Referenceable createHiveTableInstanceV1(String dbName, String tableName, Id dbId) throws Exception {
protected Referenceable createHiveTableInstanceBuiltIn(String dbName, String tableName, Id dbId) throws Exception {
Map<String, Object> values = new HashMap<>();
values.put(NAME, dbName);
values.put(DESCRIPTION, "foo database");
......@@ -426,7 +448,7 @@ public abstract class BaseResourceIT {
values.put("location", "/tmp");
Referenceable databaseInstance = new Referenceable(dbId._getId(), dbId.getTypeName(), values);
Referenceable tableInstance =
new Referenceable(HIVE_TABLE_TYPE, "classification", "pii", "phi", "pci", "sox", "sec", "finance");
new Referenceable(HIVE_TABLE_TYPE_BUILTIN, "classification", "pii", "phi", "pci", "sox", "sec", "finance");
tableInstance.set(NAME, tableName);
tableInstance.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, tableName);
tableInstance.set("db", databaseInstance);
......@@ -458,7 +480,7 @@ public abstract class BaseResourceIT {
protected AtlasEntityWithAssociations createHiveTableInstanceV2(AtlasEntity databaseInstance, String tableName) throws Exception {
AtlasEntityWithAssociations tableInstance =
new AtlasEntityWithAssociations(HIVE_TABLE_TYPE);
new AtlasEntityWithAssociations(HIVE_TABLE_TYPE_V2);
tableInstance.setClassifications(
Arrays.asList(new AtlasClassification("classification"),
new AtlasClassification("pii"),
......@@ -497,29 +519,34 @@ public abstract class BaseResourceIT {
return tableInstance;
}
protected Referenceable createHiveDBInstanceBuiltIn(String dbName) {
Referenceable databaseInstance = new Referenceable(DATABASE_TYPE_BUILTIN);
databaseInstance.set(NAME, dbName);
databaseInstance.set(QUALIFIED_NAME, dbName);
databaseInstance.set(CLUSTER_NAME, randomString());
databaseInstance.set(DESCRIPTION, "foo database");
return databaseInstance;
}
protected Referenceable createHiveDBInstanceV1(String dbName) {
Referenceable databaseInstance = new Referenceable(DATABASE_TYPE);
databaseInstance.set(NAME, dbName);
databaseInstance.set(QUALIFIED_NAME, dbName);
databaseInstance.set(CLUSTER_NAME, randomString());
databaseInstance.set(DESCRIPTION, "foo database");
return databaseInstance;
}
protected AtlasEntity createHiveDBInstanceV2(String dbName) {
AtlasEntity atlasEntity = new AtlasEntity(DATABASE_TYPE);
AtlasEntity atlasEntity = new AtlasEntity(DATABASE_TYPE_V2);
atlasEntity.setAttribute(NAME, dbName);
atlasEntity.setAttribute(QUALIFIED_NAME, dbName);
atlasEntity.setAttribute(DESCRIPTION, "foo database");
atlasEntity.setAttribute(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, dbName);
atlasEntity.setAttribute("owner", "user1");
atlasEntity.setAttribute(CLUSTER_NAME, "cl1");
atlasEntity.setAttribute("parameters", Collections.EMPTY_MAP);
atlasEntity.setAttribute("location", "/tmp");
atlasEntity.setAttribute("locationUri", "/tmp");
atlasEntity.setAttribute("createTime",1000);
return atlasEntity;
}
public interface Predicate {
/**
......
webapp/src/test/java/org/apache/atlas/web/resources/DataSetLineageJerseyResourceIT.java
......@@ -149,8 +149,7 @@ public class DataSetLineageJerseyResourceIT extends BaseResourceIT {
LOG.info("JsonRow - {}", row);
Assert.assertNotNull(row.getString("name"));
Assert.assertNotNull(row.getString("comment"));
Assert.assertNotNull(row.getString("type"));
Assert.assertEquals(row.getString("$typeName$"), "hive_column");
Assert.assertEquals(row.getString("$typeName$"), "hive_column_v1");
}
}
......@@ -168,8 +167,7 @@ public class DataSetLineageJerseyResourceIT extends BaseResourceIT {
LOG.info("JsonRow - {}", row);
Assert.assertNotNull(row.getString("name"));
Assert.assertNotNull(row.getString("comment"));
Assert.assertNotNull(row.getString("type"));
Assert.assertEquals(row.getString("$typeName$"), "hive_column");
Assert.assertEquals(row.getString("$typeName$"), "hive_column_v1");
}
}
......
webapp/src/test/java/org/apache/atlas/web/resources/EntityDiscoveryJerseyResourceIT.java
......@@ -56,12 +56,12 @@ public class EntityDiscoveryJerseyResourceIT extends BaseResourceIT {
super.setUp();
dbName = "db" + randomString();
createTypes();
createInstance(createHiveDBInstanceV1(dbName));
createInstance(createHiveDBInstanceBuiltIn(dbName));
}
@Test
public void testSearchByDSL() throws Exception {
String dslQuery = "from "+ DATABASE_TYPE + " " + QUALIFIED_NAME + "=\"" + dbName + "\"";
String dslQuery = "from "+ DATABASE_TYPE_BUILTIN + " " + QUALIFIED_NAME + "=\"" + dbName + "\"";
AtlasSearchResult searchResult = discoveryClientV2.dslSearch(dslQuery);
assertNotNull(searchResult);
......@@ -73,7 +73,7 @@ public class EntityDiscoveryJerseyResourceIT extends BaseResourceIT {
assertEquals(entities.size(), 1);
AtlasEntityHeaderWithAssociations dbEntity = entities.get(0);
assertEquals(dbEntity.getTypeName(), DATABASE_TYPE);
assertEquals(dbEntity.getTypeName(), DATABASE_TYPE_BUILTIN);
assertEquals(dbEntity.getDisplayText(), dbName);
assertEquals(dbEntity.getStatus(), Status.ACTIVE);
assertNotNull(dbEntity.getGuid());
......@@ -83,7 +83,7 @@ public class EntityDiscoveryJerseyResourceIT extends BaseResourceIT {
@Test
public void testSearchDSLLimits() throws Exception {
String dslQuery = "from "+ DATABASE_TYPE + " " + QUALIFIED_NAME + "=\"" + dbName + "\"";
String dslQuery = "from "+ DATABASE_TYPE_BUILTIN + " " + QUALIFIED_NAME + "=\"" + dbName + "\"";
AtlasSearchResult searchResult = discoveryClientV2.dslSearch(dslQuery);
assertNotNull(searchResult);
......@@ -124,7 +124,7 @@ public class EntityDiscoveryJerseyResourceIT extends BaseResourceIT {
@Test
public void testSearchUsingDSL() throws Exception {
String query = "from "+ DATABASE_TYPE + " " + QUALIFIED_NAME + "=\"" + dbName + "\"";
String query = "from "+ DATABASE_TYPE_BUILTIN + " " + QUALIFIED_NAME + "=\"" + dbName + "\"";
AtlasSearchResult searchResult = discoveryClientV2.dslSearch(query);
assertNotNull(searchResult);
......@@ -135,7 +135,7 @@ public class EntityDiscoveryJerseyResourceIT extends BaseResourceIT {
assertEquals(entities.size(), 1);
AtlasEntityHeaderWithAssociations dbEntity = entities.get(0);
assertEquals(dbEntity.getTypeName(), DATABASE_TYPE);
assertEquals(dbEntity.getTypeName(), DATABASE_TYPE_BUILTIN);
assertEquals(dbEntity.getDisplayText(), dbName);
assertEquals(dbEntity.getStatus(), Status.ACTIVE);
......@@ -166,7 +166,7 @@ public class EntityDiscoveryJerseyResourceIT extends BaseResourceIT {
AtlasFullTextResult result = fullTextResults.get(0);
assertNotNull(result.getEntity());
assertEquals(result.getEntity().getTypeName(), DATABASE_TYPE);
assertEquals(result.getEntity().getTypeName(), DATABASE_TYPE_BUILTIN);
assertNotNull(result.getScore());
//API works without limit and offset
......
webapp/src/test/java/org/apache/atlas/web/resources/EntityJerseyResourceIT.java
......@@ -85,10 +85,7 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
private final String DATABASE_NAME = "db" + randomString();
private final String TABLE_NAME = "table" + randomString();
private static final String ENTITIES = "api/atlas/entities";
private static final String TRAITS = "traits";
private static final String TRAIT_DEFINITION = "traitDefinitions";
private Referenceable tableInstance;
private Id tableId;
private Id dbId;
......@@ -103,7 +100,7 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
super.setUp();
createTypeDefinitionsV1();
Referenceable HiveDBInstance = createHiveDBInstanceV1(DATABASE_NAME);
Referenceable HiveDBInstance = createHiveDBInstanceBuiltIn(DATABASE_NAME);
dbId = createInstance(HiveDBInstance);
List<NotificationConsumer<EntityNotification>> consumers =
......@@ -114,7 +111,7 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
@Test
public void testSubmitEntity() throws Exception {
tableInstance = createHiveTableInstanceV1(DATABASE_NAME, TABLE_NAME, dbId);
tableInstance = createHiveTableInstanceBuiltIn(DATABASE_NAME, TABLE_NAME, dbId);
tableId = createInstance(tableInstance);
final String guid = tableId._getId();
......@@ -127,7 +124,7 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
@Test
public void testRequestUser() throws Exception {
Referenceable entity = new Referenceable(DATABASE_TYPE);
Referenceable entity = new Referenceable(DATABASE_TYPE_BUILTIN);
String dbName = randomString();
entity.set("name", dbName);
entity.set(QUALIFIED_NAME, dbName);
......@@ -157,7 +154,7 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
@Test
//API should accept single entity (or jsonarray of entities)
public void testSubmitSingleEntity() throws Exception {
Referenceable databaseInstance = new Referenceable(DATABASE_TYPE);
Referenceable databaseInstance = new Referenceable(DATABASE_TYPE_BUILTIN);
String dbName = randomString();
databaseInstance.set("name", dbName);
databaseInstance.set(QUALIFIED_NAME, dbName);
......@@ -181,9 +178,9 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
@Test
public void testEntityDeduping() throws Exception {
final Referenceable db = new Referenceable(DATABASE_TYPE);
final Referenceable db = new Referenceable(DATABASE_TYPE_BUILTIN);
final String dbName = "db" + randomString();
Referenceable HiveDBInstance = createHiveDBInstanceV1(dbName);
Referenceable HiveDBInstance = createHiveDBInstanceBuiltIn(dbName);
Id dbIdReference = createInstance(HiveDBInstance);
final String dbId = dbIdReference._getId();
......@@ -196,7 +193,7 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
}
});
JSONArray results = searchByDSL(String.format("%s where qualifiedName='%s'", DATABASE_TYPE, dbName));
JSONArray results = searchByDSL(String.format("%s where qualifiedName='%s'", DATABASE_TYPE_BUILTIN, dbName));
assertEquals(results.length(), 1);
//create entity again shouldn't create another instance with same unique attribute value
......@@ -214,15 +211,15 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
//expected timeout
}
results = searchByDSL(String.format("%s where qualifiedName='%s'", DATABASE_TYPE, dbName));
results = searchByDSL(String.format("%s where qualifiedName='%s'", DATABASE_TYPE_BUILTIN, dbName));
assertEquals(results.length(), 1);
//Test the same across references
Referenceable table = new Referenceable(HIVE_TABLE_TYPE);
Referenceable table = new Referenceable(HIVE_TABLE_TYPE_BUILTIN);
final String tableName = randomString();
Referenceable tableInstance = createHiveTableInstanceV1(DATABASE_NAME, tableName, dbIdReference);
Referenceable tableInstance = createHiveTableInstanceBuiltIn(DATABASE_NAME, tableName, dbIdReference);
atlasClientV1.createEntity(tableInstance);
results = searchByDSL(String.format("%s where qualifiedName='%s'", DATABASE_TYPE, dbName));
results = searchByDSL(String.format("%s where qualifiedName='%s'", DATABASE_TYPE_BUILTIN, dbName));
assertEquals(results.length(), 1);
}
......@@ -272,7 +269,7 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
@Test(dataProvider = "invalidAttrValues")
public void testEntityInvalidValue(String value) throws Exception {
Referenceable databaseInstance = new Referenceable(DATABASE_TYPE);
Referenceable databaseInstance = new Referenceable(DATABASE_TYPE_BUILTIN);
databaseInstance.set("name", randomString());
databaseInstance.set("description", value);
......@@ -286,7 +283,7 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
@Test
public void testGetEntityByAttribute() throws Exception {
Referenceable db1 = new Referenceable(DATABASE_TYPE);
Referenceable db1 = new Referenceable(DATABASE_TYPE_BUILTIN);
String dbName = randomString();
db1.set(NAME, dbName);
db1.set(DESCRIPTION, randomString());
......@@ -298,15 +295,15 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
createInstance(db1);
//get entity by attribute
Referenceable referenceable = atlasClientV1.getEntity(DATABASE_TYPE, QUALIFIED_NAME, dbName);
Assert.assertEquals(referenceable.getTypeName(), DATABASE_TYPE);
Referenceable referenceable = atlasClientV1.getEntity(DATABASE_TYPE_BUILTIN, QUALIFIED_NAME, dbName);
Assert.assertEquals(referenceable.getTypeName(), DATABASE_TYPE_BUILTIN);
Assert.assertEquals(referenceable.get(QUALIFIED_NAME), dbName);
}
@Test
public void testSubmitEntityWithBadDateFormat() throws Exception {
try {
Referenceable tableInstance = createHiveTableInstanceV1("db" + randomString(), "table" + randomString(), dbId);
Referenceable tableInstance = createHiveTableInstanceBuiltIn("db" + randomString(), "table" + randomString(), dbId);
tableInstance.set("lastAccessTime", "2014-07-11");
tableId = createInstance(tableInstance);
Assert.fail("Was expecting an exception here ");
......@@ -336,7 +333,7 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
Assert.assertEquals(e.getStatus().getStatusCode(), Response.Status.BAD_REQUEST.getStatusCode());
}
String currentTime = String.valueOf(new DateTime() );
String currentTime = String.valueOf(new DateTime());
addProperty(guid, "createTime", currentTime);
response = atlasClientV1.callAPIWithBodyAndParams(AtlasClient.API.GET_ENTITY, null, guid);
......@@ -368,7 +365,7 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
@Test(dependsOnMethods = "testSubmitEntity")
public void testAddReferenceProperty() throws Exception {
//Create new db instance
Referenceable databaseInstance = new Referenceable(DATABASE_TYPE);
Referenceable databaseInstance = new Referenceable(DATABASE_TYPE_BUILTIN);
String dbName = randomString();
databaseInstance.set(NAME, dbName);
databaseInstance.set(QUALIFIED_NAME, dbName);
......@@ -420,7 +417,7 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
@Test(dependsOnMethods = "testSubmitEntity")
public void testGetEntityList() throws Exception {
List<String> entities = atlasClientV1.listEntities(HIVE_TABLE_TYPE);
List<String> entities = atlasClientV1.listEntities(HIVE_TABLE_TYPE_BUILTIN);
Assert.assertNotNull(entities);
Assert.assertTrue(entities.contains(tableId._getId()));
}
......@@ -630,7 +627,8 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
@Test
public void testUTF8() throws Exception {
String classType = random();
//Type names cannot be arbitrary UTF8 characters. See org.apache.atlas.type.AtlasTypeUtil#validateType()
String classType = randomString();
String attrName = random();
String attrValue = random();
......@@ -651,6 +649,7 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
Assert.assertEquals(getReferenceable.get(attrName), attrValue);
}
@Test(dependsOnMethods = "testSubmitEntity")
public void testPartialUpdate() throws Exception {
String colName = "col1"+randomString();
......@@ -664,11 +663,11 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
values.put("owner", "user1");
values.put("position", 0);
values.put("description", "col1");
values.put("table", null);
values.put("table", tableId ); //table is a required reference, can't be null
Referenceable ref = new Referenceable(BaseResourceIT.COLUMN_TYPE, values);
Referenceable ref = new Referenceable(BaseResourceIT.COLUMN_TYPE_BUILTIN, values);
columns.add(ref);
Referenceable tableUpdated = new Referenceable(BaseResourceIT.HIVE_TABLE_TYPE, new HashMap<String, Object>() {{
Referenceable tableUpdated = new Referenceable(BaseResourceIT.HIVE_TABLE_TYPE_BUILTIN, new HashMap<String, Object>() {{
put("columns", columns);
}});
......@@ -685,14 +684,14 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
//Update by unique attribute
values.put("type", "int");
ref = new Referenceable(BaseResourceIT.COLUMN_TYPE, values);
ref = new Referenceable(BaseResourceIT.COLUMN_TYPE_BUILTIN, values);
columns.set(0, ref);
tableUpdated = new Referenceable(BaseResourceIT.HIVE_TABLE_TYPE, new HashMap<String, Object>() {{
tableUpdated = new Referenceable(BaseResourceIT.HIVE_TABLE_TYPE_BUILTIN, new HashMap<String, Object>() {{
put("columns", columns);
}});
LOG.debug("Updating entity= {}", tableUpdated);
entityResult = atlasClientV1.updateEntity(BaseResourceIT.HIVE_TABLE_TYPE, AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME,
entityResult = atlasClientV1.updateEntity(BaseResourceIT.HIVE_TABLE_TYPE_BUILTIN, AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME,
(String) tableInstance.get(QUALIFIED_NAME), tableUpdated);
assertEquals(entityResult.getUpdateEntities().size(), 2);
assertEquals(entityResult.getUpdateEntities().get(0), tableId._getId());
......@@ -716,7 +715,7 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
values1.put("owner", "user1");
values1.put("position", 0);
values1.put("description", "col3");
values1.put("table", null);
values1.put("table", tableId);
Map<String, Object> values2 = new HashMap<>();
......@@ -727,10 +726,10 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
values2.put("owner", "user2");
values2.put("position", 1);
values2.put("description", "col4");
values2.put("table", null);
values2.put("table", tableId);
Referenceable ref1 = new Referenceable(BaseResourceIT.COLUMN_TYPE, values1);
Referenceable ref2 = new Referenceable(BaseResourceIT.COLUMN_TYPE, values2);
Referenceable ref1 = new Referenceable(BaseResourceIT.COLUMN_TYPE_BUILTIN, values1);
Referenceable ref2 = new Referenceable(BaseResourceIT.COLUMN_TYPE_BUILTIN, values2);
columns.add(ref1);
columns.add(ref2);
tableInstance.set("columns", columns);
......@@ -774,7 +773,7 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
@Test
public void testDeleteEntitiesViaRestApi() throws Exception {
// Create 2 database entities
Referenceable db1 = new Referenceable(DATABASE_TYPE);
Referenceable db1 = new Referenceable(DATABASE_TYPE_BUILTIN);
String dbName = randomString();
db1.set(NAME, dbName);
db1.set(DESCRIPTION, randomString());
......@@ -785,7 +784,7 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
db1.set("location", "/tmp");
Id db1Id = createInstance(db1);
Referenceable db2 = new Referenceable(DATABASE_TYPE);
Referenceable db2 = new Referenceable(DATABASE_TYPE_BUILTIN);
String dbName2 = randomString();
db2.set(NAME, dbName2);
db2.set(QUALIFIED_NAME, dbName2);
......@@ -818,7 +817,7 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
@Test
public void testDeleteEntitiesViaClientApi() throws Exception {
// Create 2 database entities
Referenceable db1 = new Referenceable(DATABASE_TYPE);
Referenceable db1 = new Referenceable(DATABASE_TYPE_BUILTIN);
String dbName = randomString();
db1.set("name", dbName);
db1.set("description", randomString());
......@@ -828,7 +827,7 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
db1.set("parameters", Collections.EMPTY_MAP);
db1.set("location", "/tmp");
Id db1Id = createInstance(db1);
Referenceable db2 = new Referenceable(DATABASE_TYPE);
Referenceable db2 = new Referenceable(DATABASE_TYPE_BUILTIN);
String dbName2 = randomString();
db2.set("name", dbName2);
db2.set(QUALIFIED_NAME, dbName2);
......@@ -859,7 +858,7 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
@Test
public void testDeleteEntityByUniqAttribute() throws Exception {
// Create database entity
Referenceable db1 = new Referenceable(DATABASE_TYPE);
Referenceable db1 = new Referenceable(DATABASE_TYPE_BUILTIN);
String dbName = randomString();
db1.set(NAME, dbName);
db1.set(QUALIFIED_NAME, dbName);
......@@ -873,7 +872,7 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
Id db1Id = createInstance(db1);
// Delete the database entity
List<String> deletedGuidsList = atlasClientV1.deleteEntity(DATABASE_TYPE, QUALIFIED_NAME, dbName).getDeletedEntities();
List<String> deletedGuidsList = atlasClientV1.deleteEntity(DATABASE_TYPE_BUILTIN, QUALIFIED_NAME, dbName).getDeletedEntities();
// Verify that deleteEntities() response has database entity guids
Assert.assertEquals(deletedGuidsList.size(), 1);
......
webapp/src/test/java/org/apache/atlas/web/resources/EntityV2JerseyResourceIT.java
......@@ -71,16 +71,10 @@ public class EntityV2JerseyResourceIT extends BaseResourceIT {
private final String DATABASE_NAME = "db" + randomString();
private final String TABLE_NAME = "table" + randomString();
private static final String TRAITS = "traits";
private static final String TRAIT_DEFINITION = "traitDefinitions";
private String traitName;
private AtlasEntity dbEntity;
private AtlasEntityHeader dbEntityHeader;
private AtlasEntityWithAssociations tableEntity;
private AtlasEntityHeader tableEntityHeader;
@Inject
private NotificationInterface notificationInterface;
private NotificationConsumer<EntityNotification> notificationConsumer;
......@@ -99,7 +93,7 @@ public class EntityV2JerseyResourceIT extends BaseResourceIT {
@Test
public void testSubmitEntity() throws Exception {
TypeUtils.Pair dbAndTable = createDBAndTable(DATABASE_NAME, TABLE_NAME);
TypeUtils.Pair dbAndTable = createDBAndTable();
assertNotNull(dbAndTable);
assertNotNull(dbAndTable.left);
assertNotNull(dbAndTable.right);
......@@ -107,18 +101,18 @@ public class EntityV2JerseyResourceIT extends BaseResourceIT {
@Test
public void testRequestUser() throws Exception {
AtlasEntity hiveDBInstanceV2 = createHiveDB(DATABASE_NAME);
AtlasEntity hiveDBInstanceV2 = createHiveDB(randomString());
List<EntityAuditEvent> events = atlasClientV1.getEntityAuditEvents(hiveDBInstanceV2.getGuid(), (short) 10);
assertTrue(events.size() > 1);
assertEquals(events.size(), 1);
assertEquals(events.get(0).getUser(), "admin");
}
@Test
public void testEntityDeduping() throws Exception {
JSONArray results = searchByDSL(String.format("%s where name='%s'", DATABASE_TYPE, DATABASE_NAME));
JSONArray results = searchByDSL(String.format("%s where name='%s'", DATABASE_TYPE_V2, DATABASE_NAME));
assertEquals(results.length(), 1);
final AtlasEntity hiveDBInstanceV2 = createHiveDB(DATABASE_NAME);
final AtlasEntity hiveDBInstanceV2 = createHiveDB();
// Do the notification thing here
waitForNotification(notificationConsumer, MAX_WAIT_TIME, new NotificationPredicate() {
@Override
......@@ -128,7 +122,7 @@ public class EntityV2JerseyResourceIT extends BaseResourceIT {
});
results = searchByDSL(String.format("%s where name='%s'", DATABASE_TYPE, DATABASE_NAME));
results = searchByDSL(String.format("%s where name='%s'", DATABASE_TYPE_V2, DATABASE_NAME));
assertEquals(results.length(), 1);
//Test the same across references
......@@ -139,7 +133,7 @@ public class EntityV2JerseyResourceIT extends BaseResourceIT {
EntityMutationResponse entity = entitiesClientV2.createEntity(hiveTableInstanceV2);
assertNotNull(entity);
assertNotNull(entity.getEntitiesByOperation(EntityMutations.EntityOperation.CREATE));
results = searchByDSL(String.format("%s where name='%s'", DATABASE_TYPE, DATABASE_NAME));
results = searchByDSL(String.format("%s where name='%s'", DATABASE_TYPE_V2, DATABASE_NAME));
assertEquals(results.length(), 1);
}
......@@ -205,7 +199,7 @@ public class EntityV2JerseyResourceIT extends BaseResourceIT {
@Test(dataProvider = "invalidAttrValues")
public void testEntityInvalidValue(String value) throws Exception {
AtlasEntity databaseInstance = new AtlasEntity(DATABASE_TYPE);
AtlasEntity databaseInstance = new AtlasEntity(DATABASE_TYPE_V2);
String dbName = randomString();
databaseInstance.setAttribute("name", dbName);
databaseInstance.setAttribute("description", value);
......@@ -216,18 +210,18 @@ public class EntityV2JerseyResourceIT extends BaseResourceIT {
@Test
public void testGetEntityByAttribute() throws Exception {
AtlasEntity hiveDB = createHiveDB(DATABASE_NAME);
String qualifiedName = (String) hiveDB.getAttribute(QUALIFIED_NAME);
AtlasEntity hiveDB = createHiveDB();
String qualifiedName = (String) hiveDB.getAttribute(NAME);
//get entity by attribute
AtlasEntity byAttribute = entitiesClientV2.getEntityByAttribute(DATABASE_TYPE, QUALIFIED_NAME, qualifiedName);
assertEquals(byAttribute.getTypeName(), DATABASE_TYPE);
assertEquals(byAttribute.getAttribute(QUALIFIED_NAME), qualifiedName);
AtlasEntity byAttribute = entitiesClientV2.getEntityByAttribute(DATABASE_TYPE_V2, NAME, qualifiedName);
assertEquals(byAttribute.getTypeName(), DATABASE_TYPE_V2);
assertEquals(byAttribute.getAttribute(NAME), qualifiedName);
}
@Test
public void testSubmitEntityWithBadDateFormat() throws Exception {
AtlasEntity hiveDBInstance = createHiveDBInstanceV2("db" + randomString());
AtlasEntityHeader entity = createEntity(hiveDBInstance);
createEntity(hiveDBInstance);
AtlasEntity tableInstance = createHiveTableInstanceV2(hiveDBInstance, "table" + randomString());
tableInstance.setAttribute("lastAccessTime", "2014-07-11");
......@@ -239,9 +233,9 @@ public class EntityV2JerseyResourceIT extends BaseResourceIT {
public void testAddProperty() throws Exception {
//add property
String description = "bar table - new desc";
addProperty(tableEntity.getGuid(), "description", description);
addProperty(createHiveTable().getGuid(), "description", description);
AtlasEntity entityByGuid = entitiesClientV2.getEntityByGuid(tableEntity.getGuid());
AtlasEntity entityByGuid = entitiesClientV2.getEntityByGuid(createHiveTable().getGuid());
Assert.assertNotNull(entityByGuid);
entityByGuid.setAttribute("description", description);
......@@ -260,9 +254,9 @@ public class EntityV2JerseyResourceIT extends BaseResourceIT {
String currentTime = String.valueOf(new DateTime());
addProperty(tableEntity.getGuid(), "createTime", currentTime);
addProperty(createHiveTable().getGuid(), "createTime", currentTime);
entityByGuid = entitiesClientV2.getEntityByGuid(tableEntity.getGuid());
entityByGuid = entitiesClientV2.getEntityByGuid(createHiveTable().getGuid());
Assert.assertNotNull(entityByGuid);
}
......@@ -271,7 +265,7 @@ public class EntityV2JerseyResourceIT extends BaseResourceIT {
// FIXME: Behavior has changed between v1 and v2
//add property
// try {
addProperty(tableEntity.getGuid(), "description", null);
addProperty(createHiveTable().getGuid(), "description", null);
// Assert.fail("Expected AtlasServiceException");
// } catch(AtlasServiceException e) {
// Assert.assertEquals(e.getStatus().getStatusCode(), Response.Status.BAD_REQUEST.getStatusCode());
......@@ -329,7 +323,7 @@ public class EntityV2JerseyResourceIT extends BaseResourceIT {
@Test(dependsOnMethods = "testSubmitEntity")
public void testGetTraitNames() throws Exception {
AtlasClassifications classifications = entitiesClientV2.getClassifications(tableEntity.getGuid());
AtlasClassifications classifications = entitiesClientV2.getClassifications(createHiveTable().getGuid());
assertNotNull(classifications);
assertTrue(classifications.getList().size() > 0);
assertEquals(classifications.getList().size(), 8);
......@@ -337,7 +331,7 @@ public class EntityV2JerseyResourceIT extends BaseResourceIT {
@Test(dependsOnMethods = "testSubmitEntity")
public void testCommonAttributes() throws Exception{
AtlasEntity entity = entitiesClientV2.getEntityByGuid(tableEntity.getGuid());
AtlasEntity entity = entitiesClientV2.getEntityByGuid(createHiveTable().getGuid());
Assert.assertNotNull(entity.getStatus());
Assert.assertNotNull(entity.getVersion());
Assert.assertNotNull(entity.getCreatedBy());
......@@ -356,11 +350,10 @@ public class EntityV2JerseyResourceIT extends BaseResourceIT {
}
private AtlasEntity createHiveDB() {
if (dbEntity != null) {
return dbEntity;
} else {
return createHiveDB(DATABASE_NAME);
if (dbEntity == null) {
dbEntity = createHiveDB(DATABASE_NAME);
}
return dbEntity;
}
private AtlasEntity createHiveDB(String dbName) {
......@@ -369,23 +362,21 @@ public class EntityV2JerseyResourceIT extends BaseResourceIT {
assertNotNull(entityHeader);
assertNotNull(entityHeader.getGuid());
hiveDBInstanceV2.setGuid(entityHeader.getGuid());
dbEntity = hiveDBInstanceV2;
dbEntityHeader = entityHeader;
return hiveDBInstanceV2;
}
private TypeUtils.Pair<AtlasEntity, AtlasEntityWithAssociations> createDBAndTable(String dbName, String tableName) throws Exception {
AtlasEntity dbInstanceV2 = createHiveDB(dbName);
AtlasEntityWithAssociations hiveTableInstanceV2 = createHiveTable(dbInstanceV2, tableName);
private TypeUtils.Pair<AtlasEntity, AtlasEntityWithAssociations> createDBAndTable() throws Exception {
AtlasEntity dbInstanceV2 = createHiveDB();
AtlasEntityWithAssociations hiveTableInstanceV2 = createHiveTable();
return TypeUtils.Pair.of(dbInstanceV2, hiveTableInstanceV2);
}
private AtlasEntityWithAssociations createHiveTable() throws Exception {
if (tableEntity != null) {
return tableEntity;
} else {
return createHiveTable(createHiveDB(), TABLE_NAME);
if (tableEntity == null) {
tableEntity = createHiveTable(createHiveDB(), TABLE_NAME);
}
return tableEntity;
}
private AtlasEntityWithAssociations createHiveTable(AtlasEntity dbInstanceV2, String tableName) throws Exception {
......@@ -396,7 +387,6 @@ public class EntityV2JerseyResourceIT extends BaseResourceIT {
hiveTableInstanceV2.setGuid(createdHeader.getGuid());
entitiesClientV2.addClassifications(createdHeader.getGuid(), hiveTableInstanceV2.getClassifications());
tableEntity = hiveTableInstanceV2;
tableEntityHeader = createdHeader;
return hiveTableInstanceV2;
}
......@@ -409,9 +399,9 @@ public class EntityV2JerseyResourceIT extends BaseResourceIT {
typesDef.getClassificationDefs().add(piiTrait);
createType(typesDef);
entitiesClientV2.addClassifications(tableEntity.getGuid(), ImmutableList.of(new AtlasClassification(piiTrait.getName())));
entitiesClientV2.addClassifications(createHiveTable().getGuid(), ImmutableList.of(new AtlasClassification(piiTrait.getName())));
assertEntityAudit(tableEntity.getGuid(), EntityAuditEvent.EntityAuditAction.TAG_ADD);
assertEntityAudit(createHiveTable().getGuid(), EntityAuditEvent.EntityAuditAction.TAG_ADD);
}
@Test(dependsOnMethods = "testSubmitEntity")
......@@ -426,13 +416,14 @@ public class EntityV2JerseyResourceIT extends BaseResourceIT {
AtlasClassificationDef classificationByName = typedefClientV2.getClassificationByName(traitName);
assertNotNull(classificationByName);
assertEquals(tableEntity.getClassifications().size(), 7);
AtlasEntityWithAssociations hiveTable = createHiveTable();
assertEquals(hiveTable.getClassifications().size(), 7);
AtlasClassification piiClassification = new AtlasClassification(piiTrait.getName());
entitiesClientV2.addClassifications(tableEntity.getGuid(), Lists.newArrayList(piiClassification));
entitiesClientV2.addClassifications(hiveTable.getGuid(), Lists.newArrayList(piiClassification));
AtlasClassifications classifications = entitiesClientV2.getClassifications(tableEntity.getGuid());
AtlasClassifications classifications = entitiesClientV2.getClassifications(hiveTable.getGuid());
assertNotNull(classifications);
assertTrue(classifications.getList().size() > 0);
assertEquals(classifications.getList().size(), 8);
......@@ -452,7 +443,7 @@ public class EntityV2JerseyResourceIT extends BaseResourceIT {
AtlasClassification traitInstance = new AtlasClassification(traitName);
traitInstance.setAttribute("type", "SSN");
final String guid = tableEntity.getGuid();
final String guid = createHiveTable().getGuid();
entitiesClientV2.addClassifications(guid, ImmutableList.of(traitInstance));
// verify the response
......@@ -474,7 +465,6 @@ public class EntityV2JerseyResourceIT extends BaseResourceIT {
@Test(expectedExceptions = AtlasServiceException.class)
public void testAddTraitWithNoRegistration() throws Exception {
final String traitName = "PII_Trait" + randomString();
AtlasClassificationDef piiTrait =
AtlasTypeUtil.createTraitTypeDef(traitName, ImmutableSet.<String>of());
AtlasClassification traitInstance = new AtlasClassification(traitName);
......@@ -484,7 +474,7 @@ public class EntityV2JerseyResourceIT extends BaseResourceIT {
@Test(dependsOnMethods = "testAddTrait")
public void testDeleteTrait() throws Exception {
final String guid = tableEntity.getGuid();
final String guid = createHiveTable().getGuid();
try {
entitiesClientV2.deleteClassification(guid, traitName);
......@@ -512,7 +502,7 @@ public class EntityV2JerseyResourceIT extends BaseResourceIT {
@Test(dependsOnMethods = "testSubmitEntity")
public void testDeleteExistentTraitNonExistentForEntity() throws Exception {
final String guid = tableEntity.getGuid();
final String guid = createHiveTable().getGuid();
final String traitName = "PII_Trait" + randomString();
AtlasClassificationDef piiTrait = AtlasTypeUtil
.createTraitTypeDef(traitName, ImmutableSet.<String>of(),
......@@ -562,43 +552,44 @@ public class EntityV2JerseyResourceIT extends BaseResourceIT {
final List<AtlasEntity> columns = new ArrayList<>();
Map<String, Object> values = new HashMap<>();
values.put("name", "col1");
values.put(QUALIFIED_NAME, "qualifiedName.col1");
values.put(NAME, "qualifiedName.col1");
values.put("type", "string");
values.put("comment", "col1 comment");
AtlasEntity ref = new AtlasEntity(BaseResourceIT.COLUMN_TYPE, values);
AtlasEntity ref = new AtlasEntity(BaseResourceIT.COLUMN_TYPE_V2, values);
columns.add(ref);
AtlasEntityWithAssociations hiveTable = createHiveTable();
AtlasEntityWithAssociations tableUpdated = hiveTable;
AtlasEntityWithAssociations tableUpdated = tableEntity;
tableEntity.setAttribute("columns", columns);
hiveTable.setAttribute("columns", columns);
LOG.debug("Updating entity= " + tableUpdated);
EntityMutationResponse updateResult = entitiesClientV2.updateEntity(tableEntity.getGuid(), tableUpdated);
EntityMutationResponse updateResult = entitiesClientV2.updateEntity(hiveTable.getGuid(), tableUpdated);
assertNotNull(updateResult);
assertNotNull(updateResult.getEntitiesByOperation(EntityMutations.EntityOperation.UPDATE));
assertTrue(updateResult.getEntitiesByOperation(EntityMutations.EntityOperation.UPDATE).size() > 0);
AtlasEntity entityByGuid = entitiesClientV2.getEntityByGuid(tableEntity.getGuid());
AtlasEntity entityByGuid = entitiesClientV2.getEntityByGuid(hiveTable.getGuid());
assertNotNull(entityByGuid);
List<AtlasEntity> columns1 = (List<AtlasEntity>) entityByGuid.getAttribute("columns");
entityByGuid.getAttribute("columns");
//Update by unique attribute
values.put("type", "int");
ref = new AtlasEntity(BaseResourceIT.COLUMN_TYPE, values);
ref = new AtlasEntity(BaseResourceIT.COLUMN_TYPE_V2, values);
columns.set(0, ref);
tableUpdated = tableEntity;
tableUpdated = hiveTable;
tableUpdated.setAttribute("columns", columns);
LOG.debug("Updating entity= " + tableUpdated);
EntityMutationResponse updateResponse = entitiesClientV2.updateEntityByAttribute(BaseResourceIT.HIVE_TABLE_TYPE, AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME,
(String) tableEntity.getAttribute("name"), tableUpdated);
EntityMutationResponse updateResponse = entitiesClientV2.updateEntityByAttribute(BaseResourceIT.HIVE_TABLE_TYPE_V2, AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME,
(String) hiveTable.getAttribute("name"), tableUpdated);
assertNotNull(updateResponse);
assertNotNull(updateResponse.getEntitiesByOperation(EntityMutations.EntityOperation.UPDATE));
assertTrue(updateResponse.getEntitiesByOperation(EntityMutations.EntityOperation.UPDATE).size() > 0);
entityByGuid = entitiesClientV2.getEntityByGuid(tableEntity.getGuid());
entityByGuid = entitiesClientV2.getEntityByGuid(hiveTable.getGuid());
assertNotNull(entityByGuid);
columns1 = (List<AtlasEntity>) entityByGuid.getAttribute("columns");
entityByGuid.getAttribute("columns");
}
@Test(dependsOnMethods = "testSubmitEntity")
......@@ -606,27 +597,31 @@ public class EntityV2JerseyResourceIT extends BaseResourceIT {
final List<AtlasEntity> columns = new ArrayList<>();
Map<String, Object> values1 = new HashMap<>();
values1.put("name", "col3");
values1.put(QUALIFIED_NAME, "qualifiedName.col3");
values1.put(NAME, "qualifiedName.col3");
values1.put("type", "string");
values1.put("comment", "col3 comment");
Map<String, Object> values2 = new HashMap<>();
values2.put("name", "col4");
values2.put(QUALIFIED_NAME, "qualifiedName.col4");
values2.put(NAME, "qualifiedName.col4");
values2.put("type", "string");
values2.put("comment", "col4 comment");
AtlasEntity ref1 = new AtlasEntity(BaseResourceIT.COLUMN_TYPE, values1);
AtlasEntity ref2 = new AtlasEntity(BaseResourceIT.COLUMN_TYPE, values2);
AtlasEntity ref1 = new AtlasEntity(BaseResourceIT.COLUMN_TYPE_V2, values1);
AtlasEntity ref2 = new AtlasEntity(BaseResourceIT.COLUMN_TYPE_V2, values2);
columns.add(ref1);
columns.add(ref2);
tableEntity.setAttribute("columns", columns);
EntityMutationResponse updateEntityResult = entitiesClientV2.updateEntity(tableEntity);
AtlasEntityWithAssociations hiveTable = createHiveTable();
hiveTable.setAttribute("columns", columns);
EntityMutationResponse updateEntityResult = entitiesClientV2.updateEntity(hiveTable);
assertNotNull(updateEntityResult);
assertNotNull(updateEntityResult.getEntitiesByOperation(EntityMutations.EntityOperation.UPDATE));
assertEquals(updateEntityResult.getEntitiesByOperation(EntityMutations.EntityOperation.UPDATE).size(), 3);
assertNotNull(updateEntityResult.getEntitiesByOperation(EntityMutations.EntityOperation.CREATE));
//2 columns are being created, and 1 hiveTable is being updated
assertEquals(updateEntityResult.getEntitiesByOperation(EntityMutations.EntityOperation.UPDATE).size(), 1);
assertEquals(updateEntityResult.getEntitiesByOperation(EntityMutations.EntityOperation.CREATE).size(), 2);
AtlasEntity entityByGuid = entitiesClientV2.getEntityByGuid(tableEntity.getGuid());
AtlasEntity entityByGuid = entitiesClientV2.getEntityByGuid(hiveTable.getGuid());
List<AtlasEntity> refs = (List<AtlasEntity>) entityByGuid.getAttribute("columns");
assertEquals(refs.size(), 2);
}
......@@ -634,17 +629,17 @@ public class EntityV2JerseyResourceIT extends BaseResourceIT {
@Test
public void testDeleteEntities() throws Exception {
// Create 2 database entities
AtlasEntity db1 = new AtlasEntity(DATABASE_TYPE);
AtlasEntity db1 = new AtlasEntity(DATABASE_TYPE_V2);
String dbName1 = randomString();
db1.setAttribute("name", dbName1);
db1.setAttribute(QUALIFIED_NAME, dbName1);
db1.setAttribute(NAME, dbName1);
db1.setAttribute("clusterName", randomString());
db1.setAttribute("description", randomString());
AtlasEntityHeader entity1Header = createEntity(db1);
AtlasEntity db2 = new AtlasEntity(DATABASE_TYPE);
AtlasEntity db2 = new AtlasEntity(DATABASE_TYPE_V2);
String dbName2 = randomString();
db2.setAttribute("name", dbName2);
db2.setAttribute(QUALIFIED_NAME, dbName2);
db2.setAttribute(NAME, dbName2);
db2.setAttribute("clusterName", randomString());
db2.setAttribute("description", randomString());
AtlasEntityHeader entity2Header = createEntity(db2);
......@@ -664,10 +659,10 @@ public class EntityV2JerseyResourceIT extends BaseResourceIT {
public void testDeleteEntityByUniqAttribute() throws Exception {
// Create database entity
AtlasEntity hiveDB = createHiveDB(DATABASE_NAME + random());
String qualifiedName = (String) hiveDB.getAttribute(QUALIFIED_NAME);
String qualifiedName = (String) hiveDB.getAttribute(NAME);
// Delete the database entity
EntityMutationResponse deleteResponse = entitiesClientV2.deleteEntityByAttribute(DATABASE_TYPE, QUALIFIED_NAME, qualifiedName);
EntityMutationResponse deleteResponse = entitiesClientV2.deleteEntityByAttribute(DATABASE_TYPE_V2, NAME, qualifiedName);
// Verify that deleteEntities() response has database entity guids
assertNotNull(deleteResponse);
......
webapp/src/test/java/org/apache/atlas/web/resources/MetadataDiscoveryJerseyResourceIT.java
......@@ -66,7 +66,7 @@ public class MetadataDiscoveryJerseyResourceIT extends BaseResourceIT {
@Test
public void testSearchByDSL() throws Exception {
String dslQuery = "from "+ DATABASE_TYPE + " qualifiedName=\"" + dbName + "\"";
String dslQuery = "from "+ DATABASE_TYPE + " name=\"" + dbName + "\"";
MultivaluedMap<String, String> queryParams = new MultivaluedMapImpl();
queryParams.add("query", dslQuery);
JSONObject response = atlasClientV1.callAPIWithQueryParams(AtlasClient.API.SEARCH_DSL, queryParams);
......@@ -89,7 +89,7 @@ public class MetadataDiscoveryJerseyResourceIT extends BaseResourceIT {
public void testSearchDSLLimits() throws Exception {
//search without new parameters of limit and offset should work
String dslQuery = "from "+ DATABASE_TYPE + " qualifiedName=\"" + dbName + "\"";
String dslQuery = "from "+ DATABASE_TYPE + " name=\"" + dbName + "\"";
MultivaluedMap<String, String> queryParams = new MultivaluedMapImpl();
queryParams.add("query", dslQuery);
JSONObject response = atlasClientV1.callAPIWithQueryParams(AtlasClient.API.SEARCH_DSL, queryParams);
......@@ -146,7 +146,7 @@ public class MetadataDiscoveryJerseyResourceIT extends BaseResourceIT {
@Test
public void testSearchUsingGremlin() throws Exception {
String query = "g.V.has('type', 'hive_db').toList()";
String query = "g.V.has('type', '" + BaseResourceIT.HIVE_TABLE_TYPE + "').toList()";
MultivaluedMap<String, String> queryParams = new MultivaluedMapImpl();
queryParams.add("query", query);
......@@ -162,7 +162,7 @@ public class MetadataDiscoveryJerseyResourceIT extends BaseResourceIT {
@Test
public void testSearchUsingDSL() throws Exception {
//String query = "from dsl_test_type";
String query = "from "+ DATABASE_TYPE + " qualifiedName=\"" + dbName +"\"";
String query = "from "+ DATABASE_TYPE + " name=\"" + dbName +"\"";
MultivaluedMap<String, String> queryParams = new MultivaluedMapImpl();
queryParams.add("query", query);
JSONObject response = atlasClientV1.callAPIWithQueryParams(AtlasClient.API.SEARCH, queryParams);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment