Skip to content

A
atlas
Commit 2f1cb57a by Suma Shivaprasad Committed by Madhan Neethiraj
Options

ATLAS-1467: instance create/full-Update implementation

Signed-off-by: 's avatarMadhan Neethiraj <madhan@apache.org>
parent 511c8867
Showing with 2855 additions and 232 deletions
intg/src/main/java/org/apache/atlas/AtlasErrorCode.java
......@@ -62,6 +62,7 @@ public enum AtlasErrorCode {
INSTANCE_LINEAGE_INVALID_PARAMS(404, "ATLAS4046E", "Invalid lineage query parameters passed {0}: {1}"),
INSTANCE_LINEAGE_QUERY_FAILED(404, "ATLAS4047E", "Instance lineage query failed {0}"),
DISCOVERY_QUERY_FAILED(404, "ATLAS4048E", "Discovery query failed {0}"),
INSTANCE_CRUD_INVALID_PARAMS(404, "ATLAS4049E", "Invalid instance creation/updation parameters passed : {0}"),
// All data conflict errors go here
......@@ -72,7 +73,11 @@ public enum AtlasErrorCode {
// All internal errors go here
INTERNAL_ERROR(500, "ATLAS5001E", "Internal server error {0}"),
INDEX_CREATION_FAILED(500, "ATLAS5002E", "Index creation failed for {0}"),
INDEX_ROLLBACK_FAILED(500, "ATLAS5003E", "Index rollback failed for {0}");
INDEX_ROLLBACK_FAILED(500, "ATLAS5003E", "Index rollback failed for {0}"),
INSTANCE_BY_UNIQUE_ATTRIBUTE_NOT_FOUND(400, "ATLAS40018E", "Instance {0} with unique attribute {1} does not exist"),
UNKNOWN_ATTRIBUTE(400, "ATLAS40019E", "Attribute {0} not found for type {1}");
private String errorCode;
private String errorMessage;
......
intg/src/main/java/org/apache/atlas/model/instance/AtlasEntity.java
......@@ -55,15 +55,15 @@ public class AtlasEntity extends AtlasStruct implements Serializable {
/**
* Status of the entity - can be active or deleted. Deleted entities are not removed from Atlas store.
*/
public enum Status { STATUS_ACTIVE, STATUS_DELETED }
public enum Status { ACTIVE, DELETED }
private String guid = null;
private Status status = Status.STATUS_ACTIVE;
private Status status = Status.ACTIVE;
private String createdBy = null;
private String updatedBy = null;
private Date createTime = null;
private Date updateTime = null;
private Long version = null;
private Long version = new Long(0);
@JsonIgnore
private static AtomicLong s_nextId = new AtomicLong(System.nanoTime());
......@@ -89,7 +89,6 @@ public class AtlasEntity extends AtlasStruct implements Serializable {
setUpdatedBy(null);
setCreateTime(null);
setUpdateTime(null);
setVersion(null);
}
public AtlasEntity(AtlasEntity other) {
......@@ -247,7 +246,7 @@ public class AtlasEntity extends AtlasStruct implements Serializable {
@JsonIgnore
public boolean isUnassigned() {
return guid != null && guid.length() > 0 && guid.charAt(0) == '-';
return isUnAssigned(guid);
}
@JsonIgnore
......@@ -266,6 +265,11 @@ public class AtlasEntity extends AtlasStruct implements Serializable {
return true;
}
@JsonIgnore
public static boolean isUnAssigned(String guid) {
return guid != null && guid.length() > 0 && guid.charAt(0) == '-';
}
private String nextInternalId() {
return "-" + Long.toString(s_nextId.getAndIncrement());
}
......
intg/src/main/java/org/apache/atlas/model/instance/AtlasEntityHeader.java
......@@ -18,6 +18,7 @@
package org.apache.atlas.model.instance;
import java.io.Serializable;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
......@@ -49,7 +50,7 @@ public class AtlasEntityHeader extends AtlasStruct implements Serializable {
private static final long serialVersionUID = 1L;
private String guid = null;
private AtlasEntity.Status status = AtlasEntity.Status.STATUS_ACTIVE;
private AtlasEntity.Status status = AtlasEntity.Status.ACTIVE;
private String displayText = null;
public AtlasEntityHeader() {
......@@ -66,11 +67,15 @@ public class AtlasEntityHeader extends AtlasStruct implements Serializable {
public AtlasEntityHeader(String typeName, Map<String, Object> attributes) {
super(typeName, attributes);
}
setGuid(null);
setStatus(null);
public AtlasEntityHeader(String typeName, String guid, Map<String, Object> attributes) {
super(typeName, attributes);
setGuid(guid);
}
public AtlasEntityHeader(AtlasEntityHeader other) {
super(other);
......
intg/src/main/java/org/apache/atlas/model/instance/EntityMutations.java
......@@ -43,7 +43,8 @@ public class EntityMutations implements Serializable {
private List<EntityMutation> entityMutations = new ArrayList<>();
public enum EntityOperation {
CREATE_OR_UPDATE,
CREATE,
UPDATE,
PARTIAL_UPDATE,
DELETE,
}
......
intg/src/main/java/org/apache/atlas/model/typedef/AtlasStructDef.java
......@@ -19,6 +19,7 @@ package org.apache.atlas.model.typedef;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
......@@ -183,7 +184,7 @@ public class AtlasStructDef extends AtlasBaseTypeDef implements Serializable {
return findAttribute(attributeDefs, attrName) != null;
}
private static AtlasAttributeDef findAttribute(List<AtlasAttributeDef> attributeDefs, String attrName) {
public static AtlasAttributeDef findAttribute(Collection<AtlasAttributeDef> attributeDefs, String attrName) {
AtlasAttributeDef ret = null;
if (CollectionUtils.isNotEmpty(attributeDefs)) {
......@@ -451,6 +452,8 @@ public class AtlasStructDef extends AtlasBaseTypeDef implements Serializable {
}
}
/**
* class that captures details of a constraint.
*/
......
intg/src/main/java/org/apache/atlas/type/AtlasClassificationType.java
......@@ -46,8 +46,6 @@ public class AtlasClassificationType extends AtlasStructType {
private List<AtlasClassificationType> superTypes = Collections.emptyList();
private Set<String> allSuperTypes = Collections.emptySet();
private Map<String, AtlasAttributeDef> allAttributeDefs = Collections.emptyMap();
private Map<String, AtlasType> allAttributeTypes = new HashMap<>();
public AtlasClassificationType(AtlasClassificationDef classificationDef) {
super(classificationDef);
......@@ -72,7 +70,7 @@ public class AtlasClassificationType extends AtlasStructType {
List<AtlasClassificationType> s = new ArrayList<>();
Set<String> allS = new HashSet<>();
Map<String, AtlasAttributeDef> allA = new HashMap<>();
Map<String, AtlasAttribute> allA = new HashMap<>();
getTypeHierarchyInfo(typeRegistry, allS, allA);
......@@ -89,8 +87,7 @@ public class AtlasClassificationType extends AtlasStructType {
this.superTypes = Collections.unmodifiableList(s);
this.allSuperTypes = Collections.unmodifiableSet(allS);
this.allAttributeDefs = Collections.unmodifiableMap(allA);
this.allAttributeTypes = new HashMap<>(); // this will be rebuilt on calls to getAttributeType()
this.allAttributes = Collections.unmodifiableMap(allA);
}
public Set<String> getSuperTypes() {
......@@ -99,51 +96,6 @@ public class AtlasClassificationType extends AtlasStructType {
public Set<String> getAllSuperTypes() { return allSuperTypes; }
public Map<String, AtlasAttributeDef> getAllAttributeDefs() { return allAttributeDefs; }
@Override
public AtlasType getAttributeType(String attributeName) {
AtlasType ret = allAttributeTypes.get(attributeName);
if (ret == null) {
ret = super.getAttributeType(attributeName);
if (ret == null) {
for (AtlasClassificationType superType : superTypes) {
ret = superType.getAttributeType(attributeName);
if (ret != null) {
break;
}
}
}
if (ret != null) {
allAttributeTypes.put(attributeName, ret);
}
}
return ret;
}
@Override
public AtlasAttributeDef getAttributeDef(String attributeName) {
AtlasAttributeDef ret = super.getAttributeDef(attributeName);
if (ret == null) {
for (AtlasClassificationType superType : superTypes) {
ret = superType.getAttributeDef(attributeName);
if (ret != null) {
break;
}
}
}
return ret;
}
public boolean isSuperTypeOf(AtlasClassificationType classificationType) {
return classificationType != null && classificationType.getAllSuperTypes().contains(this.getTypeName());
}
......@@ -243,10 +195,10 @@ public class AtlasClassificationType extends AtlasStructType {
private void getTypeHierarchyInfo(AtlasTypeRegistry typeRegistry,
Set<String> allSuperTypeNames,
Map<String, AtlasAttributeDef> allAttributeDefs) throws AtlasBaseException {
Map<String, AtlasAttribute> allAttributes) throws AtlasBaseException {
List<String> visitedTypes = new ArrayList<>();
collectTypeHierarchyInfo(typeRegistry, allSuperTypeNames, allAttributeDefs, visitedTypes);
collectTypeHierarchyInfo(typeRegistry, allSuperTypeNames, allAttributes, visitedTypes);
}
/*
......@@ -255,7 +207,7 @@ public class AtlasClassificationType extends AtlasStructType {
*/
private void collectTypeHierarchyInfo(AtlasTypeRegistry typeRegistry,
Set<String> allSuperTypeNames,
Map<String, AtlasAttributeDef> allAttributeDefs,
Map<String, AtlasAttribute> allAttributes,
List<String> visitedTypes) throws AtlasBaseException {
if (visitedTypes.contains(classificationDef.getName())) {
throw new AtlasBaseException(AtlasErrorCode.CIRCULAR_REFERENCE, classificationDef.getName(),
......@@ -270,7 +222,7 @@ public class AtlasClassificationType extends AtlasStructType {
if (type instanceof AtlasClassificationType) {
AtlasClassificationType superType = (AtlasClassificationType) type;
superType.collectTypeHierarchyInfo(typeRegistry, allSuperTypeNames, allAttributeDefs, visitedTypes);
superType.collectTypeHierarchyInfo(typeRegistry, allSuperTypeNames, allAttributes, visitedTypes);
}
}
visitedTypes.remove(classificationDef.getName());
......@@ -280,7 +232,8 @@ public class AtlasClassificationType extends AtlasStructType {
if (CollectionUtils.isNotEmpty(classificationDef.getAttributeDefs())) {
for (AtlasAttributeDef attributeDef : classificationDef.getAttributeDefs()) {
allAttributeDefs.put(attributeDef.getName(), attributeDef);
AtlasType type = typeRegistry.getType(attributeDef.getTypeName());
allAttributes.put(attributeDef.getName(), new AtlasAttribute(this, classificationDef, attributeDef, type));
}
}
}
......
intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java
......@@ -21,9 +21,11 @@ package org.apache.atlas.type;
import org.apache.atlas.AtlasErrorCode;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.model.instance.AtlasObjectId;
import org.apache.atlas.model.typedef.AtlasEntityDef;
import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
......@@ -45,8 +47,6 @@ public class AtlasEntityType extends AtlasStructType {
private List<AtlasEntityType> superTypes = Collections.emptyList();
private Set<String> allSuperTypes = Collections.emptySet();
private Map<String, AtlasAttributeDef> allAttributeDefs = Collections.emptyMap();
private Map<String, AtlasType> allAttributeTypes = new HashMap<>();
public AtlasEntityType(AtlasEntityDef entityDef) {
super(entityDef);
......@@ -70,7 +70,7 @@ public class AtlasEntityType extends AtlasStructType {
List<AtlasEntityType> s = new ArrayList<>();
Set<String> allS = new HashSet<>();
Map<String, AtlasAttributeDef> allA = new HashMap<>();
Map<String, AtlasAttribute> allA = new HashMap<>();
getTypeHierarchyInfo(typeRegistry, allS, allA);
......@@ -86,8 +86,7 @@ public class AtlasEntityType extends AtlasStructType {
this.superTypes = Collections.unmodifiableList(s);
this.allSuperTypes = Collections.unmodifiableSet(allS);
this.allAttributeDefs = Collections.unmodifiableMap(allA);
this.allAttributeTypes = new HashMap<>(); // this will be rebuilt on calls to getAttributeType()
this.allAttributes = Collections.unmodifiableMap(allA);
}
public Set<String> getSuperTypes() {
......@@ -98,50 +97,6 @@ public class AtlasEntityType extends AtlasStructType {
return allSuperTypes;
}
public Map<String, AtlasAttributeDef> getAllAttributeDefs() { return allAttributeDefs; }
@Override
public AtlasType getAttributeType(String attributeName) {
AtlasType ret = allAttributeTypes.get(attributeName);
if (ret == null) {
ret = super.getAttributeType(attributeName);
if (ret == null) {
for (AtlasEntityType superType : superTypes) {
ret = superType.getAttributeType(attributeName);
if (ret != null) {
break;
}
}
}
if (ret != null) {
allAttributeTypes.put(attributeName, ret);
}
}
return ret;
}
@Override
public AtlasAttributeDef getAttributeDef(String attributeName) {
AtlasAttributeDef ret = super.getAttributeDef(attributeName);
if (ret == null) {
for (AtlasEntityType superType : superTypes) {
ret = superType.getAttributeDef(attributeName);
if (ret != null) {
break;
}
}
}
return ret;
}
public boolean isSuperTypeOf(AtlasEntityType entityType) {
return entityType != null && entityType.getAllSuperTypes().contains(this.getTypeName());
}
......@@ -150,6 +105,10 @@ public class AtlasEntityType extends AtlasStructType {
return entityType != null && allSuperTypes.contains(entityType.getTypeName());
}
public boolean isSubTypeOf(String entityTypeName) {
return StringUtils.isNotEmpty(entityTypeName) && allSuperTypes.contains(entityTypeName);
}
@Override
public AtlasEntity createDefaultValue() {
AtlasEntity ret = new AtlasEntity(entityDef.getName());
......@@ -162,14 +121,18 @@ public class AtlasEntityType extends AtlasStructType {
@Override
public boolean isValidValue(Object obj) {
if (obj != null) {
if (obj instanceof AtlasObjectId) {
AtlasObjectId objId = (AtlasObjectId ) obj;
return validateAtlasObjectId(objId);
} else {
for (AtlasEntityType superType : superTypes) {
if (!superType.isValidValue(obj)) {
return false;
}
}
return super.isValidValue(obj);
}
}
return true;
}
......@@ -186,6 +149,8 @@ public class AtlasEntityType extends AtlasStructType {
} else if (obj instanceof Map) {
normalizeAttributeValues((Map) obj);
ret = obj;
} else if (obj instanceof AtlasObjectId) {
ret = obj;
}
}
}
......@@ -194,10 +159,20 @@ public class AtlasEntityType extends AtlasStructType {
}
@Override
public AtlasAttribute getAttribute(String attributeName) {
return findAttribute(allAttributes.values(), attributeName);
}
@Override
public boolean validateValue(Object obj, String objName, List<String> messages) {
boolean ret = true;
if (obj != null) {
if (obj instanceof AtlasObjectId) {
AtlasObjectId objId = (AtlasObjectId ) obj;
return validateAtlasObjectId(objId);
}
for (AtlasEntityType superType : superTypes) {
ret = superType.validateValue(obj, objName, messages) && ret;
}
......@@ -241,10 +216,10 @@ public class AtlasEntityType extends AtlasStructType {
private void getTypeHierarchyInfo(AtlasTypeRegistry typeRegistry,
Set<String> allSuperTypeNames,
Map<String, AtlasAttributeDef> allAttributeDefs) throws AtlasBaseException {
Map<String, AtlasAttribute> allAttributes) throws AtlasBaseException {
List<String> visitedTypes = new ArrayList<>();
collectTypeHierarchyInfo(typeRegistry, allSuperTypeNames, allAttributeDefs, visitedTypes);
collectTypeHierarchyInfo(typeRegistry, allSuperTypeNames, allAttributes, visitedTypes);
}
/*
......@@ -253,7 +228,7 @@ public class AtlasEntityType extends AtlasStructType {
*/
private void collectTypeHierarchyInfo(AtlasTypeRegistry typeRegistry,
Set<String> allSuperTypeNames,
Map<String, AtlasAttributeDef> allAttributeDefs,
Map<String, AtlasAttribute> allAttributes,
List<String> visitedTypes) throws AtlasBaseException {
if (visitedTypes.contains(entityDef.getName())) {
throw new AtlasBaseException(AtlasErrorCode.CIRCULAR_REFERENCE, entityDef.getName(),
......@@ -267,19 +242,36 @@ public class AtlasEntityType extends AtlasStructType {
if (type instanceof AtlasEntityType) {
AtlasEntityType superType = (AtlasEntityType) type;
superType.collectTypeHierarchyInfo(typeRegistry, allSuperTypeNames, allAttributeDefs, visitedTypes);
superType.collectTypeHierarchyInfo(typeRegistry, allSuperTypeNames, allAttributes, visitedTypes);
}
}
visitedTypes.remove(entityDef.getName());
allSuperTypeNames.addAll(entityDef.getSuperTypes());
}
if (CollectionUtils.isNotEmpty(entityDef.getAttributeDefs())) {
for (AtlasAttributeDef attributeDef : entityDef.getAttributeDefs()) {
allAttributeDefs.put(attributeDef.getName(), attributeDef);
AtlasType type = typeRegistry.getType(attributeDef.getTypeName());
allAttributes.put(attributeDef.getName(), new AtlasAttribute(this, entityDef, attributeDef, type));
}
}
}
private boolean validateAtlasObjectId(AtlasObjectId objId) {
if (StringUtils.isEmpty(objId.getTypeName()) || StringUtils.isEmpty(objId.getGuid())) {
return false;
} else {
String typeName = objId.getTypeName();
if (!typeName.equals(getTypeName())) {
//TODO - Enable below after enabling subType check
// if ( !isSuperTypeOf(typeName)) {
// return false;
// }
return false;
}
}
return AtlasEntity.isAssigned(objId.getGuid()) || AtlasEntity.isUnAssigned((objId.getGuid()));
}
}
intg/src/test/java/org/apache/atlas/type/TestAtlasTypeRegistry.java
......@@ -355,17 +355,17 @@ public class TestAtlasTypeRegistry {
} catch (AtlasBaseException excp) {
}
Map<String, AtlasAttributeDef> attributeDefs = null;
Map<String, AtlasStructType.AtlasAttribute> attributes = null;
if (type != null) {
if (type instanceof AtlasEntityType) {
attributeDefs = ((AtlasEntityType) type).getAllAttributeDefs();
attributes = ((AtlasEntityType) type).getAllAttributes();
} else if (type instanceof AtlasClassificationType) {
attributeDefs = ((AtlasClassificationType) type).getAllAttributeDefs();
attributes = ((AtlasClassificationType) type).getAllAttributes();
}
}
assertNotNull(attributeDefs);
assertEquals(attributeDefs.keySet(), attributeNames);
assertNotNull(attributes);
assertEquals(attributes.keySet(), attributeNames);
}
}
release-log.txt
......@@ -9,7 +9,11 @@ ATLAS-1060 Add composite indexes for exact match performance improvements for al
ATLAS-1127 Modify creation and modification timestamps to Date instead of Long(sumasai)
ALL CHANGES:
ATLAS-1467 instance create/full-Update implementation (sumasai via mneethiraj)
ATLAS-1463 option to exclude specific entity attributes in audit records (sarath.kum4r@gmail.com via mneethiraj)
ATLAS-1386 Avoid uunnecessary type cache lookups (jnhagelb)
ATLAS-1000 added build instructions to README.txt (mneethiraj)
ATLAS-1471 avoid unnecessary overhead in debug log calls (mneethiraj)
ATLAS-1464 option to include only specified attributes in notification message (sarath.kum4r@gmail.com via mneethiraj)
ATLAS-1460 v2 search API updated to return name/description/owner and classification names in result (vimalsharma via mneethiraj)
ATLAS-1434 fixed unit test to use correct type names; updated error message per review comments (ashutoshm via mneethiraj)
......
repository/src/main/java/org/apache/atlas/RepositoryMetadataModule.java
......@@ -24,6 +24,7 @@ import com.google.inject.matcher.Matchers;
import com.google.inject.multibindings.Multibinder;
import org.aopalliance.intercept.MethodInterceptor;
import org.apache.atlas.discovery.AtlasDiscoveryService;
import org.apache.atlas.discovery.AtlasLineageService;
import org.apache.atlas.discovery.DataSetLineageService;
import org.apache.atlas.discovery.DiscoveryService;
......@@ -34,7 +35,6 @@ import org.apache.atlas.discovery.graph.GraphBackedDiscoveryService;
import org.apache.atlas.listener.EntityChangeListener;
import org.apache.atlas.listener.TypeDefChangeListener;
import org.apache.atlas.listener.TypesChangeListener;
import org.apache.atlas.discovery.AtlasDiscoveryService;
import org.apache.atlas.repository.MetadataRepository;
import org.apache.atlas.repository.audit.EntityAuditListener;
import org.apache.atlas.repository.audit.EntityAuditRepository;
......@@ -42,8 +42,17 @@ import org.apache.atlas.repository.graph.DeleteHandler;
import org.apache.atlas.repository.graph.GraphBackedMetadataRepository;
import org.apache.atlas.repository.graph.GraphBackedSearchIndexer;
import org.apache.atlas.repository.store.graph.AtlasEntityStore;
import org.apache.atlas.repository.store.graph.EntityGraphDiscovery;
import org.apache.atlas.repository.store.graph.EntityResolver;
import org.apache.atlas.repository.store.graph.v1.ArrayVertexMapper;
import org.apache.atlas.repository.store.graph.v1.AtlasEntityGraphDiscoveryV1;
import org.apache.atlas.repository.store.graph.v1.AtlasEntityStoreV1;
import org.apache.atlas.repository.store.graph.v1.AtlasTypeDefGraphStoreV1;
import org.apache.atlas.repository.store.graph.v1.DeleteHandlerV1;
import org.apache.atlas.repository.store.graph.v1.EntityGraphMapper;
import org.apache.atlas.repository.store.graph.v1.IDBasedEntityResolver;
import org.apache.atlas.repository.store.graph.v1.MapVertexMapper;
import org.apache.atlas.repository.store.graph.v1.UniqAttrBasedEntityResolver;
import org.apache.atlas.repository.typestore.GraphBackedTypeStore;
import org.apache.atlas.repository.typestore.ITypeStore;
import org.apache.atlas.service.Service;
......@@ -106,8 +115,21 @@ public class RepositoryMetadataModule extends com.google.inject.AbstractModule {
bind(DeleteHandler.class).to(AtlasRepositoryConfiguration.getDeleteHandlerImpl()).asEagerSingleton();
bind(DeleteHandlerV1.class).to(AtlasRepositoryConfiguration.getDeleteHandlerV1Impl()).asEagerSingleton();
bind(TypeCache.class).to(AtlasRepositoryConfiguration.getTypeCache()).asEagerSingleton();
bind(EntityGraphMapper.class);
bind(MapVertexMapper.class).asEagerSingleton();
bind(ArrayVertexMapper.class).asEagerSingleton();
Multibinder<EntityResolver> entityRefResolver =
Multibinder.newSetBinder(binder(), EntityResolver.class);
entityRefResolver.addBinding().to(IDBasedEntityResolver.class);
entityRefResolver.addBinding().to(UniqAttrBasedEntityResolver.class);
//Add EntityAuditListener as EntityChangeListener
Multibinder<EntityChangeListener> entityChangeListenerBinder =
Multibinder.newSetBinder(binder(), EntityChangeListener.class);
......@@ -116,6 +138,8 @@ public class RepositoryMetadataModule extends com.google.inject.AbstractModule {
MethodInterceptor interceptor = new GraphTransactionInterceptor();
requestInjection(interceptor);
bindInterceptor(Matchers.any(), Matchers.annotatedWith(GraphTransaction.class), interceptor);
bind(EntityGraphDiscovery.class).to(AtlasEntityGraphDiscoveryV1.class);
}
protected Configuration getConfiguration() {
......
repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java
......@@ -228,7 +228,7 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
String state = vertex.getProperty(Constants.STATE_PROPERTY_KEY, String.class);
if (state != null) {
Status status = (state.equalsIgnoreCase("ACTIVE") ? Status.STATUS_ACTIVE : Status.STATUS_DELETED);
Status status = (state.equalsIgnoreCase("ACTIVE") ? Status.ACTIVE : Status.DELETED);
ret.setStatus(status);
}
......
repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java
......@@ -184,7 +184,7 @@ public class EntityLineageService implements AtlasLineageService {
ret.setDisplayText(vertex.getProperty(Constants.QUALIFIED_NAME, String.class));
String state = vertex.getProperty(Constants.STATE_PROPERTY_KEY, String.class);
Status status = (state.equalsIgnoreCase("ACTIVE") ? Status.STATUS_ACTIVE : Status.STATUS_DELETED);
Status status = (state.equalsIgnoreCase("ACTIVE") ? Status.ACTIVE : Status.DELETED);
ret.setStatus(status);
}
......
repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java
......@@ -967,7 +967,7 @@ public final class GraphHelper {
instanceVertex.setListProperty(actualPropertyName, value);
}
public static List<String> getListProperty(AtlasVertex instanceVertex, String propertyName) throws AtlasException {
public static List<String> getListProperty(AtlasVertex instanceVertex, String propertyName) {
String actualPropertyName = GraphHelper.encodePropertyKey(propertyName);
return instanceVertex.getListProperty(actualPropertyName);
}
......
repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasEntityStore.java
......@@ -25,6 +25,7 @@ import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.model.instance.AtlasEntityWithAssociations;
import org.apache.atlas.model.instance.EntityMutations;
import org.apache.atlas.model.instance.EntityMutationResponse;
import org.apache.atlas.type.AtlasTypeRegistry;
import java.util.List;
......@@ -36,14 +37,14 @@ public interface AtlasEntityStore {
/**
* Initialization
*/
void init() throws AtlasBaseException;
void init(AtlasTypeRegistry typeRegistry, EntityGraphDiscovery graphDiscovery) throws AtlasBaseException;
/**
* Create or update an entity if it already exists.
* @param entity
* @return
*/
EntityMutationResponse createOrUpdate(AtlasEntity entity);
EntityMutationResponse createOrUpdate(AtlasEntity entity) throws AtlasBaseException;
/**
......@@ -175,4 +176,5 @@ public interface AtlasEntityStore {
* @throws AtlasBaseException
*/
AtlasEntity.AtlasEntities searchEntities(SearchFilter searchFilter) throws AtlasBaseException;
}
repository/src/main/java/org/apache/atlas/repository/store/graph/EntityGraphDiscovery.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.repository.store.graph;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.instance.AtlasEntity;
import java.util.List;
public interface EntityGraphDiscovery {
void init() throws AtlasBaseException;
/*
* Return list of resolved and unresolved references.
* Resolved references already exist in the ATLAS repository and have an assigned unique GUID
* Unresolved attribute references result in an error if they are not composite (managed by a parent entity)
*/
EntityGraphDiscoveryContext discoverEntities(List<AtlasEntity> entities) throws AtlasBaseException;
void cleanUp() throws AtlasBaseException;
}
repository/src/main/java/org/apache/atlas/repository/store/graph/EntityGraphDiscoveryContext.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.repository.store.graph;
import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.model.instance.AtlasObjectId;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.type.AtlasEntityType;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
public final class EntityGraphDiscoveryContext {
/**
* Keeps track of all the entities that need to be created/updated including its child entities *
*/
private Set<AtlasEntity> rootEntities = new LinkedHashSet<>();
//Key is a transient id/guid
/**
* These references have been resolved using a unique identifier like guid or a qualified name etc in Atlas repository
*/
private Map<String, AtlasVertex> repositoryResolvedReferences = new LinkedHashMap<>();
/**
* Unresolved entity references
*/
private List<AtlasEntity> unresolvedEntityReferences = new ArrayList<>();
/**
* Unresolved entity id references
*/
private Set<AtlasObjectId> unresolvedIdReferences = new HashSet<>();
public void addRepositoryResolvedReference(AtlasObjectId id, AtlasVertex vertex) {
repositoryResolvedReferences.put(id.getGuid(), vertex);
}
public void addUnResolvedEntityReference(AtlasEntity entity) {
this.unresolvedEntityReferences.add(entity);
}
public void addUnResolvedIdReference(AtlasEntityType entityType, String id) {
this.unresolvedIdReferences.add(new AtlasObjectId(entityType.getTypeName(), id));
}
public Set<AtlasObjectId> getUnresolvedIdReferences() {
return unresolvedIdReferences;
}
public boolean isResolved(String guid) {
return repositoryResolvedReferences.containsKey(guid);
}
public AtlasVertex getResolvedReference(AtlasObjectId ref) {
return repositoryResolvedReferences.get(ref.getGuid());
}
public Map<String, AtlasVertex> getRepositoryResolvedReferences() {
return repositoryResolvedReferences;
}
public AtlasVertex getResolvedReference(String id) {
return repositoryResolvedReferences.get(id);
}
public List<AtlasEntity> getUnResolvedEntityReferences() {
return unresolvedEntityReferences;
}
public void addRootEntity(AtlasEntity rootEntity) {
this.rootEntities.add(rootEntity);
}
public Collection<AtlasEntity> getRootEntities() {
return rootEntities;
}
public boolean removeUnResolvedEntityReference(final AtlasEntity entity) {
return unresolvedEntityReferences.remove(entity);
}
public boolean removeUnResolvedEntityReferences(final List<AtlasEntity> entities) {
return unresolvedEntityReferences.removeAll(entities);
}
public boolean removeUnResolvedIdReferences(final List<AtlasObjectId> entities) {
return unresolvedIdReferences.removeAll(entities);
}
public boolean removeUnResolvedIdReference(final AtlasObjectId entity) {
return unresolvedIdReferences.remove(entity);
}
public boolean hasUnresolvedReferences() {
return unresolvedEntityReferences.size() > 0 || unresolvedIdReferences.size() > 0;
}
@Override
public boolean equals(Object obj) {
if (obj == null) {
return false;
} else if (obj == this) {
return true;
} else if (obj.getClass() != getClass()) {
return false;
} else {
EntityGraphDiscoveryContext ctx = (EntityGraphDiscoveryContext) obj;
return Objects.equals(rootEntities, ctx.getRootEntities()) &&
Objects.equals(repositoryResolvedReferences, ctx.getRepositoryResolvedReferences()) &&
Objects.equals(unresolvedEntityReferences, ctx.getUnResolvedEntityReferences()) &&
Objects.equals(unresolvedIdReferences, ctx.getUnresolvedIdReferences());
}
}
@Override
public int hashCode() {
return Objects.hash(rootEntities, repositoryResolvedReferences, unresolvedEntityReferences, unresolvedIdReferences);
}
public StringBuilder toString(StringBuilder sb) {
if (sb == null) {
sb = new StringBuilder();
}
sb.append("EntityGraphDiscoveryCtx{");
sb.append("rootEntities='").append(rootEntities).append('\'');
sb.append(", repositoryResolvedReferences=").append(repositoryResolvedReferences);
sb.append(", unresolvedEntityReferences='").append(unresolvedEntityReferences).append('\'');
sb.append(", unresolvedIdReferences='").append(unresolvedIdReferences).append('\'');
sb.append('}');
return sb;
}
@Override
public String toString() {
return toString(new StringBuilder()).toString();
}
public void cleanUp() {
rootEntities.clear();
unresolvedEntityReferences.clear();
repositoryResolvedReferences.clear();
unresolvedIdReferences.clear();
}
}
repository/src/main/java/org/apache/atlas/repository/store/graph/EntityResolver.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.repository.store.graph;
import org.apache.atlas.exception.AtlasBaseException;
public interface EntityResolver {
void init(EntityGraphDiscoveryContext entities) throws AtlasBaseException;
EntityGraphDiscoveryContext resolveEntityReferences() throws AtlasBaseException;
void cleanUp() throws AtlasBaseException;
}
repository/src/main/java/org/apache/atlas/repository/store/graph/v1/ArrayVertexMapper.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.repository.store.graph.v1;
import com.google.common.base.Optional;
import com.google.inject.Inject;
import com.google.inject.Singleton;
import org.apache.atlas.aspect.Monitored;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.typedef.AtlasStructDef;
import org.apache.atlas.repository.graph.GraphHelper;
import org.apache.atlas.repository.graphdb.AtlasEdge;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.type.AtlasArrayType;
import org.apache.atlas.type.AtlasStructType;
import org.apache.atlas.type.AtlasType;
import org.apache.commons.collections.CollectionUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import static org.apache.atlas.repository.graph.GraphHelper.string;
@Singleton
public class ArrayVertexMapper implements InstanceGraphMapper<List> {
private static final Logger LOG = LoggerFactory.getLogger(ArrayVertexMapper.class);
protected final DeleteHandlerV1 deleteHandler;
protected StructVertexMapper structVertexMapper;
@Inject
public ArrayVertexMapper(DeleteHandlerV1 deleteHandler) {
this.deleteHandler = deleteHandler;
}
void init(StructVertexMapper structVertexMapper) {
this.structVertexMapper = structVertexMapper;
}
@Override
public List toGraph(GraphMutationContext ctx) throws AtlasBaseException {
if (LOG.isDebugEnabled()) {
LOG.debug("Mapping instance to vertex {} for array attribute {}", string(ctx.getReferringVertex()), ctx.getAttrType().getTypeName());
}
List newElements = (List) ctx.getValue();
boolean newAttributeEmpty = (newElements == null || newElements.isEmpty());
AtlasArrayType arrType = (AtlasArrayType) ctx.getAttrType();
AtlasType elementType = arrType.getElementType();
List<Object> currentElements = getArrayElementsProperty(elementType, ctx.getReferringVertex(), ctx.getVertexPropertyKey());
List<Object> newElementsCreated = new ArrayList<>();
if (!newAttributeEmpty) {
for (int index = 0; index < newElements.size(); index++) {
LOG.debug("Adding/updating element at position {}, current element {}, new element {}", index,
(currentElements != null && index < currentElements.size()) ? currentElements.get(index) : null, newElements.get(index));
Optional<AtlasEdge> existingEdge = getEdgeAt(currentElements, index, arrType.getElementType());
GraphMutationContext arrCtx = new GraphMutationContext.Builder(ctx.getAttribute(),
arrType.getElementType(), newElements.get(index))
.referringVertex(ctx.getReferringVertex())
.edge(existingEdge)
.vertexProperty(ctx.getVertexPropertyKey()).build();
Object newEntry = structVertexMapper.mapCollectionElementsToVertex(arrCtx);
newElementsCreated.add(newEntry);
}
}
if (AtlasGraphUtilsV1.isReference(elementType)) {
List<AtlasEdge> additionalEdges = removeUnusedArrayEntries(ctx.getParentType(), ctx.getAttributeDef(), (List) currentElements, (List) newElementsCreated, elementType);
newElementsCreated.addAll(additionalEdges);
}
// for dereference on way out
setArrayElementsProperty(elementType, ctx.getReferringVertex(), ctx.getVertexPropertyKey(), newElementsCreated);
return newElementsCreated;
}
@Override
public void cleanUp() throws AtlasBaseException {
}
//Removes unused edges from the old collection, compared to the new collection
private List<AtlasEdge> removeUnusedArrayEntries(
AtlasStructType entityType,
AtlasStructDef.AtlasAttributeDef attributeDef,
List<AtlasEdge> currentEntries,
List<AtlasEdge> newEntries,
AtlasType entryType) throws AtlasBaseException {
if (currentEntries != null && !currentEntries.isEmpty()) {
LOG.debug("Removing unused entries from the old collection");
if (AtlasGraphUtilsV1.isReference(entryType)) {
Collection<AtlasEdge> edgesToRemove = CollectionUtils.subtract(currentEntries, newEntries);
LOG.debug("Removing unused entries from the old collection - {}", edgesToRemove);
if (!edgesToRemove.isEmpty()) {
//Remove the edges for (current edges - new edges)
List<AtlasEdge> additionalElements = new ArrayList<>();
for (AtlasEdge edge : edgesToRemove) {
boolean deleteChildReferences = StructVertexMapper.shouldManageChildReferences(entityType, attributeDef.getName());
boolean deleted = deleteHandler.deleteEdgeReference(edge, entryType.getTypeCategory(),
deleteChildReferences, true);
if (!deleted) {
additionalElements.add(edge);
}
}
return additionalElements;
}
}
}
return Collections.emptyList();
}
public static List<Object> getArrayElementsProperty(AtlasType elementType, AtlasVertex instanceVertex, String propertyName) {
String actualPropertyName = GraphHelper.encodePropertyKey(propertyName);
if (AtlasGraphUtilsV1.isReference(elementType)) {
return (List)instanceVertex.getListProperty(actualPropertyName, AtlasEdge.class);
}
else {
return (List)instanceVertex.getListProperty(actualPropertyName);
}
}
private Optional<AtlasEdge> getEdgeAt(List<Object> currentElements, int index, AtlasType elemType) {
Optional<AtlasEdge> existingEdge = Optional.absent();
if ( AtlasGraphUtilsV1.isReference(elemType) ) {
Object currentElement = (currentElements != null && index < currentElements.size()) ?
currentElements.get(index) : null;
if ( currentElement != null) {
existingEdge = Optional.of((AtlasEdge) currentElement);
}
}
return existingEdge;
}
private void setArrayElementsProperty(AtlasType elementType, AtlasVertex instanceVertex, String propertyName, List<Object> values) {
String actualPropertyName = GraphHelper.encodePropertyKey(propertyName);
if (AtlasGraphUtilsV1.isReference(elementType)) {
GraphHelper.setListPropertyFromElementIds(instanceVertex, actualPropertyName, (List) values);
}
else {
GraphHelper.setProperty(instanceVertex, actualPropertyName, values);
}
}
}
repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityStoreV1.java
......@@ -18,25 +18,93 @@
package org.apache.atlas.repository.store.graph.v1;
import atlas.shaded.hbase.guava.common.annotations.VisibleForTesting;
import com.google.inject.Inject;
import org.apache.atlas.AtlasErrorCode;
import org.apache.atlas.GraphTransaction;
import org.apache.atlas.RequestContextV1;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.SearchFilter;
import org.apache.atlas.model.TypeCategory;
import org.apache.atlas.model.instance.AtlasClassification;
import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.model.instance.AtlasEntityWithAssociations;
import org.apache.atlas.model.instance.AtlasObjectId;
import org.apache.atlas.model.instance.EntityMutationResponse;
import org.apache.atlas.model.instance.EntityMutations;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.repository.store.graph.AtlasEntityStore;
import org.apache.atlas.repository.store.graph.EntityGraphDiscoveryContext;
import org.apache.atlas.repository.store.graph.EntityGraphDiscovery;
import org.apache.atlas.type.AtlasEntityType;
import org.apache.atlas.type.AtlasType;
import org.apache.atlas.type.AtlasTypeRegistry;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.List;
public class AtlasEntityStoreV1 implements AtlasEntityStore {
@Override
public void init() throws AtlasBaseException {
protected EntityGraphDiscovery graphDiscoverer;
protected AtlasTypeRegistry typeRegistry;
private EntityGraphMapper graphMapper;
private static final Logger LOG = LoggerFactory.getLogger(AtlasEntityStoreV1.class);
@Inject
public AtlasEntityStoreV1(EntityGraphMapper vertexMapper) {
this.graphMapper = vertexMapper;
}
@Inject
public void init(AtlasTypeRegistry typeRegistry, EntityGraphDiscovery graphDiscoverer) throws AtlasBaseException {
this.graphDiscoverer = graphDiscoverer;
this.typeRegistry = typeRegistry;
}
@Override
public EntityMutationResponse createOrUpdate(final AtlasEntity entity) {
return null;
public EntityMutationResponse createOrUpdate(final AtlasEntity entity) throws AtlasBaseException {
return createOrUpdate(new ArrayList<AtlasEntity>() {{ add(entity); }});
}
public EntityMutationContext preCreateOrUpdate(final List<AtlasEntity> atlasEntities) throws AtlasBaseException {
EntityGraphDiscoveryContext discoveredEntities = graphDiscoverer.discoverEntities(atlasEntities);
EntityMutationContext context = new EntityMutationContext(discoveredEntities);
for (AtlasEntity entity : discoveredEntities.getRootEntities()) {
AtlasVertex vertex = null;
if (LOG.isDebugEnabled()) {
LOG.debug("<== AtlasEntityStoreV1.preCreateOrUpdate({}): {}", entity);
}
AtlasEntityType entityType = (AtlasEntityType) typeRegistry.getType(entity.getTypeName());
if ( discoveredEntities.isResolved(entity.getGuid()) ) {
vertex = discoveredEntities.getResolvedReference(entity.getGuid());
context.addUpdated(entity, entityType, vertex);
String guid = AtlasGraphUtilsV1.getIdFromVertex(vertex);
RequestContextV1.get().recordEntityUpdate(guid);
} else {
//Create vertices which do not exist in the repository
vertex = graphMapper.createVertexTemplate(entity, entityType);
context.addCreated(entity, entityType, vertex);
discoveredEntities.addRepositoryResolvedReference(new AtlasObjectId(entityType.getTypeName(), entity.getGuid()), vertex);
String guid = AtlasGraphUtilsV1.getIdFromVertex(vertex);
RequestContextV1.get().recordEntityCreate(guid);
}
if (LOG.isDebugEnabled()) {
LOG.debug("<== AtlasEntityStoreV1.preCreateOrUpdate({}): {}", entity, vertex);
}
}
return context;
}
@Override
......@@ -55,8 +123,24 @@ public class AtlasEntityStoreV1 implements AtlasEntityStore {
}
@Override
@GraphTransaction
public EntityMutationResponse createOrUpdate(final List<AtlasEntity> entities) throws AtlasBaseException {
return null;
if (LOG.isDebugEnabled()) {
LOG.debug("==> AtlasEntityStoreV1.createOrUpdate({}, {})", entities);
}
//Validate
List<AtlasEntity> normalizedEntities = validateAndNormalize(entities);
//Discover entities, create vertices
EntityMutationContext ctx = preCreateOrUpdate(normalizedEntities);
if (LOG.isDebugEnabled()) {
LOG.debug("<== AtlasStructDefStoreV1.createOrUpdate({}, {}): {}", entities);
}
return graphMapper.mapAttributes(ctx);
}
@Override
......@@ -117,7 +201,48 @@ public class AtlasEntityStoreV1 implements AtlasEntityStore {
@Override
public AtlasEntity.AtlasEntities searchEntities(final SearchFilter searchFilter) throws AtlasBaseException {
// TODO: Add checks here to ensure that typename and supertype are mandatory in the requests
// TODO: Add checks here to ensure that typename and supertype are mandatory in the request
return null;
}
private List<AtlasEntity> validateAndNormalize(final List<AtlasEntity> entities) throws AtlasBaseException {
List<AtlasEntity> normalizedEntities = new ArrayList<>();
List<String> messages = new ArrayList<>();
for (AtlasEntity entity : entities) {
AtlasType type = typeRegistry.getType(entity.getTypeName());
if (type.getTypeCategory() != TypeCategory.ENTITY) {
throw new AtlasBaseException(AtlasErrorCode.TYPE_MATCH_FAILED, type.getTypeCategory().name(), TypeCategory.ENTITY.name());
}
type.validateValue(entity, entity.getTypeName(), messages);
if ( !messages.isEmpty()) {
throw new AtlasBaseException(AtlasErrorCode.INSTANCE_CRUD_INVALID_PARAMS, messages);
}
AtlasEntity normalizedEntity = (AtlasEntity) type.getNormalizedValue(entity);
if ( normalizedEntity == null) {
//TODO - Fix this. Should not come here. Should ideally fail above
throw new AtlasBaseException(AtlasErrorCode.INSTANCE_CRUD_INVALID_PARAMS, "Failed to validate entity");
}
normalizedEntities.add(normalizedEntity);
}
return normalizedEntities;
}
@VisibleForTesting
EntityGraphDiscovery getGraphDiscoverer() {
return graphDiscoverer;
}
@VisibleForTesting
void setGraphDiscoverer(EntityGraphDiscovery discoverer) {
this.graphDiscoverer = discoverer;
}
public void cleanUp() throws AtlasBaseException {
this.graphDiscoverer.cleanUp();
}
}
repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasEnumDefStoreV1.java
......@@ -283,19 +283,19 @@ public class AtlasEnumDefStoreV1 extends AtlasAbstractDefStoreV1 implements Atla
List<String> values = new ArrayList<>(enumDef.getElementDefs().size());
for (AtlasEnumElementDef element : enumDef.getElementDefs()) {
String elemKey = AtlasGraphUtilsV1.getPropertyKey(enumDef, element.getValue());
String elemKey = AtlasGraphUtilsV1.getTypeDefPropertyKey(enumDef, element.getValue());
AtlasGraphUtilsV1.setProperty(vertex, elemKey, element.getOrdinal());
if (StringUtils.isNoneBlank(element.getDescription())) {
String descKey = AtlasGraphUtilsV1.getPropertyKey(elemKey, "description");
String descKey = AtlasGraphUtilsV1.getTypeDefPropertyKey(elemKey, "description");
AtlasGraphUtilsV1.setProperty(vertex, descKey, element.getDescription());
}
values.add(element.getValue());
}
AtlasGraphUtilsV1.setProperty(vertex, AtlasGraphUtilsV1.getPropertyKey(enumDef), values);
AtlasGraphUtilsV1.setProperty(vertex, AtlasGraphUtilsV1.getTypeDefPropertyKey(enumDef), values);
}
private AtlasEnumDef toEnumDef(AtlasVertex vertex) {
......@@ -314,10 +314,10 @@ public class AtlasEnumDefStoreV1 extends AtlasAbstractDefStoreV1 implements Atla
typeDefStore.vertexToTypeDef(vertex, ret);
List<AtlasEnumElementDef> elements = new ArrayList<>();
List<String> elemValues = vertex.getProperty(AtlasGraphUtilsV1.getPropertyKey(ret), List.class);
List<String> elemValues = vertex.getProperty(AtlasGraphUtilsV1.getTypeDefPropertyKey(ret), List.class);
for (String elemValue : elemValues) {
String elemKey = AtlasGraphUtilsV1.getPropertyKey(ret, elemValue);
String descKey = AtlasGraphUtilsV1.getPropertyKey(elemKey, "description");
String elemKey = AtlasGraphUtilsV1.getTypeDefPropertyKey(ret, elemValue);
String descKey = AtlasGraphUtilsV1.getTypeDefPropertyKey(elemKey, "description");
Integer ordinal = AtlasGraphUtilsV1.getProperty(vertex, elemKey, Integer.class);
String desc = AtlasGraphUtilsV1.getProperty(vertex, descKey, String.class);
......
repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasGraphUtilsV1.java
......@@ -20,16 +20,30 @@ package org.apache.atlas.repository.store.graph.v1;
import com.google.common.collect.BiMap;
import com.google.common.collect.HashBiMap;
import org.apache.atlas.AtlasErrorCode;
import org.apache.atlas.AtlasException;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.TypeCategory;
import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.model.instance.AtlasStruct;
import org.apache.atlas.model.typedef.AtlasBaseTypeDef;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.graph.GraphHelper;
import org.apache.atlas.repository.graphdb.AtlasEdge;
import org.apache.atlas.repository.graphdb.AtlasElement;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.type.AtlasStructType;
import org.apache.atlas.type.AtlasType;
import org.apache.atlas.typesystem.persistence.Id;
import org.apache.atlas.typesystem.types.DataTypes;
import org.apache.atlas.typesystem.types.IDataType;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Date;
import java.util.HashMap;
/**
......@@ -52,19 +66,19 @@ public class AtlasGraphUtilsV1 {
}});
public static String getPropertyKey(AtlasBaseTypeDef typeDef) {
return getPropertyKey(typeDef.getName());
public static String getTypeDefPropertyKey(AtlasBaseTypeDef typeDef) {
return getTypeDefPropertyKey(typeDef.getName());
}
public static String getPropertyKey(AtlasBaseTypeDef typeDef, String child) {
return getPropertyKey(typeDef.getName(), child);
public static String getTypeDefPropertyKey(AtlasBaseTypeDef typeDef, String child) {
return getTypeDefPropertyKey(typeDef.getName(), child);
}
public static String getPropertyKey(String typeName) {
public static String getTypeDefPropertyKey(String typeName) {
return PROPERTY_PREFIX + typeName;
}
public static String getPropertyKey(String typeName, String child) {
public static String getTypeDefPropertyKey(String typeName, String child) {
return PROPERTY_PREFIX + typeName + "." + child;
}
......@@ -80,6 +94,31 @@ public class AtlasGraphUtilsV1 {
return PROPERTY_PREFIX + "edge." + fromNode + "." + toNode;
}
public static String getAttributeEdgeLabel(AtlasStructType fromType, String attributeName) throws AtlasBaseException {
return GraphHelper.EDGE_LABEL_PREFIX + getQualifiedAttributePropertyKey(fromType, attributeName);
}
public static String getQualifiedAttributePropertyKey(AtlasStructType fromType, String attributeName) throws AtlasBaseException {
switch (fromType.getTypeCategory()) {
case STRUCT:
case ENTITY:
case CLASSIFICATION:
return fromType.getQualifiedAttributeName(attributeName);
default:
throw new AtlasBaseException(AtlasErrorCode.UNKNOWN_TYPE, fromType.getTypeCategory().name());
}
}
public static boolean isReference(AtlasType type) {
return isReference(type.getTypeCategory());
}
public static boolean isReference(TypeCategory typeCategory) {
return typeCategory == TypeCategory.STRUCT ||
typeCategory == TypeCategory.ENTITY ||
typeCategory == TypeCategory.CLASSIFICATION;
}
public static String encodePropertyKey(String key) {
String ret = key;
......@@ -104,6 +143,21 @@ public class AtlasGraphUtilsV1 {
return ret;
}
/**
* Adds an additional value to a multi-property.
*
* @param propertyName
* @param value
*/
public static AtlasVertex addProperty(AtlasVertex vertex, String propertyName, Object value) {
if (LOG.isDebugEnabled()) {
LOG.debug("==> addProperty({}, {}, {})", toString(vertex), propertyName, value);
}
propertyName = encodePropertyKey(propertyName);
vertex.addProperty(propertyName, value);
return vertex;
}
public static <T extends AtlasElement> void setProperty(T element, String propertyName, Object value) {
if (LOG.isDebugEnabled()) {
LOG.debug("==> setProperty({}, {}, {})", toString(element), propertyName, value);
......@@ -127,10 +181,15 @@ public class AtlasGraphUtilsV1 {
LOG.debug("Setting property {} in {}", propertyName, toString(element));
}
if ( value instanceof Date) {
Long encodedValue = ((Date) value).getTime();
element.setProperty(propertyName, encodedValue);
} else {
element.setProperty(propertyName, value);
}
}
}
}
public static <T extends AtlasElement, O> O getProperty(T element, String propertyName, Class<O> returnType) {
Object property = element.getProperty(encodePropertyKey(propertyName), returnType);
......@@ -186,4 +245,13 @@ public class AtlasGraphUtilsV1 {
return String.format("edge[id=%s label=%s from %s -> to %s]", edge.getId(), edge.getLabel(),
toString(edge.getOutVertex()), toString(edge.getInVertex()));
}
public static AtlasEntity.Status getState(AtlasElement element) {
String state = getStateAsString(element);
return state == null ? null : AtlasEntity.Status.valueOf(state);
}
public static String getStateAsString(AtlasElement element) {
return element.getProperty(Constants.STATE_PROPERTY_KEY, String.class);
}
}
repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasStructDefStoreV1.java
......@@ -390,13 +390,13 @@ public class AtlasStructDefStoreV1 extends AtlasAbstractDefStoreV1 implements At
List<String> attrNames = new ArrayList<>(structDef.getAttributeDefs().size());
for (AtlasAttributeDef attributeDef : structDef.getAttributeDefs()) {
String propertyKey = AtlasGraphUtilsV1.getPropertyKey(structDef, attributeDef.getName());
String propertyKey = AtlasGraphUtilsV1.getTypeDefPropertyKey(structDef, attributeDef.getName());
AtlasGraphUtilsV1.setProperty(vertex, propertyKey, toJsonFromAttributeDef(attributeDef, structType));
attrNames.add(attributeDef.getName());
}
AtlasGraphUtilsV1.setProperty(vertex, AtlasGraphUtilsV1.getPropertyKey(structDef), attrNames);
AtlasGraphUtilsV1.setProperty(vertex, AtlasGraphUtilsV1.getTypeDefPropertyKey(structDef), attrNames);
}
public static void updateVertexPreUpdate(AtlasStructDef structDef, AtlasStructType structType,
......@@ -410,7 +410,7 @@ public class AtlasStructDefStoreV1 extends AtlasAbstractDefStoreV1 implements At
}
}
List<String> currAttrNames = vertex.getProperty(AtlasGraphUtilsV1.getPropertyKey(structDef), List.class);
List<String> currAttrNames = vertex.getProperty(AtlasGraphUtilsV1.getTypeDefPropertyKey(structDef), List.class);
// delete attributes that are not present in updated structDef
if (CollectionUtils.isNotEmpty(currAttrNames)) {
......@@ -434,13 +434,13 @@ public class AtlasStructDefStoreV1 extends AtlasAbstractDefStoreV1 implements At
}
}
String propertyKey = AtlasGraphUtilsV1.getPropertyKey(structDef, attributeDef.getName());
String propertyKey = AtlasGraphUtilsV1.getTypeDefPropertyKey(structDef, attributeDef.getName());
AtlasGraphUtilsV1.setProperty(vertex, propertyKey, toJsonFromAttributeDef(attributeDef, structType));
}
}
AtlasGraphUtilsV1.setProperty(vertex, AtlasGraphUtilsV1.getPropertyKey(structDef), attrNames);
AtlasGraphUtilsV1.setProperty(vertex, AtlasGraphUtilsV1.getTypeDefPropertyKey(structDef), attrNames);
}
public static void updateVertexAddReferences(AtlasStructDef structDef, AtlasVertex vertex,
......@@ -457,11 +457,11 @@ public class AtlasStructDefStoreV1 extends AtlasAbstractDefStoreV1 implements At
typeDefStore.vertexToTypeDef(vertex, ret);
List<AtlasAttributeDef> attributeDefs = new ArrayList<>();
List<String> attrNames = vertex.getProperty(AtlasGraphUtilsV1.getPropertyKey(ret), List.class);
List<String> attrNames = vertex.getProperty(AtlasGraphUtilsV1.getTypeDefPropertyKey(ret), List.class);
if (CollectionUtils.isNotEmpty(attrNames)) {
for (String attrName : attrNames) {
String propertyKey = AtlasGraphUtilsV1.getPropertyKey(ret, attrName);
String propertyKey = AtlasGraphUtilsV1.getTypeDefPropertyKey(ret, attrName);
String attribJson = vertex.getProperty(propertyKey, String.class);
attributeDefs.add(toAttributeDefFromJson(structDef, AtlasType.fromJson(attribJson, Map.class),
......@@ -586,12 +586,12 @@ public class AtlasStructDefStoreV1 extends AtlasAbstractDefStoreV1 implements At
String refAttributeName = null;
List<String> attrNames = attributeType.getProperty(
AtlasGraphUtilsV1.getPropertyKey(attrTypeName), List.class);
AtlasGraphUtilsV1.getTypeDefPropertyKey(attrTypeName), List.class);
if (CollectionUtils.isNotEmpty(attrNames)) {
for (String attrName : attrNames) {
String attribJson = attributeType.getProperty(
AtlasGraphUtilsV1.getPropertyKey(attrTypeName, attrName), String.class);
AtlasGraphUtilsV1.getTypeDefPropertyKey(attrTypeName, attrName), String.class);
Map refAttrInfo = AtlasType.fromJson(attribJson, Map.class);
String refAttribType = (String) refAttrInfo.get("dataType");
......
repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphMapper.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.repository.store.graph.v1;
import com.google.inject.Inject;
import org.apache.atlas.AtlasErrorCode;
import org.apache.atlas.RequestContextV1;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.model.instance.AtlasEntityHeader;
import org.apache.atlas.model.instance.AtlasObjectId;
import org.apache.atlas.model.instance.AtlasStruct;
import org.apache.atlas.model.instance.EntityMutationResponse;
import org.apache.atlas.model.instance.EntityMutations;
import org.apache.atlas.model.typedef.AtlasStructDef;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.RepositoryException;
import org.apache.atlas.repository.graph.GraphHelper;
import org.apache.atlas.repository.graphdb.AtlasEdge;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.type.AtlasEntityType;
import org.apache.atlas.type.AtlasStructType;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.UUID;
public class EntityGraphMapper implements InstanceGraphMapper<AtlasEdge> {
private static final Logger LOG = LoggerFactory.getLogger(EntityGraphMapper.class);
protected final GraphHelper graphHelper = GraphHelper.getInstance();
protected EntityMutationContext context;
protected final StructVertexMapper structVertexMapper;
@Inject
public EntityGraphMapper(ArrayVertexMapper arrayVertexMapper, MapVertexMapper mapVertexMapper) {
this.structVertexMapper = new StructVertexMapper(arrayVertexMapper, mapVertexMapper);
arrayVertexMapper.init(structVertexMapper);
mapVertexMapper.init(structVertexMapper);
}
public AtlasVertex createVertexTemplate(final AtlasStruct instance, final AtlasStructType structType) {
AtlasVertex vertex = structVertexMapper.createVertexTemplate(instance, structType);
AtlasEntityType entityType = (AtlasEntityType) structType;
AtlasEntity entity = (AtlasEntity) instance;
// add super types
for (String superTypeName : entityType.getAllSuperTypes()) {
AtlasGraphUtilsV1.addProperty(vertex, Constants.SUPER_TYPES_PROPERTY_KEY, superTypeName);
}
final String guid = UUID.randomUUID().toString();
// add identity
AtlasGraphUtilsV1.setProperty(vertex, Constants.GUID_PROPERTY_KEY, guid);
// add version information
AtlasGraphUtilsV1.setProperty(vertex, Constants.VERSION_PROPERTY_KEY, Integer.valueOf(entity.getVersion().intValue()));
return vertex;
}
@Override
public AtlasEdge toGraph(GraphMutationContext ctx) throws AtlasBaseException {
AtlasEdge result = null;
String guid = getId(ctx.getValue());
AtlasVertex entityVertex = context.getDiscoveryContext().getResolvedReference(guid);
String edgeLabel = AtlasGraphUtilsV1.getAttributeEdgeLabel(ctx.getParentType(), ctx.getAttributeDef().getName());
if ( ctx.getCurrentEdge().isPresent() ) {
updateEdge(ctx.getAttributeDef(), ctx.getValue(), ctx.getCurrentEdge().get(), entityVertex);
result = ctx.getCurrentEdge().get();
} else {
try {
result = graphHelper.getOrCreateEdge(ctx.getReferringVertex(), entityVertex, edgeLabel);
} catch (RepositoryException e) {
throw new AtlasBaseException(AtlasErrorCode.INTERNAL_ERROR, e);
}
}
return result;
}
@Override
public void cleanUp() throws AtlasBaseException {
}
private AtlasEdge updateEdge(AtlasStructDef.AtlasAttributeDef attributeDef, Object value, AtlasEdge currentEdge, final AtlasVertex entityVertex) throws AtlasBaseException {
LOG.debug("Updating entity reference {} for reference attribute {}", attributeDef.getName());
// Update edge if it exists
AtlasVertex currentVertex = currentEdge.getOutVertex();
String currentEntityId = AtlasGraphUtilsV1.getIdFromVertex(currentVertex);
String newEntityId = getId(value);
AtlasEdge newEdge = currentEdge;
if (!currentEntityId.equals(newEntityId)) {
// add an edge to the class vertex from the instance
if (entityVertex != null) {
try {
newEdge = graphHelper.getOrCreateEdge(currentEdge.getInVertex(), entityVertex, currentEdge.getLabel());
} catch (RepositoryException e) {
throw new AtlasBaseException(AtlasErrorCode.INTERNAL_ERROR, e);
}
}
}
return newEdge;
}
public EntityMutationResponse mapAttributes(EntityMutationContext ctx) throws AtlasBaseException {
this.context = ctx;
structVertexMapper.init(this);
EntityMutationResponse resp = new EntityMutationResponse();
//Map attributes
if (ctx.getCreatedEntities() != null) {
for (AtlasEntity createdEntity : ctx.getCreatedEntities()) {
AtlasVertex vertex = ctx.getVertex(createdEntity);
structVertexMapper.mapAttributestoVertex((AtlasStructType) ctx.getType(createdEntity), createdEntity, vertex);
resp.addEntity(EntityMutations.EntityOperation.CREATE, constructHeader(createdEntity, vertex));
}
}
if (ctx.getUpdatedEntities() != null) {
for (AtlasEntity updated : ctx.getUpdatedEntities()) {
AtlasVertex vertex = ctx.getVertex(updated);
structVertexMapper.mapAttributestoVertex((AtlasStructType) ctx.getType(updated), updated, vertex);
resp.addEntity(EntityMutations.EntityOperation.UPDATE, constructHeader(updated, vertex));
}
}
return resp;
}
public String getId(Object value) throws AtlasBaseException {
if ( value != null) {
if ( value instanceof AtlasObjectId) {
return ((AtlasObjectId) value).getGuid();
} else if (value instanceof AtlasEntity) {
return ((AtlasEntity) value).getGuid();
}
}
throw new AtlasBaseException(AtlasErrorCode.INSTANCE_GUID_NOT_FOUND, (String) value);
}
private AtlasEntityHeader constructHeader(AtlasEntity entity, AtlasVertex vertex) {
//TODO - enhance to return only selective attributes
return new AtlasEntityHeader(entity.getTypeName(), AtlasGraphUtilsV1.getIdFromVertex(vertex), entity.getAttributes());
}
public EntityMutationContext getContext() {
return context;
}
public AtlasEntityType getInstanceType(Object val) throws AtlasBaseException {
String guid = getId(val);
return (AtlasEntityType) getContext().getType(guid);
}
}
repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityMutationContext.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p/>
* http://www.apache.org/licenses/LICENSE-2.0
* <p/>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.repository.store.graph.v1;
import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.repository.store.graph.EntityGraphDiscoveryContext;
import org.apache.atlas.type.AtlasType;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
public class EntityMutationContext {
private List<AtlasEntity> entitiesCreated = new ArrayList<>();
private List<AtlasEntity> entitiesUpdated = new ArrayList<>();
private EntityGraphDiscoveryContext context;
private Map<String, AtlasType> entityVsType = new HashMap<>();
private Map<String, AtlasVertex> entityVsVertex = new HashMap<>();
public EntityMutationContext(final EntityGraphDiscoveryContext context) {
this.context = context;
}
public void addCreated(AtlasEntity entity, AtlasType type, AtlasVertex atlasVertex) {
entitiesCreated.add(entity);
entityVsVertex.put(entity.getGuid(), atlasVertex);
entityVsType.put(entity.getGuid(), type);
}
public void addUpdated(AtlasEntity entity, AtlasType type, AtlasVertex atlasVertex) {
entitiesUpdated.add(entity);
entityVsVertex.put(entity.getGuid(), atlasVertex);
entityVsType.put(entity.getGuid(), type);
}
public Collection<AtlasEntity> getCreatedEntities() {
return entitiesCreated;
}
public Collection<AtlasEntity> getUpdatedEntities() {
return entitiesUpdated;
}
public AtlasType getType(AtlasEntity entity) {
return entityVsType.get(entity.getGuid());
}
public AtlasType getType(String entityId) {
return entityVsType.get(entityId);
}
public AtlasVertex getVertex(AtlasEntity entity) {
return entityVsVertex.get(entity.getGuid());
}
public AtlasVertex getVertex(String entityId) {
return entityVsVertex.get(entityId);
}
public EntityGraphDiscoveryContext getDiscoveryContext() {
return this.context;
}
//TODO - equals/hashCode/toString
@Override
public boolean equals(final Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
final EntityMutationContext that = (EntityMutationContext) o;
if (entitiesCreated != null ? !entitiesCreated.equals(that.entitiesCreated) : that.entitiesCreated != null)
return false;
if (entitiesUpdated != null ? !entitiesUpdated.equals(that.entitiesUpdated) : that.entitiesUpdated != null)
return false;
if (context != null ? !context.equals(that.context) : that.context != null) return false;
if (entityVsType != null ? !entityVsType.equals(that.entityVsType) : that.entityVsType != null) return false;
return !(entityVsVertex != null ? !entityVsVertex.equals(that.entityVsVertex) : that.entityVsVertex != null);
}
@Override
public int hashCode() {
int result = entitiesCreated != null ? entitiesCreated.hashCode() : 0;
result = 31 * result + (entitiesUpdated != null ? entitiesUpdated.hashCode() : 0);
result = 31 * result + (context != null ? context.hashCode() : 0);
result = 31 * result + (entityVsType != null ? entityVsType.hashCode() : 0);
result = 31 * result + (entityVsVertex != null ? entityVsVertex.hashCode() : 0);
return result;
}
@Override
public String toString() {
return "EntityMutationContext{" +
"entitiesCreated=" + entitiesCreated +
", entitiesUpdated=" + entitiesUpdated +
", context=" + context +
", entityVsType=" + entityVsType +
", entityVsVertex=" + entityVsVertex +
'}';
}
}
repository/src/main/java/org/apache/atlas/repository/store/graph/v1/GraphMutationContext.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.repository.store.graph.v1;
import com.google.common.base.Optional;
import org.apache.atlas.model.typedef.AtlasStructDef;
import org.apache.atlas.repository.graphdb.AtlasEdge;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.type.AtlasStructType;
import org.apache.atlas.type.AtlasType;
import java.util.Objects;
public class GraphMutationContext {
/**
* Atlas Attribute
*/
private AtlasStructType.AtlasAttribute attribute;
/**
* Overriding type for which elements are being mapped
*/
private AtlasType currentElementType;
/**
* Current attribute value/entity/Struct instance
*/
private Object value;
/**
*
* The vertex which corresponds to the entity/struct for which we are mapping a complex attributes like struct, traits
*/
AtlasVertex referringVertex;
/**
* the vertex property that we are updating
*/
String vertexPropertyKey;
/**
* The current edge(in case of updates) from the parent entity/struct to the complex attribute like struct, trait
*/
Optional<AtlasEdge> existingEdge;
private GraphMutationContext(final Builder builder) {
this.attribute = builder.attribute;
this.currentElementType = builder.elementType;
this.existingEdge = builder.currentEdge;
this.value = builder.currentValue;
this.referringVertex = builder.referringVertex;
this.vertexPropertyKey = builder.vertexPropertyKey;
}
public String getVertexPropertyKey() {
return vertexPropertyKey;
}
@Override
public int hashCode() {
return Objects.hash(attribute, value, referringVertex, vertexPropertyKey, existingEdge);
}
@Override
public boolean equals(Object obj) {
if (obj == null) {
return false;
} else if (obj == this) {
return true;
} else if (obj.getClass() != getClass()) {
return false;
} else {
GraphMutationContext rhs = (GraphMutationContext) obj;
return Objects.equals(attribute, rhs.getAttribute())
&& Objects.equals(value, rhs.getValue())
&& Objects.equals(referringVertex, rhs.getReferringVertex())
&& Objects.equals(vertexPropertyKey, rhs.getReferringVertex())
&& Objects.equals(existingEdge, rhs.getCurrentEdge());
}
}
public static final class Builder {
private final AtlasStructType.AtlasAttribute attribute;
private final AtlasType elementType;
private final Object currentValue;
private AtlasVertex referringVertex;
private Optional<AtlasEdge> currentEdge = Optional.absent();
private String vertexPropertyKey;
public Builder(AtlasStructType.AtlasAttribute attribute, AtlasType currentElementType, Object currentValue) {
this.attribute = attribute;
this.elementType = currentElementType;
this.currentValue = currentValue;
}
public Builder(AtlasStructType.AtlasAttribute attribute, Object currentValue) {
this.attribute = attribute;
this.elementType = null;
this.currentValue = currentValue;
}
Builder referringVertex(AtlasVertex referringVertex) {
this.referringVertex = referringVertex;
return this;
}
Builder edge(AtlasEdge edge) {
this.currentEdge = Optional.of(edge);
return this;
}
Builder edge(Optional<AtlasEdge> edge) {
this.currentEdge = edge;
return this;
}
Builder vertexProperty(String propertyKey) {
this.vertexPropertyKey = propertyKey;
return this;
}
GraphMutationContext build() {
return new GraphMutationContext(this);
}
}
public AtlasStructType getParentType() {
return attribute.getStructType();
}
public AtlasStructDef getStructDef() {
return attribute.getStructDef();
}
public AtlasStructDef.AtlasAttributeDef getAttributeDef() {
return attribute.getAttributeDef();
}
public AtlasType getAttrType() {
return currentElementType == null ? attribute.getAttributeType() : currentElementType;
}
public AtlasType getCurrentElementType() {
return currentElementType;
}
public Object getValue() {
return value;
}
public AtlasVertex getReferringVertex() {
return referringVertex;
}
public Optional<AtlasEdge> getCurrentEdge() {
return existingEdge;
}
public void setElementType(final AtlasType attrType) {
this.currentElementType = attrType;
}
public AtlasStructType.AtlasAttribute getAttribute() {
return attribute;
}
}
repository/src/main/java/org/apache/atlas/repository/store/graph/v1/HardDeleteHandlerV1.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p/>
* http://www.apache.org/licenses/LICENSE-2.0
* <p/>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.repository.store.graph.v1;
import com.google.inject.Inject;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.repository.graphdb.AtlasEdge;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.type.AtlasTypeRegistry;
public class HardDeleteHandlerV1 extends DeleteHandlerV1 {
@Inject
public HardDeleteHandlerV1(AtlasTypeRegistry typeRegistry) {
super(typeRegistry, false, true);
}
@Override
protected void _deleteVertex(AtlasVertex instanceVertex, boolean force) {
graphHelper.removeVertex(instanceVertex);
}
@Override
protected void deleteEdge(AtlasEdge edge, boolean force) throws AtlasBaseException {
graphHelper.removeEdge(edge);
}
}
repository/src/main/java/org/apache/atlas/repository/store/graph/v1/IDBasedEntityResolver.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.repository.store.graph.v1;
import com.google.common.base.Optional;
import org.apache.atlas.AtlasErrorCode;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.model.instance.AtlasObjectId;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.graph.GraphHelper;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.repository.store.graph.EntityGraphDiscoveryContext;
import org.apache.atlas.repository.store.graph.EntityResolver;
import org.apache.atlas.typesystem.exception.EntityNotFoundException;
import org.apache.atlas.typesystem.persistence.Id;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
public class IDBasedEntityResolver implements EntityResolver {
private Map<String, AtlasEntity> idToEntityMap = new HashMap<>();
private final GraphHelper graphHelper = GraphHelper.getInstance();
private EntityGraphDiscoveryContext context;
@Override
public void init(EntityGraphDiscoveryContext context) throws AtlasBaseException {
this.context = context;
for (AtlasEntity entity : context.getRootEntities()) {
idToEntityMap.put(entity.getGuid(), entity);
}
}
public EntityGraphDiscoveryContext resolveEntityReferences() throws AtlasBaseException {
if ( context == null) {
throw new AtlasBaseException(AtlasErrorCode.INTERNAL_ERROR, "Entity resolver not initialized");
}
List<AtlasObjectId> resolvedReferences = new ArrayList<>();
for (AtlasObjectId typeIdPair : context.getUnresolvedIdReferences()) {
if ( AtlasEntity.isAssigned(typeIdPair.getGuid())) {
//validate in graph repo that given guid, typename exists
Optional<AtlasVertex> vertex = resolveGuid(typeIdPair);
if ( vertex.isPresent() ) {
context.addRepositoryResolvedReference(typeIdPair, vertex.get());
resolvedReferences.add(typeIdPair);
}
} else {
//check if root references have this temporary id
if (!idToEntityMap.containsKey(typeIdPair.getGuid()) ) {
throw new AtlasBaseException(AtlasErrorCode.INSTANCE_GUID_NOT_FOUND, "Could not find an entity with the specified id " + typeIdPair + " in the request");
}
}
}
context.removeUnResolvedIdReferences(resolvedReferences);
//Resolve root references
for (AtlasEntity entity : context.getRootEntities()) {
if ( !context.isResolved(entity.getGuid()) && AtlasEntity.isAssigned(entity.getGuid())) {
AtlasObjectId typeIdPair = new AtlasObjectId(entity.getTypeName(), entity.getGuid());
Optional<AtlasVertex> vertex = resolveGuid(typeIdPair);
if (vertex.isPresent()) {
context.addRepositoryResolvedReference(typeIdPair, vertex.get());
context.removeUnResolvedIdReference(typeIdPair);
}
}
}
return context;
}
private Optional<AtlasVertex> resolveGuid(AtlasObjectId typeIdPair) throws AtlasBaseException {
//validate in graph repo that given guid, typename exists
AtlasVertex vertex = null;
try {
vertex = graphHelper.findVertex(Constants.GUID_PROPERTY_KEY, typeIdPair.getGuid(),
Constants.TYPE_NAME_PROPERTY_KEY, typeIdPair.getTypeName(),
Constants.STATE_PROPERTY_KEY, Id.EntityState.ACTIVE.name());
} catch (EntityNotFoundException e) {
//Ignore
}
if ( vertex != null ) {
return Optional.of(vertex);
} else {
throw new AtlasBaseException(AtlasErrorCode.INSTANCE_GUID_NOT_FOUND, "Could not find an entity with the specified guid " + typeIdPair.getGuid() + " in Atlas respository");
}
}
@Override
public void cleanUp() throws AtlasBaseException {
idToEntityMap.clear();
this.context = null;
}
}
repository/src/main/java/org/apache/atlas/repository/store/graph/v1/InstanceGraphMapper.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.repository.store.graph.v1;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.type.AtlasEntityType;
public interface InstanceGraphMapper<T> {
/**
* Map the given type instance to the graph
*
* @param ctx
* @return the value that was mapped to the vertex
* @throws AtlasBaseException
*/
T toGraph(GraphMutationContext ctx) throws AtlasBaseException;
void cleanUp() throws AtlasBaseException;
}
repository/src/main/java/org/apache/atlas/repository/store/graph/v1/MapVertexMapper.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.repository.store.graph.v1;
import com.google.common.base.Optional;
import com.google.inject.Inject;
import com.google.inject.Singleton;
import org.apache.atlas.AtlasErrorCode;
import org.apache.atlas.AtlasException;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.TypeCategory;
import org.apache.atlas.model.typedef.AtlasStructDef;
import org.apache.atlas.repository.graph.GraphHelper;
import org.apache.atlas.repository.graphdb.AtlasEdge;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.type.AtlasEntityType;
import org.apache.atlas.type.AtlasMapType;
import org.apache.atlas.type.AtlasStructType;
import org.apache.atlas.type.AtlasType;
import org.apache.commons.collections.MapUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.inject.Provider;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import static org.apache.atlas.repository.graph.GraphHelper.string;
public class MapVertexMapper implements InstanceGraphMapper<Map> {
private DeleteHandlerV1 deleteHandler;
private static final Logger LOG = LoggerFactory.getLogger(MapVertexMapper.class);
private StructVertexMapper structVertexMapper;
@Inject
public MapVertexMapper(DeleteHandlerV1 deleteHandler) {
this.deleteHandler = deleteHandler;
}
void init(StructVertexMapper structVertexMapper) {
this.structVertexMapper = structVertexMapper;
}
@Override
public Map<String, Object> toGraph(GraphMutationContext ctx) throws AtlasBaseException {
if (LOG.isDebugEnabled()) {
LOG.debug("Mapping instance to vertex {} for map type {}", string(ctx.getReferringVertex()), ctx.getAttrType().getTypeName());
}
@SuppressWarnings("unchecked") Map<Object, Object> newVal =
(Map<Object, Object>) ctx.getValue();
boolean newAttributeEmpty = MapUtils.isEmpty(newVal);
Map<String, Object> currentMap = new HashMap<>();
Map<String, Object> newMap = new HashMap<>();
AtlasMapType mapType = (AtlasMapType) ctx.getAttrType();
try {
List<String> currentKeys = GraphHelper.getListProperty(ctx.getReferringVertex(), ctx.getVertexPropertyKey());
if (currentKeys != null && !currentKeys.isEmpty()) {
for (String key : currentKeys) {
String propertyNameForKey = GraphHelper.getQualifiedNameForMapKey(ctx.getVertexPropertyKey(), key);
Object propertyValueForKey = getMapValueProperty(mapType.getValueType(), ctx.getReferringVertex(), propertyNameForKey);
currentMap.put(key, propertyValueForKey);
}
}
if (!newAttributeEmpty) {
for (Map.Entry<Object, Object> entry : newVal.entrySet()) {
String keyStr = entry.getKey().toString();
String propertyNameForKey = GraphHelper.getQualifiedNameForMapKey(ctx.getVertexPropertyKey(), keyStr);
Optional<AtlasEdge> existingEdge = getEdgeIfExists(mapType, currentMap, keyStr);
GraphMutationContext mapCtx = new GraphMutationContext.Builder(ctx.getAttribute(), mapType.getValueType(), entry.getValue())
.referringVertex(ctx.getReferringVertex())
.edge(existingEdge)
.vertexProperty(propertyNameForKey).build();
Object newEntry = structVertexMapper.mapCollectionElementsToVertex(mapCtx);
newMap.put(keyStr, newEntry);
}
}
Map<String, Object> finalMap =
removeUnusedMapEntries(ctx.getParentType(), mapType, ctx.getAttributeDef(), ctx.getReferringVertex(), ctx.getVertexPropertyKey(), currentMap, newMap);
Set<String> newKeys = new HashSet<>(newMap.keySet());
newKeys.addAll(finalMap.keySet());
// for dereference on way out
GraphHelper.setListProperty(ctx.getReferringVertex(), ctx.getVertexPropertyKey(), new ArrayList<>(newKeys));
} catch (AtlasException e) {
throw new AtlasBaseException(AtlasErrorCode.INTERNAL_ERROR, e);
}
if (LOG.isDebugEnabled()) {
LOG.debug("Map values set in vertex {} {}", mapType.getTypeName(), newMap);
}
return newMap;
}
@Override
public void cleanUp() throws AtlasBaseException {
}
public static Object getMapValueProperty(AtlasType elementType, AtlasVertex instanceVertex, String propertyName) {
String actualPropertyName = GraphHelper.encodePropertyKey(propertyName);
if (AtlasGraphUtilsV1.isReference(elementType)) {
return instanceVertex.getProperty(actualPropertyName, AtlasEdge.class);
}
else {
return instanceVertex.getProperty(actualPropertyName, String.class).toString();
}
}
public static void setMapValueProperty(AtlasType elementType, AtlasVertex instanceVertex, String propertyName, Object value) {
String actualPropertyName = GraphHelper.encodePropertyKey(propertyName);
if (AtlasGraphUtilsV1.isReference(elementType)) {
instanceVertex.setPropertyFromElementId(actualPropertyName, (AtlasEdge)value);
}
else {
instanceVertex.setProperty(actualPropertyName, value);
}
}
//Remove unused entries from map
private Map<String, Object> removeUnusedMapEntries(
AtlasStructType entityType,
AtlasMapType mapType, AtlasStructDef.AtlasAttributeDef attributeDef,
AtlasVertex instanceVertex, String propertyName,
Map<String, Object> currentMap,
Map<String, Object> newMap)
throws AtlasException, AtlasBaseException {
Map<String, Object> additionalMap = new HashMap<>();
for (String currentKey : currentMap.keySet()) {
boolean shouldDeleteKey = !newMap.containsKey(currentKey);
if (AtlasGraphUtilsV1.isReference(mapType.getValueType())) {
//Delete the edge reference if its not part of new edges created/updated
AtlasEdge currentEdge = (AtlasEdge)currentMap.get(currentKey);
if (!newMap.values().contains(currentEdge)) {
boolean deleteChildReferences = StructVertexMapper.shouldManageChildReferences(entityType, attributeDef.getName());
boolean deleted =
deleteHandler.deleteEdgeReference(currentEdge, mapType.getValueType().getTypeCategory(), deleteChildReferences, true);
if (!deleted) {
additionalMap.put(currentKey, currentEdge);
shouldDeleteKey = false;
}
}
}
if (shouldDeleteKey) {
String propertyNameForKey = GraphHelper.getQualifiedNameForMapKey(propertyName, currentKey);
GraphHelper.setProperty(instanceVertex, propertyNameForKey, null);
}
}
return additionalMap;
}
private Optional<AtlasEdge> getEdgeIfExists(AtlasMapType mapType, Map<String, Object> currentMap, String keyStr) {
Optional<AtlasEdge> existingEdge = Optional.absent();
if ( AtlasGraphUtilsV1.isReference(mapType.getValueType()) ) {
existingEdge = Optional.of((AtlasEdge) currentMap.get(keyStr));
}
return existingEdge;
}
}
repository/src/main/java/org/apache/atlas/repository/store/graph/v1/SoftDeleteHandlerV1.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p/>
* http://www.apache.org/licenses/LICENSE-2.0
* <p/>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.repository.store.graph.v1;
import com.google.inject.Inject;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.repository.graph.GraphHelper;
import org.apache.atlas.repository.graphdb.AtlasEdge;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.RequestContext;
import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.atlas.typesystem.persistence.Id;
import static org.apache.atlas.repository.Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.MODIFIED_BY_KEY;
import static org.apache.atlas.repository.Constants.STATE_PROPERTY_KEY;
public class SoftDeleteHandlerV1 extends DeleteHandlerV1 {
@Inject
public SoftDeleteHandlerV1(AtlasTypeRegistry typeRegistry) {
super(typeRegistry, false, true);
}
@Override
protected void _deleteVertex(AtlasVertex instanceVertex, boolean force) {
if (force) {
graphHelper.removeVertex(instanceVertex);
} else {
AtlasEntity.Status state = AtlasGraphUtilsV1.getState(instanceVertex);
if (state != AtlasEntity.Status.DELETED) {
GraphHelper.setProperty(instanceVertex, STATE_PROPERTY_KEY, Id.EntityState.DELETED.name());
GraphHelper.setProperty(instanceVertex, MODIFICATION_TIMESTAMP_PROPERTY_KEY,
RequestContext.get().getRequestTime());
GraphHelper.setProperty(instanceVertex, MODIFIED_BY_KEY, RequestContext.get().getUser());
}
}
}
@Override
protected void deleteEdge(AtlasEdge edge, boolean force) throws AtlasBaseException {
if (force) {
graphHelper.removeEdge(edge);
} else {
Id.EntityState state = GraphHelper.getState(edge);
if (state != Id.EntityState.DELETED) {
GraphHelper.setProperty(edge, STATE_PROPERTY_KEY, Id.EntityState.DELETED.name());
GraphHelper
.setProperty(edge, MODIFICATION_TIMESTAMP_PROPERTY_KEY, RequestContext.get().getRequestTime());
GraphHelper.setProperty(edge, MODIFIED_BY_KEY, RequestContext.get().getUser());
}
}
}
}
repository/src/main/java/org/apache/atlas/repository/store/graph/v1/StructVertexMapper.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.repository.store.graph.v1;
import org.apache.atlas.AtlasErrorCode;
import org.apache.atlas.RequestContextV1;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.TypeCategory;
import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.model.instance.AtlasStruct;
import org.apache.atlas.model.typedef.AtlasStructDef;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.RepositoryException;
import org.apache.atlas.repository.graph.AtlasGraphProvider;
import org.apache.atlas.repository.graph.GraphHelper;
import org.apache.atlas.repository.graphdb.AtlasEdge;
import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.type.AtlasEntityType;
import org.apache.atlas.type.AtlasMapType;
import org.apache.atlas.type.AtlasStructType;
import org.apache.atlas.type.AtlasType;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class StructVertexMapper implements InstanceGraphMapper<AtlasEdge> {
private final AtlasGraph graph;
private final GraphHelper graphHelper = GraphHelper.getInstance();
private final MapVertexMapper mapVertexMapper;
private final ArrayVertexMapper arrVertexMapper;
private EntityGraphMapper entityVertexMapper;
private static final Logger LOG = LoggerFactory.getLogger(StructVertexMapper.class);
public StructVertexMapper(ArrayVertexMapper arrayVertexMapper, MapVertexMapper mapVertexMapper) {
this.graph = AtlasGraphProvider.getGraphInstance();;
this.mapVertexMapper = mapVertexMapper;
this.arrVertexMapper = arrayVertexMapper;
}
void init(final EntityGraphMapper entityVertexMapper) {
this.entityVertexMapper = entityVertexMapper;
}
@Override
public AtlasEdge toGraph(GraphMutationContext ctx) throws AtlasBaseException {
AtlasEdge result = null;
String edgeLabel = AtlasGraphUtilsV1.getAttributeEdgeLabel(ctx.getParentType(), ctx.getAttributeDef().getName());
if ( ctx.getCurrentEdge().isPresent() ) {
updateVertex(ctx.getParentType(), (AtlasStructType) ctx.getAttrType(), ctx.getAttributeDef(), (AtlasStruct) ctx.getValue(), ctx.getCurrentEdge().get().getOutVertex());
result = ctx.getCurrentEdge().get();
} else {
result = createVertex(ctx.getParentType(), (AtlasStructType) ctx.getAttrType(), ctx.getAttributeDef(), (AtlasStruct) ctx.getValue(), ctx.getReferringVertex(), edgeLabel);
}
return result;
}
@Override
public void cleanUp() throws AtlasBaseException {
}
public static boolean shouldManageChildReferences(AtlasStructType type, String attributeName) {
return type.isMappedFromRefAttribute(attributeName);
}
/**
* Map attributes for entity, struct or trait
* @param structType
* @param struct
* @param vertex
* @return
* @throws AtlasBaseException
*/
public AtlasVertex mapAttributestoVertex(AtlasStructType structType, AtlasStruct struct, AtlasVertex vertex) throws AtlasBaseException {
if (struct.getAttributes() != null) {
for (String attrName : struct.getAttributes().keySet()) {
Object value = struct.getAttribute(attrName);
AtlasType attributeType = structType.getAttributeType(attrName);
if ( attributeType != null) {
final AtlasStructType.AtlasAttribute attribute = structType.getAttribute(attrName);
GraphMutationContext ctx = new GraphMutationContext.Builder(attribute, value)
.referringVertex(vertex)
.vertexProperty(AtlasGraphUtilsV1.getQualifiedAttributePropertyKey(structType, attrName)).build();
mapToVertexByTypeCategory(ctx);
}
}
//Set updated timestamp
AtlasGraphUtilsV1.setProperty(vertex, Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY, RequestContextV1.get().getRequestTime());
GraphHelper.setProperty(vertex, Constants.MODIFIED_BY_KEY, RequestContextV1.get().getUser());
}
return vertex;
}
protected Object mapToVertexByTypeCategory(GraphMutationContext ctx) throws AtlasBaseException {
switch(ctx.getAttrType().getTypeCategory()) {
case PRIMITIVE:
case ENUM:
return primitivesToVertex(ctx);
case STRUCT:
return toGraph(ctx);
case ENTITY:
AtlasEntityType instanceType = entityVertexMapper.getInstanceType(ctx.getValue());
ctx.setElementType(instanceType);
return entityVertexMapper.toGraph(ctx);
case MAP:
return mapVertexMapper.toGraph(ctx);
case ARRAY:
return arrVertexMapper.toGraph(ctx);
default:
throw new AtlasBaseException(AtlasErrorCode.TYPE_CATEGORY_INVALID, ctx.getAttrType().getTypeCategory().name());
}
}
protected Object primitivesToVertex(GraphMutationContext ctx) {
if ( ctx.getAttrType().getTypeCategory() == TypeCategory.MAP ) {
MapVertexMapper.setMapValueProperty(((AtlasMapType) ctx.getAttrType()).getValueType(), ctx.getReferringVertex(), ctx.getVertexPropertyKey(), ctx.getValue());
} else {
AtlasGraphUtilsV1.setProperty(ctx.getReferringVertex(), ctx.getVertexPropertyKey(), ctx.getValue());
}
return ctx.getValue();
}
private AtlasEdge createVertex(AtlasStructType parentType, AtlasStructType attrType, AtlasStructDef.AtlasAttributeDef attributeDef, AtlasStruct struct, AtlasVertex referringVertex, String edgeLabel) throws AtlasBaseException {
AtlasVertex vertex = createVertexTemplate(struct, attrType);
mapAttributestoVertex(attrType, struct, vertex);
try {
//TODO - Map directly in AtlasGraphUtilsV1
return graphHelper.getOrCreateEdge(referringVertex, vertex, edgeLabel);
} catch (RepositoryException e) {
throw new AtlasBaseException(AtlasErrorCode.INTERNAL_ERROR, e);
}
}
private void updateVertex(AtlasStructType parentType, AtlasStructType structAttributeType, AtlasStructDef.AtlasAttributeDef attributeDef, AtlasStruct value, AtlasVertex structVertex) throws AtlasBaseException {
mapAttributestoVertex(structAttributeType, value, structVertex);
}
protected AtlasVertex createVertexTemplate(final AtlasStruct instance, final AtlasStructType structType) {
LOG.debug("Creating AtlasVertex for type {}", instance.getTypeName());
final AtlasVertex vertexWithoutIdentity = graph.addVertex();
// add type information
AtlasGraphUtilsV1.setProperty(vertexWithoutIdentity, Constants.ENTITY_TYPE_PROPERTY_KEY, instance.getTypeName());
// add state information
AtlasGraphUtilsV1.setProperty(vertexWithoutIdentity, Constants.STATE_PROPERTY_KEY, AtlasEntity.Status.ACTIVE.name());
// add timestamp information
AtlasGraphUtilsV1.setProperty(vertexWithoutIdentity, Constants.TIMESTAMP_PROPERTY_KEY, RequestContextV1.get().getRequestTime());
AtlasGraphUtilsV1.setProperty(vertexWithoutIdentity, Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY,
RequestContextV1.get().getRequestTime());
AtlasGraphUtilsV1.setProperty(vertexWithoutIdentity, Constants.CREATED_BY_KEY, RequestContextV1.get().getUser());
GraphHelper.setProperty(vertexWithoutIdentity, Constants.MODIFIED_BY_KEY, RequestContextV1.get().getUser());
return vertexWithoutIdentity;
}
protected Object mapCollectionElementsToVertex(GraphMutationContext ctx) throws AtlasBaseException {
switch(ctx.getAttrType().getTypeCategory()) {
case PRIMITIVE:
case ENUM:
return primitivesToVertex(ctx);
case STRUCT:
return toGraph(ctx);
case ENTITY:
AtlasEntityType instanceType = entityVertexMapper.getInstanceType(ctx.getValue());
ctx.setElementType(instanceType);
return entityVertexMapper.toGraph(ctx);
case MAP:
case ARRAY:
default:
throw new AtlasBaseException(AtlasErrorCode.TYPE_CATEGORY_INVALID, ctx.getAttrType().getTypeCategory().name());
}
}
}
repository/src/main/java/org/apache/atlas/repository/store/graph/v1/UniqAttrBasedEntityResolver.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.repository.store.graph.v1;
import com.google.common.base.Optional;
import org.apache.atlas.AtlasErrorCode;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.model.instance.AtlasObjectId;
import org.apache.atlas.model.typedef.AtlasStructDef;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.graph.GraphHelper;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.repository.store.graph.EntityGraphDiscoveryContext;
import org.apache.atlas.repository.store.graph.EntityResolver;
import org.apache.atlas.type.AtlasEntityType;
import org.apache.atlas.type.AtlasStructType;
import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.atlas.typesystem.exception.EntityNotFoundException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.inject.Inject;
import java.util.ArrayList;
import java.util.List;
public class UniqAttrBasedEntityResolver implements EntityResolver {
private static final Logger LOG = LoggerFactory.getLogger(UniqAttrBasedEntityResolver.class);
private final AtlasTypeRegistry typeRegistry;
private final GraphHelper graphHelper = GraphHelper.getInstance();
private EntityGraphDiscoveryContext context;
@Inject
public UniqAttrBasedEntityResolver(AtlasTypeRegistry typeRegistry) {
this.typeRegistry = typeRegistry;
}
@Override
public void init(EntityGraphDiscoveryContext entities) throws AtlasBaseException {
this.context = entities;
}
@Override
public EntityGraphDiscoveryContext resolveEntityReferences() throws AtlasBaseException {
if ( context == null) {
throw new AtlasBaseException(AtlasErrorCode.INTERNAL_ERROR, "Unique attribute based entity resolver not initialized");
}
//Resolve attribute references
List<AtlasEntity> resolvedReferences = new ArrayList<>();
for (AtlasEntity entity : context.getUnResolvedEntityReferences()) {
//query in graph repo that given unique attribute - check for deleted also?
Optional<AtlasVertex> vertex = resolveByUniqueAttribute(entity);
if (vertex.isPresent()) {
context.addRepositoryResolvedReference(new AtlasObjectId(entity.getTypeName(), entity.getGuid()), vertex.get());
resolvedReferences.add(entity);
}
}
context.removeUnResolvedEntityReferences(resolvedReferences);
if (context.getUnResolvedEntityReferences().size() > 0) {
throw new AtlasBaseException(AtlasErrorCode.INSTANCE_BY_UNIQUE_ATTRIBUTE_NOT_FOUND, context.getUnResolvedEntityReferences().toString());
}
//Resolve root references
for (AtlasEntity entity : context.getRootEntities()) {
if ( !context.isResolved(entity.getGuid()) ) {
Optional<AtlasVertex> vertex = resolveByUniqueAttribute(entity);
if (vertex.isPresent()) {
context.addRepositoryResolvedReference(new AtlasObjectId(entity.getTypeName(), entity.getGuid()), vertex.get());
}
}
}
return context;
}
Optional<AtlasVertex> resolveByUniqueAttribute(AtlasEntity entity) throws AtlasBaseException {
AtlasEntityType entityType = (AtlasEntityType) typeRegistry.getType(entity.getTypeName());
for (AtlasStructType.AtlasAttribute attr : entityType.getAllAttributes().values()) {
if (attr.getAttributeDef().getIsUnique()) {
Object attrVal = entity.getAttribute(attr.getAttributeDef().getName());
if (attrVal != null) {
String qualifiedAttrName = attr.getQualifiedAttributeName();
AtlasVertex vertex = null;
try {
vertex = graphHelper.findVertex(qualifiedAttrName, attrVal,
Constants.ENTITY_TYPE_PROPERTY_KEY, entityType.getTypeName(),
Constants.STATE_PROPERTY_KEY, AtlasEntity.Status.ACTIVE
.name());
if (LOG.isDebugEnabled()) {
LOG.debug("Found vertex by unique attribute : " + qualifiedAttrName + "=" + attrVal);
}
if (vertex != null) {
return Optional.of(vertex);
}
} catch (EntityNotFoundException e) {
//Ignore if not found
}
}
}
}
return Optional.absent();
}
@Override
public void cleanUp() {
//Nothing to cleanup
this.context = null;
}
}
repository/src/main/java/org/apache/atlas/util/AtlasRepositoryConfiguration.java
......@@ -27,6 +27,8 @@ import org.apache.atlas.repository.audit.HBaseBasedAuditRepository;
import org.apache.atlas.repository.graph.DeleteHandler;
import org.apache.atlas.repository.graph.SoftDeleteHandler;
import org.apache.atlas.repository.graphdb.GraphDatabase;
import org.apache.atlas.repository.store.graph.v1.DeleteHandlerV1;
import org.apache.atlas.repository.store.graph.v1.SoftDeleteHandlerV1;
import org.apache.atlas.typesystem.types.cache.DefaultTypeCache;
import org.apache.atlas.typesystem.types.cache.TypeCache;
import org.apache.commons.configuration.Configuration;
......@@ -84,6 +86,16 @@ public class AtlasRepositoryConfiguration {
}
}
public static Class<? extends DeleteHandlerV1> getDeleteHandlerV1Impl() {
try {
Configuration config = ApplicationProperties.get();
return ApplicationProperties.getClass(config,
DELETE_HANDLER_IMPLEMENTATION_PROPERTY, SoftDeleteHandlerV1.class.getName(), DeleteHandlerV1.class);
} catch (AtlasException e) {
throw new RuntimeException(e);
}
}
private static final String GRAPH_DATABASE_IMPLEMENTATION_PROPERTY = "atlas.graphdb.backend";
private static final String DEFAULT_GRAPH_DATABASE_IMPLEMENTATION_CLASS = "org.apache.atlas.repository.graphdb.titan0.Titan0GraphDatabase";
......
repository/src/test/java/org/apache/atlas/lineage/EntityLineageServiceTest.java
......@@ -258,7 +258,7 @@ public class EntityLineageServiceTest extends BaseRepositoryTest {
assertEquals(relationsInput.size(), 2);
AtlasEntityHeader tableEntityInput = entitiesInput.get(entityGuid);
assertEquals(tableEntityInput.getStatus(), Status.STATUS_ACTIVE);
assertEquals(tableEntityInput.getStatus(), Status.ACTIVE);
AtlasLineageInfo outputLineage = getOutputLineageInfo(entityGuid, 5);
assertNotNull(outputLineage);
......@@ -273,7 +273,7 @@ public class EntityLineageServiceTest extends BaseRepositoryTest {
assertEquals(relationsOutput.size(), 2);
AtlasEntityHeader tableEntityOutput = entitiesOutput.get(entityGuid);
assertEquals(tableEntityOutput.getStatus(), Status.STATUS_ACTIVE);
assertEquals(tableEntityOutput.getStatus(), Status.ACTIVE);
AtlasLineageInfo bothLineage = getBothLineageInfo(entityGuid, 5);
assertNotNull(bothLineage);
......@@ -288,7 +288,7 @@ public class EntityLineageServiceTest extends BaseRepositoryTest {
assertEquals(relationsBoth.size(), 4);
AtlasEntityHeader tableEntityBoth = entitiesBoth.get(entityGuid);
assertEquals(tableEntityBoth.getStatus(), Status.STATUS_ACTIVE);
assertEquals(tableEntityBoth.getStatus(), Status.ACTIVE);
//Delete the table entity. Lineage for entity returns the same results as before.
//Lineage for table name throws EntityNotFoundException
......@@ -297,17 +297,17 @@ public class EntityLineageServiceTest extends BaseRepositoryTest {
inputLineage = getInputLineageInfo(entityGuid, 5);
tableEntityInput = inputLineage.getGuidEntityMap().get(entityGuid);
assertEquals(tableEntityInput.getStatus(), Status.STATUS_DELETED);
assertEquals(tableEntityInput.getStatus(), Status.DELETED);
assertEquals(inputLineage.getGuidEntityMap().size(), 3);
outputLineage = getOutputLineageInfo(entityGuid, 5);
tableEntityOutput = outputLineage.getGuidEntityMap().get(entityGuid);
assertEquals(tableEntityOutput.getStatus(), Status.STATUS_DELETED);
assertEquals(tableEntityOutput.getStatus(), Status.DELETED);
assertEquals(outputLineage.getGuidEntityMap().size(), 3);
bothLineage = getBothLineageInfo(entityGuid, 5);
tableEntityBoth = bothLineage.getGuidEntityMap().get(entityGuid);
assertEquals(tableEntityBoth.getStatus(), Status.STATUS_DELETED);
assertEquals(tableEntityBoth.getStatus(), Status.DELETED);
assertEquals(bothLineage.getGuidEntityMap().size(), 5);
}
......
repository/src/test/java/org/apache/atlas/repository/store/graph/AtlasTypeDefGraphStoreTest.java
......@@ -32,6 +32,7 @@ import org.apache.atlas.repository.graph.AtlasGraphProvider;
import org.apache.atlas.store.AtlasTypeDefStore;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.testng.Assert;
import org.testng.annotations.AfterClass;
import org.testng.annotations.DataProvider;
import org.testng.annotations.Guice;
......@@ -215,7 +216,7 @@ public class AtlasTypeDefGraphStoreTest {
assertTrue(createdTypesDef.getEnumDefs().containsAll(atlasTypesDef.getEnumDefs()), "EnumDefs create failed");
assertTrue(createdTypesDef.getClassificationDefs().containsAll(atlasTypesDef.getClassificationDefs()), "ClassificationDef create failed");
assertTrue(createdTypesDef.getStructDefs().containsAll(atlasTypesDef.getStructDefs()), "StructDef creation failed");
assertTrue(createdTypesDef.getEntityDefs().containsAll(atlasTypesDef.getEntityDefs()), "EntityDef creation failed");
Assert.assertEquals(createdTypesDef.getEntityDefs(), atlasTypesDef.getEntityDefs());
} catch (AtlasBaseException e) {
fail("Creation of Types should've been a success", e);
......
server-api/src/main/java/org/apache/atlas/RequestContextV1.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p/>
* http://www.apache.org/licenses/LICENSE-2.0
* <p/>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas;
import org.apache.atlas.metrics.Metrics;
import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.model.instance.AtlasEntityHeader;
import org.apache.atlas.typesystem.ITypedReferenceableInstance;
import org.apache.atlas.typesystem.persistence.Id;
import org.apache.atlas.typesystem.types.ClassType;
import org.apache.atlas.typesystem.types.TypeSystem;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.Collection;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Set;
public class RequestContextV1 {
private static final Logger LOG = LoggerFactory.getLogger(RequestContextV1.class);
private static final ThreadLocal<RequestContextV1> CURRENT_CONTEXT = new ThreadLocal<>();
private Set<String> createdEntityIds = new LinkedHashSet<>();
private Set<String> updatedEntityIds = new LinkedHashSet<>();
private Set<String> deletedEntityIds = new LinkedHashSet<>();
private String user;
private final long requestTime;
TypeSystem typeSystem = TypeSystem.getInstance();
private Metrics metrics = new Metrics();
private RequestContextV1() {
requestTime = System.currentTimeMillis();
}
//To handle gets from background threads where createContext() is not called
//createContext called for every request in the filter
public static RequestContextV1 get() {
RequestContextV1 ret = CURRENT_CONTEXT.get();
if (ret == null) {
ret = new RequestContextV1();
CURRENT_CONTEXT.set(ret);
}
return ret;
}
public static void clear() {
CURRENT_CONTEXT.remove();
}
public String getUser() {
return user;
}
public void setUser(String user) {
this.user = user;
}
public void recordEntityCreate(Collection<String> createdEntityIds) {
this.createdEntityIds.addAll(createdEntityIds);
}
public void recordEntityCreate(String createdEntityId) {
this.createdEntityIds.add(createdEntityId);
}
public void recordEntityUpdate(Collection<String> updatedEntityIds) {
this.updatedEntityIds.addAll(updatedEntityIds);
}
public void recordEntityUpdate(String entityId) {
this.updatedEntityIds.add(entityId);
}
public void recordEntityDelete(String entityId) {
deletedEntityIds.add(entityId);
}
public Collection<String> getCreatedEntityIds() {
return createdEntityIds;
}
public Collection<String> getUpdatedEntityIds() {
return updatedEntityIds;
}
public Collection<String> getDeletedEntityIds() {
return deletedEntityIds;
}
public long getRequestTime() {
return requestTime;
}
public boolean isDeletedEntity(String entityGuid) {
return deletedEntityIds.contains(entityGuid);
}
public static Metrics getMetrics() {
return get().metrics;
}
}
webapp/src/main/java/org/apache/atlas/web/adapters/AtlasEntityFormatConverter.java
......@@ -109,9 +109,9 @@ public class AtlasEntityFormatConverter extends AtlasStructFormatConverter {
}
private AtlasEntity.Status convertState(EntityState state){
Status status = Status.STATUS_ACTIVE;
Status status = Status.ACTIVE;
if(state != null && state.equals(EntityState.DELETED)){
status = Status.STATUS_DELETED;
status = Status.DELETED;
}
LOG.debug("Setting state to {}", state);
return status;
......
webapp/src/main/java/org/apache/atlas/web/adapters/AtlasInstanceRestAdapters.java
......@@ -126,13 +126,13 @@ public class AtlasInstanceRestAdapters {
for (String guid : result.getCreatedEntities()) {
AtlasEntityHeader header = new AtlasEntityHeader();
header.setGuid(guid);
response.addEntity(EntityMutations.EntityOperation.CREATE_OR_UPDATE, header);
response.addEntity(EntityMutations.EntityOperation.CREATE, header);
}
for (String guid : result.getUpdateEntities()) {
AtlasEntityHeader header = new AtlasEntityHeader();
header.setGuid(guid);
response.addEntity(EntityMutations.EntityOperation.CREATE_OR_UPDATE, header);
response.addEntity(EntityMutations.EntityOperation.UPDATE, header);
}
for (String guid : result.getDeletedEntities()) {
......
webapp/src/main/java/org/apache/atlas/web/adapters/AtlasStructFormatConverter.java
......@@ -124,20 +124,20 @@ public class AtlasStructFormatConverter extends AtlasAbstractFormatConverter {
if (MapUtils.isNotEmpty(attributes)) {
ret = new HashMap<>();
for (AtlasStructDef.AtlasAttributeDef attrDef : getAttributeDefs(structType)) {
AtlasType attrType = structType.getAttributeType(attrDef.getName());
for (AtlasStructType.AtlasAttribute attr : getAttributes(structType)) {
AtlasType attrType = structType.getAttributeType(attr.getAttributeDef().getName());
if (attrType == null) {
LOG.warn("ignored attribute {}.{}: failed to find AtlasType", structType.getTypeName(), attrDef.getName());
LOG.warn("ignored attribute {}.{}: failed to find AtlasType", structType.getTypeName(), attr.getAttributeDef().getName());
continue;
}
AtlasFormatConverter attrConverter = converterRegistry.getConverter(attrType.getTypeCategory());
Object v2Value = attributes.get(attrDef.getName());
Object v2Value = attributes.get(attr.getAttributeDef().getName());
Object v1Value = attrConverter.fromV2ToV1(v2Value, attrType);
ret.put(attrDef.getName(), v1Value);
ret.put(attr.getAttributeDef().getName(), v1Value);
}
}
......@@ -150,29 +150,27 @@ public class AtlasStructFormatConverter extends AtlasAbstractFormatConverter {
if (MapUtils.isNotEmpty(attributes)) {
ret = new HashMap<>();
for (AtlasStructDef.AtlasAttributeDef attrDef : getAttributeDefs(structType)) {
AtlasType attrType = structType.getAttributeType(attrDef.getName());
for (AtlasStructType.AtlasAttribute attr : getAttributes(structType)) {
AtlasType attrType = structType.getAttributeType(attr.getAttributeDef().getName());
AtlasFormatConverter attrConverter = converterRegistry.getConverter(attrType.getTypeCategory());
Object v1Value = attributes.get(attrDef.getName());
Object v1Value = attributes.get(attr.getAttributeDef().getName());
Object v2Value = attrConverter.fromV1ToV2(v1Value, attrType);
ret.put(attrDef.getName(), v2Value);
ret.put(attr.getAttributeDef().getName(), v2Value);
}
}
return ret;
}
private Collection<AtlasAttributeDef> getAttributeDefs(AtlasStructType structType) {
Collection<AtlasAttributeDef> ret = null;
private Collection<AtlasStructType.AtlasAttribute> getAttributes(AtlasStructType structType) {
Collection<AtlasStructType.AtlasAttribute> ret = null;
if (structType.getTypeCategory() == TypeCategory.STRUCT) {
ret = structType.getStructDef().getAttributeDefs();
} else if (structType.getTypeCategory() == TypeCategory.CLASSIFICATION) {
ret = ((AtlasClassificationType)structType).getAllAttributeDefs().values();
} else if (structType.getTypeCategory() == TypeCategory.ENTITY) {
ret = ((AtlasEntityType)structType).getAllAttributeDefs().values();
if (structType.getTypeCategory() == TypeCategory.STRUCT
|| structType.getTypeCategory() == TypeCategory.CLASSIFICATION
|| structType.getTypeCategory() == TypeCategory.ENTITY) {
ret = structType.getAllAttributes().values();
} else {
ret = Collections.emptyList();
}
......
webapp/src/main/java/org/apache/atlas/web/util/LineageUtils.java
......@@ -69,7 +69,7 @@ public final class LineageUtils {
TypeSystem.IdType idType = TypeSystem.getInstance().getIdType();
vertexIdMap.put(idType.idAttrName(), guid);
vertexIdMap.put(idType.stateAttrName(), (entityHeader.getStatus() == AtlasEntity.Status.STATUS_ACTIVE) ? "ACTIVE" : "DELETED");
vertexIdMap.put(idType.stateAttrName(), (entityHeader.getStatus() == AtlasEntity.Status.ACTIVE) ? "ACTIVE" : "DELETED");
vertexIdMap.put(idType.typeNameAttrName(), entityHeader.getTypeName());
Map<String, Object> values = new HashMap<>();
......
webapp/src/test/java/org/apache/atlas/web/adapters/TestEntitiesREST.java
......@@ -96,7 +96,7 @@ public class TestEntitiesREST {
entities.add(tableEntity);
EntityMutationResponse response = entitiesREST.createOrUpdate(entities);
List<AtlasEntityHeader> guids = response.getEntitiesByOperation(EntityMutations.EntityOperation.CREATE_OR_UPDATE);
List<AtlasEntityHeader> guids = response.getEntitiesByOperation(EntityMutations.EntityOperation.CREATE);
Assert.assertNotNull(guids);
Assert.assertEquals(guids.size(), 3);
......@@ -125,7 +125,7 @@ public class TestEntitiesREST {
newEntities.add(newTableEntity);
EntityMutationResponse response2 = entitiesREST.createOrUpdate(newEntities);
List<AtlasEntityHeader> newGuids = response2.getEntitiesByOperation(EntityMutations.EntityOperation.CREATE_OR_UPDATE);
List<AtlasEntityHeader> newGuids = response2.getEntitiesByOperation(EntityMutations.EntityOperation.CREATE);
Assert.assertNotNull(newGuids);
Assert.assertEquals(newGuids.size(), 3);
}
......
webapp/src/test/java/org/apache/atlas/web/adapters/TestEntityREST.java
......@@ -79,7 +79,7 @@ public class TestEntityREST {
final EntityMutationResponse response = entityREST.createOrUpdate(dbEntity);
Assert.assertNotNull(response);
List<AtlasEntityHeader> entitiesMutated = response.getEntitiesByOperation(EntityMutations.EntityOperation.CREATE_OR_UPDATE);
List<AtlasEntityHeader> entitiesMutated = response.getEntitiesByOperation(EntityMutations.EntityOperation.CREATE);
Assert.assertNotNull(entitiesMutated);
Assert.assertEquals(entitiesMutated.size(), 1);
......@@ -160,7 +160,7 @@ public class TestEntityREST {
dbEntity.setAttribute(TestUtilsV2.NAME, updatedDBName);
final EntityMutationResponse response = entityREST.partialUpdateByUniqueAttribute(TestUtilsV2.DATABASE_TYPE, TestUtilsV2.NAME, prevDBName, dbEntity);
String dbGuid = response.getEntitiesByOperation(EntityMutations.EntityOperation.CREATE_OR_UPDATE).get(0).getGuid();
String dbGuid = response.getEntitiesByOperation(EntityMutations.EntityOperation.UPDATE).get(0).getGuid();
Assert.assertTrue(AtlasEntity.isAssigned(dbGuid));
//Get By unique attribute
......
webapp/src/test/java/org/apache/atlas/web/resources/BaseResourceIT.java
......@@ -225,9 +225,9 @@ public abstract class BaseResourceIT {
entity = entitiesClientV2.updateEntity(atlasEntity);
}
assertNotNull(entity);
assertNotNull(entity.getEntitiesByOperation(EntityMutations.EntityOperation.CREATE_OR_UPDATE));
assertTrue(entity.getEntitiesByOperation(EntityMutations.EntityOperation.CREATE_OR_UPDATE).size() > 0);
return entity.getEntitiesByOperation(EntityMutations.EntityOperation.CREATE_OR_UPDATE).get(0);
assertNotNull(entity.getEntitiesByOperation(EntityMutations.EntityOperation.UPDATE));
assertTrue(entity.getEntitiesByOperation(EntityMutations.EntityOperation.UPDATE).size() > 0);
return entity.getEntitiesByOperation(EntityMutations.EntityOperation.UPDATE).get(0);
} catch (AtlasServiceException e) {
LOG.error("Entity {} failed", update ? "update" : "creation", entity);
}
......
webapp/src/test/java/org/apache/atlas/web/resources/EntityDiscoveryJerseyResourceIT.java
......@@ -75,7 +75,7 @@ public class EntityDiscoveryJerseyResourceIT extends BaseResourceIT {
AtlasEntityHeaderWithAssociations dbEntity = entities.get(0);
assertEquals(dbEntity.getTypeName(), DATABASE_TYPE);
assertEquals(dbEntity.getDisplayText(), dbName);
assertEquals(dbEntity.getStatus(), Status.STATUS_ACTIVE);
assertEquals(dbEntity.getStatus(), Status.ACTIVE);
assertNotNull(dbEntity.getGuid());
assertNull(searchResult.getAttributes());
assertNull(searchResult.getFullTextResult());
......@@ -137,7 +137,7 @@ public class EntityDiscoveryJerseyResourceIT extends BaseResourceIT {
AtlasEntityHeaderWithAssociations dbEntity = entities.get(0);
assertEquals(dbEntity.getTypeName(), DATABASE_TYPE);
assertEquals(dbEntity.getDisplayText(), dbName);
assertEquals(dbEntity.getStatus(), Status.STATUS_ACTIVE);
assertEquals(dbEntity.getStatus(), Status.ACTIVE);
assertNotNull(dbEntity.getGuid());
assertNull(searchResult.getAttributes());
......
webapp/src/test/java/org/apache/atlas/web/resources/EntityV2JerseyResourceIT.java
......@@ -138,7 +138,7 @@ public class EntityV2JerseyResourceIT extends BaseResourceIT {
EntityMutationResponse entity = entitiesClientV2.createEntity(hiveTableInstanceV2);
assertNotNull(entity);
assertNotNull(entity.getEntitiesByOperation(EntityMutations.EntityOperation.CREATE_OR_UPDATE));
assertNotNull(entity.getEntitiesByOperation(EntityMutations.EntityOperation.CREATE));
results = searchByDSL(String.format("%s where name='%s'", DATABASE_TYPE, DATABASE_NAME));
assertEquals(results.length(), 1);
}
......@@ -175,9 +175,9 @@ public class EntityV2JerseyResourceIT extends BaseResourceIT {
instance.setAttribute("name", randomString());
EntityMutationResponse mutationResponse = entitiesClientV2.createEntity(instance);
assertNotNull(mutationResponse);
assertNotNull(mutationResponse.getEntitiesByOperation(EntityMutations.EntityOperation.CREATE_OR_UPDATE));
assertEquals(mutationResponse.getEntitiesByOperation(EntityMutations.EntityOperation.CREATE_OR_UPDATE).size(),1 );
String guid = mutationResponse.getEntitiesByOperation(EntityMutations.EntityOperation.CREATE_OR_UPDATE).get(0).getGuid();
assertNotNull(mutationResponse.getEntitiesByOperation(EntityMutations.EntityOperation.CREATE));
assertEquals(mutationResponse.getEntitiesByOperation(EntityMutations.EntityOperation.CREATE).size(),1 );
String guid = mutationResponse.getEntitiesByOperation(EntityMutations.EntityOperation.CREATE).get(0).getGuid();
//update type - add attribute
entityDef = AtlasTypeUtil.createClassTypeDef(entityDef.getName(), ImmutableSet.<String>of(),
......@@ -352,7 +352,7 @@ public class EntityV2JerseyResourceIT extends BaseResourceIT {
entityByGuid.setAttribute(property, value);
EntityMutationResponse response = entitiesClientV2.updateEntity(entityByGuid);
assertNotNull(response);
assertNotNull(response.getEntitiesByOperation(EntityMutations.EntityOperation.CREATE_OR_UPDATE));
assertNotNull(response.getEntitiesByOperation(EntityMutations.EntityOperation.UPDATE));
}
private AtlasEntity createHiveDB() {
......@@ -575,8 +575,8 @@ public class EntityV2JerseyResourceIT extends BaseResourceIT {
LOG.debug("Updating entity= " + tableUpdated);
EntityMutationResponse updateResult = entitiesClientV2.updateEntity(tableEntity.getGuid(), tableUpdated);
assertNotNull(updateResult);
assertNotNull(updateResult.getEntitiesByOperation(EntityMutations.EntityOperation.CREATE_OR_UPDATE));
assertTrue(updateResult.getEntitiesByOperation(EntityMutations.EntityOperation.CREATE_OR_UPDATE).size() > 0);
assertNotNull(updateResult.getEntitiesByOperation(EntityMutations.EntityOperation.UPDATE));
assertTrue(updateResult.getEntitiesByOperation(EntityMutations.EntityOperation.UPDATE).size() > 0);
AtlasEntity entityByGuid = entitiesClientV2.getEntityByGuid(tableEntity.getGuid());
assertNotNull(entityByGuid);
......@@ -593,8 +593,8 @@ public class EntityV2JerseyResourceIT extends BaseResourceIT {
EntityMutationResponse updateResponse = entitiesClientV2.updateEntityByAttribute(BaseResourceIT.HIVE_TABLE_TYPE, AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME,
(String) tableEntity.getAttribute("name"), tableUpdated);
assertNotNull(updateResponse);
assertNotNull(updateResponse.getEntitiesByOperation(EntityMutations.EntityOperation.CREATE_OR_UPDATE));
assertTrue(updateResponse.getEntitiesByOperation(EntityMutations.EntityOperation.CREATE_OR_UPDATE).size() > 0);
assertNotNull(updateResponse.getEntitiesByOperation(EntityMutations.EntityOperation.UPDATE));
assertTrue(updateResponse.getEntitiesByOperation(EntityMutations.EntityOperation.UPDATE).size() > 0);
entityByGuid = entitiesClientV2.getEntityByGuid(tableEntity.getGuid());
assertNotNull(entityByGuid);
......@@ -623,8 +623,8 @@ public class EntityV2JerseyResourceIT extends BaseResourceIT {
tableEntity.setAttribute("columns", columns);
EntityMutationResponse updateEntityResult = entitiesClientV2.updateEntity(tableEntity);
assertNotNull(updateEntityResult);
assertNotNull(updateEntityResult.getEntitiesByOperation(EntityMutations.EntityOperation.CREATE_OR_UPDATE));
assertEquals(updateEntityResult.getEntitiesByOperation(EntityMutations.EntityOperation.CREATE_OR_UPDATE).size(), 3);
assertNotNull(updateEntityResult.getEntitiesByOperation(EntityMutations.EntityOperation.UPDATE));
assertEquals(updateEntityResult.getEntitiesByOperation(EntityMutations.EntityOperation.UPDATE).size(), 3);
AtlasEntity entityByGuid = entitiesClientV2.getEntityByGuid(tableEntity.getGuid());
List<AtlasEntity> refs = (List<AtlasEntity>) entityByGuid.getAttribute("columns");
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment