Skip to content

A
atlas
Commit 3a0865ad by apoorvnaik Committed by Madhan Neethiraj
Options

ATLAS-1499: Notification processing using V2 Store

Signed-off-by: 's avatarMadhan Neethiraj <madhan@apache.org>
parent e4cc16ac
Showing with 200 additions and 547 deletions
intg/src/main/java/org/apache/atlas/AtlasErrorCode.java
......@@ -75,9 +75,9 @@ public enum AtlasErrorCode {
INSTANCE_LINEAGE_QUERY_FAILED(404, "ATLAS4047E", "Instance lineage query failed {0}"),
DISCOVERY_QUERY_FAILED(404, "ATLAS4048E", "Discovery query failed {0}"),
INSTANCE_CRUD_INVALID_PARAMS(404, "ATLAS4049E", "Invalid instance creation/updation parameters passed : {0}"),
INSTANCE_BY_UNIQUE_ATTRIBUTE_NOT_FOUND(404, "ATLAS40410E", "Instance {0} with unique attribute {1} does not exist"),
REFERENCED_ENTITY_NOT_FOUND(404, "ATLAS40411E", "Referenced entity {0} is not found"),
// All data conflict errors go here
TYPE_ALREADY_EXISTS(409, "ATLAS4091E", "Given type {0} already exists"),
TYPE_HAS_REFERENCES(409, "ATLAS4092E", "Given type {0} has references"),
......@@ -87,7 +87,8 @@ public enum AtlasErrorCode {
INTERNAL_ERROR(500, "ATLAS5001E", "Internal server error {0}"),
INDEX_CREATION_FAILED(500, "ATLAS5002E", "Index creation failed for {0}"),
INDEX_ROLLBACK_FAILED(500, "ATLAS5003E", "Index rollback failed for {0}"),
FAILED_TO_OBTAIN_TYPE_UPDATE_LOCK(500, "ATLAS5004E", "Failed to get the lock; another type update might be in progress. Please try again");
FAILED_TO_OBTAIN_TYPE_UPDATE_LOCK(500, "ATLAS5004E", "Failed to get the lock; another type update might be in progress. Please try again"),
NOTIFICATION_FAILED(500, "ATLAS5005E", "Failed to notify for change {0}");
private String errorCode;
private String errorMessage;
......
intg/src/main/java/org/apache/atlas/model/instance/AtlasEntity.java
......@@ -69,7 +69,7 @@ public class AtlasEntity extends AtlasStruct implements Serializable {
private String updatedBy = null;
private Date createTime = null;
private Date updateTime = null;
private Long version = new Long(0);
private Long version = 0L;
private List<AtlasClassification> classifications;
......
intg/src/main/java/org/apache/atlas/model/instance/EntityMutationResponse.java
......@@ -45,8 +45,8 @@ import org.codehaus.jackson.map.annotate.JsonSerialize;
@XmlAccessorType(XmlAccessType.PROPERTY)
public class EntityMutationResponse {
Map<EntityOperation, List<AtlasEntityHeader>> mutatedEntities;
Map<String, String> guidAssignments;
private Map<EntityOperation, List<AtlasEntityHeader>> mutatedEntities;
private Map<String, String> guidAssignments;
public EntityMutationResponse() {
}
......
repository/src/main/java/org/apache/atlas/repository/audit/EntityAuditListener.java
......@@ -18,7 +18,6 @@
package org.apache.atlas.repository.audit;
import com.google.inject.Inject;
import org.apache.atlas.AtlasException;
import org.apache.atlas.EntityAuditEvent;
import org.apache.atlas.EntityAuditEvent.EntityAuditAction;
......@@ -34,6 +33,7 @@ import org.apache.commons.collections.MapUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.inject.Inject;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Collection;
......@@ -109,6 +109,10 @@ public class EntityAuditListener implements EntityChangeListener {
auditRepository.putEvents(events);
}
public List<EntityAuditEvent> getAuditEvents(String guid) throws AtlasException{
return auditRepository.listEvents(guid, null, (short) 10);
}
private EntityAuditEvent createEvent(ITypedReferenceableInstance entity, long ts, EntityAuditAction action)
throws AtlasException {
String detail = getAuditEventDetail(entity, action);
......@@ -189,20 +193,21 @@ public class EntityAuditListener implements EntityChangeListener {
if (attrValue instanceof Collection) {
for (Object attribute : (Collection) attrValue) {
if (attribute instanceof ITypedReferenceableInstance) {
ITypedReferenceableInstance attrInstance = (ITypedReferenceableInstance) attribute;
Map<String, Object> prunedAttrs = pruneEntityAttributesForAudit(attrInstance);
if (MapUtils.isNotEmpty(prunedAttrs)) {
if (ret == null) {
ret = new HashMap<>();
ret = pruneAttributes(ret, (ITypedReferenceableInstance) attribute);
}
ret.put(attrInstance.getId()._getId(), prunedAttrs);
}
} else if (attrValue instanceof ITypedReferenceableInstance) {
ret = pruneAttributes(ret, (ITypedReferenceableInstance) attrValue);
}
}
}
} else if (attrValue instanceof ITypedReferenceableInstance) {
ITypedReferenceableInstance attrInstance = (ITypedReferenceableInstance) attrValue;
}
return ret;
}
private Map<String, Object> pruneAttributes(Map<String, Object> ret, ITypedReferenceableInstance attribute) throws AtlasException {
ITypedReferenceableInstance attrInstance = attribute;
Map<String, Object> prunedAttrs = pruneEntityAttributesForAudit(attrInstance);
if (MapUtils.isNotEmpty(prunedAttrs)) {
......@@ -212,11 +217,6 @@ public class EntityAuditListener implements EntityChangeListener {
ret.put(attrInstance.getId()._getId(), prunedAttrs);
}
}
}
}
}
return ret;
}
......@@ -240,24 +240,22 @@ public class EntityAuditListener implements EntityChangeListener {
if (attrValue instanceof Collection) {
for (Object attributeEntity : (Collection) attrValue) {
if (attributeEntity instanceof ITypedReferenceableInstance) {
ITypedReferenceableInstance attrInstance = (ITypedReferenceableInstance) attributeEntity;
Object obj = prunedAttributes.get(attrInstance.getId()._getId());
if (obj instanceof Map) {
restoreEntityAttributes(attrInstance, (Map) obj);
}
restoreAttributes(prunedAttributes, (ITypedReferenceableInstance) attributeEntity);
}
}
} else if (attrValue instanceof ITypedReferenceableInstance) {
ITypedReferenceableInstance attrInstance = (ITypedReferenceableInstance) attrValue;
Object obj = prunedAttributes.get(attrInstance.getId()._getId());
if (obj instanceof Map) {
restoreEntityAttributes(attrInstance, (Map) obj);
restoreAttributes(prunedAttributes, (ITypedReferenceableInstance) attrValue);
}
}
}
}
}
private void restoreAttributes(Map<String, Object> prunedAttributes, ITypedReferenceableInstance attributeEntity) throws AtlasException {
Object obj = prunedAttributes.get(attributeEntity.getId()._getId());
if (obj instanceof Map) {
restoreEntityAttributes(attributeEntity, (Map) obj);
}
}
......
webapp/src/main/java/org/apache/atlas/web/adapters/AtlasAbstractFormatConverter.java → repository/src/main/java/org/apache/atlas/repository/converters/AtlasAbstractFormatConverter.java
......@@ -15,7 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.web.adapters;
package org.apache.atlas.repository.converters;
import org.apache.atlas.model.TypeCategory;
......
webapp/src/main/java/org/apache/atlas/web/adapters/AtlasArrayFormatConverter.java → repository/src/main/java/org/apache/atlas/repository/converters/AtlasArrayFormatConverter.java
......@@ -15,7 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.web.adapters;
package org.apache.atlas.repository.converters;
import org.apache.atlas.AtlasErrorCode;
......
webapp/src/main/java/org/apache/atlas/web/adapters/AtlasClassificationFormatConverter.java → repository/src/main/java/org/apache/atlas/repository/converters/AtlasClassificationFormatConverter.java
......@@ -15,7 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.web.adapters;
package org.apache.atlas.repository.converters;
import org.apache.atlas.AtlasErrorCode;
import org.apache.atlas.AtlasException;
......
webapp/src/main/java/org/apache/atlas/web/adapters/AtlasEntityFormatConverter.java → repository/src/main/java/org/apache/atlas/repository/converters/AtlasEntityFormatConverter.java
......@@ -15,7 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.web.adapters;
package org.apache.atlas.repository.converters;
import org.apache.atlas.AtlasErrorCode;
import org.apache.atlas.AtlasException;
......@@ -51,22 +51,18 @@ public class AtlasEntityFormatConverter extends AtlasStructFormatConverter {
}
@Override
public Object fromV1ToV2(Object v1Obj, AtlasType type, ConverterContext context) throws AtlasBaseException {
AtlasObjectId ret = null;
public AtlasEntity fromV1ToV2(Object v1Obj, AtlasType type, ConverterContext context) throws AtlasBaseException {
AtlasEntity entity = null;
if (v1Obj != null) {
AtlasEntityType entityType = (AtlasEntityType) type;
if (v1Obj instanceof Id) {
Id id = (Id) v1Obj;
ret = new AtlasObjectId(id._getId(), id.getTypeName());
} else if (v1Obj instanceof IReferenceableInstance) {
if (v1Obj instanceof IReferenceableInstance) {
IReferenceableInstance entRef = (IReferenceableInstance) v1Obj;
ret = new AtlasObjectId(entRef.getId()._getId(), entRef.getTypeName());
String guid = entRef.getId()._getId();
if (!context.entityExists(ret.getGuid())) {
if (!context.entityExists(guid)) {
Map<String, Object> v1Attribs = null;
try {
......@@ -75,7 +71,7 @@ public class AtlasEntityFormatConverter extends AtlasStructFormatConverter {
LOG.error("IReferenceableInstance.getValuesMap() failed", excp);
}
AtlasEntity entity = new AtlasEntity(entRef.getTypeName(),
entity = new AtlasEntity(entRef.getTypeName(),
super.fromV1ToV2(entityType, v1Attribs, context));
entity.setGuid(entRef.getId()._getId());
entity.setStatus(convertState(entRef.getId().getState()));
......@@ -83,7 +79,7 @@ public class AtlasEntityFormatConverter extends AtlasStructFormatConverter {
entity.setCreateTime(entRef.getSystemAttributes().createdTime);
entity.setUpdatedBy(entRef.getSystemAttributes().modifiedBy);
entity.setUpdateTime(entRef.getSystemAttributes().modifiedTime);
entity.setVersion(new Long(entRef.getId().version));
entity.setVersion((long) entRef.getId().version);
if (CollectionUtils.isNotEmpty(entRef.getTraits())) {
List<AtlasClassification> classifications = new ArrayList<>();
......@@ -99,18 +95,18 @@ public class AtlasEntityFormatConverter extends AtlasStructFormatConverter {
entity.setClassifications(classifications);
}
context.addEntity(entity);
} else {
entity = context.getById(guid);
}
} else {
throw new AtlasBaseException(AtlasErrorCode.UNEXPECTED_TYPE, "IReferenceableInstance",
v1Obj.getClass().getCanonicalName());
}
}
return ret;
return entity;
}
private AtlasEntity.Status convertState(EntityState state){
private Status convertState(EntityState state){
Status status = Status.ACTIVE;
if(state != null && state.equals(EntityState.DELETED)){
status = Status.DELETED;
......@@ -160,7 +156,6 @@ public class AtlasEntityFormatConverter extends AtlasStructFormatConverter {
v2Obj.getClass().getCanonicalName());
}
}
return ret;
}
}
webapp/src/main/java/org/apache/atlas/web/adapters/AtlasEnumFormatConverter.java → repository/src/main/java/org/apache/atlas/repository/converters/AtlasEnumFormatConverter.java
......@@ -15,7 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.web.adapters;
package org.apache.atlas.repository.converters;
import org.apache.atlas.exception.AtlasBaseException;
......
webapp/src/main/java/org/apache/atlas/web/adapters/AtlasFormatConverter.java → repository/src/main/java/org/apache/atlas/repository/converters/AtlasFormatConverter.java
......@@ -15,16 +15,15 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.web.adapters;
package org.apache.atlas.repository.converters;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.TypeCategory;
import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.model.instance.AtlasEntity.AtlasEntitiesWithExtInfo;
import org.apache.atlas.type.AtlasType;
import java.util.HashMap;
import java.util.Map;
public interface AtlasFormatConverter {
Object fromV1ToV2(Object v1Obj, AtlasType type, ConverterContext context) throws AtlasBaseException;
......@@ -33,28 +32,35 @@ public interface AtlasFormatConverter {
TypeCategory getTypeCategory();
public static class ConverterContext {
class ConverterContext {
private Map<String, AtlasEntity> entities = null;
private AtlasEntity.AtlasEntitiesWithExtInfo entities = null;
public void addEntity(AtlasEntity entity) {
if (entities == null) {
entities = new HashMap<>();
entities = new AtlasEntitiesWithExtInfo();
}
entities.put(entity.getGuid(), entity);
entities.addEntity(entity);
}
public void addReferredEntity(AtlasEntity entity) {
if (entities == null) {
entities = new AtlasEntitiesWithExtInfo();
}
entities.addReferredEntity(entity);
}
public AtlasEntity getById(String guid) {
if( entities != null) {
return entities.get(guid);
return entities.getEntity(guid);
}
return null;
}
public boolean entityExists(String guid) { return entities != null && entities.containsKey(guid); }
public boolean entityExists(String guid) { return entities != null && entities.hasEntity(guid); }
public Map<String, AtlasEntity> getEntities() {
public AtlasEntitiesWithExtInfo getEntities() {
return entities;
}
}
......
webapp/src/main/java/org/apache/atlas/web/adapters/AtlasFormatConverters.java → repository/src/main/java/org/apache/atlas/repository/converters/AtlasFormatConverters.java
......@@ -15,7 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.web.adapters;
package org.apache.atlas.repository.converters;
import com.google.inject.Inject;
import com.google.inject.Singleton;
......@@ -41,6 +41,7 @@ public class AtlasFormatConverters {
registerConverter(new AtlasEntityFormatConverter(this, typeRegistry));
registerConverter(new AtlasArrayFormatConverter(this, typeRegistry));
registerConverter(new AtlasMapFormatConverter(this, typeRegistry));
registerConverter(new AtlasObjectIdConverter(this, typeRegistry));
}
private void registerConverter(AtlasFormatConverter converter) {
......
webapp/src/main/java/org/apache/atlas/web/adapters/AtlasInstanceRestAdapters.java → repository/src/main/java/org/apache/atlas/repository/converters/AtlasInstanceConverter.java
......@@ -15,8 +15,10 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.web.adapters;
package org.apache.atlas.repository.converters;
import com.google.inject.Inject;
import com.google.inject.Singleton;
import org.apache.atlas.AtlasClient;
import org.apache.atlas.AtlasErrorCode;
import org.apache.atlas.AtlasException;
......@@ -48,14 +50,12 @@ import org.slf4j.LoggerFactory;
import java.util.Collection;
import java.util.Iterator;
import com.google.inject.Inject;
import com.google.inject.Singleton;
import java.util.Map;
import java.util.List;
@Singleton
public class AtlasInstanceRestAdapters {
public class AtlasInstanceConverter {
private static final Logger LOG = LoggerFactory.getLogger(AtlasInstanceRestAdapters.class);
private static final Logger LOG = LoggerFactory.getLogger(AtlasInstanceConverter.class);
@Inject
private AtlasTypeRegistry typeRegistry;
......@@ -125,9 +125,9 @@ public class AtlasInstanceRestAdapters {
return ret;
}
public Map<String, AtlasEntity> getAtlasEntity(IReferenceableInstance referenceable) throws AtlasBaseException {
public AtlasEntity.AtlasEntitiesWithExtInfo getAtlasEntity(IReferenceableInstance referenceable) throws AtlasBaseException {
AtlasFormatConverter converter = instanceFormatters.getConverter(TypeCategory.ENTITY);
AtlasEntityFormatConverter converter = (AtlasEntityFormatConverter) instanceFormatters.getConverter(TypeCategory.ENTITY);
AtlasEntityType entityType = typeRegistry.getEntityTypeByName(referenceable.getTypeName());
if (entityType == null) {
......@@ -136,7 +136,8 @@ public class AtlasInstanceRestAdapters {
AtlasFormatConverter.ConverterContext ctx = new AtlasFormatConverter.ConverterContext();
converter.fromV1ToV2(referenceable, entityType, ctx);
AtlasEntity entity = converter.fromV1ToV2(referenceable, entityType, ctx);
ctx.addEntity(entity);
return ctx.getEntities();
}
......@@ -186,4 +187,37 @@ public class AtlasInstanceRestAdapters {
return new AtlasBaseException(e);
}
public AtlasEntity.AtlasEntitiesWithExtInfo getEntities(List<Referenceable> referenceables) throws AtlasBaseException {
if (LOG.isDebugEnabled()) {
LOG.debug("==> getEntities");
}
AtlasFormatConverter.ConverterContext context = new AtlasFormatConverter.ConverterContext();
for (Referenceable referenceable : referenceables) {
AtlasEntity entity = fromV1toV2Entity(referenceable, context);
context.addEntity(entity);
}
if (LOG.isDebugEnabled()) {
LOG.debug("<== getEntities");
}
return context.getEntities();
}
private AtlasEntity fromV1toV2Entity(Referenceable referenceable, AtlasFormatConverter.ConverterContext context) throws AtlasBaseException {
if (LOG.isDebugEnabled()) {
LOG.debug("==> fromV1toV2Entity");
}
AtlasEntityFormatConverter converter = (AtlasEntityFormatConverter) instanceFormatters.getConverter(TypeCategory.ENTITY);
AtlasEntity entity = converter.fromV1ToV2(referenceable, typeRegistry.getType(referenceable.getTypeName()), context);
if (LOG.isDebugEnabled()) {
LOG.debug("<== fromV1toV2Entity");
}
return entity;
}
}
webapp/src/main/java/org/apache/atlas/web/adapters/AtlasMapFormatConverter.java → repository/src/main/java/org/apache/atlas/repository/converters/AtlasMapFormatConverter.java
......@@ -15,7 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.web.adapters;
package org.apache.atlas.repository.converters;
import org.apache.atlas.AtlasErrorCode;
......@@ -77,12 +77,12 @@ public class AtlasMapFormatConverter extends AtlasAbstractFormatConverter {
AtlasType valueType = mapType.getValueType();
AtlasFormatConverter keyConverter = converterRegistry.getConverter(keyType.getTypeCategory());
AtlasFormatConverter valueConverter = converterRegistry.getConverter(valueType.getTypeCategory());
Map v1Map = (Map)v2Obj;
Map v2Map = (Map)v2Obj;
ret = new HashMap<>();
for (Object key : v1Map.keySet()) {
Object value = v1Map.get(key);
for (Object key : v2Map.keySet()) {
Object value = v2Map.get(key);
Object v2Key = keyConverter.fromV2ToV1(key, keyType, ctx);
Object v2Value = valueConverter.fromV2ToV1(value, valueType, ctx);
......
webapp/src/main/java/org/apache/atlas/web/adapters/AtlasObjectIdConverter.java → repository/src/main/java/org/apache/atlas/repository/converters/AtlasObjectIdConverter.java
......@@ -15,7 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.web.adapters;
package org.apache.atlas.repository.converters;
import org.apache.atlas.AtlasErrorCode;
......@@ -23,12 +23,11 @@ import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.TypeCategory;
import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.model.instance.AtlasObjectId;
import org.apache.atlas.type.AtlasEntityType;
import org.apache.atlas.type.AtlasType;
import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.atlas.typesystem.IReferenceableInstance;
import org.apache.atlas.typesystem.Referenceable;
import org.apache.atlas.typesystem.persistence.Id;
import org.apache.commons.collections.MapUtils;
import org.apache.commons.lang3.StringUtils;
import java.util.Map;
......@@ -53,8 +52,19 @@ AtlasObjectIdConverter extends AtlasAbstractFormatConverter {
Id id = (Id) v1Obj;
ret = new AtlasObjectId(id._getId(), id.getTypeName());
} else if (v1Obj instanceof IReferenceableInstance) {
IReferenceableInstance entity = (IReferenceableInstance) v1Obj;
ret = new AtlasObjectId(entity.getId()._getId(), entity.getTypeName());
IReferenceableInstance refInst = (IReferenceableInstance) v1Obj;
String guid = refInst.getId()._getId();
ret = new AtlasObjectId(guid, refInst.getTypeName());
if (!converterContext.entityExists(guid)) {
AtlasEntityType entityType = typeRegistry.getEntityTypeByName(refInst.getTypeName());
AtlasEntityFormatConverter entityFormatConverter = (AtlasEntityFormatConverter) converterRegistry.getConverter(TypeCategory.ENTITY);
AtlasEntity entity = entityFormatConverter.fromV1ToV2(v1Obj, entityType, converterContext);
converterContext.addReferredEntity(entity);
}
}
}
return ret;
......
webapp/src/main/java/org/apache/atlas/web/adapters/AtlasPrimitiveFormatConverter.java → repository/src/main/java/org/apache/atlas/repository/converters/AtlasPrimitiveFormatConverter.java
......@@ -15,7 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.web.adapters;
package org.apache.atlas.repository.converters;
import org.apache.atlas.exception.AtlasBaseException;
......
webapp/src/main/java/org/apache/atlas/web/adapters/AtlasStructFormatConverter.java → repository/src/main/java/org/apache/atlas/repository/converters/AtlasStructFormatConverter.java
......@@ -15,16 +15,16 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.web.adapters;
package org.apache.atlas.repository.converters;
import org.apache.atlas.AtlasErrorCode;
import org.apache.atlas.AtlasException;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.TypeCategory;
import org.apache.atlas.model.instance.AtlasStruct;
import org.apache.atlas.model.typedef.AtlasStructDef;
import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef;
import org.apache.atlas.type.*;
import org.apache.atlas.type.AtlasStructType;
import org.apache.atlas.type.AtlasType;
import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.atlas.typesystem.IStruct;
import org.apache.atlas.typesystem.Struct;
import org.apache.commons.collections.MapUtils;
......@@ -35,6 +35,7 @@ import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import java.util.Set;
public class AtlasStructFormatConverter extends AtlasAbstractFormatConverter {
private static final Logger LOG = LoggerFactory.getLogger(AtlasStructFormatConverter.class);
......@@ -124,7 +125,9 @@ public class AtlasStructFormatConverter extends AtlasAbstractFormatConverter {
if (MapUtils.isNotEmpty(attributes)) {
ret = new HashMap<>();
for (AtlasStructType.AtlasAttribute attr : structType.getAllAttributes().values()) {
// Only process the requested/set attributes
for (Object attribKey : attributes.keySet()) {
AtlasStructType.AtlasAttribute attr = structType.getAttribute((String) attribKey);
AtlasType attrType = attr.getAttributeType();
if (attrType == null) {
......@@ -133,16 +136,10 @@ public class AtlasStructFormatConverter extends AtlasAbstractFormatConverter {
}
Object v2Value = attributes.get(attr.getName());
Object v1Value = null;
Object v1Value;
AtlasFormatConverter attrConverter = null;
if (attrType.getTypeCategory() == TypeCategory.OBJECT_ID_TYPE && !attr.isOwnedRef()) {
attrConverter = new AtlasObjectIdConverter(converterRegistry, typeRegistry);
v1Value = attrConverter.fromV2ToV1(v2Value, attrType, context);
} else {
attrConverter = converterRegistry.getConverter(attrType.getTypeCategory());
AtlasFormatConverter attrConverter = converterRegistry.getConverter(attrType.getTypeCategory());
v1Value = attrConverter.fromV2ToV1(v2Value, attrType, context);
}
ret.put(attr.getName(), v1Value);
}
}
......@@ -156,7 +153,10 @@ public class AtlasStructFormatConverter extends AtlasAbstractFormatConverter {
if (MapUtils.isNotEmpty(attributes)) {
ret = new HashMap<>();
for (AtlasStructType.AtlasAttribute attr : structType.getAllAttributes().values()) {
// Only process the requested/set attributes
for (Object attribKey : attributes.keySet()) {
AtlasStructType.AtlasAttribute attr = structType.getAttribute((String) attribKey);
AtlasType attrType = attr.getAttributeType();
if (attrType == null) {
......
server-api/src/main/java/org/apache/atlas/RequestContextV1.java
......@@ -19,20 +19,13 @@
package org.apache.atlas;
import org.apache.atlas.metrics.Metrics;
import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.model.instance.AtlasEntityHeader;
import org.apache.atlas.model.instance.AtlasObjectId;
import org.apache.atlas.typesystem.ITypedReferenceableInstance;
import org.apache.atlas.typesystem.persistence.Id;
import org.apache.atlas.typesystem.types.ClassType;
import org.apache.atlas.typesystem.types.TypeSystem;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.Collection;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Set;
public class RequestContextV1 {
......
webapp/src/main/java/org/apache/atlas/web/resources/AdminResource.java
......@@ -60,7 +60,7 @@ import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.util.*;
import static org.apache.atlas.web.adapters.AtlasInstanceRestAdapters.toAtlasBaseException;
import static org.apache.atlas.repository.converters.AtlasInstanceConverter.toAtlasBaseException;
/**
* Jersey Resource for admin operations.
......
webapp/src/main/java/org/apache/atlas/web/rest/EntityREST.java
......@@ -23,11 +23,12 @@ import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.TypeCategory;
import org.apache.atlas.model.instance.AtlasClassification;
import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.model.instance.AtlasEntity.AtlasEntityWithExtInfo;
import org.apache.atlas.model.instance.AtlasEntity.AtlasEntitiesWithExtInfo;
import org.apache.atlas.model.instance.AtlasEntity.AtlasEntityWithExtInfo;
import org.apache.atlas.model.instance.ClassificationAssociateRequest;
import org.apache.atlas.model.instance.EntityMutationResponse;
import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef;
import org.apache.atlas.repository.converters.AtlasInstanceConverter;
import org.apache.atlas.repository.store.graph.AtlasEntityStore;
import org.apache.atlas.repository.store.graph.v1.AtlasEntityStream;
import org.apache.atlas.repository.store.graph.v1.EntityStream;
......@@ -37,7 +38,6 @@ import org.apache.atlas.type.AtlasEntityType;
import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.atlas.typesystem.IStruct;
import org.apache.atlas.typesystem.ITypedStruct;
import org.apache.atlas.web.adapters.AtlasInstanceRestAdapters;
import org.apache.atlas.web.util.Servlets;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.MapUtils;
......@@ -46,7 +46,15 @@ import org.apache.commons.lang3.StringUtils;
import javax.inject.Inject;
import javax.inject.Singleton;
import javax.servlet.http.HttpServletRequest;
import javax.ws.rs.*;
import javax.ws.rs.Consumes;
import javax.ws.rs.DELETE;
import javax.ws.rs.GET;
import javax.ws.rs.POST;
import javax.ws.rs.PUT;
import javax.ws.rs.Path;
import javax.ws.rs.PathParam;
import javax.ws.rs.Produces;
import javax.ws.rs.QueryParam;
import javax.ws.rs.core.Context;
import javax.ws.rs.core.MediaType;
import java.util.ArrayList;
......@@ -54,7 +62,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import static org.apache.atlas.web.adapters.AtlasInstanceRestAdapters.toAtlasBaseException;
import static org.apache.atlas.repository.converters.AtlasInstanceConverter.toAtlasBaseException;
/**
* REST for a single entity
......@@ -66,14 +74,15 @@ public class EntityREST {
public static final String PREFIX_ATTR = "attr:";
private final AtlasTypeRegistry typeRegistry;
private final AtlasInstanceRestAdapters restAdapters;
private final AtlasInstanceConverter restAdapters;
private final MetadataService metadataService;
private final AtlasEntityStore entitiesStore;
@Inject
public EntityREST(AtlasTypeRegistry typeRegistry, AtlasInstanceRestAdapters restAdapters, MetadataService metadataService, AtlasEntityStore entitiesStore) {
public EntityREST(AtlasTypeRegistry typeRegistry, AtlasInstanceConverter instanceConverter,
MetadataService metadataService, AtlasEntityStore entitiesStore) {
this.typeRegistry = typeRegistry;
this.restAdapters = restAdapters;
this.restAdapters = instanceConverter;
this.metadataService = metadataService;
this.entitiesStore = entitiesStore;
}
......
webapp/src/test/java/org/apache/atlas/LocalAtlasClientTest.java deleted 100644 → 0
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p/>
* http://www.apache.org/licenses/LICENSE-2.0
* <p/>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas;
import com.google.inject.Inject;
import com.sun.jersey.api.client.ClientResponse;
import org.apache.atlas.typesystem.Referenceable;
import org.apache.atlas.web.resources.EntityResource;
import org.apache.atlas.web.service.ServiceState;
import org.apache.commons.lang.RandomStringUtils;
import org.codehaus.jettison.json.JSONObject;
import org.mockito.Mock;
import org.mockito.MockitoAnnotations;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Guice;
import org.testng.annotations.Test;
import javax.servlet.http.HttpServletRequest;
import javax.ws.rs.WebApplicationException;
import javax.ws.rs.core.Response;
import java.net.URI;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import static org.apache.atlas.AtlasClient.ENTITIES;
import static org.mockito.Matchers.any;
import static org.mockito.Matchers.anyListOf;
import static org.mockito.Matchers.anyString;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertFalse;
import static org.testng.Assert.assertTrue;
import static org.testng.Assert.fail;
@Guice(modules= RepositoryMetadataModule.class)
public class LocalAtlasClientTest {
@Mock
private EntityResource mockEntityResource;
@Inject
private EntityResource entityResource;
@Mock
private ServiceState serviceState;
@BeforeMethod
public void setup() {
MockitoAnnotations.initMocks(this);
}
@Test
public void testCreateEntity() throws Exception {
Response response = mock(Response.class);
when(mockEntityResource.submit(any(HttpServletRequest.class))).thenReturn(response);
final String guid = random();
when(response.getEntity()).thenReturn(new JSONObject() {{
put(ENTITIES, new JSONObject(
new AtlasClient.EntityResult(Arrays.asList(guid), null, null).toString()).get(ENTITIES));
}});
LocalAtlasClient atlasClient = new LocalAtlasClient(serviceState, mockEntityResource);
List<String> results = atlasClient.createEntity(new Referenceable(random()));
assertEquals(results.size(), 1);
assertEquals(results.get(0), guid);
}
@Test
public void testException() throws Exception {
LocalAtlasClient atlasClient = new LocalAtlasClient(serviceState, mockEntityResource);
Response response = mock(Response.class);
when(mockEntityResource.submit(any(HttpServletRequest.class))).thenThrow(new WebApplicationException(response));
when(response.getEntity()).thenReturn(new JSONObject() {{
put("stackTrace", "stackTrace");
}});
when(response.getStatus()).thenReturn(Response.Status.BAD_REQUEST.getStatusCode());
try {
atlasClient.createEntity(new Referenceable(random()));
fail("Expected AtlasServiceException");
} catch(AtlasServiceException e) {
assertEquals(e.getStatus(), ClientResponse.Status.BAD_REQUEST);
}
when(mockEntityResource.updateByUniqueAttribute(anyString(), anyString(), anyString(),
any(HttpServletRequest.class))).thenThrow(new WebApplicationException(response));
when(response.getStatus()).thenReturn(Response.Status.NOT_FOUND.getStatusCode());
try {
atlasClient.updateEntity(random(), random(), random(), new Referenceable(random()));
fail("Expected AtlasServiceException");
} catch(AtlasServiceException e) {
assertEquals(e.getStatus(), ClientResponse.Status.NOT_FOUND);
}
}
@Test
public void testIsServerReady() throws Exception {
when(serviceState.getState()).thenReturn(ServiceState.ServiceStateValue.ACTIVE);
LocalAtlasClient atlasClient = new LocalAtlasClient(serviceState, mockEntityResource);
assertTrue(atlasClient.isServerReady());
when(serviceState.getState()).thenReturn(ServiceState.ServiceStateValue.BECOMING_ACTIVE);
assertFalse(atlasClient.isServerReady());
}
@Test
public void testUpdateEntity() throws Exception {
final String guid = random();
Response response = mock(Response.class);
when(mockEntityResource.updateByUniqueAttribute(anyString(), anyString(), anyString(),
any(HttpServletRequest.class))).thenReturn(response);
when(response.getEntity()).thenReturn(new JSONObject() {{
put(ENTITIES, new JSONObject(
new AtlasClient.EntityResult(null, Arrays.asList(guid), null).toString()).get(ENTITIES));
}});
LocalAtlasClient atlasClient = new LocalAtlasClient(serviceState, mockEntityResource);
AtlasClient.EntityResult
entityResult = atlasClient.updateEntity(random(), random(), random(), new Referenceable(random()));
assertEquals(entityResult.getUpdateEntities(), Arrays.asList(guid));
}
@Test
public void testDeleteEntity() throws Exception {
final String guid = random();
Response response = mock(Response.class);
when(response.getEntity()).thenReturn(new JSONObject() {{
put(ENTITIES, new JSONObject(
new AtlasClient.EntityResult(null, null, Arrays.asList(guid)).toString()).get(ENTITIES));
}});
when(mockEntityResource.deleteEntities(anyListOf(String.class), anyString(), anyString(), anyString())).thenReturn(response);
LocalAtlasClient atlasClient = new LocalAtlasClient(serviceState, mockEntityResource);
AtlasClient.EntityResult entityResult = atlasClient.deleteEntity(random(), random(), random());
assertEquals(entityResult.getDeletedEntities(), Arrays.asList(guid));
}
private String random() {
return RandomStringUtils.randomAlphanumeric(10);
}
@Test
@Inject
public void testGetLocationURI() {
final String guid = "123";
URI uri = entityResource.getLocationURI(new ArrayList<String>() {{ add(guid); }});
uri.getRawPath().equals(AtlasConstants.DEFAULT_ATLAS_REST_ADDRESS + "/" + AtlasClient.API.GET_ENTITY.getPath() + "/" + guid);
}
}
webapp/src/test/java/org/apache/atlas/notification/NotificationHookConsumerKafkaTest.java
......@@ -22,19 +22,30 @@ import com.google.inject.Inject;
import org.apache.atlas.AtlasClient;
import org.apache.atlas.AtlasException;
import org.apache.atlas.AtlasServiceException;
import org.apache.atlas.LocalAtlasClient;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.kafka.KafkaNotification;
import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.notification.hook.HookNotification;
import org.apache.atlas.repository.converters.AtlasInstanceConverter;
import org.apache.atlas.repository.store.graph.AtlasEntityStore;
import org.apache.atlas.repository.store.graph.v1.EntityStream;
import org.apache.atlas.type.AtlasType;
import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.atlas.typesystem.Referenceable;
import org.apache.atlas.web.service.ServiceState;
import org.apache.commons.lang.RandomStringUtils;
import org.mockito.Mock;
import org.mockito.MockitoAnnotations;
import org.testng.Assert;
import org.testng.annotations.AfterTest;
import org.testng.annotations.BeforeTest;
import org.testng.annotations.Guice;
import org.testng.annotations.Test;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
import static org.mockito.Matchers.any;
import static org.mockito.Matchers.anyBoolean;
import static org.mockito.Matchers.anyString;
import static org.mockito.Mockito.*;
@Guice(modules = NotificationModule.class)
public class NotificationHookConsumerKafkaTest {
......@@ -45,10 +56,28 @@ public class NotificationHookConsumerKafkaTest {
@Inject
private NotificationInterface notificationInterface;
@Mock
private AtlasEntityStore atlasEntityStore;
@Mock
private ServiceState serviceState;
@Mock
private AtlasInstanceConverter instanceConverter;
@Mock
private AtlasTypeRegistry typeRegistry;
private KafkaNotification kafkaNotification;
@BeforeTest
public void setup() throws AtlasException, InterruptedException {
public void setup() throws AtlasException, InterruptedException, AtlasBaseException {
MockitoAnnotations.initMocks(this);
AtlasType mockType = mock(AtlasType.class);
when(typeRegistry.getType(anyString())).thenReturn(mockType);
AtlasEntity.AtlasEntitiesWithExtInfo mockEntity = mock(AtlasEntity.AtlasEntitiesWithExtInfo.class);
when(instanceConverter.getEntities(anyList())).thenReturn(mockEntity);
kafkaNotification = startKafkaServer();
}
......@@ -58,25 +87,25 @@ public class NotificationHookConsumerKafkaTest {
}
@Test
public void testConsumerConsumesNewMessageWithAutoCommitDisabled() throws AtlasException, InterruptedException {
public void testConsumerConsumesNewMessageWithAutoCommitDisabled() throws AtlasException, InterruptedException, AtlasBaseException {
try {
produceMessage(new HookNotification.EntityCreateRequest("test_user1", createEntity()));
NotificationConsumer<HookNotification.HookNotificationMessage> consumer =
createNewConsumer(kafkaNotification, false);
LocalAtlasClient localAtlasClient = mock(LocalAtlasClient.class);
NotificationHookConsumer notificationHookConsumer =
new NotificationHookConsumer(kafkaNotification, localAtlasClient);
new NotificationHookConsumer(notificationInterface, atlasEntityStore, serviceState, instanceConverter, typeRegistry);
NotificationHookConsumer.HookConsumer hookConsumer =
notificationHookConsumer.new HookConsumer(consumer);
consumeOneMessage(consumer, hookConsumer);
verify(localAtlasClient).setUser("test_user1");
verify(atlasEntityStore).createOrUpdate(any(EntityStream.class), anyBoolean());
// produce another message, and make sure it moves ahead. If commit succeeded, this would work.
produceMessage(new HookNotification.EntityCreateRequest("test_user2", createEntity()));
consumeOneMessage(consumer, hookConsumer);
verify(localAtlasClient).setUser("test_user2");
verify(atlasEntityStore, times(2)).createOrUpdate(any(EntityStream.class), anyBoolean());
reset(atlasEntityStore);
}
finally {
kafkaNotification.close();
......@@ -90,20 +119,19 @@ public class NotificationHookConsumerKafkaTest {
NotificationConsumer<HookNotification.HookNotificationMessage> consumer =
createNewConsumer(kafkaNotification, true);
LocalAtlasClient localAtlasClient = mock(LocalAtlasClient.class);
NotificationHookConsumer notificationHookConsumer =
new NotificationHookConsumer(kafkaNotification, localAtlasClient);
new NotificationHookConsumer(notificationInterface, atlasEntityStore, serviceState, instanceConverter, typeRegistry);
NotificationHookConsumer.HookConsumer hookConsumer =
notificationHookConsumer.new HookConsumer(consumer);
consumeOneMessage(consumer, hookConsumer);
verify(localAtlasClient).setUser("test_user3");
verify(atlasEntityStore).createOrUpdate(any(EntityStream.class), anyBoolean());
// produce another message, but this will not be consumed, as commit code is not executed in hook consumer.
produceMessage(new HookNotification.EntityCreateRequest("test_user4", createEntity()));
consumeOneMessage(consumer, hookConsumer);
verify(localAtlasClient).setUser("test_user3");
verify(atlasEntityStore, times(2)).createOrUpdate(any(EntityStream.class), anyBoolean());
}
finally {
kafkaNotification.close();
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment