ATLAS-1499: Notification processing using V2 Store

intg/src/main/java/org/apache/atlas/AtlasErrorCode.java

@@ -75,9 +75,9 @@ public enum AtlasErrorCode {
     INSTANCE_LINEAGE_QUERY_FAILED(404, "ATLAS4047E", "Instance lineage query failed {0}"),
     DISCOVERY_QUERY_FAILED(404, "ATLAS4048E", "Discovery query failed {0}"),
     INSTANCE_CRUD_INVALID_PARAMS(404, "ATLAS4049E", "Invalid instance creation/updation parameters passed : {0}"),
+
     INSTANCE_BY_UNIQUE_ATTRIBUTE_NOT_FOUND(404, "ATLAS40410E", "Instance {0} with unique attribute {1} does not exist"),
     REFERENCED_ENTITY_NOT_FOUND(404, "ATLAS40411E", "Referenced entity {0} is not found"),
-
     // All data conflict errors go here
     TYPE_ALREADY_EXISTS(409, "ATLAS4091E", "Given type {0} already exists"),
     TYPE_HAS_REFERENCES(409, "ATLAS4092E", "Given type {0} has references"),
@@ -87,7 +87,8 @@ public enum AtlasErrorCode {
     INTERNAL_ERROR(500, "ATLAS5001E", "Internal server error {0}"),
     INDEX_CREATION_FAILED(500, "ATLAS5002E", "Index creation failed for {0}"),
     INDEX_ROLLBACK_FAILED(500, "ATLAS5003E", "Index rollback failed for {0}"),
-    FAILED_TO_OBTAIN_TYPE_UPDATE_LOCK(500, "ATLAS5004E", "Failed to get the lock; another type update might be in progress. Please try again");
+    FAILED_TO_OBTAIN_TYPE_UPDATE_LOCK(500, "ATLAS5004E", "Failed to get the lock; another type update might be in progress. Please try again"),
+    NOTIFICATION_FAILED(500, "ATLAS5005E", "Failed to notify for change {0}");
 
     private String errorCode;
     private String errorMessage;

intg/src/main/java/org/apache/atlas/model/instance/AtlasEntity.java

@@ -69,7 +69,7 @@ public class AtlasEntity extends AtlasStruct implements Serializable {
     private String updatedBy  = null;
     private Date   createTime = null;
     private Date   updateTime = null;
-    private Long   version    = new Long(0);
+    private Long   version    = 0L;
 
     private List<AtlasClassification> classifications;
 

intg/src/main/java/org/apache/atlas/model/instance/EntityMutationResponse.java

@@ -45,8 +45,8 @@ import org.codehaus.jackson.map.annotate.JsonSerialize;
 @XmlAccessorType(XmlAccessType.PROPERTY)
 public class EntityMutationResponse {
 
-    Map<EntityOperation, List<AtlasEntityHeader>> mutatedEntities;
-    Map<String, String>                           guidAssignments;
+    private Map<EntityOperation, List<AtlasEntityHeader>> mutatedEntities;
+    private Map<String, String>                           guidAssignments;
 
     public EntityMutationResponse() {
     }

repository/src/main/java/org/apache/atlas/repository/audit/EntityAuditListener.java

@@ -18,7 +18,6 @@
 
 package org.apache.atlas.repository.audit;
 
-import com.google.inject.Inject;
 import org.apache.atlas.AtlasException;
 import org.apache.atlas.EntityAuditEvent;
 import org.apache.atlas.EntityAuditEvent.EntityAuditAction;
@@ -34,6 +33,7 @@ import org.apache.commons.collections.MapUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import javax.inject.Inject;
 import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.Collection;
@@ -109,6 +109,10 @@ public class EntityAuditListener implements EntityChangeListener {
         auditRepository.putEvents(events);
     }
 
+    public List<EntityAuditEvent> getAuditEvents(String guid) throws AtlasException{
+        return auditRepository.listEvents(guid, null, (short) 10);
+    }
+
     private EntityAuditEvent createEvent(ITypedReferenceableInstance entity, long ts, EntityAuditAction action)
             throws AtlasException {
         String detail = getAuditEventDetail(entity, action);
@@ -189,29 +193,11 @@ public class EntityAuditListener implements EntityChangeListener {
                     if (attrValue instanceof Collection) {
                         for (Object attribute : (Collection) attrValue) {
                             if (attribute instanceof ITypedReferenceableInstance) {
-                                ITypedReferenceableInstance attrInstance = (ITypedReferenceableInstance) attribute;
-                                Map<String, Object>         prunedAttrs  = pruneEntityAttributesForAudit(attrInstance);
-
-                                if (MapUtils.isNotEmpty(prunedAttrs)) {
-                                    if (ret == null) {
-                                        ret = new HashMap<>();
-                                    }
-
-                                    ret.put(attrInstance.getId()._getId(), prunedAttrs);
-                                }
+                                ret = pruneAttributes(ret, (ITypedReferenceableInstance) attribute);
                             }
                         }
                     } else if (attrValue instanceof ITypedReferenceableInstance) {
-                        ITypedReferenceableInstance attrInstance = (ITypedReferenceableInstance) attrValue;
-                        Map<String, Object>         prunedAttrs  = pruneEntityAttributesForAudit(attrInstance);
-
-                        if (MapUtils.isNotEmpty(prunedAttrs)) {
-                            if (ret == null) {
-                                ret = new HashMap<>();
-                            }
-
-                            ret.put(attrInstance.getId()._getId(), prunedAttrs);
-                        }
+                        ret = pruneAttributes(ret, (ITypedReferenceableInstance) attrValue);
                     }
                 }
             }
@@ -220,6 +206,20 @@ public class EntityAuditListener implements EntityChangeListener {
         return ret;
     }
 
+    private Map<String, Object> pruneAttributes(Map<String, Object> ret, ITypedReferenceableInstance attribute) throws AtlasException {
+        ITypedReferenceableInstance attrInstance = attribute;
+        Map<String, Object>         prunedAttrs  = pruneEntityAttributesForAudit(attrInstance);
+
+        if (MapUtils.isNotEmpty(prunedAttrs)) {
+            if (ret == null) {
+                ret = new HashMap<>();
+            }
+
+            ret.put(attrInstance.getId()._getId(), prunedAttrs);
+        }
+        return ret;
+    }
+
     private void restoreEntityAttributes(ITypedReferenceableInstance entity, Map<String, Object> prunedAttributes) throws AtlasException {
         if (MapUtils.isEmpty(prunedAttributes)) {
             return;
@@ -240,27 +240,25 @@ public class EntityAuditListener implements EntityChangeListener {
                     if (attrValue instanceof Collection) {
                         for (Object attributeEntity : (Collection) attrValue) {
                             if (attributeEntity instanceof ITypedReferenceableInstance) {
-                                ITypedReferenceableInstance attrInstance = (ITypedReferenceableInstance) attributeEntity;
-                                Object                      obj          = prunedAttributes.get(attrInstance.getId()._getId());
-
-                                if (obj instanceof Map) {
-                                    restoreEntityAttributes(attrInstance, (Map) obj);
-                                }
+                                restoreAttributes(prunedAttributes, (ITypedReferenceableInstance) attributeEntity);
                             }
                         }
                     } else if (attrValue instanceof ITypedReferenceableInstance) {
-                        ITypedReferenceableInstance attrInstance = (ITypedReferenceableInstance) attrValue;
-                        Object                      obj          = prunedAttributes.get(attrInstance.getId()._getId());
-
-                        if (obj instanceof Map) {
-                            restoreEntityAttributes(attrInstance, (Map) obj);
-                        }
+                        restoreAttributes(prunedAttributes, (ITypedReferenceableInstance) attrValue);
                     }
                 }
             }
         }
     }
 
+    private void restoreAttributes(Map<String, Object> prunedAttributes, ITypedReferenceableInstance attributeEntity) throws AtlasException {
+        Object                      obj          = prunedAttributes.get(attributeEntity.getId()._getId());
+
+        if (obj instanceof Map) {
+            restoreEntityAttributes(attributeEntity, (Map) obj);
+        }
+    }
+
     private String getAuditPrefix(EntityAuditAction action) {
         final String ret;
 

webapp/src/main/java/org/apache/atlas/web/adapters/AtlasAbstractFormatConverter.java → repository/src/main/java/org/apache/atlas/repository/converters/AtlasAbstractFormatConverter.java

@@ -15,7 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.atlas.web.adapters;
+package org.apache.atlas.repository.converters;
 
 
 import org.apache.atlas.model.TypeCategory;

webapp/src/main/java/org/apache/atlas/web/adapters/AtlasArrayFormatConverter.java → repository/src/main/java/org/apache/atlas/repository/converters/AtlasArrayFormatConverter.java

@@ -15,7 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.atlas.web.adapters;
+package org.apache.atlas.repository.converters;
 
 
 import org.apache.atlas.AtlasErrorCode;

webapp/src/main/java/org/apache/atlas/web/adapters/AtlasClassificationFormatConverter.java → repository/src/main/java/org/apache/atlas/repository/converters/AtlasClassificationFormatConverter.java

@@ -15,7 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.atlas.web.adapters;
+package org.apache.atlas.repository.converters;
 
 import org.apache.atlas.AtlasErrorCode;
 import org.apache.atlas.AtlasException;

webapp/src/main/java/org/apache/atlas/web/adapters/AtlasEntityFormatConverter.java → repository/src/main/java/org/apache/atlas/repository/converters/AtlasEntityFormatConverter.java

@@ -15,7 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.atlas.web.adapters;
+package org.apache.atlas.repository.converters;
 
 import org.apache.atlas.AtlasErrorCode;
 import org.apache.atlas.AtlasException;
@@ -51,22 +51,18 @@ public class AtlasEntityFormatConverter extends AtlasStructFormatConverter {
     }
 
     @Override
-    public Object fromV1ToV2(Object v1Obj, AtlasType type, ConverterContext context) throws AtlasBaseException {
-        AtlasObjectId ret = null;
+    public AtlasEntity fromV1ToV2(Object v1Obj, AtlasType type, ConverterContext context) throws AtlasBaseException {
+        AtlasEntity entity = null;
 
         if (v1Obj != null) {
             AtlasEntityType entityType = (AtlasEntityType) type;
 
-            if (v1Obj instanceof Id) {
-                Id id = (Id) v1Obj;
-
-                ret = new AtlasObjectId(id._getId(), id.getTypeName());
-            } else if (v1Obj instanceof IReferenceableInstance) {
+            if (v1Obj instanceof IReferenceableInstance) {
                 IReferenceableInstance entRef = (IReferenceableInstance) v1Obj;
 
-                ret = new AtlasObjectId(entRef.getId()._getId(), entRef.getTypeName());
+                String guid = entRef.getId()._getId();
 
-                if (!context.entityExists(ret.getGuid())) {
+                if (!context.entityExists(guid)) {
                     Map<String, Object> v1Attribs = null;
 
                     try {
@@ -75,7 +71,7 @@ public class AtlasEntityFormatConverter extends AtlasStructFormatConverter {
                         LOG.error("IReferenceableInstance.getValuesMap() failed", excp);
                     }
 
-                    AtlasEntity entity = new AtlasEntity(entRef.getTypeName(),
+                    entity = new AtlasEntity(entRef.getTypeName(),
                                                          super.fromV1ToV2(entityType, v1Attribs, context));
                     entity.setGuid(entRef.getId()._getId());
                     entity.setStatus(convertState(entRef.getId().getState()));
@@ -83,7 +79,7 @@ public class AtlasEntityFormatConverter extends AtlasStructFormatConverter {
                     entity.setCreateTime(entRef.getSystemAttributes().createdTime);
                     entity.setUpdatedBy(entRef.getSystemAttributes().modifiedBy);
                     entity.setUpdateTime(entRef.getSystemAttributes().modifiedTime);
-                    entity.setVersion(new Long(entRef.getId().version));
+                    entity.setVersion((long) entRef.getId().version);
 
                     if (CollectionUtils.isNotEmpty(entRef.getTraits())) {
                         List<AtlasClassification> classifications = new ArrayList<>();
@@ -99,18 +95,18 @@ public class AtlasEntityFormatConverter extends AtlasStructFormatConverter {
 
                         entity.setClassifications(classifications);
                     }
-
-                    context.addEntity(entity);
+                } else {
+                    entity = context.getById(guid);
                 }
             } else {
                 throw new AtlasBaseException(AtlasErrorCode.UNEXPECTED_TYPE, "IReferenceableInstance",
                                              v1Obj.getClass().getCanonicalName());
             }
         }
-        return ret;
+        return entity;
     }
 
-    private AtlasEntity.Status convertState(EntityState state){
+    private Status convertState(EntityState state){
         Status status = Status.ACTIVE;
         if(state != null && state.equals(EntityState.DELETED)){
             status = Status.DELETED;
@@ -160,7 +156,6 @@ public class AtlasEntityFormatConverter extends AtlasStructFormatConverter {
                                              v2Obj.getClass().getCanonicalName());
             }
         }
-
         return ret;
     }
 }

webapp/src/main/java/org/apache/atlas/web/adapters/AtlasEnumFormatConverter.java → repository/src/main/java/org/apache/atlas/repository/converters/AtlasEnumFormatConverter.java

@@ -15,7 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.atlas.web.adapters;
+package org.apache.atlas.repository.converters;
 
 
 import org.apache.atlas.exception.AtlasBaseException;

webapp/src/main/java/org/apache/atlas/web/adapters/AtlasFormatConverter.java → repository/src/main/java/org/apache/atlas/repository/converters/AtlasFormatConverter.java

@@ -15,16 +15,15 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.atlas.web.adapters;
+package org.apache.atlas.repository.converters;
 
 
 import org.apache.atlas.exception.AtlasBaseException;
 import org.apache.atlas.model.TypeCategory;
 import org.apache.atlas.model.instance.AtlasEntity;
+import org.apache.atlas.model.instance.AtlasEntity.AtlasEntitiesWithExtInfo;
 import org.apache.atlas.type.AtlasType;
 
-import java.util.HashMap;
-import java.util.Map;
 
 public interface AtlasFormatConverter {
     Object fromV1ToV2(Object v1Obj, AtlasType type, ConverterContext context) throws AtlasBaseException;
@@ -33,28 +32,35 @@ public interface AtlasFormatConverter {
 
     TypeCategory getTypeCategory();
 
-    public static class ConverterContext {
+    class ConverterContext {
 
-        private Map<String, AtlasEntity> entities = null;
+        private AtlasEntity.AtlasEntitiesWithExtInfo entities = null;
 
         public void addEntity(AtlasEntity entity) {
             if (entities == null) {
-                entities = new HashMap<>();
+                entities = new AtlasEntitiesWithExtInfo();
             }
-            entities.put(entity.getGuid(), entity);
+            entities.addEntity(entity);
+        }
+
+        public void addReferredEntity(AtlasEntity entity) {
+            if (entities == null) {
+                entities = new AtlasEntitiesWithExtInfo();
+            }
+            entities.addReferredEntity(entity);
         }
 
         public AtlasEntity getById(String guid) {
             if( entities != null) {
-                return entities.get(guid);
+                return entities.getEntity(guid);
             }
 
             return null;
         }
 
-        public boolean entityExists(String guid) { return entities != null && entities.containsKey(guid); }
+        public boolean entityExists(String guid) { return entities != null && entities.hasEntity(guid); }
 
-        public Map<String, AtlasEntity> getEntities() {
+        public AtlasEntitiesWithExtInfo getEntities() {
             return entities;
         }
     }

webapp/src/main/java/org/apache/atlas/web/adapters/AtlasFormatConverters.java → repository/src/main/java/org/apache/atlas/repository/converters/AtlasFormatConverters.java

@@ -15,7 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.atlas.web.adapters;
+package org.apache.atlas.repository.converters;
 
 import com.google.inject.Inject;
 import com.google.inject.Singleton;
@@ -41,6 +41,7 @@ public class AtlasFormatConverters {
         registerConverter(new AtlasEntityFormatConverter(this, typeRegistry));
         registerConverter(new AtlasArrayFormatConverter(this, typeRegistry));
         registerConverter(new AtlasMapFormatConverter(this, typeRegistry));
+        registerConverter(new AtlasObjectIdConverter(this, typeRegistry));
     }
 
     private void registerConverter(AtlasFormatConverter converter) {

webapp/src/main/java/org/apache/atlas/web/adapters/AtlasInstanceRestAdapters.java → repository/src/main/java/org/apache/atlas/repository/converters/AtlasInstanceConverter.java

@@ -15,8 +15,10 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.atlas.web.adapters;
+package org.apache.atlas.repository.converters;
 
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
 import org.apache.atlas.AtlasClient;
 import org.apache.atlas.AtlasErrorCode;
 import org.apache.atlas.AtlasException;
@@ -48,14 +50,12 @@ import org.slf4j.LoggerFactory;
 
 import java.util.Collection;
 import java.util.Iterator;
-import com.google.inject.Inject;
-import com.google.inject.Singleton;
-import java.util.Map;
+import java.util.List;
 
 @Singleton
-public class AtlasInstanceRestAdapters {
+public class AtlasInstanceConverter {
 
-    private static final Logger LOG = LoggerFactory.getLogger(AtlasInstanceRestAdapters.class);
+    private static final Logger LOG = LoggerFactory.getLogger(AtlasInstanceConverter.class);
 
     @Inject
     private AtlasTypeRegistry typeRegistry;
@@ -125,9 +125,9 @@ public class AtlasInstanceRestAdapters {
         return ret;
     }
 
-    public Map<String, AtlasEntity> getAtlasEntity(IReferenceableInstance referenceable) throws AtlasBaseException {
+    public AtlasEntity.AtlasEntitiesWithExtInfo getAtlasEntity(IReferenceableInstance referenceable) throws AtlasBaseException {
 
-        AtlasFormatConverter converter  = instanceFormatters.getConverter(TypeCategory.ENTITY);
+        AtlasEntityFormatConverter converter  = (AtlasEntityFormatConverter) instanceFormatters.getConverter(TypeCategory.ENTITY);
         AtlasEntityType      entityType = typeRegistry.getEntityTypeByName(referenceable.getTypeName());
 
         if (entityType == null) {
@@ -136,7 +136,8 @@ public class AtlasInstanceRestAdapters {
 
         AtlasFormatConverter.ConverterContext ctx = new AtlasFormatConverter.ConverterContext();
 
-        converter.fromV1ToV2(referenceable, entityType, ctx);
+        AtlasEntity entity = converter.fromV1ToV2(referenceable, entityType, ctx);
+        ctx.addEntity(entity);
 
         return ctx.getEntities();
     }
@@ -186,4 +187,37 @@ public class AtlasInstanceRestAdapters {
         return new AtlasBaseException(e);
     }
 
+    public AtlasEntity.AtlasEntitiesWithExtInfo getEntities(List<Referenceable> referenceables) throws AtlasBaseException {
+        if (LOG.isDebugEnabled()) {
+            LOG.debug("==> getEntities");
+        }
+
+        AtlasFormatConverter.ConverterContext context = new AtlasFormatConverter.ConverterContext();
+        for (Referenceable referenceable : referenceables) {
+            AtlasEntity entity = fromV1toV2Entity(referenceable, context);
+
+            context.addEntity(entity);
+        }
+        if (LOG.isDebugEnabled()) {
+            LOG.debug("<== getEntities");
+        }
+
+        return context.getEntities();
+    }
+
+    private AtlasEntity fromV1toV2Entity(Referenceable referenceable, AtlasFormatConverter.ConverterContext context) throws AtlasBaseException {
+        if (LOG.isDebugEnabled()) {
+            LOG.debug("==> fromV1toV2Entity");
+        }
+
+        AtlasEntityFormatConverter converter = (AtlasEntityFormatConverter) instanceFormatters.getConverter(TypeCategory.ENTITY);
+
+        AtlasEntity entity = converter.fromV1ToV2(referenceable, typeRegistry.getType(referenceable.getTypeName()), context);
+
+        if (LOG.isDebugEnabled()) {
+            LOG.debug("<== fromV1toV2Entity");
+        }
+        return entity;
+    }
+
 }

webapp/src/main/java/org/apache/atlas/web/adapters/AtlasMapFormatConverter.java → repository/src/main/java/org/apache/atlas/repository/converters/AtlasMapFormatConverter.java

@@ -15,7 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.atlas.web.adapters;
+package org.apache.atlas.repository.converters;
 
 
 import org.apache.atlas.AtlasErrorCode;
@@ -77,12 +77,12 @@ public class AtlasMapFormatConverter extends AtlasAbstractFormatConverter {
                 AtlasType            valueType      = mapType.getValueType();
                 AtlasFormatConverter keyConverter   = converterRegistry.getConverter(keyType.getTypeCategory());
                 AtlasFormatConverter valueConverter = converterRegistry.getConverter(valueType.getTypeCategory());
-                Map                  v1Map          = (Map)v2Obj;
+                Map                  v2Map          = (Map)v2Obj;
 
                 ret = new HashMap<>();
 
-                for (Object key : v1Map.keySet()) {
-                    Object value = v1Map.get(key);
+                for (Object key : v2Map.keySet()) {
+                    Object value = v2Map.get(key);
 
                     Object v2Key   = keyConverter.fromV2ToV1(key, keyType, ctx);
                     Object v2Value = valueConverter.fromV2ToV1(value, valueType, ctx);

webapp/src/main/java/org/apache/atlas/web/adapters/AtlasObjectIdConverter.java → repository/src/main/java/org/apache/atlas/repository/converters/AtlasObjectIdConverter.java

@@ -15,7 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.atlas.web.adapters;
+package org.apache.atlas.repository.converters;
 
 
 import org.apache.atlas.AtlasErrorCode;
@@ -23,12 +23,11 @@ import org.apache.atlas.exception.AtlasBaseException;
 import org.apache.atlas.model.TypeCategory;
 import org.apache.atlas.model.instance.AtlasEntity;
 import org.apache.atlas.model.instance.AtlasObjectId;
+import org.apache.atlas.type.AtlasEntityType;
 import org.apache.atlas.type.AtlasType;
 import org.apache.atlas.type.AtlasTypeRegistry;
 import org.apache.atlas.typesystem.IReferenceableInstance;
-import org.apache.atlas.typesystem.Referenceable;
 import org.apache.atlas.typesystem.persistence.Id;
-import org.apache.commons.collections.MapUtils;
 import org.apache.commons.lang3.StringUtils;
 
 import java.util.Map;
@@ -53,8 +52,19 @@ AtlasObjectIdConverter extends  AtlasAbstractFormatConverter {
                 Id id = (Id) v1Obj;
                 ret = new AtlasObjectId(id._getId(), id.getTypeName());
             } else if (v1Obj instanceof IReferenceableInstance) {
-                IReferenceableInstance entity = (IReferenceableInstance) v1Obj;
-                ret = new AtlasObjectId(entity.getId()._getId(), entity.getTypeName());
+                IReferenceableInstance refInst = (IReferenceableInstance) v1Obj;
+
+                String guid = refInst.getId()._getId();
+                ret = new AtlasObjectId(guid, refInst.getTypeName());
+
+                if (!converterContext.entityExists(guid)) {
+                    AtlasEntityType entityType = typeRegistry.getEntityTypeByName(refInst.getTypeName());
+                    AtlasEntityFormatConverter entityFormatConverter = (AtlasEntityFormatConverter) converterRegistry.getConverter(TypeCategory.ENTITY);
+
+                    AtlasEntity entity = entityFormatConverter.fromV1ToV2(v1Obj, entityType, converterContext);
+
+                    converterContext.addReferredEntity(entity);
+                }
             }
         }
         return ret;

webapp/src/main/java/org/apache/atlas/web/adapters/AtlasPrimitiveFormatConverter.java → repository/src/main/java/org/apache/atlas/repository/converters/AtlasPrimitiveFormatConverter.java

@@ -15,7 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.atlas.web.adapters;
+package org.apache.atlas.repository.converters;
 
 
 import org.apache.atlas.exception.AtlasBaseException;

webapp/src/main/java/org/apache/atlas/web/adapters/AtlasStructFormatConverter.java → repository/src/main/java/org/apache/atlas/repository/converters/AtlasStructFormatConverter.java

@@ -15,16 +15,16 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.atlas.web.adapters;
+package org.apache.atlas.repository.converters;
 
 import org.apache.atlas.AtlasErrorCode;
 import org.apache.atlas.AtlasException;
 import org.apache.atlas.exception.AtlasBaseException;
 import org.apache.atlas.model.TypeCategory;
 import org.apache.atlas.model.instance.AtlasStruct;
-import org.apache.atlas.model.typedef.AtlasStructDef;
-import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef;
-import org.apache.atlas.type.*;
+import org.apache.atlas.type.AtlasStructType;
+import org.apache.atlas.type.AtlasType;
+import org.apache.atlas.type.AtlasTypeRegistry;
 import org.apache.atlas.typesystem.IStruct;
 import org.apache.atlas.typesystem.Struct;
 import org.apache.commons.collections.MapUtils;
@@ -35,6 +35,7 @@ import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
+import java.util.Set;
 
 public class AtlasStructFormatConverter extends AtlasAbstractFormatConverter {
     private static final Logger LOG = LoggerFactory.getLogger(AtlasStructFormatConverter.class);
@@ -124,7 +125,9 @@ public class AtlasStructFormatConverter extends AtlasAbstractFormatConverter {
         if (MapUtils.isNotEmpty(attributes)) {
             ret = new HashMap<>();
 
-            for (AtlasStructType.AtlasAttribute attr : structType.getAllAttributes().values()) {
+            // Only process the requested/set attributes
+            for (Object attribKey : attributes.keySet()) {
+                AtlasStructType.AtlasAttribute attr = structType.getAttribute((String) attribKey);
                 AtlasType attrType = attr.getAttributeType();
 
                 if (attrType == null) {
@@ -133,16 +136,10 @@ public class AtlasStructFormatConverter extends AtlasAbstractFormatConverter {
                 }
 
                 Object v2Value = attributes.get(attr.getName());
-                Object v1Value = null;
+                Object v1Value;
 
-                AtlasFormatConverter attrConverter = null;
-                if (attrType.getTypeCategory() == TypeCategory.OBJECT_ID_TYPE && !attr.isOwnedRef()) {
-                    attrConverter = new AtlasObjectIdConverter(converterRegistry, typeRegistry);
-                    v1Value = attrConverter.fromV2ToV1(v2Value, attrType, context);
-                } else {
-                    attrConverter = converterRegistry.getConverter(attrType.getTypeCategory());
-                    v1Value = attrConverter.fromV2ToV1(v2Value, attrType, context);
-                }
+                AtlasFormatConverter attrConverter = converterRegistry.getConverter(attrType.getTypeCategory());
+                v1Value = attrConverter.fromV2ToV1(v2Value, attrType, context);
                 ret.put(attr.getName(), v1Value);
             }
         }
@@ -156,7 +153,10 @@ public class AtlasStructFormatConverter extends AtlasAbstractFormatConverter {
         if (MapUtils.isNotEmpty(attributes)) {
             ret = new HashMap<>();
 
-            for (AtlasStructType.AtlasAttribute attr : structType.getAllAttributes().values()) {
+            // Only process the requested/set attributes
+            for (Object attribKey : attributes.keySet()) {
+                AtlasStructType.AtlasAttribute attr = structType.getAttribute((String) attribKey);
+
                 AtlasType attrType = attr.getAttributeType();
 
                 if (attrType == null) {

server-api/src/main/java/org/apache/atlas/RequestContextV1.java

@@ -19,20 +19,13 @@
 package org.apache.atlas;
 
 import org.apache.atlas.metrics.Metrics;
-import org.apache.atlas.model.instance.AtlasEntity;
-import org.apache.atlas.model.instance.AtlasEntityHeader;
 import org.apache.atlas.model.instance.AtlasObjectId;
-import org.apache.atlas.typesystem.ITypedReferenceableInstance;
-import org.apache.atlas.typesystem.persistence.Id;
-import org.apache.atlas.typesystem.types.ClassType;
 import org.apache.atlas.typesystem.types.TypeSystem;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.util.ArrayList;
 import java.util.Collection;
 import java.util.LinkedHashSet;
-import java.util.List;
 import java.util.Set;
 
 public class RequestContextV1 {

webapp/src/main/java/org/apache/atlas/web/resources/AdminResource.java

@@ -60,7 +60,7 @@ import java.io.ByteArrayInputStream;
 import java.io.IOException;
 import java.util.*;
 
-import static org.apache.atlas.web.adapters.AtlasInstanceRestAdapters.toAtlasBaseException;
+import static org.apache.atlas.repository.converters.AtlasInstanceConverter.toAtlasBaseException;
 
 /**
  * Jersey Resource for admin operations.

webapp/src/main/java/org/apache/atlas/web/rest/EntityREST.java

@@ -23,11 +23,12 @@ import org.apache.atlas.exception.AtlasBaseException;
 import org.apache.atlas.model.TypeCategory;
 import org.apache.atlas.model.instance.AtlasClassification;
 import org.apache.atlas.model.instance.AtlasEntity;
-import org.apache.atlas.model.instance.AtlasEntity.AtlasEntityWithExtInfo;
 import org.apache.atlas.model.instance.AtlasEntity.AtlasEntitiesWithExtInfo;
+import org.apache.atlas.model.instance.AtlasEntity.AtlasEntityWithExtInfo;
 import org.apache.atlas.model.instance.ClassificationAssociateRequest;
 import org.apache.atlas.model.instance.EntityMutationResponse;
 import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef;
+import org.apache.atlas.repository.converters.AtlasInstanceConverter;
 import org.apache.atlas.repository.store.graph.AtlasEntityStore;
 import org.apache.atlas.repository.store.graph.v1.AtlasEntityStream;
 import org.apache.atlas.repository.store.graph.v1.EntityStream;
@@ -37,7 +38,6 @@ import org.apache.atlas.type.AtlasEntityType;
 import org.apache.atlas.type.AtlasTypeRegistry;
 import org.apache.atlas.typesystem.IStruct;
 import org.apache.atlas.typesystem.ITypedStruct;
-import org.apache.atlas.web.adapters.AtlasInstanceRestAdapters;
 import org.apache.atlas.web.util.Servlets;
 import org.apache.commons.collections.CollectionUtils;
 import org.apache.commons.collections.MapUtils;
@@ -46,7 +46,15 @@ import org.apache.commons.lang3.StringUtils;
 import javax.inject.Inject;
 import javax.inject.Singleton;
 import javax.servlet.http.HttpServletRequest;
-import javax.ws.rs.*;
+import javax.ws.rs.Consumes;
+import javax.ws.rs.DELETE;
+import javax.ws.rs.GET;
+import javax.ws.rs.POST;
+import javax.ws.rs.PUT;
+import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
+import javax.ws.rs.Produces;
+import javax.ws.rs.QueryParam;
 import javax.ws.rs.core.Context;
 import javax.ws.rs.core.MediaType;
 import java.util.ArrayList;
@@ -54,7 +62,7 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
-import static org.apache.atlas.web.adapters.AtlasInstanceRestAdapters.toAtlasBaseException;
+import static org.apache.atlas.repository.converters.AtlasInstanceConverter.toAtlasBaseException;
 
 /**
  * REST for a single entity
@@ -66,14 +74,15 @@ public class EntityREST {
     public static final String PREFIX_ATTR = "attr:";
 
     private final AtlasTypeRegistry         typeRegistry;
-    private final AtlasInstanceRestAdapters restAdapters;
+    private final AtlasInstanceConverter restAdapters;
     private final MetadataService           metadataService;
     private final AtlasEntityStore          entitiesStore;
 
     @Inject
-    public EntityREST(AtlasTypeRegistry typeRegistry, AtlasInstanceRestAdapters restAdapters, MetadataService metadataService, AtlasEntityStore entitiesStore) {
+    public EntityREST(AtlasTypeRegistry typeRegistry, AtlasInstanceConverter instanceConverter,
+                      MetadataService metadataService, AtlasEntityStore entitiesStore) {
         this.typeRegistry    = typeRegistry;
-        this.restAdapters    = restAdapters;
+        this.restAdapters    = instanceConverter;
         this.metadataService = metadataService;
         this.entitiesStore   = entitiesStore;
     }
@@ -437,4 +446,4 @@ public class EntityREST {
             }
         }
     }
-}
+}
\ No newline at end of file

webapp/src/test/java/org/apache/atlas/LocalAtlasClientTest.java

-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * <p/>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.atlas;
-
-import com.google.inject.Inject;
-import com.sun.jersey.api.client.ClientResponse;
-import org.apache.atlas.typesystem.Referenceable;
-import org.apache.atlas.web.resources.EntityResource;
-import org.apache.atlas.web.service.ServiceState;
-import org.apache.commons.lang.RandomStringUtils;
-import org.codehaus.jettison.json.JSONObject;
-import org.mockito.Mock;
-import org.mockito.MockitoAnnotations;
-import org.testng.annotations.BeforeMethod;
-import org.testng.annotations.Guice;
-import org.testng.annotations.Test;
-
-import javax.servlet.http.HttpServletRequest;
-import javax.ws.rs.WebApplicationException;
-import javax.ws.rs.core.Response;
-import java.net.URI;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-
-import static org.apache.atlas.AtlasClient.ENTITIES;
-import static org.mockito.Matchers.any;
-import static org.mockito.Matchers.anyListOf;
-import static org.mockito.Matchers.anyString;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-import static org.testng.Assert.assertEquals;
-import static org.testng.Assert.assertFalse;
-import static org.testng.Assert.assertTrue;
-import static org.testng.Assert.fail;
-
-@Guice(modules= RepositoryMetadataModule.class)
-public class LocalAtlasClientTest {
-    @Mock
-    private EntityResource mockEntityResource;
-
-    @Inject
-    private EntityResource entityResource;
-
-    @Mock
-    private ServiceState serviceState;
-
-    @BeforeMethod
-    public void setup() {
-        MockitoAnnotations.initMocks(this);
-    }
-
-    @Test
-    public void testCreateEntity() throws Exception {
-        Response response = mock(Response.class);
-        when(mockEntityResource.submit(any(HttpServletRequest.class))).thenReturn(response);
-        final String guid = random();
-        when(response.getEntity()).thenReturn(new JSONObject() {{
-            put(ENTITIES, new JSONObject(
-                    new AtlasClient.EntityResult(Arrays.asList(guid), null, null).toString()).get(ENTITIES));
-        }});
-
-        LocalAtlasClient atlasClient = new LocalAtlasClient(serviceState, mockEntityResource);
-        List<String> results = atlasClient.createEntity(new Referenceable(random()));
-        assertEquals(results.size(), 1);
-        assertEquals(results.get(0), guid);
-    }
-
-    @Test
-    public void testException() throws Exception {
-        LocalAtlasClient atlasClient = new LocalAtlasClient(serviceState, mockEntityResource);
-
-        Response response = mock(Response.class);
-        when(mockEntityResource.submit(any(HttpServletRequest.class))).thenThrow(new WebApplicationException(response));
-        when(response.getEntity()).thenReturn(new JSONObject() {{
-            put("stackTrace", "stackTrace");
-        }});
-        when(response.getStatus()).thenReturn(Response.Status.BAD_REQUEST.getStatusCode());
-        try {
-            atlasClient.createEntity(new Referenceable(random()));
-            fail("Expected AtlasServiceException");
-        } catch(AtlasServiceException e) {
-            assertEquals(e.getStatus(), ClientResponse.Status.BAD_REQUEST);
-        }
-
-        when(mockEntityResource.updateByUniqueAttribute(anyString(), anyString(), anyString(),
-                any(HttpServletRequest.class))).thenThrow(new WebApplicationException(response));
-        when(response.getStatus()).thenReturn(Response.Status.NOT_FOUND.getStatusCode());
-        try {
-            atlasClient.updateEntity(random(), random(), random(), new Referenceable(random()));
-            fail("Expected AtlasServiceException");
-        } catch(AtlasServiceException e) {
-            assertEquals(e.getStatus(), ClientResponse.Status.NOT_FOUND);
-        }
-
-    }
-
-    @Test
-    public void testIsServerReady() throws Exception {
-        when(serviceState.getState()).thenReturn(ServiceState.ServiceStateValue.ACTIVE);
-        LocalAtlasClient atlasClient = new LocalAtlasClient(serviceState, mockEntityResource);
-        assertTrue(atlasClient.isServerReady());
-
-        when(serviceState.getState()).thenReturn(ServiceState.ServiceStateValue.BECOMING_ACTIVE);
-        assertFalse(atlasClient.isServerReady());
-    }
-
-    @Test
-    public void testUpdateEntity() throws Exception {
-        final String guid = random();
-        Response response = mock(Response.class);
-        when(mockEntityResource.updateByUniqueAttribute(anyString(), anyString(), anyString(),
-                any(HttpServletRequest.class))).thenReturn(response);
-        when(response.getEntity()).thenReturn(new JSONObject() {{
-            put(ENTITIES, new JSONObject(
-                    new AtlasClient.EntityResult(null, Arrays.asList(guid), null).toString()).get(ENTITIES));
-        }});
-
-        LocalAtlasClient atlasClient = new LocalAtlasClient(serviceState, mockEntityResource);
-        AtlasClient.EntityResult
-                entityResult = atlasClient.updateEntity(random(), random(), random(), new Referenceable(random()));
-        assertEquals(entityResult.getUpdateEntities(), Arrays.asList(guid));
-    }
-
-    @Test
-    public void testDeleteEntity() throws Exception {
-        final String guid = random();
-        Response response = mock(Response.class);
-        when(response.getEntity()).thenReturn(new JSONObject() {{
-            put(ENTITIES, new JSONObject(
-                    new AtlasClient.EntityResult(null, null, Arrays.asList(guid)).toString()).get(ENTITIES));
-        }});
-
-        when(mockEntityResource.deleteEntities(anyListOf(String.class), anyString(), anyString(), anyString())).thenReturn(response);
-        LocalAtlasClient atlasClient = new LocalAtlasClient(serviceState, mockEntityResource);
-        AtlasClient.EntityResult entityResult = atlasClient.deleteEntity(random(), random(), random());
-        assertEquals(entityResult.getDeletedEntities(), Arrays.asList(guid));
-    }
-
-    private String random() {
-        return RandomStringUtils.randomAlphanumeric(10);
-    }
-
-    @Test
-    @Inject
-    public void testGetLocationURI() {
-        final String guid = "123";
-        URI uri = entityResource.getLocationURI(new ArrayList<String>() {{ add(guid); }});
-        uri.getRawPath().equals(AtlasConstants.DEFAULT_ATLAS_REST_ADDRESS + "/" + AtlasClient.API.GET_ENTITY.getPath() + "/" + guid);
-    }
-}

webapp/src/test/java/org/apache/atlas/notification/NotificationHookConsumerKafkaTest.java

@@ -22,19 +22,30 @@ import com.google.inject.Inject;
 import org.apache.atlas.AtlasClient;
 import org.apache.atlas.AtlasException;
 import org.apache.atlas.AtlasServiceException;
-import org.apache.atlas.LocalAtlasClient;
+import org.apache.atlas.exception.AtlasBaseException;
 import org.apache.atlas.kafka.KafkaNotification;
+import org.apache.atlas.model.instance.AtlasEntity;
 import org.apache.atlas.notification.hook.HookNotification;
+import org.apache.atlas.repository.converters.AtlasInstanceConverter;
+import org.apache.atlas.repository.store.graph.AtlasEntityStore;
+import org.apache.atlas.repository.store.graph.v1.EntityStream;
+import org.apache.atlas.type.AtlasType;
+import org.apache.atlas.type.AtlasTypeRegistry;
 import org.apache.atlas.typesystem.Referenceable;
+import org.apache.atlas.web.service.ServiceState;
 import org.apache.commons.lang.RandomStringUtils;
+import org.mockito.Mock;
+import org.mockito.MockitoAnnotations;
 import org.testng.Assert;
 import org.testng.annotations.AfterTest;
 import org.testng.annotations.BeforeTest;
 import org.testng.annotations.Guice;
 import org.testng.annotations.Test;
 
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.verify;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.anyBoolean;
+import static org.mockito.Matchers.anyString;
+import static org.mockito.Mockito.*;
 
 @Guice(modules = NotificationModule.class)
 public class NotificationHookConsumerKafkaTest {
@@ -45,10 +56,28 @@ public class NotificationHookConsumerKafkaTest {
     @Inject
     private NotificationInterface notificationInterface;
 
+
+    @Mock
+    private AtlasEntityStore atlasEntityStore;
+
+    @Mock
+    private ServiceState serviceState;
+
+    @Mock
+    private AtlasInstanceConverter instanceConverter;
+
+    @Mock
+    private AtlasTypeRegistry typeRegistry;
+
     private KafkaNotification kafkaNotification;
 
     @BeforeTest
-    public void setup() throws AtlasException, InterruptedException {
+    public void setup() throws AtlasException, InterruptedException, AtlasBaseException {
+        MockitoAnnotations.initMocks(this);
+        AtlasType mockType = mock(AtlasType.class);
+        when(typeRegistry.getType(anyString())).thenReturn(mockType);
+        AtlasEntity.AtlasEntitiesWithExtInfo mockEntity = mock(AtlasEntity.AtlasEntitiesWithExtInfo.class);
+        when(instanceConverter.getEntities(anyList())).thenReturn(mockEntity);
         kafkaNotification = startKafkaServer();
     }
 
@@ -58,25 +87,25 @@ public class NotificationHookConsumerKafkaTest {
     }
 
     @Test
-    public void testConsumerConsumesNewMessageWithAutoCommitDisabled() throws AtlasException, InterruptedException {
+    public void testConsumerConsumesNewMessageWithAutoCommitDisabled() throws AtlasException, InterruptedException, AtlasBaseException {
         try {
             produceMessage(new HookNotification.EntityCreateRequest("test_user1", createEntity()));
     
             NotificationConsumer<HookNotification.HookNotificationMessage> consumer =
                     createNewConsumer(kafkaNotification, false);
-            LocalAtlasClient localAtlasClient = mock(LocalAtlasClient.class);
             NotificationHookConsumer notificationHookConsumer =
-                    new NotificationHookConsumer(kafkaNotification, localAtlasClient);
+                    new NotificationHookConsumer(notificationInterface, atlasEntityStore, serviceState, instanceConverter, typeRegistry);
             NotificationHookConsumer.HookConsumer hookConsumer =
                     notificationHookConsumer.new HookConsumer(consumer);
     
             consumeOneMessage(consumer, hookConsumer);
-            verify(localAtlasClient).setUser("test_user1");
-    
+            verify(atlasEntityStore).createOrUpdate(any(EntityStream.class), anyBoolean());
+
             // produce another message, and make sure it moves ahead. If commit succeeded, this would work.
             produceMessage(new HookNotification.EntityCreateRequest("test_user2", createEntity()));
             consumeOneMessage(consumer, hookConsumer);
-            verify(localAtlasClient).setUser("test_user2");
+            verify(atlasEntityStore, times(2)).createOrUpdate(any(EntityStream.class), anyBoolean());
+            reset(atlasEntityStore);
         }
         finally {
             kafkaNotification.close();
@@ -90,20 +119,19 @@ public class NotificationHookConsumerKafkaTest {
     
             NotificationConsumer<HookNotification.HookNotificationMessage> consumer =
                     createNewConsumer(kafkaNotification, true);
-            LocalAtlasClient localAtlasClient = mock(LocalAtlasClient.class);
             NotificationHookConsumer notificationHookConsumer =
-                    new NotificationHookConsumer(kafkaNotification, localAtlasClient);
+                    new NotificationHookConsumer(notificationInterface, atlasEntityStore, serviceState, instanceConverter, typeRegistry);
             NotificationHookConsumer.HookConsumer hookConsumer =
                     notificationHookConsumer.new HookConsumer(consumer);
     
             consumeOneMessage(consumer, hookConsumer);
-            verify(localAtlasClient).setUser("test_user3");
+            verify(atlasEntityStore).createOrUpdate(any(EntityStream.class), anyBoolean());
     
             // produce another message, but this will not be consumed, as commit code is not executed in hook consumer.
             produceMessage(new HookNotification.EntityCreateRequest("test_user4", createEntity()));
     
             consumeOneMessage(consumer, hookConsumer);
-            verify(localAtlasClient).setUser("test_user3");
+            verify(atlasEntityStore, times(2)).createOrUpdate(any(EntityStream.class), anyBoolean());
         }
         finally {
             kafkaNotification.close();