Skip to content

A
atlas
Commit 5f508c97 by Jeff Hagelberg
Options

ATLAS-746 : After updating a set of entities, response contains only the first entity definition

parent acfe9a48
Showing with 1417 additions and 302 deletions
catalog/src/main/java/org/apache/atlas/catalog/DefaultTypeSystem.java
......@@ -105,7 +105,8 @@ public class DefaultTypeSystem implements AtlasTypeSystem {
entity.set(TaxonomyResourceProvider.NAMESPACE_ATTRIBUTE_NAME, TaxonomyResourceProvider.TAXONOMY_NS);
ITypedReferenceableInstance typedInstance = metadataService.getTypedReferenceableInstance(entity);
final List<String> entities = metadataService.createEntities(Collections.singletonList(typedInstance).toArray(new ITypedReferenceableInstance[1]));
ITypedReferenceableInstance[] entitiesToCreate = Collections.singletonList(typedInstance).toArray(new ITypedReferenceableInstance[1]);
final List<String> entities = metadataService.createEntities(entitiesToCreate).getCreatedEntities();
return entities != null && entities.size() > 0 ? entities.get(0) : null;
} catch (EntityExistsException e) {
throw new ResourceAlreadyExistsException(
......
client/src/main/java/org/apache/atlas/AtlasBaseClient.java
......@@ -17,15 +17,19 @@
*/
package org.apache.atlas;
import com.google.common.annotations.VisibleForTesting;
import com.sun.jersey.api.client.Client;
import com.sun.jersey.api.client.ClientHandlerException;
import com.sun.jersey.api.client.ClientResponse;
import com.sun.jersey.api.client.WebResource;
import com.sun.jersey.api.client.config.DefaultClientConfig;
import com.sun.jersey.api.client.filter.HTTPBasicAuthFilter;
import com.sun.jersey.api.json.JSONConfiguration;
import com.sun.jersey.client.urlconnection.URLConnectionClientHandler;
import static org.apache.atlas.security.SecurityProperties.TLS_ENABLED;
import java.io.IOException;
import java.net.ConnectException;
import java.util.List;
import java.util.Map;
import javax.ws.rs.HttpMethod;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.MultivaluedMap;
import javax.ws.rs.core.Response;
import javax.ws.rs.core.UriBuilder;
import org.apache.atlas.model.metrics.AtlasMetrics;
import org.apache.atlas.security.SecureClientUtils;
import org.apache.atlas.type.AtlasType;
......@@ -38,17 +42,16 @@ import org.codehaus.jettison.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.ws.rs.HttpMethod;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.MultivaluedMap;
import javax.ws.rs.core.Response;
import javax.ws.rs.core.UriBuilder;
import java.io.IOException;
import java.net.ConnectException;
import java.util.List;
import java.util.Map;
import static org.apache.atlas.security.SecurityProperties.TLS_ENABLED;
import com.google.common.annotations.VisibleForTesting;
import com.sun.jersey.api.client.Client;
import com.sun.jersey.api.client.ClientHandlerException;
import com.sun.jersey.api.client.ClientResponse;
import com.sun.jersey.api.client.GenericType;
import com.sun.jersey.api.client.WebResource;
import com.sun.jersey.api.client.config.DefaultClientConfig;
import com.sun.jersey.api.client.filter.HTTPBasicAuthFilter;
import com.sun.jersey.api.json.JSONConfiguration;
import com.sun.jersey.client.urlconnection.URLConnectionClientHandler;
public abstract class AtlasBaseClient {
public static final String BASE_URI = "api/atlas/";
......@@ -277,6 +280,14 @@ public abstract class AtlasBaseClient {
}
protected <T> T callAPIWithResource(APIInfo api, WebResource resource, Object requestObject, Class<T> responseType) throws AtlasServiceException {
GenericType<T> genericType = null;
if(responseType != null) {
genericType = new GenericType<>(responseType);
}
return callAPIWithResource(api, resource, requestObject, genericType);
}
protected <T> T callAPIWithResource(APIInfo api, WebResource resource, Object requestObject, GenericType<T> responseType) throws AtlasServiceException {
ClientResponse clientResponse = null;
int i = 0;
do {
......@@ -297,7 +308,7 @@ public abstract class AtlasBaseClient {
return null;
}
try {
if (responseType == JSONObject.class) {
if (responseType.getRawClass() == JSONObject.class) {
String stringEntity = clientResponse.getEntity(String.class);
try {
JSONObject jsonObject = new JSONObject(stringEntity);
......@@ -419,6 +430,12 @@ public abstract class AtlasBaseClient {
return callAPIWithResource(api, getResource(api, params), requestObject, responseType);
}
public <T> T callAPI(APIInfo api, Object requestObject, GenericType<T> responseType, String... params)
throws AtlasServiceException {
return callAPIWithResource(api, getResource(api, params), requestObject, responseType);
}
public <T> T callAPI(APIInfo api, Object requestBody, Class<T> responseType,
MultivaluedMap<String, String> queryParams, String... params) throws AtlasServiceException {
WebResource resource = getResource(api, queryParams, params);
......@@ -431,6 +448,12 @@ public abstract class AtlasBaseClient {
return callAPIWithResource(api, resource, null, responseType);
}
public <T> T callAPI(APIInfo api, GenericType<T> responseType, MultivaluedMap<String, String> queryParams, String... params)
throws AtlasServiceException {
WebResource resource = getResource(api, queryParams, params);
return callAPIWithResource(api, resource, null, responseType);
}
protected WebResource getResource(APIInfo api, String... pathParams) {
return getResource(service, api, pathParams);
}
......
client/src/main/java/org/apache/atlas/AtlasClient.java
......@@ -61,6 +61,7 @@ public class AtlasClient extends AtlasBaseClient {
public static final String TYPENAME = "typeName";
public static final String GUID = "GUID";
public static final String ENTITIES = "entities";
public static final String GUID_ASSIGNMENTS = "guidAssignments";
public static final String DEFINITION = "definition";
public static final String ERROR = "error";
......
client/src/main/java/org/apache/atlas/AtlasEntitiesClientV2.java
......@@ -17,29 +17,32 @@
*/
package org.apache.atlas;
import com.google.common.annotations.VisibleForTesting;
import com.sun.jersey.api.client.WebResource;
import com.sun.jersey.core.util.MultivaluedMapImpl;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import javax.ws.rs.HttpMethod;
import javax.ws.rs.core.MultivaluedMap;
import javax.ws.rs.core.Response;
import org.apache.atlas.model.SearchFilter;
import org.apache.atlas.model.instance.AtlasClassification;
import org.apache.atlas.model.instance.AtlasClassification.AtlasClassifications;
import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.model.instance.AtlasEntity.AtlasEntities;
import org.apache.atlas.model.instance.AtlasEntityWithAssociations;
import org.apache.atlas.model.instance.EntityMutationResponse;
import org.apache.commons.configuration.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
import javax.ws.rs.HttpMethod;
import javax.ws.rs.core.MultivaluedMap;
import javax.ws.rs.core.Response;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import static org.apache.atlas.model.instance.AtlasEntity.AtlasEntities;
import com.google.common.annotations.VisibleForTesting;
import com.sun.jersey.api.client.GenericType;
import com.sun.jersey.api.client.WebResource;
import com.sun.jersey.core.util.MultivaluedMapImpl;
public class AtlasEntitiesClientV2 extends AtlasBaseClient {
private static final GenericType<List<AtlasEntityWithAssociations>> ENTITY_WITH_ASSOCIATIONS_LIST_TYPE = new GenericType<List<AtlasEntityWithAssociations>>(){};
public static final String ENTITY_API = BASE_URI + "v2/entity/";
public static final String ENTITIES_API = BASE_URI + "v2/entities/";
......@@ -84,22 +87,25 @@ public class AtlasEntitiesClientV2 extends AtlasBaseClient {
super(service, configuration);
}
public AtlasEntity getEntityByGuid(String guid) throws AtlasServiceException {
return callAPI(GET_ENTITY_BY_GUID, null, AtlasEntity.class, guid);
public List<AtlasEntityWithAssociations> getEntityByGuid(String guid) throws AtlasServiceException {
return callAPI(GET_ENTITY_BY_GUID, null, ENTITY_WITH_ASSOCIATIONS_LIST_TYPE, guid);
}
public AtlasEntities getEntityByGuids(List<String> guids) throws AtlasServiceException {
return callAPI(GET_ENTITY_BY_GUID, AtlasEntities.class, "guid", guids);
}
public AtlasEntityWithAssociations getEntityWithAssociationByGuid(String guid) throws AtlasServiceException {
return callAPI(formatPathForPathParams(GET_ENTITY_WITH_ASSOCIATION_BY_GUID, guid), null, AtlasEntityWithAssociations.class);
public List<AtlasEntityWithAssociations> getEntityWithAssociationByGuid(String guid) throws AtlasServiceException {
return callAPI(formatPathForPathParams(GET_ENTITY_WITH_ASSOCIATION_BY_GUID, guid), null, ENTITY_WITH_ASSOCIATIONS_LIST_TYPE);
}
public AtlasEntity getEntityByAttribute(String type, String attribute, String value) throws AtlasServiceException {
public List<AtlasEntityWithAssociations> getEntityByAttribute(String type, String attribute, String value) throws AtlasServiceException {
MultivaluedMap<String, String> queryParams = new MultivaluedMapImpl();
queryParams.add("value", value);
return callAPI(formatPathForPathParams(GET_ENTITY_BY_ATTRIBUTE, type, attribute), AtlasEntity.class, queryParams);
return callAPI(formatPathForPathParams(GET_ENTITY_BY_ATTRIBUTE, type, attribute), ENTITY_WITH_ASSOCIATIONS_LIST_TYPE, queryParams);
}
public EntityMutationResponse updateEntityByAttribute(String type, String attribute, String value, AtlasEntity entity) throws AtlasServiceException {
......@@ -115,11 +121,11 @@ public class AtlasEntitiesClientV2 extends AtlasBaseClient {
}
public EntityMutationResponse createEntity(final AtlasEntity atlasEntity) throws AtlasServiceException {
return callAPI(CREATE_ENTITY, new HashMap<String, AtlasEntity>() {{ put(atlasEntity.getGuid(), atlasEntity); }}, EntityMutationResponse.class);
return callAPI(CREATE_ENTITY, new HashMap<String, AtlasEntity>(1) {{ put(atlasEntity.getGuid(), atlasEntity); }}, EntityMutationResponse.class);
}
public EntityMutationResponse updateEntity(final AtlasEntity atlasEntity) throws AtlasServiceException {
return callAPI(UPDATE_ENTITY, new HashMap<String, AtlasEntity>() {{ put(atlasEntity.getGuid(), atlasEntity); }}, EntityMutationResponse.class);
return callAPI(UPDATE_ENTITY, new HashMap<String, AtlasEntity>(1) {{ put(atlasEntity.getGuid(), atlasEntity); }}, EntityMutationResponse.class);
}
public AtlasEntity deleteEntityByGuid(String guid) throws AtlasServiceException {
......@@ -135,7 +141,7 @@ public class AtlasEntitiesClientV2 extends AtlasBaseClient {
}
public void addClassifications(String guid, List<AtlasClassification> classifications) throws AtlasServiceException {
callAPI(formatPathForPathParams(ADD_CLASSIFICATIONS, guid), classifications, null, (String[]) null);
callAPI(formatPathForPathParams(ADD_CLASSIFICATIONS, guid), classifications, (Class<?>)null, (String[]) null);
}
public void updateClassifications(String guid, List<AtlasClassification> classifications) throws AtlasServiceException {
......@@ -156,11 +162,24 @@ public class AtlasEntitiesClientV2 extends AtlasBaseClient {
return null;
}
public List<AtlasEntity> createEntities(Map<String, AtlasEntity> atlasEntities) throws AtlasServiceException {
return (List<AtlasEntity>)callAPI(CREATE_ENTITIES, atlasEntities, List.class);
public EntityMutationResponse createEntities(List<AtlasEntity> atlasEntities) throws AtlasServiceException {
return callAPI(CREATE_ENTITIES, entityListToMap(atlasEntities), EntityMutationResponse.class);
}
private Map<String, AtlasEntity> entityListToMap(List<AtlasEntity> atlasEntities) {
Map<String,AtlasEntity> toSend = new HashMap<String, AtlasEntity>(atlasEntities.size());
for(AtlasEntity entity : atlasEntities) {
toSend.put(entity.getGuid(), entity);
}
return toSend;
}
public EntityMutationResponse updateEntities(List<AtlasEntity> atlasEntities) throws AtlasServiceException {
return callAPI(UPDATE_ENTITIES, entityListToMap(atlasEntities), EntityMutationResponse.class);
}
public List<AtlasEntity> updateEntities(Map<String, AtlasEntity> atlasEntities) throws AtlasServiceException {
return (List<AtlasEntity>)callAPI(UPDATE_ENTITIES, atlasEntities, List.class);
public AtlasEntity.AtlasEntities searchEntities(SearchFilter searchFilter) throws AtlasServiceException {
return callAPI(GET_ENTITIES, AtlasEntity.AtlasEntities.class, searchFilter.getParams());
}
}
client/src/main/java/org/apache/atlas/CreateUpdateEntitiesResult.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas;
import java.util.Collections;
import java.util.List;
import org.apache.atlas.AtlasClient.EntityResult;
import org.apache.atlas.model.instance.GuidMapping;
/**
* Result from creating or updating entities.
*/
public class CreateUpdateEntitiesResult {
/**
* Guid mapping for the entities that were created/updated
*/
private GuidMapping guidMapping;
/**
* Entity result
*/
private EntityResult entityResult;
/**
* Gets the guid mapping
*/
public GuidMapping getGuidMapping() {
return guidMapping;
}
/**
* Sets the guid mapping
*/
public void setGuidMapping(GuidMapping guidMapping) {
this.guidMapping = guidMapping;
}
/**
* Gets the entity result
*/
public EntityResult getEntityResult() {
return entityResult;
}
/**
* Sets the entity result
*/
public void setEntityResult(EntityResult entityResult) {
this.entityResult = entityResult;
}
/**
* Deserializes the given json into an instance of
* CreateUpdateEntitiesResult.
*
* @param json
* the (unmodified) json that comes back from Atlas.
* @return
* @throws AtlasServiceException
*/
public static CreateUpdateEntitiesResult fromJson(String json) throws AtlasServiceException {
GuidMapping guidMapping = GuidMapping.fromString(json);
EntityResult entityResult = EntityResult.fromString(json);
CreateUpdateEntitiesResult result = new CreateUpdateEntitiesResult();
result.setEntityResult(entityResult);
result.setGuidMapping(guidMapping);
return result;
}
/**
* Convenience method to get the guids of the created entities from
* the EntityResult.
*/
public List<String> getCreatedEntities() {
if(entityResult == null) {
return Collections.emptyList();
}
return getEntityResult().getCreatedEntities();
}
/**
* Convenience method to get the guids of the updated entities from
* the EntityResult.
*/
public List<String> getUpdatedEntities() {
if(entityResult == null) {
return Collections.emptyList();
}
return getEntityResult().getUpdateEntities();
}
/**
* Convenience method to get the guids of the deleted entities
* from the EntityResult.
*/
public List<String> getDeletedEntities() {
if (entityResult == null) {
return Collections.emptyList();
}
return getEntityResult().getDeletedEntities();
}
}
intg/src/main/java/org/apache/atlas/model/instance/EntityMutationResponse.java
......@@ -18,25 +18,24 @@
package org.apache.atlas.model.instance;
import org.apache.atlas.model.typedef.AtlasBaseTypeDef;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.MapUtils;
import org.codehaus.jackson.annotate.JsonAutoDetect;
import org.codehaus.jackson.annotate.JsonIgnore;
import org.codehaus.jackson.annotate.JsonIgnoreProperties;
import org.codehaus.jackson.map.annotate.JsonSerialize;
import static org.codehaus.jackson.annotate.JsonAutoDetect.Visibility.NONE;
import static org.codehaus.jackson.annotate.JsonAutoDetect.Visibility.PUBLIC_ONLY;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlRootElement;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import static org.codehaus.jackson.annotate.JsonAutoDetect.Visibility.NONE;
import static org.codehaus.jackson.annotate.JsonAutoDetect.Visibility.PUBLIC_ONLY;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlRootElement;
import org.apache.atlas.model.typedef.AtlasBaseTypeDef;
import org.codehaus.jackson.annotate.JsonAutoDetect;
import org.codehaus.jackson.annotate.JsonIgnore;
import org.codehaus.jackson.annotate.JsonIgnoreProperties;
import org.codehaus.jackson.map.annotate.JsonSerialize;
@JsonAutoDetect(getterVisibility=PUBLIC_ONLY, setterVisibility=PUBLIC_ONLY, fieldVisibility=NONE)
@JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL)
......@@ -46,6 +45,7 @@ import static org.codehaus.jackson.annotate.JsonAutoDetect.Visibility.PUBLIC_ONL
public class EntityMutationResponse {
Map<EntityMutations.EntityOperation, List<AtlasEntityHeader>> entitiesMutated;
Map<String,String> guidAssignments;
public EntityMutationResponse() {
}
......@@ -148,16 +148,25 @@ public class EntityMutationResponse {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
EntityMutationResponse that = (EntityMutationResponse) o;
return Objects.equals(entitiesMutated, that.entitiesMutated);
return Objects.equals(entitiesMutated, that.entitiesMutated) &&
Objects.equals(guidAssignments, that.guidAssignments);
}
@Override
public int hashCode() {
return Objects.hash(entitiesMutated);
return Objects.hash(entitiesMutated, guidAssignments);
}
@Override
public String toString() {
return toString(new StringBuilder()).toString();
}
public void setGuidAssignments(Map<String,String> guidAssignments) {
this.guidAssignments = guidAssignments;
}
public Map<String,String> getGuidAssignments() {
return guidAssignments;
}
}
intg/src/main/java/org/apache/atlas/model/instance/GuidMapping.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.model.instance;
import static org.codehaus.jackson.annotate.JsonAutoDetect.Visibility.NONE;
import static org.codehaus.jackson.annotate.JsonAutoDetect.Visibility.PUBLIC_ONLY;
import java.util.Map;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlRootElement;
import org.codehaus.jackson.annotate.JsonAutoDetect;
import org.codehaus.jackson.annotate.JsonIgnore;
import org.codehaus.jackson.annotate.JsonIgnoreProperties;
import org.codehaus.jackson.map.annotate.JsonSerialize;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
/**
* This stores a mapping of guid assignments that were made during the processing
* of a create or update entity request.
*.
*/
@JsonAutoDetect(getterVisibility=PUBLIC_ONLY, setterVisibility=PUBLIC_ONLY, fieldVisibility=NONE)
@JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL)
@JsonIgnoreProperties(ignoreUnknown=true)
@XmlRootElement
@XmlAccessorType(XmlAccessType.PROPERTY)
public class GuidMapping {
@JsonIgnore
private static final Gson gson = new GsonBuilder().setPrettyPrinting().create();
private Map<String,String> guidAssignments;
public GuidMapping() {
}
public GuidMapping(Map<String,String> guidAssignments) {
this.guidAssignments = guidAssignments;
}
public Map<String,String> getGuidAssignments() {
return guidAssignments;
}
public void setGuidAssignments(Map<String,String> guidAssignments) {
this.guidAssignments = guidAssignments;
}
/**
* Converts the GuidMapping to json
*/
@Override
public String toString() {
return gson.toJson(this);
}
@JsonIgnore
public static GuidMapping fromString(String json) {
return gson.fromJson(json, GuidMapping.class);
}
}
\ No newline at end of file
release-log.txt
......@@ -9,6 +9,7 @@ ATLAS-1060 Add composite indexes for exact match performance improvements for al
ATLAS-1127 Modify creation and modification timestamps to Date instead of Long(sumasai)
ALL CHANGES:
ATLAS-746 After updating a set of entities, response contains only the first entity definition (jnhagelb)
ATLAS-1510 Consolidate/batch calls to GraphBackedTypeStore.findVertex() (jnhagelb)
ATLAS-1388 Cache entities that are created/updated (jnhagelb)
ATLAS-1369 Optimize Gremlin queries generated by DSL translator (jnhagelb)
......
repository/src/main/java/org/apache/atlas/repository/MetadataRepository.java
......@@ -20,6 +20,7 @@ package org.apache.atlas.repository;
import org.apache.atlas.AtlasClient;
import org.apache.atlas.AtlasException;
import org.apache.atlas.CreateUpdateEntitiesResult;
import org.apache.atlas.typesystem.ITypedReferenceableInstance;
import org.apache.atlas.typesystem.ITypedStruct;
import org.apache.atlas.typesystem.exception.EntityExistsException;
......@@ -91,11 +92,11 @@ public interface MetadataRepository {
* Creates an entity definition (instance) corresponding to a given type.
*
* @param entities entity (typed instance)
* @return a globally unique identifier
* @return CreateOrUpdateEntitiesResult with the guids of the entities that were created
* @throws RepositoryException
* @throws EntityExistsException
*/
List<String> createEntities(ITypedReferenceableInstance... entities) throws RepositoryException, EntityExistsException;
CreateUpdateEntitiesResult createEntities(ITypedReferenceableInstance... entities) throws RepositoryException, EntityExistsException;
/**
* Fetch the complete definition of an entity given its GUID.
......@@ -166,13 +167,13 @@ public interface MetadataRepository {
* Adds/Updates the property to the entity that corresponds to the GUID
* Supports only primitive attribute/Class Id updations.
*/
AtlasClient.EntityResult updatePartial(ITypedReferenceableInstance entity) throws RepositoryException;
CreateUpdateEntitiesResult updatePartial(ITypedReferenceableInstance entity) throws RepositoryException;
/**
* Adds the property to the entity that corresponds to the GUID
* @param entitiesToBeUpdated The entities to be updated
*/
AtlasClient.EntityResult updateEntities(ITypedReferenceableInstance... entitiesToBeUpdated) throws RepositoryException;
CreateUpdateEntitiesResult updateEntities(ITypedReferenceableInstance... entitiesToBeUpdated) throws RepositoryException;
/**
* Returns the entity for the given type and qualified name
......
repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepository.java
......@@ -27,8 +27,10 @@ import java.util.Map;
import org.apache.atlas.AtlasClient;
import org.apache.atlas.AtlasException;
import org.apache.atlas.CreateUpdateEntitiesResult;
import org.apache.atlas.GraphTransaction;
import org.apache.atlas.RequestContext;
import org.apache.atlas.model.instance.GuidMapping;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.MetadataRepository;
import org.apache.atlas.repository.RepositoryException;
......@@ -143,7 +145,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
@Override
@GraphTransaction
public List<String> createEntities(ITypedReferenceableInstance... entities) throws RepositoryException,
public CreateUpdateEntitiesResult createEntities(ITypedReferenceableInstance... entities) throws RepositoryException,
EntityExistsException {
if (LOG.isDebugEnabled()) {
LOG.debug("adding entities={}", entities);
......@@ -152,7 +154,13 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
try {
TypedInstanceToGraphMapper instanceToGraphMapper = new TypedInstanceToGraphMapper(graphToInstanceMapper, deleteHandler);
instanceToGraphMapper.mapTypedInstanceToGraph(TypedInstanceToGraphMapper.Operation.CREATE, entities);
return RequestContext.get().getCreatedEntityIds();
List<String> createdGuids = RequestContext.get().getCreatedEntityIds();
CreateUpdateEntitiesResult result = new CreateUpdateEntitiesResult();
AtlasClient.EntityResult entityResult = new AtlasClient.EntityResult(createdGuids, null, null);
GuidMapping mapping = instanceToGraphMapper.createGuidMapping();
result.setEntityResult(entityResult);
result.setGuidMapping(mapping);
return result;
} catch (EntityExistsException e) {
throw e;
} catch (AtlasException e) {
......@@ -360,7 +368,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
@Override
@GraphTransaction
public AtlasClient.EntityResult updateEntities(ITypedReferenceableInstance... entitiesUpdated) throws RepositoryException {
public CreateUpdateEntitiesResult updateEntities(ITypedReferenceableInstance... entitiesUpdated) throws RepositoryException {
if (LOG.isDebugEnabled()) {
LOG.debug("updating entity {}", entitiesUpdated);
}
......@@ -369,8 +377,12 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
TypedInstanceToGraphMapper instanceToGraphMapper = new TypedInstanceToGraphMapper(graphToInstanceMapper, deleteHandler);
instanceToGraphMapper.mapTypedInstanceToGraph(TypedInstanceToGraphMapper.Operation.UPDATE_FULL,
entitiesUpdated);
CreateUpdateEntitiesResult result = new CreateUpdateEntitiesResult();
RequestContext requestContext = RequestContext.get();
return createEntityResultFromContext(requestContext);
result.setEntityResult(createEntityResultFromContext(requestContext));
GuidMapping mapping = instanceToGraphMapper.createGuidMapping();
result.setGuidMapping(mapping);
return result;
} catch (AtlasException e) {
throw new RepositoryException(e);
}
......@@ -378,7 +390,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
@Override
@GraphTransaction
public AtlasClient.EntityResult updatePartial(ITypedReferenceableInstance entity) throws RepositoryException {
public CreateUpdateEntitiesResult updatePartial(ITypedReferenceableInstance entity) throws RepositoryException {
if (LOG.isDebugEnabled()) {
LOG.debug("updating entity {}", entity);
}
......@@ -387,7 +399,11 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
TypedInstanceToGraphMapper instanceToGraphMapper = new TypedInstanceToGraphMapper(graphToInstanceMapper, deleteHandler);
instanceToGraphMapper.mapTypedInstanceToGraph(TypedInstanceToGraphMapper.Operation.UPDATE_PARTIAL, entity);
RequestContext requestContext = RequestContext.get();
return createEntityResultFromContext(requestContext);
CreateUpdateEntitiesResult result = new CreateUpdateEntitiesResult();
GuidMapping mapping = instanceToGraphMapper.createGuidMapping();
result.setEntityResult(createEntityResultFromContext(requestContext));
result.setGuidMapping(mapping);
return result;
} catch (AtlasException e) {
throw new RepositoryException(e);
}
......
repository/src/main/java/org/apache/atlas/repository/graph/TypedInstanceToGraphMapper.java
......@@ -32,6 +32,7 @@ import java.util.Set;
import org.apache.atlas.AtlasException;
import org.apache.atlas.RequestContext;
import org.apache.atlas.model.instance.GuidMapping;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.RepositoryException;
import org.apache.atlas.repository.graphdb.AtlasEdge;
......@@ -847,4 +848,16 @@ public final class TypedInstanceToGraphMapper {
context.cache(instance);
}
}
public GuidMapping createGuidMapping() {
Map<String,String> mapping = new HashMap<>(idToVertexMap.size());
for(Map.Entry<Id, AtlasVertex> entry : idToVertexMap.entrySet()) {
Id id = entry.getKey();
if (id.isUnassigned()) {
AtlasVertex classVertex = entry.getValue();
mapping.put(id._getId(), GraphHelper.getGuid(classVertex));
}
}
return new GuidMapping(mapping);
}
}
\ No newline at end of file
repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasStructDefStoreV1.java
......@@ -43,6 +43,8 @@ import org.codehaus.jettison.json.JSONException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.annotations.VisibleForTesting;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
......@@ -498,7 +500,8 @@ public class AtlasStructDefStoreV1 extends AtlasAbstractDefStoreV1 implements At
}
}
private static String toJsonFromAttribute(AtlasAttribute attribute) {
@VisibleForTesting
public static String toJsonFromAttribute(AtlasAttribute attribute) {
AtlasAttributeDef attributeDef = attribute.getAttributeDef();
boolean isComposite = attribute.legacyIsComposite();
String reverseAttribName = attribute.legacyReverseAttribute();
......@@ -539,7 +542,8 @@ public class AtlasStructDefStoreV1 extends AtlasAbstractDefStoreV1 implements At
return AtlasType.toJson(attribInfo);
}
private static AtlasAttributeDef toAttributeDefFromJson(AtlasStructDef structDef,
@VisibleForTesting
public static AtlasAttributeDef toAttributeDefFromJson(AtlasStructDef structDef,
Map attribInfo,
AtlasTypeDefGraphStoreV1 typeDefStore)
throws AtlasBaseException {
......
repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasTypeDefGraphStoreV1.java
......@@ -17,6 +17,7 @@
*/
package org.apache.atlas.repository.store.graph.v1;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.inject.Inject;
......@@ -110,7 +111,8 @@ public class AtlasTypeDefGraphStoreV1 extends AtlasTypeDefGraphStore {
AtlasGraph getAtlasGraph() { return atlasGraph; }
AtlasVertex findTypeVertexByName(String typeName) {
@VisibleForTesting
public AtlasVertex findTypeVertexByName(String typeName) {
Iterator results = atlasGraph.query().has(VERTEX_TYPE_PROPERTY_KEY, VERTEX_TYPE)
.has(Constants.TYPENAME_PROPERTY_KEY, typeName)
.vertices().iterator();
......@@ -276,7 +278,8 @@ public class AtlasTypeDefGraphStoreV1 extends AtlasTypeDefGraphStore {
return VERTEX_TYPE.equals(vertexType);
}
boolean isTypeVertex(AtlasVertex vertex, TypeCategory category) {
@VisibleForTesting
public boolean isTypeVertex(AtlasVertex vertex, TypeCategory category) {
boolean ret = false;
if (isTypeVertex(vertex)) {
......
repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java
......@@ -26,6 +26,7 @@ import org.apache.atlas.ApplicationProperties;
import org.apache.atlas.AtlasClient;
import org.apache.atlas.AtlasErrorCode;
import org.apache.atlas.AtlasException;
import org.apache.atlas.CreateUpdateEntitiesResult;
import org.apache.atlas.EntityAuditEvent;
import org.apache.atlas.RequestContext;
import org.apache.atlas.exception.AtlasBaseException;
......@@ -293,7 +294,7 @@ public class DefaultMetadataService implements MetadataService, ActiveStateChang
* @return guids - list of guids
*/
@Override
public List<String> createEntities(String entityInstanceDefinition) throws AtlasException {
public CreateUpdateEntitiesResult createEntities(String entityInstanceDefinition) throws AtlasException {
entityInstanceDefinition = ParamChecker.notEmpty(entityInstanceDefinition, "Entity instance definition");
ITypedReferenceableInstance[] typedInstances = deserializeClassInstances(entityInstanceDefinition);
......@@ -301,10 +302,10 @@ public class DefaultMetadataService implements MetadataService, ActiveStateChang
return createEntities(typedInstances);
}
public List<String> createEntities(ITypedReferenceableInstance[] typedInstances) throws AtlasException {
final List<String> guids = repository.createEntities(typedInstances);
onEntitiesAdded(guids);
return guids;
public CreateUpdateEntitiesResult createEntities(ITypedReferenceableInstance[] typedInstances) throws AtlasException {
final CreateUpdateEntitiesResult result = repository.createEntities(typedInstances);
onEntitiesAdded(result.getCreatedEntities());
return result;
}
private ITypedReferenceableInstance[] deserializeClassInstances(String entityInstanceDefinition) throws AtlasException {
......@@ -397,13 +398,13 @@ public class DefaultMetadataService implements MetadataService, ActiveStateChang
* @return guids - json array of guids
*/
@Override
public AtlasClient.EntityResult updateEntities(String entityInstanceDefinition) throws AtlasException {
public CreateUpdateEntitiesResult updateEntities(String entityInstanceDefinition) throws AtlasException {
entityInstanceDefinition = ParamChecker.notEmpty(entityInstanceDefinition, "Entity instance definition");
ITypedReferenceableInstance[] typedInstances = deserializeClassInstances(entityInstanceDefinition);
AtlasClient.EntityResult entityResult = repository.updateEntities(typedInstances);
onEntitiesAddedUpdated(entityResult);
return entityResult;
CreateUpdateEntitiesResult result = repository.updateEntities(typedInstances);
onEntitiesAddedUpdated(result.getEntityResult());
return result;
}
/**
......@@ -413,10 +414,10 @@ public class DefaultMetadataService implements MetadataService, ActiveStateChang
* @return guids - json array of guids
*/
@Override
public AtlasClient.EntityResult updateEntities(ITypedReferenceableInstance[] entityInstanceDefinitions) throws AtlasException {
AtlasClient.EntityResult entityResult = repository.updateEntities(entityInstanceDefinitions);
onEntitiesAddedUpdated(entityResult);
return entityResult;
public CreateUpdateEntitiesResult updateEntities(ITypedReferenceableInstance[] entityInstanceDefinitions) throws AtlasException {
CreateUpdateEntitiesResult result = repository.updateEntities(entityInstanceDefinitions);
onEntitiesAddedUpdated(result.getEntityResult());
return result;
}
private void onEntitiesAddedUpdated(AtlasClient.EntityResult entityResult) throws AtlasException {
......@@ -427,7 +428,7 @@ public class DefaultMetadataService implements MetadataService, ActiveStateChang
}
@Override
public AtlasClient.EntityResult updateEntityAttributeByGuid(String guid, String attributeName,
public CreateUpdateEntitiesResult updateEntityAttributeByGuid(String guid, String attributeName,
String value) throws AtlasException {
guid = ParamChecker.notEmpty(guid, "entity id");
attributeName = ParamChecker.notEmpty(attributeName, "attribute name");
......@@ -457,9 +458,9 @@ public class DefaultMetadataService implements MetadataService, ActiveStateChang
}
((ReferenceableInstance)newInstance).replaceWithNewId(new Id(guid, 0, newInstance.getTypeName()));
AtlasClient.EntityResult entityResult = repository.updatePartial(newInstance);
onEntitiesAddedUpdated(entityResult);
return entityResult;
CreateUpdateEntitiesResult result = repository.updatePartial(newInstance);
onEntitiesAddedUpdated(result.getEntityResult());
return result;
}
private ITypedReferenceableInstance validateEntityExists(String guid)
......@@ -472,7 +473,7 @@ public class DefaultMetadataService implements MetadataService, ActiveStateChang
}
@Override
public AtlasClient.EntityResult updateEntityPartialByGuid(String guid, Referenceable newEntity)
public CreateUpdateEntitiesResult updateEntityPartialByGuid(String guid, Referenceable newEntity)
throws AtlasException {
guid = ParamChecker.notEmpty(guid, "guid cannot be null");
newEntity = ParamChecker.notNull(newEntity, "updatedEntity cannot be null");
......@@ -481,9 +482,9 @@ public class DefaultMetadataService implements MetadataService, ActiveStateChang
ITypedReferenceableInstance newInstance = convertToTypedInstance(newEntity, existInstance.getTypeName());
((ReferenceableInstance)newInstance).replaceWithNewId(new Id(guid, 0, newInstance.getTypeName()));
AtlasClient.EntityResult entityResult = repository.updatePartial(newInstance);
onEntitiesAddedUpdated(entityResult);
return entityResult;
CreateUpdateEntitiesResult result = repository.updatePartial(newInstance);
onEntitiesAddedUpdated(result.getEntityResult());
return result;
}
private ITypedReferenceableInstance convertToTypedInstance(Referenceable updatedEntity, String typeName)
......@@ -530,7 +531,7 @@ public class DefaultMetadataService implements MetadataService, ActiveStateChang
}
@Override
public AtlasClient.EntityResult updateEntityByUniqueAttribute(String typeName, String uniqueAttributeName,
public CreateUpdateEntitiesResult updateEntityByUniqueAttribute(String typeName, String uniqueAttributeName,
String attrValue,
Referenceable updatedEntity) throws AtlasException {
typeName = ParamChecker.notEmpty(typeName, "typeName");
......@@ -543,9 +544,9 @@ public class DefaultMetadataService implements MetadataService, ActiveStateChang
final ITypedReferenceableInstance newInstance = convertToTypedInstance(updatedEntity, typeName);
((ReferenceableInstance)newInstance).replaceWithNewId(oldInstance.getId());
AtlasClient.EntityResult entityResult = repository.updatePartial(newInstance);
onEntitiesAddedUpdated(entityResult);
return entityResult;
CreateUpdateEntitiesResult result = repository.updatePartial(newInstance);
onEntitiesAddedUpdated(result.getEntityResult());
return result;
}
private void validateTypeExists(String entityType) throws AtlasException {
......
repository/src/test/java/org/apache/atlas/BaseRepositoryTest.java
......@@ -395,7 +395,7 @@ public class BaseRepositoryTest {
}
private Id createInstance(Referenceable referenceable, ClassType clsType) throws Exception {
ITypedReferenceableInstance typedInstance = clsType.convert(referenceable, Multiplicity.REQUIRED);
List<String> guids = repository.createEntities(typedInstance);
List<String> guids = repository.createEntities(typedInstance).getCreatedEntities();
// return the reference to created instance with guid
return new Id(guids.get(guids.size() - 1), 0, referenceable.getTypeName());
......
repository/src/test/java/org/apache/atlas/TestUtils.java
......@@ -24,6 +24,7 @@ import static org.apache.atlas.typesystem.types.utils.TypesUtil.createRequiredAt
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createStructTypeDef;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createTraitTypeDef;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createUniqueRequiredAttrDef;
import static org.testng.Assert.assertEquals;
import java.io.File;
import java.io.FileOutputStream;
......@@ -38,7 +39,11 @@ import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.apache.atlas.listener.EntityChangeListener;
import org.apache.atlas.listener.TypesChangeListener;
......@@ -53,14 +58,17 @@ import org.apache.atlas.repository.typestore.ITypeStore;
import org.apache.atlas.services.DefaultMetadataService;
import org.apache.atlas.services.MetadataService;
import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.atlas.typesystem.IInstance;
import org.apache.atlas.typesystem.ITypedReferenceableInstance;
import org.apache.atlas.typesystem.Referenceable;
import org.apache.atlas.typesystem.TypesDef;
import org.apache.atlas.typesystem.json.InstanceSerialization;
import org.apache.atlas.typesystem.persistence.Id;
import org.apache.atlas.typesystem.types.AttributeDefinition;
import org.apache.atlas.typesystem.types.AttributeInfo;
import org.apache.atlas.typesystem.types.ClassType;
import org.apache.atlas.typesystem.types.DataTypes;
import org.apache.atlas.typesystem.types.DataTypes.TypeCategory;
import org.apache.atlas.typesystem.types.EnumTypeDefinition;
import org.apache.atlas.typesystem.types.EnumValue;
import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition;
......@@ -89,6 +97,32 @@ public final class TestUtils {
public static final long TEST_DATE_IN_LONG = 1418265358440L;
public static final String EMPLOYEES_ATTR = "employees";
public static final String DEPARTMENT_ATTR = "department";
public static final String ASSETS_ATTR = "assets";
public static final String POSITIONS_ATTR = "positions";
public static final String ASSET_TYPE = "TestAsset";
public static final String DATABASE_TYPE = "hive_database";
public static final String DATABASE_NAME = "foo";
public static final String TABLE_TYPE = "hive_table";
public static final String PROCESS_TYPE = "hive_process";
public static final String COLUMN_TYPE = "column_type";
public static final String TABLE_NAME = "bar";
public static final String CLASSIFICATION = "classification";
public static final String PII = "PII";
public static final String SUPER_TYPE_NAME = "Base";
public static final String STORAGE_DESC_TYPE = "hive_storagedesc";
public static final String PARTITION_STRUCT_TYPE = "partition_struct_type";
public static final String PARTITION_CLASS_TYPE = "partition_class_type";
public static final String SERDE_TYPE = "serdeType";
public static final String COLUMNS_MAP = "columnsMap";
public static final String COLUMNS_ATTR_NAME = "columns";
public static final String NAME = "name";
private TestUtils() {
}
......@@ -141,17 +175,21 @@ public final class TestUtils {
createRequiredAttrDef("city", DataTypes.STRING_TYPE));
HierarchicalTypeDefinition<ClassType> deptTypeDef = createClassTypeDef(DEPARTMENT_TYPE, "Department"+_description, ImmutableSet.<String>of(),
createRequiredAttrDef("name", DataTypes.STRING_TYPE),
new AttributeDefinition("employees", String.format("array<%s>", "Person"), Multiplicity.OPTIONAL,
true, "department"));
createRequiredAttrDef(NAME, DataTypes.STRING_TYPE),
new AttributeDefinition(EMPLOYEES_ATTR, String.format("array<%s>", "Person"), Multiplicity.OPTIONAL,
true, DEPARTMENT_ATTR),
new AttributeDefinition(POSITIONS_ATTR, String.format("map<%s,%s>", DataTypes.STRING_TYPE.getName(), "Person"), Multiplicity.OPTIONAL,
false, null)
);
HierarchicalTypeDefinition<ClassType> personTypeDef = createClassTypeDef("Person", "Person"+_description, ImmutableSet.<String>of(),
createRequiredAttrDef("name", DataTypes.STRING_TYPE),
createRequiredAttrDef(NAME, DataTypes.STRING_TYPE),
createOptionalAttrDef("orgLevel", "OrgLevel"),
createOptionalAttrDef("address", "Address"),
new AttributeDefinition("department", "Department", Multiplicity.REQUIRED, false, "employees"),
new AttributeDefinition(DEPARTMENT_ATTR, "Department", Multiplicity.REQUIRED, false, EMPLOYEES_ATTR),
new AttributeDefinition("manager", "Manager", Multiplicity.OPTIONAL, false, "subordinates"),
new AttributeDefinition("mentor", "Person", Multiplicity.OPTIONAL, false, null),
new AttributeDefinition(ASSETS_ATTR, String.format("array<%s>", ASSET_TYPE) , Multiplicity.OPTIONAL, false, null),
createOptionalAttrDef("birthday", DataTypes.DATE_TYPE),
createOptionalAttrDef("hasPets", DataTypes.BOOLEAN_TYPE),
createOptionalAttrDef("numberOfCars", DataTypes.BYTE_TYPE),
......@@ -165,6 +203,12 @@ public final class TestUtils {
createOptionalAttrDef("isOrganDonor", DataTypes.BOOLEAN_TYPE)
);
HierarchicalTypeDefinition<ClassType> assetTypeDef = createClassTypeDef(ASSET_TYPE, "Asset"+_description, ImmutableSet.<String>of(),
createRequiredAttrDef(NAME, DataTypes.STRING_TYPE),
new AttributeDefinition("childAssets", String.format("array<%s>", ASSET_TYPE) , Multiplicity.OPTIONAL, false, null)
);
HierarchicalTypeDefinition<ClassType> managerTypeDef = createClassTypeDef("Manager", "Manager"+_description, ImmutableSet.of("Person"),
new AttributeDefinition("subordinates", String.format("array<%s>", "Person"), Multiplicity.COLLECTION,
false, "manager"));
......@@ -175,7 +219,7 @@ public final class TestUtils {
ts.defineTypes(ImmutableList.of(orgLevelEnum), ImmutableList.of(addressDetails),
ImmutableList.of(securityClearanceTypeDef),
ImmutableList.of(deptTypeDef, personTypeDef, managerTypeDef));
ImmutableList.of(deptTypeDef, personTypeDef, managerTypeDef, assetTypeDef));
}
public static final String DEPARTMENT_TYPE = "Department";
......@@ -193,9 +237,9 @@ public final class TestUtils {
Referenceable max = new Referenceable("Person");
Referenceable maxAddr = new Referenceable("Address");
hrDept.set("name", "hr");
john.set("name", "John");
john.set("department", hrDept);
hrDept.set(NAME, "hr");
john.set(NAME, "John");
john.set(DEPARTMENT_ATTR, hrDept);
johnAddr.set("street", "Stewart Drive");
johnAddr.set("city", "Sunnyvale");
john.set("address", johnAddr);
......@@ -212,22 +256,22 @@ public final class TestUtils {
john.set("numberOfStarsEstimate", new BigInteger("1000000000000000000000"));
john.set("approximationOfPi", new BigDecimal("3.141592653589793238462643383279502884197169399375105820974944592307816406286"));
jane.set("name", "Jane");
jane.set("department", hrDept);
jane.set(NAME, "Jane");
jane.set(DEPARTMENT_ATTR, hrDept);
janeAddr.set("street", "Great America Parkway");
janeAddr.set("city", "Santa Clara");
jane.set("address", janeAddr);
janeAddr.set("street", "Great America Parkway");
julius.set("name", "Julius");
julius.set("department", hrDept);
julius.set(NAME, "Julius");
julius.set(DEPARTMENT_ATTR, hrDept);
juliusAddr.set("street", "Madison Ave");
juliusAddr.set("city", "Newtonville");
julius.set("address", juliusAddr);
julius.set("subordinates", ImmutableList.<Referenceable>of());
max.set("name", "Max");
max.set("department", hrDept);
max.set(NAME, "Max");
max.set(DEPARTMENT_ATTR, hrDept);
maxAddr.set("street", "Ripley St");
maxAddr.set("city", "Newton");
max.set("address", maxAddr);
......@@ -247,7 +291,7 @@ public final class TestUtils {
john.set("manager", jane);
john.set("mentor", max);
hrDept.set("employees", ImmutableList.of(john, jane, julius, max));
hrDept.set(EMPLOYEES_ATTR, ImmutableList.of(john, jane, julius, max));
jane.set("subordinates", ImmutableList.of(john, max));
......@@ -260,23 +304,7 @@ public final class TestUtils {
return hrDept2;
}
public static final String DATABASE_TYPE = "hive_database";
public static final String DATABASE_NAME = "foo";
public static final String TABLE_TYPE = "hive_table";
public static final String PROCESS_TYPE = "hive_process";
public static final String COLUMN_TYPE = "column_type";
public static final String TABLE_NAME = "bar";
public static final String CLASSIFICATION = "classification";
public static final String PII = "PII";
public static final String SUPER_TYPE_NAME = "Base";
public static final String STORAGE_DESC_TYPE = "hive_storagedesc";
public static final String PARTITION_STRUCT_TYPE = "partition_struct_type";
public static final String PARTITION_CLASS_TYPE = "partition_class_type";
public static final String SERDE_TYPE = "serdeType";
public static final String COLUMNS_MAP = "columnsMap";
public static final String COLUMNS_ATTR_NAME = "columns";
public static final String NAME = "name";
public static TypesDef simpleType(){
HierarchicalTypeDefinition<ClassType> superTypeDefinition =
......@@ -284,7 +312,7 @@ public final class TestUtils {
createOptionalAttrDef("attr", DataTypes.STRING_TYPE));
StructTypeDefinition structTypeDefinition = new StructTypeDefinition("s_type", "structType",
new AttributeDefinition[]{createRequiredAttrDef("name", DataTypes.STRING_TYPE)});
new AttributeDefinition[]{createRequiredAttrDef(NAME, DataTypes.STRING_TYPE)});
HierarchicalTypeDefinition<TraitType> traitTypeDefinition =
createTraitTypeDef("t_type", "traitType", ImmutableSet.<String>of());
......@@ -306,7 +334,7 @@ public final class TestUtils {
createOptionalAttrDef("attr", DataTypes.STRING_TYPE));
StructTypeDefinition structTypeDefinition = new StructTypeDefinition("s_type", "structType",
new AttributeDefinition[]{createRequiredAttrDef("name", DataTypes.STRING_TYPE)});
new AttributeDefinition[]{createRequiredAttrDef(NAME, DataTypes.STRING_TYPE)});
HierarchicalTypeDefinition<TraitType> traitTypeDefinition =
createTraitTypeDef("t_type", "traitType", ImmutableSet.<String>of());
......@@ -344,7 +372,7 @@ public final class TestUtils {
StructTypeDefinition structTypeDefinition = new StructTypeDefinition("serdeType", "serdeType" + _description,
new AttributeDefinition[]{createRequiredAttrDef("name", DataTypes.STRING_TYPE),
new AttributeDefinition[]{createRequiredAttrDef(NAME, DataTypes.STRING_TYPE),
createRequiredAttrDef("serde", DataTypes.STRING_TYPE),
createOptionalAttrDef("description", DataTypes.STRING_TYPE)});
......@@ -354,11 +382,11 @@ public final class TestUtils {
HierarchicalTypeDefinition<ClassType> columnsDefinition =
createClassTypeDef(COLUMN_TYPE, ImmutableSet.<String>of(),
createUniqueRequiredAttrDef("name", DataTypes.STRING_TYPE),
createUniqueRequiredAttrDef(NAME, DataTypes.STRING_TYPE),
createRequiredAttrDef("type", DataTypes.STRING_TYPE));
StructTypeDefinition partitionDefinition = new StructTypeDefinition("partition_struct_type", "partition_struct_type" + _description,
new AttributeDefinition[]{createRequiredAttrDef("name", DataTypes.STRING_TYPE),});
new AttributeDefinition[]{createRequiredAttrDef(NAME, DataTypes.STRING_TYPE),});
AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
new AttributeDefinition("location", DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false,
......@@ -403,7 +431,7 @@ public final class TestUtils {
HierarchicalTypeDefinition<ClassType> tableTypeDefinition =
createClassTypeDef(TABLE_TYPE, TABLE_TYPE + _description, ImmutableSet.of(SUPER_TYPE_NAME),
TypesUtil.createUniqueRequiredAttrDef("name", DataTypes.STRING_TYPE),
TypesUtil.createUniqueRequiredAttrDef(NAME, DataTypes.STRING_TYPE),
createRequiredAttrDef("description", DataTypes.STRING_TYPE),
createRequiredAttrDef("type", DataTypes.STRING_TYPE),
createOptionalAttrDef("created", DataTypes.DATE_TYPE),
......@@ -498,19 +526,75 @@ public final class TestUtils {
return entity;
}
/**
* Creates an entity in the graph and does basic validation
* of the GuidMapping that was created in the process.
*
*/
public static String createInstance(MetadataService metadataService, Referenceable entity) throws Exception {
RequestContext.createContext();
String entityjson = InstanceSerialization.toJson(entity, true);
JSONArray entitiesJson = new JSONArray();
entitiesJson.put(entityjson);
List<String> guids = metadataService.createEntities(entitiesJson.toString());
CreateUpdateEntitiesResult creationResult = metadataService.createEntities(entitiesJson.toString());
Map<String,String> guidMap = creationResult.getGuidMapping().getGuidAssignments();
Map<Id, Referenceable> referencedObjects = findReferencedObjects(entity);
for(Map.Entry<Id,Referenceable> entry : referencedObjects.entrySet()) {
Id foundId = entry.getKey();
if(foundId.isUnassigned()) {
String guid = guidMap.get(entry.getKey()._getId());
Referenceable obj = entry.getValue();
loadAndDoSimpleValidation(guid,obj, metadataService);
}
}
List<String> guids = creationResult.getCreatedEntities();
if (guids != null && guids.size() > 0) {
return guids.get(guids.size() - 1);
}
return null;
}
private static Map<Id,Referenceable> findReferencedObjects(Referenceable ref) {
Map<Id, Referenceable> result = new HashMap<>();
findReferencedObjects(ref, result);
return result;
}
private static void findReferencedObjects(Referenceable ref, Map<Id, Referenceable> seen) {
Id guid = ref.getId();
if(seen.containsKey(guid)) {
return;
}
seen.put(guid, ref);
for(Map.Entry<String, Object> attr : ref.getValuesMap().entrySet()) {
Object value = attr.getValue();
if(value instanceof Referenceable) {
findReferencedObjects((Referenceable)value, seen);
}
else if(value instanceof List) {
for(Object o : (List)value) {
if(o instanceof Referenceable) {
findReferencedObjects((Referenceable)o, seen);
}
}
}
else if(value instanceof Map) {
for(Object o : ((Map)value).values()) {
if(o instanceof Referenceable) {
findReferencedObjects((Referenceable)o, seen);
}
}
}
}
}
/**
* Clears the state in the request context.
*
*/
public static void resetRequestContext() {
//reset the context while preserving the user
String user = RequestContext.get().getUser();
......@@ -518,6 +602,10 @@ public final class TestUtils {
RequestContext.get().setUser(user);
}
/**
* Triggers the Atlas initialization process using the specified MetadataRepository.
* This causes the built-in types and their indices to be created.
*/
public static void setupGraphProvider(MetadataRepository repo) throws AtlasException {
TypeCache typeCache = null;
try {
......@@ -634,4 +722,60 @@ public final class TestUtils {
});
}
/**
* Loads the entity and does sanity testing of the GuidMapping that was
* created during the operation.
*
*/
public static ITypedReferenceableInstance loadAndDoSimpleValidation(String guid, Referenceable original, MetadataRepository repositoryService) throws AtlasException {
ITypedReferenceableInstance loaded = repositoryService.getEntityDefinition(guid);
doSimpleValidation(original, loaded);
return loaded;
}
/**
* Loads the entity and does sanity testing of the GuidMapping that was
* created during the operation.
*
*/
public static ITypedReferenceableInstance loadAndDoSimpleValidation(String guid, Referenceable original, MetadataService repositoryService) throws AtlasException {
ITypedReferenceableInstance loaded = repositoryService.getEntityDefinition(guid);
doSimpleValidation(original, loaded);
return loaded;
}
private static void doSimpleValidation(Referenceable original, IInstance loaded) throws AtlasException {
assertEquals(loaded.getTypeName(), original.getTypeName());
ClassType ct = TypeSystem.getInstance().getDataType(ClassType.class, loaded.getTypeName());
//compare primitive fields
for(AttributeInfo field : ct.fieldMapping.fields.values()) {
if(field.dataType().getTypeCategory() == TypeCategory.PRIMITIVE) {
if(original.get(field.name) != null) {
Object rawLoadedValue = loaded.get(field.name);
Object rawProvidedValue = original.get(field.name);
Object convertedLoadedValue = field.dataType().convert(rawLoadedValue, Multiplicity.REQUIRED);
Object convertedProvidedValue = field.dataType().convert(rawProvidedValue, Multiplicity.REQUIRED);
assertEquals(convertedLoadedValue, convertedProvidedValue);
}
}
}
}
/**
* Validates that the two String Collections contain the same items, without
* regard to order.
*
*/
public static void assertContentsSame(Collection<String> actual, Collection<String> expected) {
assertEquals(actual.size(), expected.size());
Set<String> checker = new HashSet<>();
checker.addAll(expected);
checker.removeAll(actual);
assertEquals(checker.size(), 0);
}
}
repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java
......@@ -1207,6 +1207,27 @@ public class GraphBackedDiscoveryServiceTest extends BaseRepositoryTest {
}
@Test
public void testSearchForTypeWithNoInstances() throws Exception {
HierarchicalTypeDefinition EMPTY = createClassTypeDef("EmptyType", null,
createRequiredAttrDef("a", DataTypes.INT_TYPE));
TypeSystem.getInstance().defineClassTypes(EMPTY);
String dslQuery = "EmptyType";
String jsonResults = searchByDSL(dslQuery);
assertNotNull(jsonResults);
JSONObject results = new JSONObject(jsonResults);
assertEquals(results.length(), 3);
JSONArray rows = results.getJSONArray("rows");
assertNotNull(rows);
// query should not return any rows
assertEquals(rows.length(), 0);
}
private FieldValueValidator makeCountValidator(int count) {
return new FieldValueValidator().withFieldNames("count()").withExpectedValues(count);
}
......
repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryDeleteTestBase.java
......@@ -257,7 +257,7 @@ public abstract class GraphBackedMetadataRepositoryDeleteTestBase {
private String createInstance(Referenceable entity) throws Exception {
ClassType dataType = typeSystem.getDataType(ClassType.class, entity.getTypeName());
ITypedReferenceableInstance instance = dataType.convert(entity, Multiplicity.REQUIRED);
List<String> results = repositoryService.createEntities(instance);
List<String> results = repositoryService.createEntities(instance).getCreatedEntities();
return results.get(results.size() - 1);
}
......@@ -403,7 +403,7 @@ public abstract class GraphBackedMetadataRepositoryDeleteTestBase {
ITypedReferenceableInstance mapValueInstance = compositeMapValueType.createInstance();
mapValueInstance.set(NAME, TestUtils.randomString());
mapOwnerInstance.set("map", Collections.singletonMap("value1", mapValueInstance));
List<String> createEntitiesResult = repositoryService.createEntities(mapOwnerInstance, mapValueInstance);
List<String> createEntitiesResult = repositoryService.createEntities(mapOwnerInstance, mapValueInstance).getCreatedEntities();
Assert.assertEquals(createEntitiesResult.size(), 2);
ITypedReferenceableInstance entityDefinition = repositoryService.getEntityDefinition("CompositeMapOwner",
NAME, mapOwnerInstance.get(NAME));
......@@ -412,7 +412,7 @@ public abstract class GraphBackedMetadataRepositoryDeleteTestBase {
private AtlasClient.EntityResult updatePartial(ITypedReferenceableInstance entity) throws RepositoryException {
RequestContext.createContext();
return repositoryService.updatePartial(entity);
return repositoryService.updatePartial(entity).getEntityResult();
}
@Test
......@@ -641,7 +641,7 @@ public abstract class GraphBackedMetadataRepositoryDeleteTestBase {
structContainerType.convert(structContainerEntity, Multiplicity.REQUIRED);
List<String> guids = repositoryService.createEntities(
structTargetConvertedEntity, traitTargetConvertedEntity, structContainerConvertedEntity);
structTargetConvertedEntity, traitTargetConvertedEntity, structContainerConvertedEntity).getCreatedEntities();
Assert.assertEquals(guids.size(), 3);
guids = repositoryService.getEntityList("StructTarget");
......@@ -746,7 +746,7 @@ public abstract class GraphBackedMetadataRepositoryDeleteTestBase {
mapOwnerInstance.set("biMap", Collections.singletonMap("value1", mapValueInstance));
// Set biMapOwner reverse reference on MapValue.
mapValueInstance.set("biMapOwner", mapOwnerInstance);
List<String> createEntitiesResult = repositoryService.createEntities(mapOwnerInstance, mapValueInstance);
List<String> createEntitiesResult = repositoryService.createEntities(mapOwnerInstance, mapValueInstance).getCreatedEntities();
Assert.assertEquals(createEntitiesResult.size(), 2);
List<String> guids = repositoryService.getEntityList("MapOwner");
Assert.assertEquals(guids.size(), 1);
......@@ -856,7 +856,7 @@ public abstract class GraphBackedMetadataRepositoryDeleteTestBase {
ITypedReferenceableInstance mapOwnerInstance = mapOwnerType.createInstance();
ITypedReferenceableInstance mapValueInstance = mapValueType.createInstance();
mapOwnerInstance.set("map", Collections.singletonMap("value1", mapValueInstance));
List<String> createEntitiesResult = repositoryService.createEntities(mapOwnerInstance, mapValueInstance);
List<String> createEntitiesResult = repositoryService.createEntities(mapOwnerInstance, mapValueInstance).getCreatedEntities();
Assert.assertEquals(createEntitiesResult.size(), 2);
List<String> guids = repositoryService.getEntityList("RequiredMapOwner");
Assert.assertEquals(guids.size(), 1);
......@@ -985,7 +985,7 @@ public abstract class GraphBackedMetadataRepositoryDeleteTestBase {
// Create instance of MapValueReferencerContainer
RequestContext.createContext();
ITypedReferenceableInstance mapValueReferencerContainer = mapValueReferencerContainerType.createInstance();
List<String> createdEntities = repositoryService.createEntities(mapValueReferencerContainer);
List<String> createdEntities = repositoryService.createEntities(mapValueReferencerContainer).getCreatedEntities();
Assert.assertEquals(createdEntities.size(), 1);
String mapValueReferencerContainerGuid = createdEntities.get(0);
mapValueReferencerContainer = repositoryService.getEntityDefinition(createdEntities.get(0));
......@@ -997,7 +997,7 @@ public abstract class GraphBackedMetadataRepositoryDeleteTestBase {
mapValueReferencer.set("refToMapValue", mapValueInstance.getId());
RequestContext.createContext();
EntityResult updateEntitiesResult = repositoryService.updateEntities(mapValueReferencerContainer);
EntityResult updateEntitiesResult = repositoryService.updateEntities(mapValueReferencerContainer).getEntityResult();
Assert.assertEquals(updateEntitiesResult.getCreatedEntities().size(), 1);
Assert.assertEquals(updateEntitiesResult.getUpdateEntities().size(), 1);
Assert.assertEquals(updateEntitiesResult.getUpdateEntities().get(0), mapValueReferencerContainerGuid);
......@@ -1016,7 +1016,7 @@ public abstract class GraphBackedMetadataRepositoryDeleteTestBase {
public void testDeleteMixOfExistentAndNonExistentEntities() throws Exception {
ITypedReferenceableInstance entity1 = compositeMapValueType.createInstance();
ITypedReferenceableInstance entity2 = compositeMapValueType.createInstance();
List<String> createEntitiesResult = repositoryService.createEntities(entity1, entity2);
List<String> createEntitiesResult = repositoryService.createEntities(entity1, entity2).getCreatedEntities();
Assert.assertEquals(createEntitiesResult.size(), 2);
List<String> guids = Arrays.asList(createEntitiesResult.get(0), "non-existent-guid1", "non-existent-guid2", createEntitiesResult.get(1));
EntityResult deleteEntitiesResult = repositoryService.deleteEntities(guids);
......@@ -1028,7 +1028,7 @@ public abstract class GraphBackedMetadataRepositoryDeleteTestBase {
public void testDeleteMixOfNullAndNonNullGuids() throws Exception {
ITypedReferenceableInstance entity1 = compositeMapValueType.createInstance();
ITypedReferenceableInstance entity2 = compositeMapValueType.createInstance();
List<String> createEntitiesResult = repositoryService.createEntities(entity1, entity2);
List<String> createEntitiesResult = repositoryService.createEntities(entity1, entity2).getCreatedEntities();
Assert.assertEquals(createEntitiesResult.size(), 2);
List<String> guids = Arrays.asList(createEntitiesResult.get(0), null, null, createEntitiesResult.get(1));
EntityResult deleteEntitiesResult = repositoryService.deleteEntities(guids);
......@@ -1076,7 +1076,7 @@ public abstract class GraphBackedMetadataRepositoryDeleteTestBase {
ClassType dataType = typeSystem.getDataType(ClassType.class, table1Entity.getTypeName());
ITypedReferenceableInstance instance = dataType.convert(table1Entity, Multiplicity.REQUIRED);
TestUtils.resetRequestContext();
List<String> result = repositoryService.createEntities(instance);
List<String> result = repositoryService.createEntities(instance).getCreatedEntities();
Assert.assertEquals(result.size(), 3);
ITypedReferenceableInstance entityDefinition = repositoryService.getEntityDefinition(TABLE_TYPE, NAME, tableName);
String tableGuid = entityDefinition.getId()._getId();
......@@ -1106,7 +1106,7 @@ public abstract class GraphBackedMetadataRepositoryDeleteTestBase {
private String createHrDeptGraph() throws Exception {
ITypedReferenceableInstance hrDept = TestUtils.createDeptEg1(typeSystem);
List<String> guids = repositoryService.createEntities(hrDept);
List<String> guids = repositoryService.createEntities(hrDept).getCreatedEntities();
Assert.assertNotNull(guids);
Assert.assertEquals(guids.size(), 5);
......
repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryTest.java
......@@ -18,9 +18,29 @@
package org.apache.atlas.repository.graph;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createClassTypeDef;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createUniqueRequiredAttrDef;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertNotEquals;
import static org.testng.Assert.assertNotNull;
import static org.testng.Assert.assertTrue;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import javax.inject.Inject;
import org.apache.atlas.AtlasException;
import org.apache.atlas.CreateUpdateEntitiesResult;
import org.apache.atlas.GraphTransaction;
import org.apache.atlas.RepositoryMetadataModule;
import org.apache.atlas.RequestContext;
......@@ -64,28 +84,11 @@ import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Guice;
import org.testng.annotations.Test;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.UUID;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import javax.inject.Inject;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import java.util.Arrays;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createClassTypeDef;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createUniqueRequiredAttrDef;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertNotEquals;
import static org.testng.Assert.assertNotNull;
import static org.testng.Assert.assertTrue;
/**
* GraphBackedMetadataRepository test
*
......@@ -197,13 +200,190 @@ public class GraphBackedMetadataRepositoryTest {
public void testSubmitEntity() throws Exception {
ITypedReferenceableInstance hrDept = TestUtils.createDeptEg1(typeSystem);
List<String> guids = repositoryService.createEntities(hrDept);
List<String> guids = repositoryService.createEntities(hrDept).getCreatedEntities();
Assert.assertNotNull(guids);
Assert.assertEquals(guids.size(), 5);
guid = guids.get(4);
Assert.assertNotNull(guid);
}
@Test
public void testCreateEntityWithOneNestingLevel() throws AtlasException {
List<Referenceable> toValidate = new ArrayList<>();
Referenceable dept = new Referenceable(TestUtils.DEPARTMENT_TYPE);
toValidate.add(dept);
dept.set(TestUtils.NAME, "test1");
Referenceable mike = new Referenceable(TestUtils.PERSON_TYPE);
toValidate.add(mike);
mike.set(TestUtils.NAME, "Mike");
mike.set(TestUtils.DEPARTMENT_ATTR, dept);
Referenceable mark = new Referenceable(TestUtils.PERSON_TYPE);
toValidate.add(mark);
mark.set(TestUtils.NAME, "Mark");
mark.set(TestUtils.DEPARTMENT_ATTR, dept);
dept.set(TestUtils.EMPLOYEES_ATTR, ImmutableList.of(mike, mark));
Map<String,Referenceable> positions = new HashMap<>();
final String JANITOR = "janitor";
final String RECEPTIONIST = "receptionist";
positions.put(JANITOR, mike);
positions.put(RECEPTIONIST, mark);
dept.set(TestUtils.POSITIONS_ATTR, positions);
ClassType deptType = TypeSystem.getInstance().getDataType(ClassType.class, TestUtils.DEPARTMENT_TYPE);
ITypedReferenceableInstance deptInstance = deptType.convert(dept, Multiplicity.REQUIRED);
CreateUpdateEntitiesResult result = repositoryService.createEntities(deptInstance);
System.out.println(result.getGuidMapping().toString());
validateGuidMapping(toValidate, result);
}
@Test
public void testCreateEntityWithTwoNestingLevels() throws AtlasException {
List<Referenceable> toVerify = new ArrayList<>();
Referenceable dept = new Referenceable(TestUtils.DEPARTMENT_TYPE);
toVerify.add(dept);
dept.set(TestUtils.NAME, "test2");
Referenceable wallace = new Referenceable(TestUtils.PERSON_TYPE);
toVerify.add(wallace);
wallace.set(TestUtils.NAME, "Wallace");
wallace.set(TestUtils.DEPARTMENT_ATTR, dept);
Referenceable wallaceComputer = new Referenceable(TestUtils.ASSET_TYPE);
toVerify.add(wallaceComputer);
wallaceComputer.set("name", "wallaceComputer");
wallace.set(TestUtils.ASSETS_ATTR, ImmutableList.of(wallaceComputer));
Referenceable jordan = new Referenceable(TestUtils.PERSON_TYPE);
toVerify.add(jordan);
jordan.set(TestUtils.NAME, "Jordan");
jordan.set(TestUtils.DEPARTMENT_ATTR, dept);
Referenceable jordanComputer = new Referenceable(TestUtils.ASSET_TYPE);
toVerify.add(jordanComputer);
jordanComputer.set("name", "jordanComputer");
jordan.set(TestUtils.ASSETS_ATTR, ImmutableList.of(jordanComputer));
dept.set(TestUtils.EMPLOYEES_ATTR, ImmutableList.of(wallace, jordan));
Map<String,Referenceable> positions = new HashMap<>();
final String JANITOR = "janitor";
final String RECEPTIONIST = "receptionist";
positions.put(JANITOR, wallace);
positions.put(RECEPTIONIST, jordan);
dept.set(TestUtils.POSITIONS_ATTR, positions);
ClassType deptType = TypeSystem.getInstance().getDataType(ClassType.class, TestUtils.DEPARTMENT_TYPE);
ITypedReferenceableInstance deptInstance = deptType.convert(dept, Multiplicity.REQUIRED);
CreateUpdateEntitiesResult result = repositoryService.createEntities(deptInstance);
validateGuidMapping(toVerify, result);
}
@Test
public void testCreateEntityWithThreeNestingLevels() throws AtlasException {
List<Referenceable> toVerify = new ArrayList<>();
Referenceable dept = new Referenceable(TestUtils.DEPARTMENT_TYPE);
toVerify.add(dept);
dept.set(TestUtils.NAME, "test3");
Referenceable barry = new Referenceable(TestUtils.PERSON_TYPE);
toVerify.add(barry);
barry.set(TestUtils.NAME, "barry");
barry.set(TestUtils.DEPARTMENT_ATTR, dept);
Referenceable barryComputer = new Referenceable(TestUtils.ASSET_TYPE);
toVerify.add(barryComputer);
barryComputer.set("name", "barryComputer");
barry.set(TestUtils.ASSETS_ATTR, ImmutableList.of(barryComputer));
Referenceable barryHardDrive = new Referenceable(TestUtils.ASSET_TYPE);
toVerify.add(barryHardDrive);
barryHardDrive.set("name", "barryHardDrive");
Referenceable barryCpuFan = new Referenceable(TestUtils.ASSET_TYPE);
toVerify.add(barryCpuFan);
barryCpuFan.set("name", "barryCpuFan");
Referenceable barryVideoCard = new Referenceable(TestUtils.ASSET_TYPE);
toVerify.add(barryVideoCard);
barryVideoCard.set("name", "barryVideoCard");
barryComputer.set("childAssets", ImmutableList.of(barryHardDrive, barryVideoCard, barryCpuFan));
Referenceable jacob = new Referenceable(TestUtils.PERSON_TYPE);
toVerify.add(jacob);
jacob.set(TestUtils.NAME, "jacob");
jacob.set(TestUtils.DEPARTMENT_ATTR, dept);
Referenceable jacobComputer = new Referenceable(TestUtils.ASSET_TYPE);
toVerify.add(jacobComputer);
jacobComputer.set("name", "jacobComputer");
jacob.set(TestUtils.ASSETS_ATTR, ImmutableList.of(jacobComputer));
Referenceable jacobHardDrive = new Referenceable(TestUtils.ASSET_TYPE);
toVerify.add(jacobHardDrive);
jacobHardDrive.set("name", "jacobHardDrive");
Referenceable jacobCpuFan = new Referenceable(TestUtils.ASSET_TYPE);
toVerify.add(jacobCpuFan);
jacobCpuFan.set("name", "jacobCpuFan");
Referenceable jacobVideoCard = new Referenceable(TestUtils.ASSET_TYPE);
toVerify.add(jacobVideoCard);
jacobVideoCard.set("name", "jacobVideoCard");
jacobComputer.set("childAssets", ImmutableList.of(jacobHardDrive, jacobVideoCard, jacobCpuFan));
dept.set(TestUtils.EMPLOYEES_ATTR, ImmutableList.of(barry, jacob));
Map<String,Referenceable> positions = new HashMap<>();
final String JANITOR = "janitor";
final String RECEPTIONIST = "receptionist";
positions.put(JANITOR, barry);
positions.put(RECEPTIONIST, jacob);
dept.set(TestUtils.POSITIONS_ATTR, positions);
ClassType deptType = TypeSystem.getInstance().getDataType(ClassType.class, TestUtils.DEPARTMENT_TYPE);
ITypedReferenceableInstance deptInstance = deptType.convert(dept, Multiplicity.REQUIRED);
CreateUpdateEntitiesResult result = repositoryService.createEntities(deptInstance);
System.out.println(result.getGuidMapping().toString());
assertEquals(result.getCreatedEntities().size(), toVerify.size());
validateGuidMapping(toVerify, result);
}
private void validateGuidMapping(List<Referenceable> toVerify, CreateUpdateEntitiesResult result)
throws AtlasException {
Map<String,String> guids = result.getGuidMapping().getGuidAssignments();
TestUtils.assertContentsSame(result.getCreatedEntities(), guids.values());
assertEquals(guids.size(), toVerify.size());
for(Referenceable r : toVerify) {
loadAndDoSimpleValidation(guids.get(r.getId()._getId()), r);
}
}
private ITypedReferenceableInstance loadAndDoSimpleValidation(String guid, Referenceable inst) throws AtlasException {
return TestUtils.loadAndDoSimpleValidation(guid, inst, repositoryService);
}
@Test(dependsOnMethods = "testSubmitEntity")
public void testGetEntityDefinitionForDepartment() throws Exception {
ITypedReferenceableInstance entity = repositoryService.getEntityDefinition(guid);
......@@ -279,7 +459,7 @@ public class GraphBackedMetadataRepositoryTest {
private List<String> createEntities(ITypedReferenceableInstance... instances) throws Exception {
RequestContext.createContext();
return repositoryService.createEntities(instances);
return repositoryService.createEntities(instances).getCreatedEntities();
}
private List<String> createEntity(Referenceable entity) throws Exception {
......@@ -729,7 +909,7 @@ public class GraphBackedMetadataRepositoryTest {
ClassType deptType = typeSystem.getDataType(ClassType.class, "Department");
ITypedReferenceableInstance hrDept2 = deptType.convert(hrDept, Multiplicity.REQUIRED);
List<String> guids = repositoryService.createEntities(hrDept2);
List<String> guids = repositoryService.createEntities(hrDept2).getCreatedEntities();
Assert.assertNotNull(guids);
Assert.assertEquals(guids.size(), 2);
Assert.assertNotNull(guids.get(0));
......
repository/src/test/java/org/apache/atlas/repository/graph/GraphHelperTest.java
......@@ -143,7 +143,7 @@ public class GraphHelperTest {
@Test
public void testGetVerticesForGUIDSWithDuplicates() throws Exception {
ITypedReferenceableInstance hrDept = TestUtils.createDeptEg1(TypeSystem.getInstance());
List<String> result = repositoryService.createEntities(hrDept);
List<String> result = repositoryService.createEntities(hrDept).getCreatedEntities();
String guid = result.get(0);
Map<String, AtlasVertex> verticesForGUIDs = GraphHelper.getInstance().getVerticesForGUIDs(Arrays.asList(guid, guid));
Assert.assertEquals(verticesForGUIDs.size(), 1);
......@@ -152,7 +152,7 @@ public class GraphHelperTest {
@Test
public void testGetCompositeGuidsAndVertices() throws Exception {
ITypedReferenceableInstance hrDept = TestUtils.createDeptEg1(typeSystem);
List<String> createdGuids = repositoryService.createEntities(hrDept);
List<String> createdGuids = repositoryService.createEntities(hrDept).getCreatedEntities();
String deptGuid = null;
Set<String> expectedGuids = new HashSet<>();
......@@ -214,7 +214,7 @@ public class GraphHelperTest {
String entityjson = InstanceSerialization.toJson(entity, true);
JSONArray entitiesJson = new JSONArray();
entitiesJson.put(entityjson);
List<String> guids = metadataService.createEntities(entitiesJson.toString());
List<String> guids = metadataService.createEntities(entitiesJson.toString()).getCreatedEntities();
if (guids != null && guids.size() > 0) {
return guids.get(guids.size() - 1);
}
......
repository/src/test/java/org/apache/atlas/repository/graph/GraphRepoMapperScaleTest.java
......@@ -19,10 +19,12 @@
package org.apache.atlas.repository.graph;
import org.apache.atlas.ApplicationProperties;
import org.apache.atlas.CreateUpdateEntitiesResult;
import org.apache.atlas.GraphTransaction;
import org.apache.atlas.RepositoryMetadataModule;
import org.apache.atlas.TestUtils;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.RepositoryException;
import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.repository.graphdb.AtlasGraphQuery;
import org.apache.atlas.repository.graphdb.AtlasGraphQuery.ComparisionOperator;
......@@ -32,6 +34,7 @@ import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.atlas.typesystem.ITypedReferenceableInstance;
import org.apache.atlas.typesystem.Referenceable;
import org.apache.atlas.typesystem.Struct;
import org.apache.atlas.typesystem.exception.EntityExistsException;
import org.apache.atlas.typesystem.persistence.Id;
import org.apache.atlas.typesystem.types.ClassType;
import org.apache.atlas.typesystem.types.IDataType;
......@@ -100,16 +103,21 @@ public class GraphRepoMapperScaleTest {
ClassType dbType = typeSystem.getDataType(ClassType.class, TestUtils.DATABASE_TYPE);
ITypedReferenceableInstance db = dbType.convert(databaseInstance, Multiplicity.REQUIRED);
dbGUID = repositoryService.createEntities(db).get(0);
dbGUID = result(db).getCreatedEntities().get(0);
Referenceable dbInstance = new Referenceable(dbGUID, TestUtils.DATABASE_TYPE, databaseInstance.getValuesMap());
for (int index = 0; index < 1000; index++) {
ITypedReferenceableInstance table = createHiveTableInstance(dbInstance, index);
repositoryService.createEntities(table);
result(table);
}
}
private CreateUpdateEntitiesResult result(ITypedReferenceableInstance db)
throws RepositoryException, EntityExistsException {
return repositoryService.createEntities(db);
}
@Test(dependsOnMethods = "testSubmitEntity")
public void testSearchIndex() throws Exception {
......
repository/src/test/java/org/apache/atlas/repository/typestore/GraphBackedTypeStoreTest.java
......@@ -165,7 +165,8 @@ public class GraphBackedTypeStoreTest {
HierarchicalTypeDefinition<ClassType> deptTypeDef = createClassTypeDef("Department", "Department"+_description,
ImmutableSet.<String>of(), createRequiredAttrDef("name", DataTypes.STRING_TYPE),
new AttributeDefinition("employees", String.format("array<%s>", "Person"), Multiplicity.OPTIONAL,
true, "department"));
true, "department"),
new AttributeDefinition("positions", String.format("map<%s,%s>", DataTypes.STRING_TYPE.getName(), "Person"), Multiplicity.OPTIONAL, false, null));
TypesDef typesDef = TypesUtil.getTypesDef(ImmutableList.of(orgLevelEnum), ImmutableList.of(addressDetails),
ImmutableList.<HierarchicalTypeDefinition<TraitType>>of(),
ImmutableList.of(deptTypeDef));
......
repository/src/test/java/org/apache/atlas/service/DefaultMetadataServiceTest.java
......@@ -170,7 +170,7 @@ public class DefaultMetadataServiceTest {
String entityjson = InstanceSerialization.toJson(entity, true);
JSONArray entitiesJson = new JSONArray();
entitiesJson.put(entityjson);
return metadataService.updateEntities(entitiesJson.toString());
return metadataService.updateEntities(entitiesJson.toString()).getEntityResult();
}
@Test(expectedExceptions = TypeNotFoundException.class)
......@@ -544,7 +544,7 @@ public class DefaultMetadataServiceTest {
private AtlasClient.EntityResult updateEntityPartial(String guid, Referenceable entity) throws AtlasException {
RequestContext.createContext();
return metadataService.updateEntityPartialByGuid(guid, entity);
return metadataService.updateEntityPartialByGuid(guid, entity).getEntityResult();
}
@Test
......
server-api/src/main/java/org/apache/atlas/services/MetadataService.java
......@@ -20,6 +20,7 @@ package org.apache.atlas.services;
import org.apache.atlas.AtlasClient;
import org.apache.atlas.AtlasException;
import org.apache.atlas.CreateUpdateEntitiesResult;
import org.apache.atlas.EntityAuditEvent;
import org.apache.atlas.listener.EntityChangeListener;
import org.apache.atlas.typesystem.ITypedReferenceableInstance;
......@@ -78,9 +79,9 @@ public interface MetadataService {
* Creates an entity, instance of the type.
*
* @param entityDefinition definition
* @return json array of guids of entities created
* @return CreateUpdateEntitiesResult with the guids of the entities created
*/
List<String> createEntities(String entityDefinition) throws AtlasException;
CreateUpdateEntitiesResult createEntities(String entityDefinition) throws AtlasException;
/**
* Get a typed entity instance.
......@@ -96,11 +97,11 @@ public interface MetadataService {
* Create entity instances.
*
* @param typedInstances instance to create
* @return collection of guids for created entities
* @return CreateUpdateEntitiesResult with the guids of the entities created
*
* @throws AtlasException if unable to create the entities
*/
List<String> createEntities(ITypedReferenceableInstance[] typedInstances) throws AtlasException;
CreateUpdateEntitiesResult createEntities(ITypedReferenceableInstance[] typedInstances) throws AtlasException;
/**
......@@ -148,36 +149,36 @@ public interface MetadataService {
* @param guid entity id
* @param attribute property name
* @param value property value
* @return json array of guids of entities created/updated
* @return {@link CreateUpdateEntitiesResult} with the guids of the entities that were created/updated
*/
AtlasClient.EntityResult updateEntityAttributeByGuid(String guid, String attribute, String value) throws AtlasException;
CreateUpdateEntitiesResult updateEntityAttributeByGuid(String guid, String attribute, String value) throws AtlasException;
/**
* Supports Partial updates of an entity. Users can update a subset of attributes for an entity identified by its guid
* Note however that it cannot be used to set attribute values to null or delete attrbute values
* @param guid entity id
* @param entity
* @return json array of guids of entities created/updated
* @return {@link CreateUpdateEntitiesResult} with the guids of the entities that were created/updated
* @throws AtlasException
*/
AtlasClient.EntityResult updateEntityPartialByGuid(String guid, Referenceable entity) throws AtlasException;
CreateUpdateEntitiesResult updateEntityPartialByGuid(String guid, Referenceable entity) throws AtlasException;
/**
* Batch API - Adds/Updates the given entity id(guid).
*
* @param entityJson entity json
* @return json array of guids of entities created/updated
* @return {@link CreateUpdateEntitiesResult} with the guids of the entities that were created/updated
*/
AtlasClient.EntityResult updateEntities(String entityJson) throws AtlasException;
CreateUpdateEntitiesResult updateEntities(String entityJson) throws AtlasException;
/**
* Batch API - Adds/Updates the given entity id(guid).
*
* @param entityJson entity json
* @return json array of guids of entities created/updated
* @return {@link CreateUpdateEntitiesResult} with the guids of the entities that were created/updated
*/
AtlasClient.EntityResult updateEntities(ITypedReferenceableInstance[] iTypedReferenceableInstances) throws AtlasException;
CreateUpdateEntitiesResult updateEntities(ITypedReferenceableInstance[] iTypedReferenceableInstances) throws AtlasException;
// Trait management functions
......@@ -191,7 +192,7 @@ public interface MetadataService {
* @return Guid of updated entity
* @throws AtlasException
*/
AtlasClient.EntityResult updateEntityByUniqueAttribute(String typeName, String uniqueAttributeName,
CreateUpdateEntitiesResult updateEntityByUniqueAttribute(String typeName, String uniqueAttributeName,
String attrValue,
Referenceable updatedEntity) throws AtlasException;
......
typesystem/src/main/java/org/apache/atlas/typesystem/types/TypeSystem.java
......@@ -618,7 +618,7 @@ public class TypeSystem {
try {
oldType = TypeSystem.this.getDataType(IDataType.class, newType.getName());
} catch (TypeNotFoundException e) {
LOG.debug("No existing type %s found - update OK", newType.getName());
LOG.debug(String.format("No existing type %s found - update OK", newType.getName()));
}
if (oldType != null) {
oldType.validateUpdate(newType);
......
webapp/src/main/java/org/apache/atlas/examples/QuickStartV2.java
......@@ -356,7 +356,8 @@ public class QuickStartV2 {
List<AtlasEntityHeader> entities = response.getEntitiesByOperation(EntityOperation.CREATE);
if (CollectionUtils.isNotEmpty(entities)) {
ret = entitiesClient.getEntityByGuid(entities.get(0).getGuid());
List<AtlasEntityWithAssociations> getByGuidResponse = entitiesClient.getEntityByGuid(entities.get(0).getGuid());
ret = getByGuidResponse.get(0);
System.out.println("Created entity of type [" + ret.getTypeName() + "], guid: " + ret.getGuid());
}
......@@ -567,7 +568,7 @@ public class QuickStartV2 {
}
private String getTableId(String tableName) throws AtlasServiceException {
AtlasEntity tableEntity = entitiesClient.getEntityByAttribute(TABLE_TYPE, AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, tableName);
AtlasEntity tableEntity = entitiesClient.getEntityByAttribute(TABLE_TYPE, AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, tableName).get(0);
return tableEntity.getGuid();
}
}
webapp/src/main/java/org/apache/atlas/util/RestUtils.java
......@@ -18,10 +18,22 @@
package org.apache.atlas.util;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import static org.apache.atlas.AtlasErrorCode.INVALID_TYPE_DEFINITION;
import static org.apache.atlas.model.typedef.AtlasStructDef.AtlasConstraintDef.CONSTRAINT_PARAM_ON_DELETE;
import static org.apache.atlas.model.typedef.AtlasStructDef.AtlasConstraintDef.CONSTRAINT_PARAM_VAL_CASCADE;
import static org.apache.atlas.model.typedef.AtlasStructDef.AtlasConstraintDef.CONSTRAINT_TYPE_FOREIGN_KEY;
import static org.apache.atlas.model.typedef.AtlasStructDef.AtlasConstraintDef.CONSTRAINT_TYPE_MAPPED_FROM_REF;
import static org.apache.atlas.type.AtlasTypeUtil.isArrayType;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.TypeCategory;
import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.model.typedef.AtlasClassificationDef;
import org.apache.atlas.model.typedef.AtlasEntityDef;
import org.apache.atlas.model.typedef.AtlasEnumDef;
......@@ -55,21 +67,12 @@ import org.apache.atlas.typesystem.types.TraitType;
import org.apache.atlas.typesystem.types.utils.TypesUtil;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang.StringUtils;
import org.codehaus.jackson.map.ObjectMapper;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import static org.apache.atlas.AtlasErrorCode.INVALID_TYPE_DEFINITION;
import static org.apache.atlas.model.typedef.AtlasStructDef.AtlasConstraintDef.CONSTRAINT_PARAM_ON_DELETE;
import static org.apache.atlas.model.typedef.AtlasStructDef.AtlasConstraintDef.CONSTRAINT_PARAM_VAL_CASCADE;
import static org.apache.atlas.model.typedef.AtlasStructDef.AtlasConstraintDef.CONSTRAINT_TYPE_FOREIGN_KEY;
import static org.apache.atlas.model.typedef.AtlasStructDef.AtlasConstraintDef.CONSTRAINT_TYPE_MAPPED_FROM_REF;
import static org.apache.atlas.type.AtlasTypeUtil.isArrayType;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
public final class RestUtils {
......@@ -500,4 +503,5 @@ public final class RestUtils {
private static boolean isEntity(AtlasType type) {
return type.getTypeCategory() == TypeCategory.ENTITY;
}
}
\ No newline at end of file
webapp/src/main/java/org/apache/atlas/web/adapters/AtlasInstanceRestAdapters.java
......@@ -17,11 +17,12 @@
*/
package org.apache.atlas.web.adapters;
import com.google.inject.Inject;
import com.google.inject.Singleton;
import java.util.List;
import org.apache.atlas.AtlasClient;
import org.apache.atlas.AtlasErrorCode;
import org.apache.atlas.AtlasException;
import org.apache.atlas.CreateUpdateEntitiesResult;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.TypeCategory;
import org.apache.atlas.model.instance.AtlasClassification;
......@@ -30,6 +31,7 @@ import org.apache.atlas.model.instance.AtlasEntityHeader;
import org.apache.atlas.model.instance.AtlasEntityWithAssociations;
import org.apache.atlas.model.instance.EntityMutationResponse;
import org.apache.atlas.model.instance.EntityMutations;
import org.apache.atlas.model.instance.GuidMapping;
import org.apache.atlas.services.MetadataService;
import org.apache.atlas.type.AtlasClassificationType;
import org.apache.atlas.type.AtlasEntityType;
......@@ -50,7 +52,8 @@ import org.slf4j.LoggerFactory;
import java.util.Collection;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import com.google.inject.Inject;
import com.google.inject.Singleton;
import java.util.Map;
@Singleton
......@@ -142,8 +145,14 @@ public class AtlasInstanceRestAdapters {
return ctx.getEntities();
}
public static EntityMutationResponse toEntityMutationResponse(AtlasClient.EntityResult entityResult) {
public static EntityMutationResponse toEntityMutationResponse(AtlasClient.EntityResult result) {
CreateUpdateEntitiesResult result = new CreateUpdateEntitiesResult();
result.setEntityResult(entityResult);
return toEntityMutationResponse(result);
}
public static EntityMutationResponse toEntityMutationResponse(CreateUpdateEntitiesResult result) {
EntityMutationResponse response = new EntityMutationResponse();
for (String guid : result.getCreatedEntities()) {
AtlasEntityHeader header = new AtlasEntityHeader();
......@@ -151,7 +160,7 @@ public class AtlasInstanceRestAdapters {
response.addEntity(EntityMutations.EntityOperation.CREATE, header);
}
for (String guid : result.getUpdateEntities()) {
for (String guid : result.getUpdatedEntities()) {
AtlasEntityHeader header = new AtlasEntityHeader();
header.setGuid(guid);
response.addEntity(EntityMutations.EntityOperation.UPDATE, header);
......@@ -162,6 +171,10 @@ public class AtlasInstanceRestAdapters {
header.setGuid(guid);
response.addEntity(EntityMutations.EntityOperation.DELETE, header);
}
GuidMapping guidMapping = result.getGuidMapping();
if(guidMapping != null) {
response.setGuidAssignments(guidMapping.getGuidAssignments());
}
return response;
}
......
webapp/src/main/java/org/apache/atlas/web/resources/EntityResource.java
......@@ -23,7 +23,10 @@ import com.google.common.base.Preconditions;
import org.apache.atlas.AtlasClient;
import org.apache.atlas.AtlasConstants;
import org.apache.atlas.AtlasException;
import org.apache.atlas.CreateUpdateEntitiesResult;
import org.apache.atlas.EntityAuditEvent;
import org.apache.atlas.AtlasClient.EntityResult;
import org.apache.atlas.model.instance.GuidMapping;
import org.apache.atlas.services.MetadataService;
import org.apache.atlas.typesystem.IStruct;
import org.apache.atlas.typesystem.Referenceable;
......@@ -127,13 +130,13 @@ public class EntityResource {
LOG.debug("submitting entities {} ", entityJson);
}
final List<String> guids = metadataService.createEntities(entities);
final CreateUpdateEntitiesResult result = metadataService.createEntities(entities);
final List<String> guids = result.getEntityResult().getCreatedEntities();
if (LOG.isDebugEnabled()) {
LOG.debug("Created entities {}", guids);
}
JSONObject response = getResponse(new AtlasClient.EntityResult(guids, null, null));
JSONObject response = getResponse(result);
URI locationURI = getLocationURI(guids);
......@@ -179,14 +182,27 @@ public class EntityResource {
}
private JSONObject getResponse(AtlasClient.EntityResult entityResult) throws AtlasException, JSONException {
CreateUpdateEntitiesResult result = new CreateUpdateEntitiesResult();
result.setEntityResult(entityResult);
return getResponse(result);
}
private JSONObject getResponse(CreateUpdateEntitiesResult result) throws AtlasException, JSONException {
JSONObject response = new JSONObject();
EntityResult entityResult = result.getEntityResult();
GuidMapping mapping = result.getGuidMapping();
response.put(AtlasClient.REQUEST_ID, Servlets.getRequestId());
if(entityResult != null) {
response.put(AtlasClient.ENTITIES, new JSONObject(entityResult.toString()).get(AtlasClient.ENTITIES));
String sampleEntityId = getSample(entityResult);
String sampleEntityId = getSample(result.getEntityResult());
if (sampleEntityId != null) {
String entityDefinition = metadataService.getEntityDefinitionJson(sampleEntityId);
response.put(AtlasClient.DEFINITION, new JSONObject(entityDefinition));
}
}
if(mapping != null) {
response.put(AtlasClient.GUID_ASSIGNMENTS, new JSONObject(mapping.toString()).get(AtlasClient.GUID_ASSIGNMENTS));
}
return response;
}
......@@ -218,13 +234,13 @@ public class EntityResource {
LOG.info("updating entities {} ", entityJson);
}
AtlasClient.EntityResult entityResult = metadataService.updateEntities(entities);
CreateUpdateEntitiesResult result = metadataService.updateEntities(entities);
if (LOG.isDebugEnabled()) {
LOG.debug("Updated entities: {}", entityResult);
LOG.debug("Updated entities: {}", result.getEntityResult());
}
JSONObject response = getResponse(entityResult);
JSONObject response = getResponse(result);
return Response.ok(response).build();
} catch(EntityExistsException e) {
LOG.error("Unique constraint violation for entityDef={}", entityJson, e);
......@@ -303,14 +319,14 @@ public class EntityResource {
Referenceable updatedEntity =
InstanceSerialization.fromJsonReferenceable(entityJson, true);
AtlasClient.EntityResult entityResult =
CreateUpdateEntitiesResult result =
metadataService.updateEntityByUniqueAttribute(entityType, attribute, value, updatedEntity);
if (LOG.isDebugEnabled()) {
LOG.debug("Updated entities: {}", entityResult);
LOG.debug("Updated entities: {}", result.getEntityResult());
}
JSONObject response = getResponse(entityResult);
JSONObject response = getResponse(result);
return Response.ok(response).build();
} catch (ValueConversionException ve) {
LOG.error("Unable to persist entity instance due to a deserialization error {} ", entityJson, ve);
......@@ -388,13 +404,13 @@ public class EntityResource {
Referenceable updatedEntity =
InstanceSerialization.fromJsonReferenceable(entityJson, true);
AtlasClient.EntityResult entityResult = metadataService.updateEntityPartialByGuid(guid, updatedEntity);
CreateUpdateEntitiesResult result = metadataService.updateEntityPartialByGuid(guid, updatedEntity);
if (LOG.isDebugEnabled()) {
LOG.debug("Updated entities: {}", entityResult);
LOG.debug("Updated entities: {}", result.getEntityResult());
}
JSONObject response = getResponse(entityResult);
JSONObject response = getResponse(result);
return Response.ok(response).build();
} catch (EntityNotFoundException e) {
LOG.error("An entity with GUID={} does not exist {} ", guid, entityJson, e);
......@@ -429,13 +445,13 @@ public class EntityResource {
LOG.debug("Updating entity {} for property {} = {}", guid, property, value);
}
AtlasClient.EntityResult entityResult = metadataService.updateEntityAttributeByGuid(guid, property, value);
CreateUpdateEntitiesResult result = metadataService.updateEntityAttributeByGuid(guid, property, value);
if (LOG.isDebugEnabled()) {
LOG.debug("Updated entities: {}", entityResult);
LOG.debug("Updated entities: {}", result.getEntityResult());
}
JSONObject response = getResponse(entityResult);
JSONObject response = getResponse(result);
return Response.ok(response).build();
} catch (EntityNotFoundException e) {
LOG.error("An entity with GUID={} does not exist {} ", guid, value, e);
......
webapp/src/main/java/org/apache/atlas/web/rest/EntitiesREST.java
......@@ -20,6 +20,7 @@ package org.apache.atlas.web.rest;
import org.apache.atlas.AtlasClient;
import org.apache.atlas.AtlasErrorCode;
import org.apache.atlas.AtlasException;
import org.apache.atlas.CreateUpdateEntitiesResult;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.SearchFilter;
import org.apache.atlas.model.instance.AtlasClassification;
......@@ -100,7 +101,7 @@ public class EntitiesREST {
ITypedReferenceableInstance[] entitiesInOldFormat = restAdapters.getITypedReferenceables(entities.values());
try {
final AtlasClient.EntityResult result = metadataService.updateEntities(entitiesInOldFormat);
final CreateUpdateEntitiesResult result = metadataService.updateEntities(entitiesInOldFormat);
response = toEntityMutationResponse(result);
} catch (AtlasException e) {
LOG.error("Exception while getting a typed reference for the entity ", e);
......
webapp/src/main/java/org/apache/atlas/web/rest/EntityREST.java
......@@ -20,6 +20,7 @@ package org.apache.atlas.web.rest;
import org.apache.atlas.AtlasClient;
import org.apache.atlas.AtlasErrorCode;
import org.apache.atlas.AtlasException;
import org.apache.atlas.CreateUpdateEntitiesResult;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.TypeCategory;
import org.apache.atlas.model.instance.AtlasClassification;
......@@ -151,7 +152,7 @@ public class EntityREST {
AtlasFormatConverter.ConverterContext ctx = new AtlasFormatConverter.ConverterContext();
ctx.addEntity(entity);
Referenceable ref = restAdapters.getReferenceable(entity, ctx);
AtlasClient.EntityResult result = metadataService.updateEntityByUniqueAttribute(entityType, attribute, value, ref);
CreateUpdateEntitiesResult result = metadataService.updateEntityByUniqueAttribute(entityType, attribute, value, ref);
return toEntityMutationResponse(result);
}
......
webapp/src/test/java/org/apache/atlas/examples/QuickStartV2IT.java
......@@ -56,7 +56,7 @@ public class QuickStartV2IT extends BaseResourceIT {
}
private AtlasEntity getDB(String dbName) throws AtlasServiceException, JSONException {
AtlasEntity dbEntity = entitiesClientV2.getEntityByAttribute(QuickStartV2.DATABASE_TYPE, "name", dbName);
AtlasEntity dbEntity = entitiesClientV2.getEntityByAttribute(QuickStartV2.DATABASE_TYPE, "name", dbName).get(0);
return dbEntity;
}
......@@ -73,15 +73,16 @@ public class QuickStartV2IT extends BaseResourceIT {
}
private AtlasEntity getTable(String tableName) throws AtlasServiceException {
AtlasEntity tableEntity = entitiesClientV2.getEntityByAttribute(QuickStartV2.TABLE_TYPE, AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, tableName);
AtlasEntity tableEntity = entitiesClientV2.getEntityByAttribute(QuickStartV2.TABLE_TYPE, AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, tableName).get(0);
return tableEntity;
}
private AtlasEntity getProcess(String processName) throws AtlasServiceException {
AtlasEntity processEntity = entitiesClientV2.getEntityByAttribute(QuickStartV2.LOAD_PROCESS_TYPE, AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, processName);
AtlasEntity processEntity = entitiesClientV2.getEntityByAttribute(QuickStartV2.LOAD_PROCESS_TYPE, AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, processName).get(0);
return processEntity;
}
private void verifyTrait(AtlasEntity table) throws AtlasServiceException {
AtlasClassification.AtlasClassifications classfications = entitiesClientV2.getClassifications(table.getGuid());
List<AtlasClassification> traits = classfications.getList();
......@@ -115,7 +116,7 @@ public class QuickStartV2IT extends BaseResourceIT {
@Test
public void testProcessIsAdded() throws AtlasServiceException, JSONException {
AtlasEntity loadProcess = entitiesClientV2.getEntityByAttribute(QuickStartV2.LOAD_PROCESS_TYPE, AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME,
QuickStartV2.LOAD_SALES_DAILY_PROCESS);
QuickStartV2.LOAD_SALES_DAILY_PROCESS).get(0);
Map loadProcessAttribs = loadProcess.getAttributes();
assertEquals(QuickStartV2.LOAD_SALES_DAILY_PROCESS, loadProcessAttribs.get(AtlasClient.NAME));
......@@ -168,7 +169,7 @@ public class QuickStartV2IT extends BaseResourceIT {
@Test
public void testViewIsAdded() throws AtlasServiceException, JSONException {
AtlasEntity view = entitiesClientV2.getEntityByAttribute(QuickStartV2.VIEW_TYPE, AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, QuickStartV2.PRODUCT_DIM_VIEW);
AtlasEntity view = entitiesClientV2.getEntityByAttribute(QuickStartV2.VIEW_TYPE, AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, QuickStartV2.PRODUCT_DIM_VIEW).get(0);
Map<String, Object> viewAttributes = view.getAttributes();
assertEquals(QuickStartV2.PRODUCT_DIM_VIEW, viewAttributes.get(AtlasClient.NAME));
......
webapp/src/test/java/org/apache/atlas/util/RestUtilsTest.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.util;
import static org.mockito.Matchers.any;
import static org.mockito.Matchers.eq;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.model.typedef.AtlasEntityDef;
import org.apache.atlas.model.typedef.AtlasStructDef;
import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef;
import org.apache.atlas.model.typedef.AtlasTypesDef;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.repository.store.graph.v1.AtlasGraphUtilsV1;
import org.apache.atlas.repository.store.graph.v1.AtlasStructDefStoreV1;
import org.apache.atlas.repository.store.graph.v1.AtlasTypeDefGraphStoreV1;
import org.apache.atlas.type.AtlasEntityType;
import org.apache.atlas.type.AtlasStructType;
import org.apache.atlas.type.AtlasStructType.AtlasAttribute;
import org.apache.atlas.type.AtlasType;
import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.atlas.type.AtlasTypeRegistry.AtlasTransientTypeRegistry;
import org.apache.atlas.typesystem.TypesDef;
import org.apache.atlas.typesystem.json.TypesSerialization;
import org.apache.atlas.typesystem.types.AttributeDefinition;
import org.apache.atlas.typesystem.types.ClassType;
import org.apache.atlas.typesystem.types.DataTypes;
import org.apache.atlas.typesystem.types.DataTypes.TypeCategory;
import org.apache.atlas.typesystem.types.EnumTypeDefinition;
import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition;
import org.apache.atlas.typesystem.types.Multiplicity;
import org.apache.atlas.typesystem.types.StructTypeDefinition;
import org.apache.atlas.typesystem.types.TraitType;
import org.apache.atlas.typesystem.types.utils.TypesUtil;
import org.testng.Assert;
import org.testng.annotations.Test;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
/**
* Validates that conversion from V1 to legacy types (and back) is consistent. This also tests
* that the conversion logic in AtlasStructDefStoreV1 is consistent with the conversion logic
* in RestUtils. This tests particularly focuses on composite attributes, since a defect was
* found in that area.
*/
public class RestUtilsTest {
@Test(enabled=false)
// FIXME: On conversion back to V1, reverse attribute name
// "containingDatabase"
// in tables attribute in "database" type is lost. See ATLAS-1528.
public void testBidirectonalCompositeMappingConsistent() throws AtlasBaseException {
HierarchicalTypeDefinition<ClassType> dbV1Type = TypesUtil.createClassTypeDef("database",
ImmutableSet.<String> of(), new AttributeDefinition("tables", DataTypes.arrayTypeName("table"),
Multiplicity.OPTIONAL, true, "containingDatabase"));
HierarchicalTypeDefinition<ClassType> tableV1Type = TypesUtil.createClassTypeDef("table",
ImmutableSet.<String> of(),
new AttributeDefinition("containingDatabase", "database", Multiplicity.OPTIONAL, false, "tables"));
testV1toV2toV1Conversion(Arrays.asList(dbV1Type, tableV1Type), new boolean[] { true, false });
}
@Test(enabled=false)
// FIXME: On conversion back to V1, reverse attribute name
// "containingDatabase" is lost
// in "table" attribute in "database". See ATLAS-1528.
public void testBidirectonalNonCompositeMappingConsistent() throws AtlasBaseException {
HierarchicalTypeDefinition<ClassType> dbV1Type = TypesUtil.createClassTypeDef("database",
ImmutableSet.<String> of(), new AttributeDefinition("tables", DataTypes.arrayTypeName("table"),
Multiplicity.OPTIONAL, false, "containingDatabase"));
HierarchicalTypeDefinition<ClassType> tableV1Type = TypesUtil.createClassTypeDef("table",
ImmutableSet.<String> of(),
new AttributeDefinition("containingDatabase", "database", Multiplicity.OPTIONAL, false, "tables"));
testV1toV2toV1Conversion(Arrays.asList(dbV1Type, tableV1Type), new boolean[] { false, false });
}
private AtlasTypeDefGraphStoreV1 makeTypeStore(AtlasTypeRegistry reg) {
AtlasTypeDefGraphStoreV1 result = mock(AtlasTypeDefGraphStoreV1.class);
for (AtlasEntityType type : reg.getAllEntityTypes()) {
String typeName = type.getTypeName();
AtlasVertex typeVertex = mock(AtlasVertex.class);
when(result.isTypeVertex(eq(typeVertex), any(TypeCategory.class))).thenReturn(true);
when(typeVertex.getProperty(eq(Constants.TYPE_CATEGORY_PROPERTY_KEY), eq(TypeCategory.class)))
.thenReturn(TypeCategory.CLASS);
String attributeListPropertyKey = AtlasGraphUtilsV1.getTypeDefPropertyKey(typeName);
when(typeVertex.getProperty(eq(attributeListPropertyKey), eq(List.class)))
.thenReturn(new ArrayList<>(type.getAllAttributes().keySet()));
for (AtlasAttribute attribute : type.getAllAttributes().values()) {
String attributeDefPropertyKey = AtlasGraphUtilsV1.getTypeDefPropertyKey(typeName, attribute.getName());
String attributeJson = AtlasStructDefStoreV1.toJsonFromAttribute(attribute);
when(typeVertex.getProperty(eq(attributeDefPropertyKey), eq(String.class))).thenReturn(attributeJson);
}
when(result.findTypeVertexByName(eq(typeName))).thenReturn(typeVertex);
}
return result;
}
private AtlasAttributeDef convertToJsonAndBack(AtlasTypeRegistry registry, AtlasStructDef structDef,
AtlasAttributeDef attributeDef, boolean compositeExpected) throws AtlasBaseException {
AtlasTypeDefGraphStoreV1 typeDefStore = makeTypeStore(registry);
AtlasStructType structType = (AtlasStructType) registry.getType(structDef.getName());
AtlasAttribute attribute = structType.getAttribute(attributeDef.getName());
String attribJson = AtlasStructDefStoreV1.toJsonFromAttribute(attribute);
Map attrInfo = AtlasType.fromJson(attribJson, Map.class);
Assert.assertEquals(attrInfo.get("isComposite"), compositeExpected);
return AtlasStructDefStoreV1.toAttributeDefFromJson(structDef, attrInfo, typeDefStore);
}
private void testV1toV2toV1Conversion(List<HierarchicalTypeDefinition<ClassType>> typesToTest,
boolean[] compositeExpected) throws AtlasBaseException {
List<AtlasEntityDef> convertedEntityDefs = convertV1toV2(typesToTest);
AtlasTypeRegistry registry = createRegistry(convertedEntityDefs);
for(int i = 0 ; i < convertedEntityDefs.size(); i++) {
AtlasEntityDef def = convertedEntityDefs.get(i);
for (AtlasAttributeDef attrDef : def.getAttributeDefs()) {
AtlasAttributeDef converted = convertToJsonAndBack(registry, def, attrDef, compositeExpected[i]);
Assert.assertEquals(converted, attrDef);
}
}
List<HierarchicalTypeDefinition<ClassType>> convertedBackTypeDefs = convertV2toV1(convertedEntityDefs);
for (int i = 0; i < typesToTest.size(); i++) {
HierarchicalTypeDefinition<ClassType> convertedBack = convertedBackTypeDefs.get(i);
Assert.assertEquals(convertedBack, typesToTest.get(i));
AttributeDefinition[] attributeDefinitions = convertedBack.attributeDefinitions;
if (attributeDefinitions.length > 0) {
Assert.assertEquals(attributeDefinitions[0].isComposite, compositeExpected[i]);
}
}
}
private List<HierarchicalTypeDefinition<ClassType>> convertV2toV1(List<AtlasEntityDef> toConvert)
throws AtlasBaseException {
AtlasTypeRegistry reg = createRegistry(toConvert);
List<HierarchicalTypeDefinition<ClassType>> result = new ArrayList<>(toConvert.size());
for (int i = 0; i < toConvert.size(); i++) {
AtlasEntityDef entityDef = toConvert.get(i);
AtlasEntityType entity = reg.getEntityTypeByName(entityDef.getName());
HierarchicalTypeDefinition<ClassType> converted = RestUtils.toTypesDef(entity, reg)
.classTypesAsJavaList().get(0);
result.add(converted);
}
return result;
}
private AtlasTypeRegistry createRegistry(List<AtlasEntityDef> toConvert) throws AtlasBaseException {
AtlasTypeRegistry reg = new AtlasTypeRegistry();
AtlasTransientTypeRegistry tmp = reg.lockTypeRegistryForUpdate();
tmp.addTypes(toConvert);
reg.releaseTypeRegistryForUpdate(tmp, true);
return reg;
}
private List<AtlasEntityDef> convertV1toV2(List<HierarchicalTypeDefinition<ClassType>> types)
throws AtlasBaseException {
ImmutableList<HierarchicalTypeDefinition<ClassType>> classTypeList = ImmutableList
.<HierarchicalTypeDefinition<ClassType>> builder().addAll(types).build();
TypesDef toConvert = TypesUtil.getTypesDef(ImmutableList.<EnumTypeDefinition> of(),
ImmutableList.<StructTypeDefinition> of(), ImmutableList.<HierarchicalTypeDefinition<TraitType>> of(),
classTypeList);
String json = TypesSerialization.toJson(toConvert);
AtlasTypeRegistry emptyRegistry = new AtlasTypeRegistry();
AtlasTypesDef converted = RestUtils.toAtlasTypesDef(json, emptyRegistry);
List<AtlasEntityDef> convertedEntityDefs = converted.getEntityDefs();
return convertedEntityDefs;
}
}
webapp/src/test/java/org/apache/atlas/web/resources/BaseResourceIT.java
......@@ -18,10 +18,18 @@
package org.apache.atlas.web.resources;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import kafka.consumer.ConsumerTimeoutException;
import static org.apache.atlas.model.typedef.AtlasStructDef.AtlasConstraintDef.CONSTRAINT_PARAM_ON_DELETE;
import static org.apache.atlas.model.typedef.AtlasStructDef.AtlasConstraintDef.CONSTRAINT_PARAM_VAL_CASCADE;
import static org.apache.atlas.model.typedef.AtlasStructDef.AtlasConstraintDef.CONSTRAINT_TYPE_FOREIGN_KEY;
import static org.testng.Assert.assertNotNull;
import static org.testng.Assert.assertTrue;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.atlas.ApplicationProperties;
import org.apache.atlas.AtlasClient;
import org.apache.atlas.AtlasDiscoveryClientV2;
......@@ -40,6 +48,9 @@ import org.apache.atlas.model.typedef.AtlasClassificationDef;
import org.apache.atlas.model.typedef.AtlasEntityDef;
import org.apache.atlas.model.typedef.AtlasEnumDef;
import org.apache.atlas.model.typedef.AtlasStructDef;
import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef;
import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef.Cardinality;
import org.apache.atlas.model.typedef.AtlasStructDef.AtlasConstraintDef;
import org.apache.atlas.model.typedef.AtlasTypesDef;
import org.apache.atlas.notification.NotificationConsumer;
import org.apache.atlas.notification.entity.EntityNotification;
......@@ -50,7 +61,17 @@ import org.apache.atlas.typesystem.TypesDef;
import org.apache.atlas.typesystem.json.InstanceSerialization;
import org.apache.atlas.typesystem.json.TypesSerialization;
import org.apache.atlas.typesystem.persistence.Id;
import org.apache.atlas.typesystem.types.*;
import org.apache.atlas.typesystem.types.AttributeDefinition;
import org.apache.atlas.typesystem.types.ClassType;
import org.apache.atlas.typesystem.types.DataTypes;
import org.apache.atlas.typesystem.types.EnumTypeDefinition;
import org.apache.atlas.typesystem.types.EnumValue;
import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition;
import org.apache.atlas.typesystem.types.IDataType;
import org.apache.atlas.typesystem.types.Multiplicity;
import org.apache.atlas.typesystem.types.StructTypeDefinition;
import org.apache.atlas.typesystem.types.TraitType;
import org.apache.atlas.typesystem.types.TypeUtils;
import org.apache.atlas.typesystem.types.utils.TypesUtil;
import org.apache.atlas.utils.AuthenticationUtil;
import org.apache.atlas.utils.ParamChecker;
......@@ -62,14 +83,11 @@ import org.slf4j.LoggerFactory;
import org.testng.Assert;
import org.testng.annotations.BeforeClass;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import static org.testng.Assert.assertNotNull;
import static org.testng.Assert.assertTrue;
import kafka.consumer.ConsumerTimeoutException;
/**
* Base class for integration tests.
......@@ -125,6 +143,45 @@ public abstract class BaseResourceIT {
}
}
protected void batchCreateTypes(AtlasTypesDef typesDef) throws AtlasServiceException {
for (AtlasEnumDef enumDef : typesDef.getEnumDefs()) {
try {
typedefClientV2.createEnumDef(enumDef);
} catch (AtlasServiceException ex) {
LOG.warn("EnumDef creation failed for {}", enumDef.getName());
}
}
for (AtlasStructDef structDef : typesDef.getStructDefs()) {
try {
typedefClientV2.createStructDef(structDef);
} catch (AtlasServiceException ex) {
LOG.warn("StructDef creation failed for {}", structDef.getName());
}
}
AtlasTypesDef entityDefs = new AtlasTypesDef(
Collections.<AtlasEnumDef>emptyList(),
Collections.<AtlasStructDef>emptyList(),
Collections.<AtlasClassificationDef>emptyList(),
typesDef.getEntityDefs());
try {
typedefClientV2.createAtlasTypeDefs(entityDefs);
}
catch(AtlasServiceException e) {
LOG.warn("Type creation failed for {}", typesDef.toString());
LOG.warn(e.toString());
}
for (AtlasClassificationDef classificationDef : typesDef.getClassificationDefs()) {
try {
typedefClientV2.createClassificationDef(classificationDef);
} catch (AtlasServiceException ex) {
LOG.warn("ClassificationDef creation failed for {}", classificationDef.getName());
}
}
}
protected void createType(AtlasTypesDef typesDef) {
// Since the bulk create bails out on a single failure, this has to be done as a workaround
for (AtlasEnumDef enumDef : typesDef.getEnumDefs()) {
......@@ -182,7 +239,7 @@ public abstract class BaseResourceIT {
}
LOG.info("Types already exist. Skipping type creation");
} catch(AtlasServiceException ase) {
//Expected if type doesnt exist
//Expected if type doesn't exist
String typesAsJSON = TypesSerialization.toJson(typesDef);
createType(typesAsJSON);
}
......@@ -278,7 +335,10 @@ public abstract class BaseResourceIT {
TypesUtil.createUniqueRequiredAttrDef(NAME, DataTypes.STRING_TYPE),
TypesUtil.createRequiredAttrDef(DESCRIPTION, DataTypes.STRING_TYPE),
attrDef("locationUri", DataTypes.STRING_TYPE),
attrDef("owner", DataTypes.STRING_TYPE), attrDef("createTime", DataTypes.INT_TYPE));
attrDef("owner", DataTypes.STRING_TYPE), attrDef("createTime", DataTypes.INT_TYPE),
new AttributeDefinition("tables", DataTypes.arrayTypeName(HIVE_TABLE_TYPE),
Multiplicity.OPTIONAL, false, "db")
);
HierarchicalTypeDefinition<ClassType> columnClsDef = TypesUtil
.createClassTypeDef(COLUMN_TYPE, null, attrDef(NAME, DataTypes.STRING_TYPE),
......@@ -297,7 +357,7 @@ public abstract class BaseResourceIT {
attrDef("owner", DataTypes.STRING_TYPE), attrDef("createTime", DataTypes.LONG_TYPE),
attrDef("lastAccessTime", DataTypes.DATE_TYPE),
attrDef("temporary", DataTypes.BOOLEAN_TYPE),
new AttributeDefinition("db", DATABASE_TYPE, Multiplicity.REQUIRED, true, null),
new AttributeDefinition("db", DATABASE_TYPE, Multiplicity.OPTIONAL, true, "tables"),
new AttributeDefinition("columns", DataTypes.arrayTypeName(COLUMN_TYPE),
Multiplicity.OPTIONAL, true, null),
new AttributeDefinition("tableType", "tableType", Multiplicity.OPTIONAL, false, null),
......@@ -347,6 +407,13 @@ public abstract class BaseResourceIT {
}
protected void createTypeDefinitionsV2() throws Exception {
AtlasConstraintDef isCompositeSourceConstraint = new AtlasConstraintDef(CONSTRAINT_TYPE_FOREIGN_KEY,
Collections.<String, Object>singletonMap(CONSTRAINT_PARAM_ON_DELETE, CONSTRAINT_PARAM_VAL_CASCADE));
AtlasConstraintDef isCompositeTargetConstraint = new AtlasConstraintDef(CONSTRAINT_TYPE_FOREIGN_KEY,
Collections.<String, Object>emptyMap());
AtlasEntityDef dbClsTypeDef = AtlasTypeUtil.createClassTypeDef(
DATABASE_TYPE_V2,
null,
......@@ -354,7 +421,15 @@ public abstract class BaseResourceIT {
AtlasTypeUtil.createRequiredAttrDef(DESCRIPTION, "string"),
AtlasTypeUtil.createOptionalAttrDef("locationUri", "string"),
AtlasTypeUtil.createOptionalAttrDef("owner", "string"),
AtlasTypeUtil.createOptionalAttrDef("createTime", "int"));
AtlasTypeUtil.createOptionalAttrDef("createTime", "int"),
AtlasTypeUtil.createOptionalAttrDef("createTime", "int"),
//there is a serializ
new AtlasAttributeDef("randomTable",
DataTypes.arrayTypeName(HIVE_TABLE_TYPE_V2),
true,
Cardinality.SET,
0, -1, false, true, Collections.singletonList(isCompositeSourceConstraint))
);
AtlasEntityDef columnClsDef = AtlasTypeUtil
.createClassTypeDef(COLUMN_TYPE_V2, null,
......@@ -379,7 +454,12 @@ public abstract class BaseResourceIT {
AtlasTypeUtil.createOptionalAttrDef("createTime", "long"),
AtlasTypeUtil.createOptionalAttrDef("lastAccessTime", "date"),
AtlasTypeUtil.createOptionalAttrDef("temporary", "boolean"),
AtlasTypeUtil.createRequiredAttrDef("db", DATABASE_TYPE_V2),
new AtlasAttributeDef("db",
DATABASE_TYPE_V2,
true,
Cardinality.SINGLE,
0, 1, false, true, Collections.singletonList(isCompositeTargetConstraint)),
//some tests don't set the columns field or set it to null...
AtlasTypeUtil.createOptionalAttrDef("columns", DataTypes.arrayTypeName(COLUMN_TYPE_V2)),
AtlasTypeUtil.createOptionalAttrDef("tableType", "tableType"),
......@@ -418,7 +498,7 @@ public abstract class BaseResourceIT {
ImmutableList.of(classificationTrait, piiTrait, phiTrait, pciTrait, soxTrait, secTrait, financeTrait),
ImmutableList.of(dbClsTypeDef, columnClsDef, tblClsDef, loadProcessClsDef));
createType(typesDef);
batchCreateTypes(typesDef);
}
AttributeDefinition attrDef(String name, IDataType dT) {
......
webapp/src/test/java/org/apache/atlas/web/resources/EntityJerseyResourceIT.java
......@@ -18,16 +18,24 @@
package org.apache.atlas.web.resources;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import com.google.gson.Gson;
import com.google.gson.JsonSyntaxException;
import com.google.inject.Inject;
import com.sun.jersey.api.client.ClientResponse;
import com.sun.jersey.core.util.MultivaluedMapImpl;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertNotNull;
import static org.testng.Assert.fail;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import javax.ws.rs.core.MultivaluedMap;
import javax.ws.rs.core.Response;
import org.apache.atlas.AtlasClient;
import org.apache.atlas.AtlasServiceException;
import org.apache.atlas.EntityAuditEvent;
import org.apache.atlas.model.instance.GuidMapping;
import org.apache.atlas.notification.NotificationConsumer;
import org.apache.atlas.notification.NotificationInterface;
import org.apache.atlas.notification.NotificationModule;
......@@ -61,18 +69,13 @@ import org.testng.annotations.DataProvider;
import org.testng.annotations.Guice;
import org.testng.annotations.Test;
import javax.ws.rs.core.MultivaluedMap;
import javax.ws.rs.core.Response;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertNotNull;
import static org.testng.Assert.fail;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import com.google.gson.Gson;
import com.google.gson.JsonSyntaxException;
import com.google.inject.Inject;
import com.sun.jersey.api.client.ClientResponse;
import com.sun.jersey.core.util.MultivaluedMapImpl;
/**
......@@ -110,6 +113,66 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
}
@Test
public void testCreateNestedEntities() throws Exception {
Referenceable databaseInstance = new Referenceable(DATABASE_TYPE);
databaseInstance.set("name", "db1");
databaseInstance.set("description", "foo database");
int nTables = 5;
int colsPerTable=3;
List<Referenceable> tables = new ArrayList<>();
List<Referenceable> allColumns = new ArrayList<>();
for(int i = 0; i < nTables; i++) {
String tableName = "db1-table-" + i;
Referenceable tableInstance =
new Referenceable(HIVE_TABLE_TYPE);
tableInstance.set("name", tableName);
tableInstance.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, tableName);
tableInstance.set("db", databaseInstance);
tableInstance.set("description", tableName + " table");
tables.add(tableInstance);
List<Referenceable> columns = new ArrayList<>();
for(int j = 0; j < colsPerTable; j++) {
Referenceable columnInstance = new Referenceable(COLUMN_TYPE);
columnInstance.set("name", tableName + "-col-" + j);
columnInstance.set("dataType", "String");
columnInstance.set("comment", "column " + j + " for table " + i);
allColumns.add(columnInstance);
columns.add(columnInstance);
}
tableInstance.set("columns", columns);
}
//Create the tables. The database and columns should be created automatically, since
//the tables reference them.
JSONArray entityArray = new JSONArray(tables.size());
for(int i = 0; i < tables.size(); i++) {
Referenceable table = tables.get(i);
entityArray.put(InstanceSerialization.toJson(table, true));
}
String json = entityArray.toString();
JSONObject response = atlasClientV1.callAPIWithBodyAndParams(AtlasClient.API.CREATE_ENTITY, json);
GuidMapping guidMapping = GuidMapping.fromString(response.toString());
Map<String,String> guidsCreated = guidMapping.getGuidAssignments();
assertEquals(guidsCreated.size(), nTables * colsPerTable + nTables + 1);
assertNotNull(guidsCreated.get(databaseInstance.getId()._getId()));
for(Referenceable r : allColumns) {
assertNotNull(guidsCreated.get(r.getId()._getId()));
}
for(Referenceable r : tables) {
assertNotNull(guidsCreated.get(r.getId()._getId()));
}
}
@Test
public void testSubmitEntity() throws Exception {
tableInstance = createHiveTableInstanceBuiltIn(DATABASE_NAME, TABLE_NAME, dbId);
tableId = createInstance(tableInstance);
......
webapp/src/test/java/org/apache/atlas/web/resources/EntityV2JerseyResourceIT.java
......@@ -18,11 +18,18 @@
package org.apache.atlas.web.resources;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Lists;
import com.google.inject.Inject;
import com.sun.jersey.api.client.ClientResponse;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertFalse;
import static org.testng.Assert.assertNotNull;
import static org.testng.Assert.assertNull;
import static org.testng.Assert.assertTrue;
import static org.testng.Assert.fail;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.atlas.AtlasClient;
import org.apache.atlas.AtlasServiceException;
import org.apache.atlas.EntityAuditEvent;
......@@ -53,12 +60,11 @@ import org.testng.annotations.DataProvider;
import org.testng.annotations.Guice;
import org.testng.annotations.Test;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import static org.testng.Assert.*;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Lists;
import com.google.inject.Inject;
import com.sun.jersey.api.client.ClientResponse;
/**
......@@ -100,6 +106,60 @@ public class EntityV2JerseyResourceIT extends BaseResourceIT {
}
@Test
public void testCreateNestedEntities() throws Exception {
AtlasEntity databaseInstance = new AtlasEntity(DATABASE_TYPE_V2);
databaseInstance.setAttribute("name", "db1");
databaseInstance.setAttribute("description", "foo database");
databaseInstance.setAttribute("owner", "user1");
databaseInstance.setAttribute("locationUri", "/tmp");
databaseInstance.setAttribute("createTime",1000);
int nTables = 5;
int colsPerTable=3;
List<AtlasEntity> tables = new ArrayList<>();
List<AtlasEntity> allColumns = new ArrayList<>();
for(int i = 0; i < nTables; i++) {
String tableName = "db1-table-" + i;
AtlasEntity tableInstance =
new AtlasEntity(HIVE_TABLE_TYPE_V2);
tableInstance.setAttribute("name", tableName);
tableInstance.setAttribute(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, tableName);
tableInstance.setAttribute("db", databaseInstance);
tableInstance.setAttribute("description", tableName + " table");
tables.add(tableInstance);
List<AtlasEntity> columns = new ArrayList<>();
for(int j = 0; j < colsPerTable; j++) {
AtlasEntity columnInstance = new AtlasEntity(COLUMN_TYPE_V2);
columnInstance.setAttribute("name", tableName + "-col-" + j);
columnInstance.setAttribute("dataType", "String");
columnInstance.setAttribute("comment", "column " + j + " for table " + i);
allColumns.add(columnInstance);
columns.add(columnInstance);
}
tableInstance.setAttribute("columns", ImmutableList.builder().addAll(columns).build());
}
//Create the tables. The database and columns should be created automatically, since
//the tables reference them.
EntityMutationResponse response = entitiesClientV2.createEntities(tables);
Assert.assertNotNull(response);
Map<String,String> guidsCreated = response.getGuidAssignments();
assertEquals(guidsCreated.size(), nTables * colsPerTable + nTables + 1);
assertNotNull(guidsCreated.get(databaseInstance.getGuid()));
for(AtlasEntity r : allColumns) {
assertNotNull(guidsCreated.get(r.getGuid()));
}
for(AtlasEntity r : tables) {
assertNotNull(guidsCreated.get(r.getGuid()));
}
}
@Test
public void testRequestUser() throws Exception {
AtlasEntity hiveDBInstanceV2 = createHiveDB(randomString());
List<EntityAuditEvent> events = atlasClientV1.getEntityAuditEvents(hiveDBInstanceV2.getGuid(), (short) 10);
......@@ -187,7 +247,7 @@ public class EntityV2JerseyResourceIT extends BaseResourceIT {
assertEquals(updated.getEntityDefs().size(), 1);
//Get definition after type update - new attributes should be null
AtlasEntity entityByGuid = entitiesClientV2.getEntityByGuid(guid);
AtlasEntity entityByGuid = getEntityByGuid(guid);
assertNull(entityByGuid.getAttribute("description"));
assertEquals(entityByGuid.getAttribute("name"), instance.getAttribute("name"));
}
......@@ -213,7 +273,7 @@ public class EntityV2JerseyResourceIT extends BaseResourceIT {
AtlasEntity hiveDB = createHiveDB();
String qualifiedName = (String) hiveDB.getAttribute(NAME);
//get entity by attribute
AtlasEntity byAttribute = entitiesClientV2.getEntityByAttribute(DATABASE_TYPE_V2, NAME, qualifiedName);
AtlasEntity byAttribute = entitiesClientV2.getEntityByAttribute(DATABASE_TYPE_V2, NAME, qualifiedName).get(0);
assertEquals(byAttribute.getTypeName(), DATABASE_TYPE_V2);
assertEquals(byAttribute.getAttribute(NAME), qualifiedName);
}
......@@ -224,9 +284,11 @@ public class EntityV2JerseyResourceIT extends BaseResourceIT {
createEntity(hiveDBInstance);
AtlasEntity tableInstance = createHiveTableInstanceV2(hiveDBInstance, "table" + randomString());
//Dates with an invalid format are simply nulled out. This does not produce
//an error. See AtlasBuiltInTypes.AtlasDateType.getNormalizedValue().
tableInstance.setAttribute("lastAccessTime", "2014-07-11");
AtlasEntityHeader tableEntityHeader = createEntity(tableInstance);
assertNull(tableEntityHeader);
assertNotNull(tableEntityHeader);
}
@Test(dependsOnMethods = "testSubmitEntity")
......@@ -235,7 +297,7 @@ public class EntityV2JerseyResourceIT extends BaseResourceIT {
String description = "bar table - new desc";
addProperty(createHiveTable().getGuid(), "description", description);
AtlasEntity entityByGuid = entitiesClientV2.getEntityByGuid(createHiveTable().getGuid());
AtlasEntity entityByGuid = getEntityByGuid(createHiveTable().getGuid());
Assert.assertNotNull(entityByGuid);
entityByGuid.setAttribute("description", description);
......@@ -256,7 +318,7 @@ public class EntityV2JerseyResourceIT extends BaseResourceIT {
addProperty(createHiveTable().getGuid(), "createTime", currentTime);
entityByGuid = entitiesClientV2.getEntityByGuid(createHiveTable().getGuid());
entityByGuid = getEntityByGuid(createHiveTable().getGuid());
Assert.assertNotNull(entityByGuid);
}
......@@ -274,7 +336,7 @@ public class EntityV2JerseyResourceIT extends BaseResourceIT {
@Test(expectedExceptions = AtlasServiceException.class)
public void testGetInvalidEntityDefinition() throws Exception {
entitiesClientV2.getEntityByGuid("blah");
getEntityByGuid("blah");
}
@Test(dependsOnMethods = "testSubmitEntity", enabled = false)
......@@ -331,7 +393,7 @@ public class EntityV2JerseyResourceIT extends BaseResourceIT {
@Test(dependsOnMethods = "testSubmitEntity")
public void testCommonAttributes() throws Exception{
AtlasEntity entity = entitiesClientV2.getEntityByGuid(createHiveTable().getGuid());
AtlasEntity entity = getEntityByGuid(createHiveTable().getGuid());
Assert.assertNotNull(entity.getStatus());
Assert.assertNotNull(entity.getVersion());
Assert.assertNotNull(entity.getCreatedBy());
......@@ -342,7 +404,7 @@ public class EntityV2JerseyResourceIT extends BaseResourceIT {
private void addProperty(String guid, String property, String value) throws AtlasServiceException {
AtlasEntity entityByGuid = entitiesClientV2.getEntityByGuid(guid);
AtlasEntity entityByGuid = getEntityByGuid(guid);
entityByGuid.setAttribute(property, value);
EntityMutationResponse response = entitiesClientV2.updateEntity(entityByGuid);
assertNotNull(response);
......@@ -447,7 +509,7 @@ public class EntityV2JerseyResourceIT extends BaseResourceIT {
entitiesClientV2.addClassifications(guid, ImmutableList.of(traitInstance));
// verify the response
AtlasEntityWithAssociations withAssociationByGuid = entitiesClientV2.getEntityWithAssociationByGuid(guid);
AtlasEntityWithAssociations withAssociationByGuid = entitiesClientV2.getEntityWithAssociationByGuid(guid).get(0);
assertNotNull(withAssociationByGuid);
assertFalse(withAssociationByGuid.getClassifications().isEmpty());
......@@ -543,7 +605,7 @@ public class EntityV2JerseyResourceIT extends BaseResourceIT {
assertNotNull(entity);
assertNotNull(entity.getGuid());
AtlasEntity entityByGuid = entitiesClientV2.getEntityByGuid(entity.getGuid());
AtlasEntity entityByGuid = getEntityByGuid(entity.getGuid());
assertEquals(entityByGuid.getAttribute(attrName), attrValue);
}
......@@ -569,7 +631,8 @@ public class EntityV2JerseyResourceIT extends BaseResourceIT {
assertNotNull(updateResult.getEntitiesByOperation(EntityMutations.EntityOperation.UPDATE));
assertTrue(updateResult.getEntitiesByOperation(EntityMutations.EntityOperation.UPDATE).size() > 0);
AtlasEntity entityByGuid = entitiesClientV2.getEntityByGuid(hiveTable.getGuid());
String guid = hiveTable.getGuid();
AtlasEntity entityByGuid = getEntityByGuid(guid);
assertNotNull(entityByGuid);
entityByGuid.getAttribute("columns");
......@@ -587,11 +650,15 @@ public class EntityV2JerseyResourceIT extends BaseResourceIT {
assertNotNull(updateResponse.getEntitiesByOperation(EntityMutations.EntityOperation.UPDATE));
assertTrue(updateResponse.getEntitiesByOperation(EntityMutations.EntityOperation.UPDATE).size() > 0);
entityByGuid = entitiesClientV2.getEntityByGuid(hiveTable.getGuid());
entityByGuid = getEntityByGuid(guid);
assertNotNull(entityByGuid);
entityByGuid.getAttribute("columns");
}
private AtlasEntity getEntityByGuid(String guid) throws AtlasServiceException {
return entitiesClientV2.getEntityByGuid(guid).get(0);
}
@Test(dependsOnMethods = "testSubmitEntity")
public void testCompleteUpdate() throws Exception {
final List<AtlasEntity> columns = new ArrayList<>();
......@@ -621,7 +688,7 @@ public class EntityV2JerseyResourceIT extends BaseResourceIT {
assertEquals(updateEntityResult.getEntitiesByOperation(EntityMutations.EntityOperation.UPDATE).size(), 1);
assertEquals(updateEntityResult.getEntitiesByOperation(EntityMutations.EntityOperation.CREATE).size(), 2);
AtlasEntity entityByGuid = entitiesClientV2.getEntityByGuid(hiveTable.getGuid());
AtlasEntity entityByGuid = getEntityByGuid(hiveTable.getGuid());
List<AtlasEntity> refs = (List<AtlasEntity>) entityByGuid.getAttribute("columns");
assertEquals(refs.size(), 2);
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment