Skip to content

A
atlas
Commit 51656991 by Shwetha GS
Options

ATLAS-47 Entity mutations for complex types (sumasai via shwethags)

parent 6c3f0964
Showing with 3137 additions and 1354 deletions
addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java
......@@ -479,12 +479,13 @@ public class HiveMetaStoreBridge {
HiveDataModelGenerator dataModelGenerator = new HiveDataModelGenerator();
AtlasClient dgiClient = getAtlasClient();
//Register hive data model if its not already registered
if (dgiClient.getType(HiveDataTypes.HIVE_PROCESS.getName()) == null) {
try {
dgiClient.getType(HiveDataTypes.HIVE_PROCESS.getName());
LOG.info("Hive data model is already registered!");
} catch(AtlasServiceException ase) {
//Expected in case types do not exist
LOG.info("Registering Hive data model");
dgiClient.createType(dataModelGenerator.getModelAsJson());
} else {
LOG.info("Hive data model is already registered!");
}
}
......
addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java
......@@ -20,7 +20,7 @@ package org.apache.atlas.hive.hook;
import org.apache.atlas.ApplicationProperties;
import org.apache.atlas.AtlasClient;
import org.apache.atlas.ParamChecker;
import org.apache.atlas.utils.ParamChecker;
import org.apache.atlas.hive.bridge.HiveMetaStoreBridge;
import org.apache.atlas.hive.model.HiveDataModelGenerator;
import org.apache.atlas.hive.model.HiveDataTypes;
......
client/src/main/java/org/apache/atlas/AtlasClient.java
......@@ -133,40 +133,43 @@ public class AtlasClient {
enum API {
//Type operations
CREATE_TYPE(BASE_URI + TYPES, HttpMethod.POST),
UPDATE_TYPE(BASE_URI + TYPES, HttpMethod.PUT),
GET_TYPE(BASE_URI + TYPES, HttpMethod.GET),
LIST_TYPES(BASE_URI + TYPES, HttpMethod.GET),
LIST_TRAIT_TYPES(BASE_URI + TYPES + "?type=trait", HttpMethod.GET),
CREATE_TYPE(BASE_URI + TYPES, HttpMethod.POST, Response.Status.CREATED),
UPDATE_TYPE(BASE_URI + TYPES, HttpMethod.PUT, Response.Status.OK),
GET_TYPE(BASE_URI + TYPES, HttpMethod.GET, Response.Status.OK),
LIST_TYPES(BASE_URI + TYPES, HttpMethod.GET, Response.Status.OK),
LIST_TRAIT_TYPES(BASE_URI + TYPES + "?type=trait", HttpMethod.GET, Response.Status.OK),
//Entity operations
CREATE_ENTITY(BASE_URI + URI_ENTITY, HttpMethod.POST),
GET_ENTITY(BASE_URI + URI_ENTITY, HttpMethod.GET),
UPDATE_ENTITY(BASE_URI + URI_ENTITY, HttpMethod.PUT),
LIST_ENTITIES(BASE_URI + URI_ENTITY, HttpMethod.GET),
CREATE_ENTITY(BASE_URI + URI_ENTITY, HttpMethod.POST, Response.Status.CREATED),
GET_ENTITY(BASE_URI + URI_ENTITY, HttpMethod.GET, Response.Status.OK),
UPDATE_ENTITY(BASE_URI + URI_ENTITY, HttpMethod.PUT, Response.Status.OK),
UPDATE_ENTITY_PARTIAL(BASE_URI + URI_ENTITY, HttpMethod.POST, Response.Status.OK),
LIST_ENTITIES(BASE_URI + URI_ENTITY, HttpMethod.GET, Response.Status.OK),
//Trait operations
ADD_TRAITS(BASE_URI + URI_ENTITY, HttpMethod.POST),
DELETE_TRAITS(BASE_URI + URI_ENTITY, HttpMethod.DELETE),
LIST_TRAITS(BASE_URI + URI_ENTITY, HttpMethod.GET),
ADD_TRAITS(BASE_URI + URI_ENTITY, HttpMethod.POST, Response.Status.CREATED),
DELETE_TRAITS(BASE_URI + URI_ENTITY, HttpMethod.DELETE, Response.Status.OK),
LIST_TRAITS(BASE_URI + URI_ENTITY, HttpMethod.GET, Response.Status.OK),
//Search operations
SEARCH(BASE_URI + URI_SEARCH, HttpMethod.GET),
SEARCH_DSL(BASE_URI + URI_SEARCH + "/dsl", HttpMethod.GET),
SEARCH_GREMLIN(BASE_URI + URI_SEARCH + "/gremlin", HttpMethod.GET),
SEARCH_FULL_TEXT(BASE_URI + URI_SEARCH + "/fulltext", HttpMethod.GET),
SEARCH(BASE_URI + URI_SEARCH, HttpMethod.GET, Response.Status.OK),
SEARCH_DSL(BASE_URI + URI_SEARCH + "/dsl", HttpMethod.GET, Response.Status.OK),
SEARCH_GREMLIN(BASE_URI + URI_SEARCH + "/gremlin", HttpMethod.GET, Response.Status.OK),
SEARCH_FULL_TEXT(BASE_URI + URI_SEARCH + "/fulltext", HttpMethod.GET, Response.Status.OK),
//Lineage operations
LINEAGE_INPUTS_GRAPH(BASE_URI + URI_LINEAGE, HttpMethod.GET),
LINEAGE_OUTPUTS_GRAPH(BASE_URI + URI_LINEAGE, HttpMethod.GET),
LINEAGE_SCHEMA(BASE_URI + URI_LINEAGE, HttpMethod.GET);
LINEAGE_INPUTS_GRAPH(BASE_URI + URI_LINEAGE, HttpMethod.GET, Response.Status.OK),
LINEAGE_OUTPUTS_GRAPH(BASE_URI + URI_LINEAGE, HttpMethod.GET, Response.Status.OK),
LINEAGE_SCHEMA(BASE_URI + URI_LINEAGE, HttpMethod.GET, Response.Status.OK);
private final String method;
private final String path;
private final Response.Status status;
API(String path, String method) {
API(String path, String method, Response.Status status) {
this.path = path;
this.method = method;
this.status = status;
}
public String getMethod() {
......@@ -176,6 +179,8 @@ public class AtlasClient {
public String getPath() {
return path;
}
public Response.Status getExpectedStatus() { return status; }
}
/**
......@@ -231,7 +236,7 @@ public class AtlasClient {
JSONObject response = callAPIWithResource(API.GET_TYPE, resource);
return response.getString(DEFINITION);
} catch (AtlasServiceException e) {
if (e.getStatus() == ClientResponse.Status.NOT_FOUND) {
if (Response.Status.NOT_FOUND.equals(e.getStatus())) {
return null;
}
throw e;
......@@ -266,11 +271,82 @@ public class AtlasClient {
}
public JSONArray createEntity(Referenceable... entities) throws AtlasServiceException {
JSONArray entityArray = getEntitiesArray(entities);
return createEntity(entityArray);
}
private JSONArray getEntitiesArray(Referenceable[] entities) {
JSONArray entityArray = new JSONArray(entities.length);
for (Referenceable entity : entities) {
entityArray.put(InstanceSerialization.toJson(entity, true));
}
return createEntity(entityArray);
return entityArray;
}
/**
* Replaces entity definitions identified by their guid or unique attribute
* Updates properties set in the definition for the entity corresponding to guid
* @param entities entities to be updated
* @return json array of guids which were updated/created
* @throws AtlasServiceException
*/
public JSONArray updateEntities(Referenceable... entities) throws AtlasServiceException {
JSONArray entitiesArray = getEntitiesArray(entities);
JSONObject response = callAPI(API.UPDATE_ENTITY, entitiesArray.toString());
try {
return response.getJSONArray(GUID);
} catch (JSONException e) {
throw new AtlasServiceException(API.UPDATE_ENTITY, e);
}
}
/**
* Supports Partial updates
* Updates property for the entity corresponding to guid
* @param guid guid
* @param attribute property key
* @param value property value
*/
public void updateEntityAttribute(String guid, String attribute, String value) throws AtlasServiceException {
API api = API.UPDATE_ENTITY_PARTIAL;
WebResource resource = getResource(api, guid);
resource = resource.queryParam(ATTRIBUTE_NAME, attribute);
callAPIWithResource(api, resource, value);
}
/**
* Supports Partial updates
* Updates properties set in the definition for the entity corresponding to guid
* @param guid guid
* @param entity entity definition
*/
public void updateEntity(String guid, Referenceable entity) throws AtlasServiceException {
String entityJson = InstanceSerialization.toJson(entity, true);
callAPI(API.UPDATE_ENTITY_PARTIAL, entityJson, guid);
}
/**
* Supports Partial updates
* Updates properties set in the definition for the entity corresponding to guid
* @param entityType Type of the entity being updated
* @param uniqueAttributeName Attribute Name that uniquely identifies the entity
* @param uniqueAttributeValue Attribute Value that uniquely identifies the entity
* @param entity entity definition
*/
public String updateEntity(String entityType, String uniqueAttributeName, String uniqueAttributeValue,
Referenceable entity) throws AtlasServiceException {
API api = API.UPDATE_ENTITY_PARTIAL;
WebResource resource = getResource(api, "qualifiedName");
resource = resource.queryParam(TYPE, entityType);
resource = resource.queryParam(ATTRIBUTE_NAME, uniqueAttributeName);
resource = resource.queryParam(ATTRIBUTE_VALUE, uniqueAttributeValue);
String entityJson = InstanceSerialization.toJson(entity, true);
JSONObject response = callAPIWithResource(api, resource, entityJson);
try {
return response.getString(GUID);
} catch (JSONException e) {
throw new AtlasServiceException(api, e);
}
}
/**
......@@ -351,19 +427,6 @@ public class AtlasClient {
}
/**
* Updates property for the entity corresponding to guid
* @param guid guid
* @param property property key
* @param value property value
*/
public JSONObject updateEntity(String guid, String property, String value) throws AtlasServiceException {
WebResource resource = getResource(API.UPDATE_ENTITY, guid);
resource = resource.queryParam(ATTRIBUTE_NAME, property);
resource = resource.queryParam(ATTRIBUTE_VALUE, value);
return callAPIWithResource(API.UPDATE_ENTITY, resource);
}
/**
* Search using gremlin/dsl/full text
* @param searchQuery
* @return
......@@ -488,13 +551,11 @@ public class AtlasClient {
}
private JSONObject callAPIWithResource(API api, WebResource resource, Object requestObject)
throws AtlasServiceException {
throws AtlasServiceException {
ClientResponse clientResponse = resource.accept(JSON_MEDIA_TYPE).type(JSON_MEDIA_TYPE)
.method(api.getMethod(), ClientResponse.class, requestObject);
.method(api.getMethod(), ClientResponse.class, requestObject);
Response.Status expectedStatus =
HttpMethod.POST.equals(api.getMethod()) ? Response.Status.CREATED : Response.Status.OK;
if (clientResponse.getStatus() == expectedStatus.getStatusCode()) {
if (clientResponse.getStatus() == api.getExpectedStatus().getStatusCode()) {
String responseAsString = clientResponse.getEntity(String.class);
try {
return new JSONObject(responseAsString);
......
common/pom.xml
......@@ -33,13 +33,19 @@
<dependencies>
<dependency>
<groupId>org.apache.atlas</groupId>
<artifactId>atlas-typesystem</artifactId>
<groupId>org.testng</groupId>
<artifactId>testng</artifactId>
</dependency>
<dependency>
<groupId>org.testng</groupId>
<artifactId>testng</artifactId>
<groupId>com.google.inject</groupId>
<artifactId>guice</artifactId>
</dependency>
<dependency>
<groupId>commons-configuration</groupId>
<artifactId>commons-configuration</artifactId>
</dependency>
</dependencies>
</project>
common/src/main/java/org/apache/atlas/utils/MD5Utils.java 0 → 100644
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.utils;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
public class MD5Utils {
private static final ThreadLocal<MessageDigest> DIGESTER_FACTORY =
new ThreadLocal<MessageDigest>() {
@Override
protected MessageDigest initialValue() {
try {
return MessageDigest.getInstance("MD5");
} catch (NoSuchAlgorithmException e) {
throw new RuntimeException(e);
}
}
};
/**
* Create a thread local MD5 digester
*/
public static MessageDigest getDigester() {
MessageDigest digester = DIGESTER_FACTORY.get();
digester.reset();
return digester;
}
private static final char[] HEX_DIGITS =
{'0','1','2','3','4','5','6','7','8','9','a','b','c','d','e','f'};
public static String toString(byte[] digest) {
StringBuilder buf = new StringBuilder(MD5_LEN*2);
for (int i = 0; i < MD5_LEN; i++) {
int b = digest[i];
buf.append(HEX_DIGITS[(b >> 4) & 0xf]);
buf.append(HEX_DIGITS[b & 0xf]);
}
return buf.toString();
}
public static final int MD5_LEN = 16;
}
typesystem/src/main/java/org/apache/atlas/ParamChecker.java → common/src/main/java/org/apache/atlas/utils/ParamChecker.java
......@@ -15,7 +15,7 @@
* limitations under the License.
*/
package org.apache.atlas;
package org.apache.atlas.utils;
import java.util.Arrays;
import java.util.Collection;
......
docs/src/site/twiki/Configuration.twiki
......@@ -60,8 +60,9 @@ Without Ranger, HBase shell can be used to set the permissions.
</verbatim>
---++++ Graph Search Index
This section sets up the graph db - titan - to use an search indexing system. The example
configuration below setsup to use an embedded Elastic search indexing system.
configuration below sets up to use an embedded Elastic search indexing system.
<verbatim>
atlas.graph.index.search.backend=elasticsearch
......@@ -72,6 +73,7 @@ atlas.graph.index.search.elasticsearch.create.sleep=2000
</verbatim>
---++++ Graph Search Index - Solr
Please note that Solr installation in Cloud mode is a prerequisite before configuring Solr as the search indexing backend. Refer InstallationSteps section for Solr installation/configuration.
<verbatim>
atlas.graph.index.search.backend=solr5
......
docs/src/site/twiki/InstallationSteps.twiki
......@@ -124,7 +124,7 @@ export METADATA_SERVER_OPTS="-Djava.awt.headless=true -Djava.security.krb5.realm
* Hbase as the Storage Backend for the Graph Repository
By default, Atlas uses Titan as the graph repository and is the only graph repository implementation available currently.
The HBase versions currently supported are 0.98.x, 1.0.x, 1.1.x. For configuring ATLAS graph persistence on HBase, please go through the "Configuration - Graph persistence engine - HBase" section
The HBase versions currently supported are 1.1.x. For configuring ATLAS graph persistence on HBase, please go through the "Configuration - Graph persistence engine - HBase" section
for more details.
Pre-requisites for running HBase as a distributed cluster
......
notification/pom.xml
......@@ -48,6 +48,11 @@
</dependency>
<dependency>
<groupId>org.apache.atlas</groupId>
<artifactId>atlas-server-api</artifactId>
</dependency>
<dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka-clients</artifactId>
</dependency>
......
notification/src/main/java/org/apache/atlas/notification/entity/NotificationEntityChangeListener.java
......@@ -73,8 +73,8 @@ public class NotificationEntityChangeListener implements EntityChangeListener {
}
@Override
public void onEntityUpdated(ITypedReferenceableInstance entity) throws AtlasException {
notifyOfEntityEvent(Collections.singleton(entity), EntityNotification.OperationType.ENTITY_UPDATE);
public void onEntitiesUpdated(Collection<ITypedReferenceableInstance> entities) throws AtlasException {
notifyOfEntityEvent(entities, EntityNotification.OperationType.ENTITY_UPDATE);
}
@Override
......
pom.xml
......@@ -407,6 +407,7 @@
<modules>
<module>common</module>
<module>typesystem</module>
<module>server-api</module>
<module>notification</module>
<module>client</module>
<module>titan</module>
......@@ -932,6 +933,12 @@
<dependency>
<groupId>org.apache.atlas</groupId>
<artifactId>atlas-server-api</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.atlas</groupId>
<artifactId>atlas-repository</artifactId>
<version>${project.version}</version>
</dependency>
......
release-log.txt
......@@ -14,6 +14,7 @@ ATLAS-54 Rename configs in hive hook (shwethags)
ATLAS-3 Mixed Index creation fails with Date types (sumasai via shwethags)
ALL CHANGES:
ATLAS-47 Entity mutations for complex types (sumasai via shwethags)
ATLAS-345 UI: Should allow tag addition on any search result that returns a reference-able entity (darshankumar89 via shwethags)
ATLAS-279 UI not displaying results for certain successful "select" search queries (anilsg via shwethags)
ATLAS-242 The qualified name for hive entities should be backward compatible (shwethags)
......
repository/pom.xml
......@@ -41,7 +41,7 @@
<dependency>
<groupId>org.apache.atlas</groupId>
<artifactId>atlas-common</artifactId>
<artifactId>atlas-server-api</artifactId>
</dependency>
<dependency>
......
repository/src/main/java/org/apache/atlas/RepositoryMetadataModule.java
......@@ -49,11 +49,13 @@ public class RepositoryMetadataModule extends com.google.inject.AbstractModule {
@Override
protected void configure() {
// special wiring for Titan Graph
ThrowingProviderBinder.create(binder()).bind(GraphProvider.class, TitanGraph.class).to(TitanGraphProvider.class)
.asEagerSingleton();
// allow for dynamic binding of the metadata repo & graph service
// bind the MetadataRepositoryService interface to an implementation
bind(MetadataRepository.class).to(GraphBackedMetadataRepository.class).asEagerSingleton();
......
repository/src/main/java/org/apache/atlas/discovery/HiveLineageService.java
......@@ -22,14 +22,14 @@ import com.thinkaurelius.titan.core.TitanGraph;
import org.apache.atlas.ApplicationProperties;
import org.apache.atlas.AtlasException;
import org.apache.atlas.GraphTransaction;
import org.apache.atlas.ParamChecker;
import org.apache.atlas.typesystem.exception.EntityNotFoundException;
import org.apache.atlas.utils.ParamChecker;
import org.apache.atlas.discovery.graph.DefaultGraphPersistenceStrategy;
import org.apache.atlas.discovery.graph.GraphBackedDiscoveryService;
import org.apache.atlas.query.Expressions;
import org.apache.atlas.query.GremlinQueryResult;
import org.apache.atlas.query.HiveLineageQuery;
import org.apache.atlas.query.HiveWhereUsedQuery;
import org.apache.atlas.repository.EntityNotFoundException;
import org.apache.atlas.repository.MetadataRepository;
import org.apache.atlas.repository.graph.GraphProvider;
import org.apache.atlas.typesystem.persistence.ReferenceableInstance;
......
repository/src/main/java/org/apache/atlas/discovery/graph/DefaultGraphPersistenceStrategy.java
......@@ -30,6 +30,7 @@ import org.apache.atlas.query.TypeUtils;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.MetadataRepository;
import org.apache.atlas.repository.graph.GraphBackedMetadataRepository;
import org.apache.atlas.repository.graph.GraphHelper;
import org.apache.atlas.typesystem.ITypedReferenceableInstance;
import org.apache.atlas.typesystem.ITypedStruct;
import org.apache.atlas.typesystem.persistence.Id;
......@@ -43,7 +44,7 @@ import org.apache.atlas.typesystem.types.TypeSystem;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import javax.inject.Inject;
import java.util.List;
/**
......@@ -55,6 +56,7 @@ public class DefaultGraphPersistenceStrategy implements GraphPersistenceStrategi
private final GraphBackedMetadataRepository metadataRepository;
@Inject
public DefaultGraphPersistenceStrategy(MetadataRepository metadataRepository) {
this.metadataRepository = (GraphBackedMetadataRepository) metadataRepository;
}
......@@ -71,7 +73,11 @@ public class DefaultGraphPersistenceStrategy implements GraphPersistenceStrategi
@Override
public String edgeLabel(IDataType<?> dataType, AttributeInfo aInfo) {
return metadataRepository.getEdgeLabel(dataType, aInfo);
try {
return metadataRepository.getEdgeLabel(dataType, aInfo);
} catch (AtlasException e) {
throw new RuntimeException(e);
}
}
@Override
......@@ -90,7 +96,7 @@ public class DefaultGraphPersistenceStrategy implements GraphPersistenceStrategi
@Override
public List<String> traitNames(TitanVertex vertex) {
return metadataRepository.getTraitNames(vertex);
return GraphHelper.getTraitNames(vertex);
}
@Override
......@@ -100,7 +106,7 @@ public class DefaultGraphPersistenceStrategy implements GraphPersistenceStrategi
@Override
public Id getIdFromVertex(String dataTypeName, TitanVertex vertex) {
return metadataRepository.getIdFromVertex(dataTypeName, vertex);
return GraphHelper.getIdFromVertex(dataTypeName, vertex);
}
@Override
......
repository/src/main/java/org/apache/atlas/repository/MetadataRepository.java
......@@ -19,6 +19,8 @@
package org.apache.atlas.repository;
import org.apache.atlas.AtlasException;
import org.apache.atlas.typesystem.exception.EntityExistsException;
import org.apache.atlas.typesystem.exception.EntityNotFoundException;
import org.apache.atlas.typesystem.ITypedReferenceableInstance;
import org.apache.atlas.typesystem.ITypedStruct;
import org.apache.atlas.typesystem.types.AttributeInfo;
......@@ -70,7 +72,7 @@ public interface MetadataRepository {
* @param aInfo attribute info
* @return edge label for a given attribute
*/
String getEdgeLabel(IDataType<?> dataType, AttributeInfo aInfo);
String getEdgeLabel(IDataType<?> dataType, AttributeInfo aInfo) throws AtlasException;
/**
* Creates an entity definition (instance) corresponding to a given type.
......@@ -89,7 +91,7 @@ public interface MetadataRepository {
* @return entity (typed instance) definition
* @throws RepositoryException
*/
ITypedReferenceableInstance getEntityDefinition(String guid) throws RepositoryException;
ITypedReferenceableInstance getEntityDefinition(String guid) throws RepositoryException, EntityNotFoundException;
/**
* Gets the list of entities for a given entity type.
......@@ -108,20 +110,6 @@ public interface MetadataRepository {
* @throws RepositoryException
*/
// boolean deleteEntity(String guid) throws RepositoryException;
/**
* Updates an entity given its GUID with the attribute name and value.
*
* @param guid globally unique identifier for the entity
* @param attributeName name of the attribute
* @param attributeValue value of the attribute
* @return an entity instance with updated state
* @throws RepositoryException
*/
//ITypedReferenceableInstance updateEntity(String guid, String attributeName,
// String attributeValue) throws RepositoryException;
// Trait management functions
/**
......@@ -149,15 +137,19 @@ public interface MetadataRepository {
* @param traitNameToBeDeleted name of the trait
* @throws RepositoryException
*/
void deleteTrait(String guid, String traitNameToBeDeleted) throws RepositoryException;
void deleteTrait(String guid, String traitNameToBeDeleted) throws EntityNotFoundException, RepositoryException;
/**
* Adds/Updates the property to the entity that corresponds to the GUID
* Supports only primitive attribute/Class Id updations.
*/
void updatePartial(ITypedReferenceableInstance entity) throws RepositoryException;
/**
* Adds/Updates the property to/in the entity that corresponds to the GUID
* @param guid entity id
* @param property property name
* @param value property value
* Adds the property to the entity that corresponds to the GUID
* @param entitiesToBeUpdated The entities to be updated
*/
void updateEntity(String guid, String property, String value) throws RepositoryException;
String[] updateEntities(ITypedReferenceableInstance... entitiesToBeUpdated) throws RepositoryException;
/**
* Returns the entity for the given type and qualified name
......@@ -166,5 +158,5 @@ public interface MetadataRepository {
* @param value
* @return entity instance
*/
ITypedReferenceableInstance getEntityDefinition(String entityType, String attribute, String value) throws AtlasException;
ITypedReferenceableInstance getEntityDefinition(String entityType, String attribute, Object value) throws AtlasException;
}
repository/src/main/java/org/apache/atlas/repository/graph/EntityProcessor.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.repository.graph;
import org.apache.atlas.AtlasException;
import org.apache.atlas.repository.RepositoryException;
import org.apache.atlas.typesystem.IReferenceableInstance;
import org.apache.atlas.typesystem.ITypedReferenceableInstance;
import org.apache.atlas.typesystem.persistence.Id;
import org.apache.atlas.typesystem.types.DataTypes;
import org.apache.atlas.typesystem.types.ObjectGraphWalker;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.LinkedHashMap;
import java.util.Map;
public final class EntityProcessor implements ObjectGraphWalker.NodeProcessor {
private final Map<Id, IReferenceableInstance> idToInstanceMap;
public EntityProcessor() {
idToInstanceMap = new LinkedHashMap<>();
}
public Collection<IReferenceableInstance> getInstances() {
ArrayList<IReferenceableInstance> instances = new ArrayList<IReferenceableInstance>(idToInstanceMap.values());
Collections.reverse(instances);
return instances;
}
@Override
public void processNode(ObjectGraphWalker.Node nd) throws AtlasException {
IReferenceableInstance ref = null;
Id id = null;
if (nd.attributeName == null) {
ref = (IReferenceableInstance) nd.instance;
id = ref.getId();
} else if (nd.aInfo.dataType().getTypeCategory() == DataTypes.TypeCategory.CLASS) {
if (nd.value != null && (nd.value instanceof Id)) {
id = (Id) nd.value;
}
}
if (id != null) {
if (id.isUnassigned()) {
if (ref != null) {
if (idToInstanceMap.containsKey(id)) { // Oops
throw new RepositoryException(
String.format("Unexpected internal error: Id %s processed again", id));
}
idToInstanceMap.put(id, ref);
}
}
}
}
public void addInstanceIfNotExists(ITypedReferenceableInstance ref) {
if(!idToInstanceMap.containsKey(ref.getId())) {
idToInstanceMap.put(ref.getId(), ref);
}
}
}
repository/src/main/java/org/apache/atlas/repository/graph/FullTextMapper.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.repository.graph;
import com.tinkerpop.blueprints.Vertex;
import org.apache.atlas.AtlasException;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.typesystem.ITypedInstance;
import org.apache.atlas.typesystem.ITypedReferenceableInstance;
import org.apache.atlas.typesystem.types.AttributeInfo;
import org.apache.atlas.typesystem.types.DataTypes;
import org.apache.atlas.typesystem.types.EnumValue;
import org.apache.atlas.typesystem.types.IDataType;
import org.apache.commons.lang.StringUtils;
import java.util.List;
import java.util.Map;
public class FullTextMapper {
private final GraphToTypedInstanceMapper graphToTypedInstanceMapper;
private static final GraphHelper graphHelper = GraphHelper.getInstance();
private static final String FULL_TEXT_DELIMITER = " ";
FullTextMapper(GraphToTypedInstanceMapper graphToTypedInstanceMapper) {
this.graphToTypedInstanceMapper = graphToTypedInstanceMapper;
}
public String mapRecursive(Vertex instanceVertex, boolean followReferences) throws AtlasException {
String guid = instanceVertex.getProperty(Constants.GUID_PROPERTY_KEY);
ITypedReferenceableInstance typedReference =
graphToTypedInstanceMapper.mapGraphToTypedInstance(guid, instanceVertex);
String fullText = forInstance(typedReference, followReferences);
StringBuilder fullTextBuilder =
new StringBuilder(typedReference.getTypeName()).append(FULL_TEXT_DELIMITER).append(fullText);
List<String> traits = typedReference.getTraits();
for (String traitName : traits) {
String traitText = forInstance((ITypedInstance) typedReference.getTrait(traitName), false);
fullTextBuilder.append(FULL_TEXT_DELIMITER).append(traitName).append(FULL_TEXT_DELIMITER)
.append(traitText);
}
return fullTextBuilder.toString();
}
private String forAttribute(IDataType type, Object value, boolean followReferences)
throws AtlasException {
if (value == null) {
return null;
}
switch (type.getTypeCategory()) {
case PRIMITIVE:
return String.valueOf(value);
case ENUM:
return ((EnumValue) value).value;
case ARRAY:
StringBuilder fullText = new StringBuilder();
IDataType elemType = ((DataTypes.ArrayType) type).getElemType();
List list = (List) value;
for (Object element : list) {
String elemFullText = forAttribute(elemType, element, false);
if (StringUtils.isNotEmpty(elemFullText)) {
fullText = fullText.append(FULL_TEXT_DELIMITER).append(elemFullText);
}
}
return fullText.toString();
case MAP:
fullText = new StringBuilder();
IDataType keyType = ((DataTypes.MapType) type).getKeyType();
IDataType valueType = ((DataTypes.MapType) type).getValueType();
Map map = (Map) value;
for (Object entryObj : map.entrySet()) {
Map.Entry entry = (Map.Entry) entryObj;
String keyFullText = forAttribute(keyType, entry.getKey(), false);
if (StringUtils.isNotEmpty(keyFullText)) {
fullText = fullText.append(FULL_TEXT_DELIMITER).append(keyFullText);
}
String valueFullText = forAttribute(valueType, entry.getValue(), false);
if (StringUtils.isNotEmpty(valueFullText)) {
fullText = fullText.append(FULL_TEXT_DELIMITER).append(valueFullText);
}
}
return fullText.toString();
case CLASS:
if (followReferences) {
String refGuid = ((ITypedReferenceableInstance) value).getId()._getId();
Vertex refVertex = graphHelper.getVertexForGUID(refGuid);
return mapRecursive(refVertex, false);
}
break;
case STRUCT:
if (followReferences) {
return forInstance((ITypedInstance) value, true);
}
break;
default:
throw new IllegalStateException("Unhandled type category " + type.getTypeCategory());
}
return null;
}
private String forInstance(ITypedInstance typedInstance, boolean followReferences)
throws AtlasException {
StringBuilder fullText = new StringBuilder();
for (AttributeInfo attributeInfo : typedInstance.fieldMapping().fields.values()) {
Object attrValue = typedInstance.get(attributeInfo.name);
if (attrValue == null) {
continue;
}
String attrFullText = forAttribute(attributeInfo.dataType(), attrValue, followReferences);
if (StringUtils.isNotEmpty(attrFullText)) {
fullText =
fullText.append(FULL_TEXT_DELIMITER).append(attributeInfo.name).append(FULL_TEXT_DELIMITER)
.append(attrFullText);
}
}
return fullText.toString();
}
}
repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepository.java
......@@ -19,11 +19,7 @@
package org.apache.atlas.repository.graph;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
import com.thinkaurelius.titan.core.SchemaViolationException;
import com.thinkaurelius.titan.core.TitanGraph;
import com.thinkaurelius.titan.core.TitanProperty;
import com.thinkaurelius.titan.core.TitanVertex;
import com.tinkerpop.blueprints.Direction;
import com.tinkerpop.blueprints.Edge;
import com.tinkerpop.blueprints.GraphQuery;
......@@ -31,42 +27,25 @@ import com.tinkerpop.blueprints.Vertex;
import org.apache.atlas.AtlasException;
import org.apache.atlas.GraphTransaction;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.EntityExistsException;
import org.apache.atlas.repository.EntityNotFoundException;
import org.apache.atlas.repository.MetadataRepository;
import org.apache.atlas.repository.RepositoryException;
import org.apache.atlas.typesystem.IReferenceableInstance;
import org.apache.atlas.typesystem.ITypedInstance;
import org.apache.atlas.typesystem.ITypedReferenceableInstance;
import org.apache.atlas.typesystem.ITypedStruct;
import org.apache.atlas.typesystem.persistence.Id;
import org.apache.atlas.typesystem.exception.EntityExistsException;
import org.apache.atlas.typesystem.exception.EntityNotFoundException;
import org.apache.atlas.typesystem.types.AttributeInfo;
import org.apache.atlas.typesystem.types.ClassType;
import org.apache.atlas.typesystem.types.DataTypes;
import org.apache.atlas.typesystem.types.EnumValue;
import org.apache.atlas.typesystem.types.HierarchicalType;
import org.apache.atlas.typesystem.types.IDataType;
import org.apache.atlas.typesystem.types.Multiplicity;
import org.apache.atlas.typesystem.types.ObjectGraphWalker;
import org.apache.atlas.typesystem.types.StructType;
import org.apache.atlas.typesystem.types.TraitType;
import org.apache.atlas.typesystem.types.TypeSystem;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.inject.Inject;
import javax.inject.Singleton;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
/**
* An implementation backed by a Graph database provided
......@@ -76,19 +55,19 @@ import java.util.Map;
public class GraphBackedMetadataRepository implements MetadataRepository {
private static final Logger LOG = LoggerFactory.getLogger(GraphBackedMetadataRepository.class);
private static final String FULL_TEXT_DELIMITER = " ";
private static final String EDGE_LABEL_PREFIX = "__";
private final TypedInstanceToGraphMapper instanceToGraphMapper = new TypedInstanceToGraphMapper();
private final GraphToTypedInstanceMapper graphToInstanceMapper = new GraphToTypedInstanceMapper();
private final GraphToTypedInstanceMapper graphToInstanceMapper;
private static TypeSystem typeSystem = TypeSystem.getInstance();
private static final GraphHelper graphHelper = GraphHelper.getInstance();
private final TypeSystem typeSystem;
private final TitanGraph titanGraph;
@Inject
public GraphBackedMetadataRepository(GraphProvider<TitanGraph> graphProvider) throws AtlasException {
this.typeSystem = TypeSystem.getInstance();
public GraphBackedMetadataRepository(GraphProvider<TitanGraph> graphProvider) {
this.titanGraph = graphProvider.get();
this.graphToInstanceMapper = new GraphToTypedInstanceMapper(titanGraph);
}
public GraphToTypedInstanceMapper getGraphToInstanceMapper() {
......@@ -116,42 +95,31 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
@Override
public String getTraitLabel(IDataType<?> dataType, String traitName) {
return getTraitLabel(dataType.getName(), traitName);
return GraphHelper.getTraitLabel(dataType.getName(), traitName);
}
@Override
public String getFieldNameInVertex(IDataType<?> dataType, AttributeInfo aInfo) throws AtlasException {
return getQualifiedName(dataType, aInfo.name);
return GraphHelper.getQualifiedFieldName(dataType, aInfo.name);
}
public String getFieldNameInVertex(IDataType<?> dataType, String attrName) throws AtlasException {
return getQualifiedName(dataType, attrName);
}
@Override
public String getEdgeLabel(IDataType<?> dataType, AttributeInfo aInfo) {
return getEdgeLabel(dataType.getName(), aInfo.name);
return GraphHelper.getQualifiedFieldName(dataType, attrName);
}
public String getEdgeLabel(String typeName, String attrName) {
return EDGE_LABEL_PREFIX + typeName + "." + attrName;
}
public String getTraitLabel(String typeName, String attrName) {
return typeName + "." + attrName;
}
public String getEdgeLabel(ITypedInstance typedInstance, AttributeInfo aInfo) throws AtlasException {
IDataType dataType = typeSystem.getDataType(IDataType.class, typedInstance.getTypeName());
return getEdgeLabel(dataType, aInfo);
@Override
public String getEdgeLabel(IDataType<?> dataType, AttributeInfo aInfo) throws AtlasException {
return GraphHelper.getEdgeLabel(dataType, aInfo);
}
@Override
@GraphTransaction
public String[] createEntities(ITypedReferenceableInstance... entities) throws RepositoryException,
EntityExistsException {
EntityExistsException {
LOG.info("adding entities={}", entities);
try {
return instanceToGraphMapper.mapTypedInstanceToGraph(entities);
TypedInstanceToGraphMapper instanceToGraphMapper = new TypedInstanceToGraphMapper(graphToInstanceMapper);
return instanceToGraphMapper.mapTypedInstanceToGraph(TypedInstanceToGraphMapper.Operation.CREATE, entities);
} catch (EntityExistsException e) {
throw e;
} catch (AtlasException e) {
......@@ -161,10 +129,10 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
@Override
@GraphTransaction
public ITypedReferenceableInstance getEntityDefinition(String guid) throws RepositoryException {
public ITypedReferenceableInstance getEntityDefinition(String guid) throws RepositoryException, EntityNotFoundException {
LOG.info("Retrieving entity with guid={}", guid);
Vertex instanceVertex = getVertexForGUID(guid);
Vertex instanceVertex = graphHelper.getVertexForGUID(guid);
try {
return graphToInstanceMapper.mapGraphToTypedInstance(guid, instanceVertex);
......@@ -175,35 +143,17 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
@Override
@GraphTransaction
public ITypedReferenceableInstance getEntityDefinition(String entityType, String attribute, String value)
public ITypedReferenceableInstance getEntityDefinition(String entityType, String attribute, Object value)
throws AtlasException {
LOG.info("Retrieving entity with type={} and {}={}", entityType, attribute, value);
IDataType type = typeSystem.getDataType(IDataType.class, entityType);
String propertyKey = getFieldNameInVertex(type, attribute);
Vertex instanceVertex = getVertexForProperty(propertyKey, value);
Vertex instanceVertex = graphHelper.getVertexForProperty(propertyKey, value);
String guid = instanceVertex.getProperty(Constants.GUID_PROPERTY_KEY);
return graphToInstanceMapper.mapGraphToTypedInstance(guid, instanceVertex);
}
private Vertex getVertexForGUID(String guid) throws EntityNotFoundException {
return getVertexForProperty(Constants.GUID_PROPERTY_KEY, guid);
}
private Vertex getVertexForProperty(String propertyKey, Object value) throws EntityNotFoundException {
Vertex instanceVertex = GraphHelper.findVertex(titanGraph, propertyKey, value);
if (instanceVertex == null) {
LOG.debug("Could not find a vertex with {}={}", propertyKey, value);
throw new EntityNotFoundException("Could not find an entity in the repository with " + propertyKey + "="
+ value);
} else {
LOG.debug("Found a vertex {} with {}={}", instanceVertex, propertyKey, value);
}
return instanceVertex;
}
@Override
@GraphTransaction
public List<String> getEntityList(String entityType) throws RepositoryException {
......@@ -234,18 +184,10 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
@GraphTransaction
public List<String> getTraitNames(String guid) throws AtlasException {
LOG.info("Retrieving trait names for entity={}", guid);
Vertex instanceVertex = getVertexForGUID(guid);
return getTraitNames(instanceVertex);
Vertex instanceVertex = graphHelper.getVertexForGUID(guid);
return GraphHelper.getTraitNames(instanceVertex);
}
public List<String> getTraitNames(Vertex entityVertex) {
ArrayList<String> traits = new ArrayList<>();
for (TitanProperty property : ((TitanVertex) entityVertex).getProperties(Constants.TRAIT_NAMES_PROPERTY_KEY)) {
traits.add((String) property.getValue());
}
return traits;
}
/**
* Adds a new trait to an existing entity represented by a guid.
......@@ -262,14 +204,15 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
LOG.info("Adding a new trait={} for entity={}", traitName, guid);
try {
Vertex instanceVertex = getVertexForGUID(guid);
Vertex instanceVertex = graphHelper.getVertexForGUID(guid);
// add the trait instance as a new vertex
final String typeName = getTypeName(instanceVertex);
final String typeName = GraphHelper.getTypeName(instanceVertex);
TypedInstanceToGraphMapper instanceToGraphMapper = new TypedInstanceToGraphMapper(graphToInstanceMapper);
instanceToGraphMapper.mapTraitInstanceToVertex(traitInstance,
typeSystem.getDataType(ClassType.class, typeName), instanceVertex);
instanceToGraphMapper
.mapTraitInstanceToVertex(traitInstance, getIdFromVertex(typeName, instanceVertex), typeSystem.getDataType(ClassType.class, typeName),
instanceVertex, Collections.<Id, Vertex>emptyMap());
// update the traits in entity once adding trait instance is successful
GraphHelper.addProperty(instanceVertex, Constants.TRAIT_NAMES_PROPERTY_KEY, traitName);
......@@ -290,19 +233,19 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
*/
@Override
@GraphTransaction
public void deleteTrait(String guid, String traitNameToBeDeleted) throws RepositoryException {
public void deleteTrait(String guid, String traitNameToBeDeleted) throws EntityNotFoundException, RepositoryException {
LOG.info("Deleting trait={} from entity={}", traitNameToBeDeleted, guid);
try {
Vertex instanceVertex = getVertexForGUID(guid);
Vertex instanceVertex = graphHelper.getVertexForGUID(guid);
List<String> traitNames = getTraitNames(instanceVertex);
List<String> traitNames = GraphHelper.getTraitNames(instanceVertex);
if (!traitNames.contains(traitNameToBeDeleted)) {
throw new EntityNotFoundException(
"Could not find trait=" + traitNameToBeDeleted + " in the repository for entity: " + guid);
}
final String entityTypeName = getTypeName(instanceVertex);
String relationshipLabel = getTraitLabel(entityTypeName, traitNameToBeDeleted);
final String entityTypeName = GraphHelper.getTypeName(instanceVertex);
String relationshipLabel = GraphHelper.getTraitLabel(entityTypeName, traitNameToBeDeleted);
Iterator<Edge> results = instanceVertex.getEdges(Direction.OUT, relationshipLabel).iterator();
if (results.hasNext()) { // there should only be one edge for this label
final Edge traitEdge = results.next();
......@@ -319,8 +262,6 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
updateTraits(instanceVertex, traitNames);
}
}
} catch (RepositoryException e) {
throw e;
} catch (Exception e) {
throw new RepositoryException(e);
}
......@@ -338,961 +279,26 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
@Override
@GraphTransaction
public void updateEntity(String guid, String property, String value) throws RepositoryException {
LOG.info("Adding property {} for entity guid {}", property, guid);
public String[] updateEntities(ITypedReferenceableInstance... entitiesUpdated) throws RepositoryException {
LOG.info("updating entity {}", entitiesUpdated);
try {
Vertex instanceVertex = getVertexForGUID(guid);
LOG.debug("Found a vertex {} for guid {}", instanceVertex, guid);
String typeName = instanceVertex.getProperty(Constants.ENTITY_TYPE_PROPERTY_KEY);
ClassType type = typeSystem.getDataType(ClassType.class, typeName);
AttributeInfo attributeInfo = type.fieldMapping.fields.get(property);
if (attributeInfo == null) {
throw new AtlasException("Invalid property " + property + " for entity " + typeName);
}
DataTypes.TypeCategory attrTypeCategory = attributeInfo.dataType().getTypeCategory();
ITypedReferenceableInstance instance = type.createInstance();
if (attrTypeCategory == DataTypes.TypeCategory.PRIMITIVE) {
instance.set(property, value);
} else if (attrTypeCategory == DataTypes.TypeCategory.CLASS) {
// Disconnect any existing reference to the previous reference target.
disconnectClassReference(instanceVertex, attributeInfo, instance);
Id id = new Id(value, 0, attributeInfo.dataType().getName());
instance.set(property, id);
} else {
throw new RepositoryException("Update of " + attrTypeCategory + " is not supported");
}
instanceToGraphMapper
.mapAttributesToVertex(getIdFromVertex(typeName, instanceVertex), instance, instanceVertex,
new HashMap<Id, Vertex>(), attributeInfo, attributeInfo.dataType());
} catch (RepositoryException e) {
throw e;
} catch (Exception e) {
TypedInstanceToGraphMapper instanceToGraphMapper = new TypedInstanceToGraphMapper(graphToInstanceMapper);
return instanceToGraphMapper.mapTypedInstanceToGraph(TypedInstanceToGraphMapper.Operation.UPDATE_FULL,
entitiesUpdated);
} catch (AtlasException e) {
throw new RepositoryException(e);
}
}
private void disconnectClassReference(Vertex instanceVertex, AttributeInfo attributeInfo,
ITypedReferenceableInstance instance) throws AtlasException {
String edgeLabel = getEdgeLabel(instance, attributeInfo);
Iterable<Edge> edges = instanceVertex.getEdges(Direction.OUT, edgeLabel);
if (edges != null) {
Iterator<Edge> it = edges.iterator();
if (it.hasNext()) {
titanGraph.removeEdge(it.next());
}
}
}
public Id getIdFromVertex(String dataTypeName, Vertex vertex) {
return new Id(vertex.<String>getProperty(Constants.GUID_PROPERTY_KEY),
vertex.<Integer>getProperty(Constants.VERSION_PROPERTY_KEY), dataTypeName);
}
String getTypeName(Vertex instanceVertex) {
return instanceVertex.getProperty(Constants.ENTITY_TYPE_PROPERTY_KEY);
}
String getQualifiedName(ITypedInstance typedInstance, AttributeInfo attributeInfo) throws AtlasException {
IDataType dataType = typeSystem.getDataType(IDataType.class, typedInstance.getTypeName());
return getQualifiedName(dataType, attributeInfo.name);
}
public static String getQualifiedName(IDataType dataType, String attributeName) throws AtlasException {
return dataType.getTypeCategory() == DataTypes.TypeCategory.STRUCT ? dataType.getName() + "." + attributeName
// else class or trait
: ((HierarchicalType) dataType).getQualifiedName(attributeName);
}
private final class EntityProcessor implements ObjectGraphWalker.NodeProcessor {
public final Map<Id, IReferenceableInstance> idToInstanceMap;
public final Map<Id, Vertex> idToVertexMap;
public EntityProcessor() {
idToInstanceMap = new LinkedHashMap<>();
idToVertexMap = new HashMap<>();
}
public void cleanUp() {
idToInstanceMap.clear();
}
@Override
public void processNode(ObjectGraphWalker.Node nd) throws AtlasException {
IReferenceableInstance ref = null;
Id id = null;
if (nd.attributeName == null) {
ref = (IReferenceableInstance) nd.instance;
id = ref.getId();
} else if (nd.aInfo.dataType().getTypeCategory() == DataTypes.TypeCategory.CLASS) {
if (nd.value != null && (nd.value instanceof Id)) {
id = (Id) nd.value;
}
}
if (id != null) {
if (id.isUnassigned()) {
if (ref != null) {
if (idToInstanceMap.containsKey(id)) { // Oops
throw new RepositoryException(
String.format("Unexpected internal error: Id %s processed again", id));
}
idToInstanceMap.put(id, ref);
}
}
}
}
private List<ITypedReferenceableInstance> createVerticesForClassType(
List<ITypedReferenceableInstance> typedInstances) throws AtlasException {
List<ITypedReferenceableInstance> instancesCreated = new ArrayList<>();
for (ITypedReferenceableInstance typedInstance : typedInstances) {
final Id id = typedInstance.getId();
if (!idToVertexMap.containsKey(id)) {
Vertex instanceVertex;
if (id.isAssigned()) { // has a GUID
instanceVertex = getVertexForGUID(id.id);
} else {
//Check if there is already an instance with the same unique attribute value
ClassType classType = typeSystem.getDataType(ClassType.class, typedInstance.getTypeName());
instanceVertex = instanceToGraphMapper.getVertexForInstanceByUniqueAttribute(classType, typedInstance);
if (instanceVertex == null) {
instanceVertex = GraphHelper.createVertexWithIdentity(titanGraph, typedInstance,
classType.getAllSuperTypeNames());
instancesCreated.add(typedInstance);
instanceToGraphMapper.mapInstanceToVertex(id, typedInstance, instanceVertex,
classType.fieldMapping().fields, idToVertexMap, true);
}
}
idToVertexMap.put(id, instanceVertex);
}
}
return instancesCreated;
}
}
private final class TypedInstanceToGraphMapper {
private String[] mapTypedInstanceToGraph(ITypedReferenceableInstance[] typedInstances)
throws AtlasException {
EntityProcessor entityProcessor = new EntityProcessor();
List<String> guids = new ArrayList<>();
for (ITypedReferenceableInstance typedInstance : typedInstances) {
try {
LOG.debug("Walking the object graph for instance {}", typedInstance.getTypeName());
entityProcessor.cleanUp();
new ObjectGraphWalker(typeSystem, entityProcessor, typedInstance).walk();
} catch (AtlasException me) {
throw new RepositoryException("TypeSystem error when walking the ObjectGraph", me);
}
List<ITypedReferenceableInstance> newTypedInstances = discoverInstances(entityProcessor);
List<ITypedReferenceableInstance> instancesCreated =
entityProcessor.createVerticesForClassType(newTypedInstances);
for (ITypedReferenceableInstance instance : instancesCreated) {
try {
//new vertex, set all the properties
addDiscoveredInstance(entityProcessor, instance);
} catch(SchemaViolationException e) {
throw new EntityExistsException(typedInstance, e);
}
}
addFullTextProperty(entityProcessor, instancesCreated);
//Return guid for
Vertex instanceVertex = entityProcessor.idToVertexMap.get(typedInstance.getId());
String guid = instanceVertex.getProperty(Constants.GUID_PROPERTY_KEY);
guids.add(guid);
}
return guids.toArray(new String[guids.size()]);
}
private void addFullTextProperty(EntityProcessor entityProcessor,
List<ITypedReferenceableInstance> newTypedInstances) throws AtlasException {
for (ITypedReferenceableInstance typedInstance : newTypedInstances) { // Traverse
Id id = typedInstance.getId();
Vertex instanceVertex = entityProcessor.idToVertexMap.get(id);
String fullText = getFullTextForVertex(instanceVertex, true);
GraphHelper.setProperty(instanceVertex, Constants.ENTITY_TEXT_PROPERTY_KEY, fullText);
}
}
private String getFullTextForVertex(Vertex instanceVertex, boolean followReferences) throws AtlasException {
String guid = instanceVertex.getProperty(Constants.GUID_PROPERTY_KEY);
ITypedReferenceableInstance typedReference =
graphToInstanceMapper.mapGraphToTypedInstance(guid, instanceVertex);
String fullText = getFullTextForInstance(typedReference, followReferences);
StringBuilder fullTextBuilder =
new StringBuilder(typedReference.getTypeName()).append(FULL_TEXT_DELIMITER).append(fullText);
List<String> traits = typedReference.getTraits();
for (String traitName : traits) {
String traitText = getFullTextForInstance((ITypedInstance) typedReference.getTrait(traitName), false);
fullTextBuilder.append(FULL_TEXT_DELIMITER).append(traitName).append(FULL_TEXT_DELIMITER)
.append(traitText);
}
return fullTextBuilder.toString();
}
private String getFullTextForAttribute(IDataType type, Object value, boolean followReferences)
throws AtlasException {
switch (type.getTypeCategory()) {
case PRIMITIVE:
return String.valueOf(value);
case ENUM:
return ((EnumValue) value).value;
case ARRAY:
StringBuilder fullText = new StringBuilder();
IDataType elemType = ((DataTypes.ArrayType) type).getElemType();
List list = (List) value;
for (Object element : list) {
String elemFullText = getFullTextForAttribute(elemType, element, false);
if (StringUtils.isNotEmpty(elemFullText)) {
fullText = fullText.append(FULL_TEXT_DELIMITER).append(elemFullText);
}
}
return fullText.toString();
case MAP:
fullText = new StringBuilder();
IDataType keyType = ((DataTypes.MapType) type).getKeyType();
IDataType valueType = ((DataTypes.MapType) type).getValueType();
Map map = (Map) value;
for (Object entryObj : map.entrySet()) {
Map.Entry entry = (Map.Entry) entryObj;
String keyFullText = getFullTextForAttribute(keyType, entry.getKey(), false);
if (StringUtils.isNotEmpty(keyFullText)) {
fullText = fullText.append(FULL_TEXT_DELIMITER).append(keyFullText);
}
String valueFullText = getFullTextForAttribute(valueType, entry.getValue(), false);
if (StringUtils.isNotEmpty(valueFullText)) {
fullText = fullText.append(FULL_TEXT_DELIMITER).append(valueFullText);
}
}
return fullText.toString();
case CLASS:
if (followReferences) {
String refGuid = ((ITypedReferenceableInstance) value).getId()._getId();
Vertex refVertex = getVertexForGUID(refGuid);
return getFullTextForVertex(refVertex, false);
}
break;
case STRUCT:
if (followReferences) {
return getFullTextForInstance((ITypedInstance) value, false);
}
break;
default:
throw new IllegalStateException("Unhandled type category " + type.getTypeCategory());
}
return null;
}
private String getFullTextForInstance(ITypedInstance typedInstance, boolean followReferences)
throws AtlasException {
StringBuilder fullText = new StringBuilder();
for (AttributeInfo attributeInfo : typedInstance.fieldMapping().fields.values()) {
Object attrValue = typedInstance.get(attributeInfo.name);
if (attrValue == null) {
continue;
}
String attrFullText = getFullTextForAttribute(attributeInfo.dataType(), attrValue, followReferences);
if (StringUtils.isNotEmpty(attrFullText)) {
fullText =
fullText.append(FULL_TEXT_DELIMITER).append(attributeInfo.name).append(FULL_TEXT_DELIMITER)
.append(attrFullText);
}
}
return fullText.toString();
}
/**
* Step 2: Traverse oldIdToInstance map create newInstances :
* List[ITypedReferenceableInstance]
* - create a ITypedReferenceableInstance.
* replace any old References ( ids or object references) with new Ids.
*/
private List<ITypedReferenceableInstance> discoverInstances(EntityProcessor entityProcessor)
throws RepositoryException {
List<ITypedReferenceableInstance> newTypedInstances = new ArrayList<>();
for (IReferenceableInstance transientInstance : entityProcessor.idToInstanceMap.values()) {
LOG.debug("Discovered instance {}", transientInstance.getTypeName());
try {
ClassType cT = typeSystem.getDataType(ClassType.class, transientInstance.getTypeName());
ITypedReferenceableInstance newInstance = cT.convert(transientInstance, Multiplicity.REQUIRED);
newTypedInstances.add(newInstance);
} catch (AtlasException me) {
throw new RepositoryException(
String.format("Failed to create Instance(id = %s", transientInstance.getId()), me);
}
}
//Reverse the list to create the entities in dependency order
return Lists.reverse(newTypedInstances);
}
/**
* For the given type, finds an unique attribute and checks if there is an existing instance with the same
* unique value
* @param classType
* @param instance
* @return
* @throws AtlasException
*/
Vertex getVertexForInstanceByUniqueAttribute(ClassType classType, IReferenceableInstance instance)
throws AtlasException {
for (AttributeInfo attributeInfo : classType.fieldMapping().fields.values()) {
if (attributeInfo.isUnique) {
String propertyKey = getFieldNameInVertex(classType, attributeInfo);
try {
return getVertexForProperty(propertyKey, instance.get(attributeInfo.name));
} catch(EntityNotFoundException e) {
//Its ok if there is no entity with the same unique value
}
}
}
return null;
}
private void addDiscoveredInstance(EntityProcessor entityProcessor, ITypedReferenceableInstance typedInstance)
throws AtlasException {
LOG.debug("Adding typed instance {}", typedInstance.getTypeName());
Id id = typedInstance.getId();
if (id == null) { // oops
throw new RepositoryException("id cannot be null");
}
Vertex instanceVertex = entityProcessor.idToVertexMap.get(id);
// add the attributes for the instance
ClassType classType = typeSystem.getDataType(ClassType.class, typedInstance.getTypeName());
final Map<String, AttributeInfo> fields = classType.fieldMapping().fields;
mapInstanceToVertex(id, typedInstance, instanceVertex, fields, entityProcessor.idToVertexMap, false);
for (String traitName : typedInstance.getTraits()) {
LOG.debug("mapping trait {}", traitName);
GraphHelper.addProperty(instanceVertex, Constants.TRAIT_NAMES_PROPERTY_KEY, traitName);
ITypedStruct traitInstance = (ITypedStruct) typedInstance.getTrait(traitName);
// add the attributes for the trait instance
mapTraitInstanceToVertex(traitInstance, typedInstance.getId(), classType, instanceVertex,
entityProcessor.idToVertexMap);
}
}
private void mapInstanceToVertex(Id id, ITypedInstance typedInstance, Vertex instanceVertex,
Map<String, AttributeInfo> fields, Map<Id, Vertex> idToVertexMap, boolean mapOnlyUniqueAttributes)
throws AtlasException {
LOG.debug("Mapping instance {} of {} to vertex {}", typedInstance, typedInstance.getTypeName(),
instanceVertex);
for (AttributeInfo attributeInfo : fields.values()) {
if (mapOnlyUniqueAttributes && !attributeInfo.isUnique) {
continue;
}
final IDataType dataType = attributeInfo.dataType();
mapAttributesToVertex(id, typedInstance, instanceVertex, idToVertexMap, attributeInfo, dataType);
}
}
private void mapAttributesToVertex(Id id, ITypedInstance typedInstance, Vertex instanceVertex,
Map<Id, Vertex> idToVertexMap, AttributeInfo attributeInfo, IDataType dataType) throws AtlasException {
Object attrValue = typedInstance.get(attributeInfo.name);
LOG.debug("mapping attribute {} = {}", attributeInfo.name, attrValue);
final String propertyName = getQualifiedName(typedInstance, attributeInfo);
String edgeLabel = getEdgeLabel(typedInstance, attributeInfo);
if (attrValue == null) {
return;
}
switch (dataType.getTypeCategory()) {
case PRIMITIVE:
mapPrimitiveToVertex(typedInstance, instanceVertex, attributeInfo);
break;
case ENUM:
//handles both int and string for enum
EnumValue enumValue =
(EnumValue) dataType.convert(typedInstance.get(attributeInfo.name), Multiplicity.REQUIRED);
GraphHelper.setProperty(instanceVertex, propertyName, enumValue.value);
break;
case ARRAY:
mapArrayCollectionToVertex(id, typedInstance, instanceVertex, attributeInfo, idToVertexMap);
break;
case MAP:
mapMapCollectionToVertex(id, typedInstance, instanceVertex, attributeInfo, idToVertexMap);
break;
case STRUCT:
Vertex structInstanceVertex =
mapStructInstanceToVertex(id, (ITypedStruct) typedInstance.get(attributeInfo.name),
attributeInfo, idToVertexMap);
// add an edge to the newly created vertex from the parent
GraphHelper.addEdge(titanGraph, instanceVertex, structInstanceVertex, edgeLabel);
break;
case TRAIT:
// do NOTHING - this is taken care of earlier
break;
case CLASS:
mapClassReferenceAsEdge(instanceVertex, idToVertexMap, edgeLabel, (ITypedReferenceableInstance) attrValue);
break;
default:
throw new IllegalArgumentException("Unknown type category: " + dataType.getTypeCategory());
}
}
private void mapArrayCollectionToVertex(Id id, ITypedInstance typedInstance, Vertex instanceVertex,
AttributeInfo attributeInfo, Map<Id, Vertex> idToVertexMap) throws AtlasException {
LOG.debug("Mapping instance {} to vertex {} for name {}", typedInstance.getTypeName(), instanceVertex,
attributeInfo.name);
List list = (List) typedInstance.get(attributeInfo.name);
if (list == null || list.isEmpty()) {
return;
}
String propertyName = getQualifiedName(typedInstance, attributeInfo);
IDataType elementType = ((DataTypes.ArrayType) attributeInfo.dataType()).getElemType();
List<String> values = new ArrayList<>(list.size());
for (int index = 0; index < list.size(); index++) {
String entryId =
mapCollectionEntryToVertex(id, instanceVertex, attributeInfo, idToVertexMap, elementType,
list.get(index), propertyName);
values.add(entryId);
}
// for dereference on way out
GraphHelper.setProperty(instanceVertex, propertyName, values);
}
private void mapMapCollectionToVertex(Id id, ITypedInstance typedInstance, Vertex instanceVertex,
AttributeInfo attributeInfo, Map<Id, Vertex> idToVertexMap) throws AtlasException {
LOG.debug("Mapping instance {} to vertex {} for name {}", typedInstance.getTypeName(), instanceVertex,
attributeInfo.name);
@SuppressWarnings("unchecked") Map<Object, Object> collection =
(Map<Object, Object>) typedInstance.get(attributeInfo.name);
if (collection == null || collection.isEmpty()) {
return;
}
String propertyName = getQualifiedName(typedInstance, attributeInfo);
IDataType elementType = ((DataTypes.MapType) attributeInfo.dataType()).getValueType();
for (Map.Entry entry : collection.entrySet()) {
String myPropertyName = propertyName + "." + entry.getKey().toString();
String value = mapCollectionEntryToVertex(id, instanceVertex, attributeInfo, idToVertexMap, elementType,
entry.getValue(), myPropertyName);
GraphHelper.setProperty(instanceVertex, myPropertyName, value);
}
// for dereference on way out
GraphHelper.setProperty(instanceVertex, propertyName, new ArrayList(collection.keySet()));
}
private String mapCollectionEntryToVertex(Id id, Vertex instanceVertex, AttributeInfo attributeInfo,
Map<Id, Vertex> idToVertexMap, IDataType elementType, Object value, String propertyName)
throws AtlasException {
final String edgeLabel = EDGE_LABEL_PREFIX + propertyName;
switch (elementType.getTypeCategory()) {
case PRIMITIVE:
case ENUM:
return value.toString();
case ARRAY:
case MAP:
case TRAIT:
// do nothing
return null;
case STRUCT:
Vertex structInstanceVertex =
mapStructInstanceToVertex(id, (ITypedStruct) value, attributeInfo, idToVertexMap);
// add an edge to the newly created vertex from the parent
Edge structElementEdge =
GraphHelper.addEdge(titanGraph, instanceVertex, structInstanceVertex, edgeLabel);
return structElementEdge.getId().toString();
case CLASS:
return mapClassReferenceAsEdge(instanceVertex, idToVertexMap, edgeLabel,
(ITypedReferenceableInstance) value);
default:
throw new IllegalArgumentException("Unknown type category: " + elementType.getTypeCategory());
}
}
private String mapClassReferenceAsEdge(Vertex instanceVertex, Map<Id, Vertex> idToVertexMap, String propertyKey,
ITypedReferenceableInstance typedReference) throws AtlasException {
if (typedReference != null) {
Vertex referenceVertex;
Id id = typedReference instanceof Id ? (Id) typedReference : typedReference.getId();
if (id.isAssigned()) {
referenceVertex = getVertexForGUID(id.id);
} else {
referenceVertex = idToVertexMap.get(id);
}
if (referenceVertex != null) {
// Add an edge to the class vertex from the instance.
Edge edge = GraphHelper.addEdge(titanGraph, instanceVertex, referenceVertex, propertyKey);
return String.valueOf(edge.getId());
}
}
return null;
}
private Vertex mapStructInstanceToVertex(Id id, ITypedStruct structInstance, AttributeInfo attributeInfo,
Map<Id, Vertex> idToVertexMap) throws AtlasException {
// add a new vertex for the struct or trait instance
Vertex structInstanceVertex = GraphHelper
.createVertexWithoutIdentity(titanGraph, structInstance.getTypeName(), id,
Collections.<String>emptySet()); // no super types for struct type
LOG.debug("created vertex {} for struct {} value {}", structInstanceVertex, attributeInfo.name,
structInstance);
// map all the attributes to this newly created vertex
mapInstanceToVertex(id, structInstance, structInstanceVertex, structInstance.fieldMapping().fields,
idToVertexMap, false);
return structInstanceVertex;
}
private void mapTraitInstanceToVertex(ITypedStruct traitInstance, Id typedInstanceId,
IDataType entityType, Vertex parentInstanceVertex, Map<Id, Vertex> idToVertexMap)
throws AtlasException {
// add a new vertex for the struct or trait instance
final String traitName = traitInstance.getTypeName();
Vertex traitInstanceVertex = GraphHelper
.createVertexWithoutIdentity(titanGraph, traitInstance.getTypeName(), typedInstanceId,
typeSystem.getDataType(TraitType.class, traitName).getAllSuperTypeNames());
LOG.debug("created vertex {} for trait {}", traitInstanceVertex, traitName);
// map all the attributes to this newly created vertex
mapInstanceToVertex(typedInstanceId, traitInstance, traitInstanceVertex,
traitInstance.fieldMapping().fields, idToVertexMap, false);
// add an edge to the newly created vertex from the parent
String relationshipLabel = getTraitLabel(entityType, traitName);
GraphHelper.addEdge(titanGraph, parentInstanceVertex, traitInstanceVertex, relationshipLabel);
}
private void mapPrimitiveToVertex(ITypedInstance typedInstance, Vertex instanceVertex,
AttributeInfo attributeInfo) throws AtlasException {
Object attrValue = typedInstance.get(attributeInfo.name);
if (attrValue == null) {
return; // add only if instance has this attribute
}
final String vertexPropertyName = getQualifiedName(typedInstance, attributeInfo);
Object propertyValue = null;
if (attributeInfo.dataType() == DataTypes.STRING_TYPE) {
propertyValue = typedInstance.getString(attributeInfo.name);
} else if (attributeInfo.dataType() == DataTypes.SHORT_TYPE) {
propertyValue = typedInstance.getShort(attributeInfo.name);
} else if (attributeInfo.dataType() == DataTypes.INT_TYPE) {
propertyValue = typedInstance.getInt(attributeInfo.name);
} else if (attributeInfo.dataType() == DataTypes.BIGINTEGER_TYPE) {
propertyValue = typedInstance.getBigInt(attributeInfo.name);
} else if (attributeInfo.dataType() == DataTypes.BOOLEAN_TYPE) {
propertyValue = typedInstance.getBoolean(attributeInfo.name);
} else if (attributeInfo.dataType() == DataTypes.BYTE_TYPE) {
propertyValue = typedInstance.getByte(attributeInfo.name);
} else if (attributeInfo.dataType() == DataTypes.LONG_TYPE) {
propertyValue = typedInstance.getLong(attributeInfo.name);
} else if (attributeInfo.dataType() == DataTypes.FLOAT_TYPE) {
propertyValue = typedInstance.getFloat(attributeInfo.name);
} else if (attributeInfo.dataType() == DataTypes.DOUBLE_TYPE) {
propertyValue = typedInstance.getDouble(attributeInfo.name);
} else if (attributeInfo.dataType() == DataTypes.BIGDECIMAL_TYPE) {
propertyValue = typedInstance.getBigDecimal(attributeInfo.name);
} else if (attributeInfo.dataType() == DataTypes.DATE_TYPE) {
final Date dateVal = typedInstance.getDate(attributeInfo.name);
//Convert Property value to Long while persisting
propertyValue = dateVal.getTime();
}
GraphHelper.setProperty(instanceVertex, vertexPropertyName, propertyValue);
}
}
public final class GraphToTypedInstanceMapper {
public ITypedReferenceableInstance mapGraphToTypedInstance(String guid, Vertex instanceVertex)
throws AtlasException {
LOG.debug("Mapping graph root vertex {} to typed instance for guid {}", instanceVertex, guid);
String typeName = instanceVertex.getProperty(Constants.ENTITY_TYPE_PROPERTY_KEY);
List<String> traits = getTraitNames(instanceVertex);
Id id = new Id(guid, instanceVertex.<Integer>getProperty(Constants.VERSION_PROPERTY_KEY), typeName);
LOG.debug("Created id {} for instance type {}", id, typeName);
ClassType classType = typeSystem.getDataType(ClassType.class, typeName);
ITypedReferenceableInstance typedInstance =
classType.createInstance(id, traits.toArray(new String[traits.size()]));
mapVertexToInstance(instanceVertex, typedInstance, classType.fieldMapping().fields);
mapVertexToInstanceTraits(instanceVertex, typedInstance, traits);
return typedInstance;
}
private void mapVertexToInstanceTraits(Vertex instanceVertex, ITypedReferenceableInstance typedInstance,
List<String> traits) throws AtlasException {
for (String traitName : traits) {
LOG.debug("mapping trait {} to instance", traitName);
TraitType traitType = typeSystem.getDataType(TraitType.class, traitName);
mapVertexToTraitInstance(instanceVertex, typedInstance, traitName, traitType);
}
}
public void mapVertexToInstance(Vertex instanceVertex, ITypedInstance typedInstance,
Map<String, AttributeInfo> fields) throws AtlasException {
LOG.debug("Mapping vertex {} to instance {} for fields", instanceVertex, typedInstance.getTypeName(),
fields);
for (AttributeInfo attributeInfo : fields.values()) {
mapVertexToAttribute(instanceVertex, typedInstance, attributeInfo);
}
}
private void mapVertexToAttribute(Vertex instanceVertex, ITypedInstance typedInstance,
AttributeInfo attributeInfo) throws AtlasException {
LOG.debug("Mapping attributeInfo {}", attributeInfo.name);
final IDataType dataType = attributeInfo.dataType();
final String vertexPropertyName = getQualifiedName(typedInstance, attributeInfo);
switch (dataType.getTypeCategory()) {
case PRIMITIVE:
mapVertexToPrimitive(instanceVertex, typedInstance, attributeInfo);
break; // add only if vertex has this attribute
case ENUM:
if (instanceVertex.getProperty(vertexPropertyName) == null) {
return;
}
typedInstance.set(attributeInfo.name,
dataType.convert(instanceVertex.<String>getProperty(vertexPropertyName),
Multiplicity.REQUIRED));
break;
case ARRAY:
mapVertexToArrayInstance(instanceVertex, typedInstance, attributeInfo, vertexPropertyName);
break;
case MAP:
mapVertexToMapInstance(instanceVertex, typedInstance, attributeInfo, vertexPropertyName);
break;
case STRUCT:
mapVertexToStructInstance(instanceVertex, typedInstance, attributeInfo);
break;
case TRAIT:
// do NOTHING - handled in class
break;
case CLASS:
String relationshipLabel = getEdgeLabel(typedInstance, attributeInfo);
Object idOrInstance = mapVertexToClassReference(instanceVertex, attributeInfo, relationshipLabel,
attributeInfo.dataType());
typedInstance.set(attributeInfo.name, idOrInstance);
break;
default:
break;
}
}
private Object mapVertexToClassReference(Vertex instanceVertex, AttributeInfo attributeInfo,
String relationshipLabel, IDataType dataType) throws AtlasException {
LOG.debug("Finding edge for {} -> label {} ", instanceVertex, relationshipLabel);
Iterator<Edge> results = instanceVertex.getEdges(Direction.OUT, relationshipLabel).iterator();
if (results.hasNext()) {
final Vertex referenceVertex = results.next().getVertex(Direction.IN);
if (referenceVertex != null) {
final String guid = referenceVertex.getProperty(Constants.GUID_PROPERTY_KEY);
LOG.debug("Found vertex {} for label {} with guid {}", referenceVertex, relationshipLabel, guid);
if (attributeInfo.isComposite) {
//Also, when you retrieve a type's instance, you get the complete object graph of the composites
LOG.debug("Found composite, mapping vertex to instance");
return mapGraphToTypedInstance(guid, referenceVertex);
} else {
Id referenceId =
new Id(guid, referenceVertex.<Integer>getProperty(Constants.VERSION_PROPERTY_KEY),
dataType.getName());
LOG.debug("Found non-composite, adding id {} ", referenceId);
return referenceId;
}
}
}
return null;
}
@SuppressWarnings("unchecked")
private void mapVertexToArrayInstance(Vertex instanceVertex, ITypedInstance typedInstance,
AttributeInfo attributeInfo, String propertyName) throws AtlasException {
LOG.debug("mapping vertex {} to array {}", instanceVertex, attributeInfo.name);
List list = instanceVertex.getProperty(propertyName);
if (list == null || list.size() == 0) {
return;
}
DataTypes.ArrayType arrayType = (DataTypes.ArrayType) attributeInfo.dataType();
final IDataType elementType = arrayType.getElemType();
ArrayList values = new ArrayList();
for (Object listElement : list) {
values.add(mapVertexToCollectionEntry(instanceVertex, attributeInfo, elementType, listElement,
propertyName));
}
typedInstance.set(attributeInfo.name, values);
}
private Object mapVertexToCollectionEntry(Vertex instanceVertex, AttributeInfo attributeInfo,
IDataType elementType, Object value, String propertyName) throws AtlasException {
String edgeLabel = EDGE_LABEL_PREFIX + propertyName;
switch (elementType.getTypeCategory()) {
case PRIMITIVE:
case ENUM:
return value;
case ARRAY:
case MAP:
case TRAIT:
// do nothing
break;
case STRUCT:
return getStructInstanceFromVertex(instanceVertex, elementType, attributeInfo.name, edgeLabel,
(String) value);
case CLASS:
return mapVertexToClassReference(instanceVertex, attributeInfo, edgeLabel, elementType, (String) value);
default:
break;
}
throw new IllegalArgumentException();
}
@SuppressWarnings("unchecked")
private void mapVertexToMapInstance(Vertex instanceVertex, ITypedInstance typedInstance,
AttributeInfo attributeInfo, final String propertyName) throws AtlasException {
LOG.debug("mapping vertex {} to array {}", instanceVertex, attributeInfo.name);
List<String> keys = instanceVertex.getProperty(propertyName);
if (keys == null || keys.size() == 0) {
return;
}
DataTypes.MapType mapType = (DataTypes.MapType) attributeInfo.dataType();
final IDataType valueType = mapType.getValueType();
HashMap values = new HashMap();
for (String key : keys) {
String keyPropertyName = propertyName + "." + key;
Object keyValue = instanceVertex.getProperty(keyPropertyName);
values.put(key,
mapVertexToCollectionEntry(instanceVertex, attributeInfo, valueType, keyValue, propertyName));
}
typedInstance.set(attributeInfo.name, values);
}
private ITypedStruct getStructInstanceFromVertex(Vertex instanceVertex, IDataType elemType,
String attributeName, String relationshipLabel, String edgeId) throws AtlasException {
LOG.debug("Finding edge for {} -> label {} ", instanceVertex, relationshipLabel);
for (Edge edge : instanceVertex.getEdges(Direction.OUT, relationshipLabel)) {
if (edgeId.equals(String.valueOf(edge.getId()))) {
Vertex structInstanceVertex = edge.getVertex(Direction.IN);
LOG.debug("mapping vertex {} to struct {}", structInstanceVertex, attributeName);
if (structInstanceVertex != null) {
LOG.debug("Found struct instance vertex {}, mapping to instance {} ", structInstanceVertex,
elemType.getName());
StructType structType = typeSystem.getDataType(StructType.class, elemType.getName());
ITypedStruct structInstance = structType.createInstance();
mapVertexToInstance(structInstanceVertex, structInstance, structType.fieldMapping().fields);
return structInstance;
}
break;
}
}
return null;
}
private Object mapVertexToClassReference(Vertex instanceVertex, AttributeInfo attributeInfo,
String relationshipLabel, IDataType dataType, String edgeId) throws AtlasException {
LOG.debug("Finding edge for {} -> label {} ", instanceVertex, relationshipLabel);
for (Edge edge : instanceVertex.getEdges(Direction.OUT, relationshipLabel)) {
if (edgeId.equals(String.valueOf(edge.getId()))) {
final Vertex referenceVertex = edge.getVertex(Direction.IN);
if (referenceVertex != null) {
final String guid = referenceVertex.getProperty(Constants.GUID_PROPERTY_KEY);
LOG.debug("Found vertex {} for label {} with guid {}", referenceVertex, relationshipLabel,
guid);
if (attributeInfo.isComposite) {
//Also, when you retrieve a type's instance, you get the complete object graph of the composites
LOG.debug("Found composite, mapping vertex to instance");
return mapGraphToTypedInstance(guid, referenceVertex);
} else {
Id referenceId =
new Id(guid, referenceVertex.<Integer>getProperty(Constants.VERSION_PROPERTY_KEY),
dataType.getName());
LOG.debug("Found non-composite, adding id {} ", referenceId);
return referenceId;
}
}
break;
}
}
return null;
}
private void mapVertexToStructInstance(Vertex instanceVertex, ITypedInstance typedInstance,
AttributeInfo attributeInfo) throws AtlasException {
LOG.debug("mapping vertex {} to struct {}", instanceVertex, attributeInfo.name);
StructType structType = typeSystem.getDataType(StructType.class, attributeInfo.dataType().getName());
ITypedStruct structInstance = structType.createInstance();
typedInstance.set(attributeInfo.name, structInstance);
String relationshipLabel = getEdgeLabel(typedInstance, attributeInfo);
LOG.debug("Finding edge for {} -> label {} ", instanceVertex, relationshipLabel);
for (Edge edge : instanceVertex.getEdges(Direction.OUT, relationshipLabel)) {
final Vertex structInstanceVertex = edge.getVertex(Direction.IN);
if (structInstanceVertex != null) {
LOG.debug("Found struct instance vertex {}, mapping to instance {} ", structInstanceVertex,
structInstance.getTypeName());
mapVertexToInstance(structInstanceVertex, structInstance, structType.fieldMapping().fields);
break;
}
}
}
private void mapVertexToTraitInstance(Vertex instanceVertex, ITypedReferenceableInstance typedInstance,
String traitName, TraitType traitType) throws AtlasException {
ITypedStruct traitInstance = (ITypedStruct) typedInstance.getTrait(traitName);
mapVertexToTraitInstance(instanceVertex, typedInstance.getTypeName(), traitName, traitType, traitInstance);
}
private void mapVertexToTraitInstance(Vertex instanceVertex, String typedInstanceTypeName, String traitName,
TraitType traitType, ITypedStruct traitInstance) throws AtlasException {
String relationshipLabel = getTraitLabel(typedInstanceTypeName, traitName);
LOG.debug("Finding edge for {} -> label {} ", instanceVertex, relationshipLabel);
for (Edge edge : instanceVertex.getEdges(Direction.OUT, relationshipLabel)) {
final Vertex traitInstanceVertex = edge.getVertex(Direction.IN);
if (traitInstanceVertex != null) {
LOG.debug("Found trait instance vertex {}, mapping to instance {} ", traitInstanceVertex,
traitInstance.getTypeName());
mapVertexToInstance(traitInstanceVertex, traitInstance, traitType.fieldMapping().fields);
break;
}
}
}
private void mapVertexToPrimitive(Vertex instanceVertex, ITypedInstance typedInstance,
AttributeInfo attributeInfo) throws AtlasException {
LOG.debug("Adding primitive {} from vertex {}", attributeInfo, instanceVertex);
final String vertexPropertyName = getQualifiedName(typedInstance, attributeInfo);
if (instanceVertex.getProperty(vertexPropertyName) == null) {
return;
}
if (attributeInfo.dataType() == DataTypes.STRING_TYPE) {
typedInstance.setString(attributeInfo.name, instanceVertex.<String>getProperty(vertexPropertyName));
} else if (attributeInfo.dataType() == DataTypes.SHORT_TYPE) {
typedInstance.setShort(attributeInfo.name, instanceVertex.<Short>getProperty(vertexPropertyName));
} else if (attributeInfo.dataType() == DataTypes.INT_TYPE) {
typedInstance.setInt(attributeInfo.name, instanceVertex.<Integer>getProperty(vertexPropertyName));
} else if (attributeInfo.dataType() == DataTypes.BIGINTEGER_TYPE) {
typedInstance.setBigInt(attributeInfo.name, instanceVertex.<BigInteger>getProperty(vertexPropertyName));
} else if (attributeInfo.dataType() == DataTypes.BOOLEAN_TYPE) {
typedInstance.setBoolean(attributeInfo.name, instanceVertex.<Boolean>getProperty(vertexPropertyName));
} else if (attributeInfo.dataType() == DataTypes.BYTE_TYPE) {
typedInstance.setByte(attributeInfo.name, instanceVertex.<Byte>getProperty(vertexPropertyName));
} else if (attributeInfo.dataType() == DataTypes.LONG_TYPE) {
typedInstance.setLong(attributeInfo.name, instanceVertex.<Long>getProperty(vertexPropertyName));
} else if (attributeInfo.dataType() == DataTypes.FLOAT_TYPE) {
typedInstance.setFloat(attributeInfo.name, instanceVertex.<Float>getProperty(vertexPropertyName));
} else if (attributeInfo.dataType() == DataTypes.DOUBLE_TYPE) {
typedInstance.setDouble(attributeInfo.name, instanceVertex.<Double>getProperty(vertexPropertyName));
} else if (attributeInfo.dataType() == DataTypes.BIGDECIMAL_TYPE) {
typedInstance
.setBigDecimal(attributeInfo.name, instanceVertex.<BigDecimal>getProperty(vertexPropertyName));
} else if (attributeInfo.dataType() == DataTypes.DATE_TYPE) {
final Long dateVal = instanceVertex.<Long>getProperty(vertexPropertyName);
typedInstance.setDate(attributeInfo.name, new Date(dateVal));
}
}
public ITypedInstance getReferredEntity(String edgeId, IDataType<?> referredType) throws AtlasException {
final Edge edge = titanGraph.getEdge(edgeId);
if(edge != null) {
final Vertex referredVertex = edge.getVertex(Direction.IN);
if (referredVertex != null) {
switch (referredType.getTypeCategory()) {
case STRUCT:
LOG.debug("Found struct instance vertex {}, mapping to instance {} ", referredVertex,
referredType.getName());
StructType structType = (StructType)referredType;
ITypedStruct instance = structType.createInstance();
Map<String, AttributeInfo> fields = structType.fieldMapping().fields;
mapVertexToInstance(referredVertex, instance, fields);
return instance;
case CLASS:
//TODO isComposite handling for class loads
final String guid = referredVertex.getProperty(Constants.GUID_PROPERTY_KEY);
Id referenceId =
new Id(guid, referredVertex.<Integer>getProperty(Constants.VERSION_PROPERTY_KEY),
referredType.getName());
return referenceId;
default:
throw new UnsupportedOperationException("Loading " + referredType.getTypeCategory() + " is not supported");
}
}
}
return null;
@Override
@GraphTransaction
public void updatePartial(ITypedReferenceableInstance entity) throws RepositoryException {
LOG.info("updating entity {}", entity);
try {
TypedInstanceToGraphMapper instanceToGraphMapper = new TypedInstanceToGraphMapper(graphToInstanceMapper);
instanceToGraphMapper.mapTypedInstanceToGraph(TypedInstanceToGraphMapper.Operation.UPDATE_PARTIAL, entity);
} catch (AtlasException e) {
throw new RepositoryException(e);
}
}
}
repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java
......@@ -19,19 +19,33 @@
package org.apache.atlas.repository.graph;
import com.thinkaurelius.titan.core.TitanGraph;
import com.thinkaurelius.titan.core.TitanProperty;
import com.thinkaurelius.titan.core.TitanVertex;
import com.tinkerpop.blueprints.Direction;
import com.tinkerpop.blueprints.Edge;
import com.tinkerpop.blueprints.Graph;
import com.tinkerpop.blueprints.GraphQuery;
import com.tinkerpop.blueprints.Vertex;
import org.apache.atlas.AtlasException;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.typesystem.IReferenceableInstance;
import org.apache.atlas.typesystem.ITypedInstance;
import org.apache.atlas.typesystem.ITypedReferenceableInstance;
import org.apache.atlas.typesystem.exception.EntityNotFoundException;
import org.apache.atlas.typesystem.persistence.Id;
import org.apache.atlas.typesystem.types.AttributeInfo;
import org.apache.atlas.typesystem.types.ClassType;
import org.apache.atlas.typesystem.types.DataTypes;
import org.apache.atlas.typesystem.types.HierarchicalType;
import org.apache.atlas.typesystem.types.IDataType;
import org.apache.atlas.typesystem.types.TypeSystem;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
import java.util.UUID;
......@@ -41,26 +55,41 @@ import java.util.UUID;
public final class GraphHelper {
private static final Logger LOG = LoggerFactory.getLogger(GraphHelper.class);
public static final String EDGE_LABEL_PREFIX = "__";
private GraphHelper() {
private static final TypeSystem typeSystem = TypeSystem.getInstance();
private static final GraphHelper INSTANCE = new GraphHelper(TitanGraphProvider.getGraphInstance());
private TitanGraph titanGraph;
private GraphHelper(TitanGraph titanGraph) {
this.titanGraph = titanGraph;
}
public static Vertex createVertexWithIdentity(Graph graph, ITypedReferenceableInstance typedInstance,
public static GraphHelper getInstance() {
return INSTANCE;
}
public Vertex createVertexWithIdentity(ITypedReferenceableInstance typedInstance,
Set<String> superTypeNames) {
final Vertex vertexWithIdentity = createVertexWithoutIdentity(graph, typedInstance.getTypeName(),
final Vertex vertexWithIdentity = createVertexWithoutIdentity(typedInstance.getTypeName(),
typedInstance.getId(), superTypeNames);
// add identity
final String guid = UUID.randomUUID().toString();
setProperty(vertexWithIdentity, Constants.GUID_PROPERTY_KEY, guid);
// add version information
setProperty(vertexWithIdentity, Constants.VERSION_PROPERTY_KEY, typedInstance.getId().version);
return vertexWithIdentity;
}
public static Vertex createVertexWithoutIdentity(Graph graph, String typeName, Id typedInstanceId,
Set<String> superTypeNames) {
LOG.debug("Creating vertex for type {} id {}", typeName, typedInstanceId._getId());
final Vertex vertexWithoutIdentity = graph.addVertex(null);
public Vertex createVertexWithoutIdentity(String typeName, Id typedInstanceId, Set<String> superTypeNames) {
LOG.debug("Creating vertex for type {} id {}", typeName,
typedInstanceId != null ? typedInstanceId._getId() : null);
final Vertex vertexWithoutIdentity = titanGraph.addVertex(null);
// add type information
setProperty(vertexWithoutIdentity, Constants.ENTITY_TYPE_PROPERTY_KEY, typeName);
......@@ -70,22 +99,19 @@ public final class GraphHelper {
addProperty(vertexWithoutIdentity, Constants.SUPER_TYPES_PROPERTY_KEY, superTypeName);
}
// add version information
setProperty(vertexWithoutIdentity, Constants.VERSION_PROPERTY_KEY, typedInstanceId.version);
// add timestamp information
setProperty(vertexWithoutIdentity, Constants.TIMESTAMP_PROPERTY_KEY, System.currentTimeMillis());
return vertexWithoutIdentity;
}
public static Edge addEdge(TitanGraph titanGraph, Vertex fromVertex, Vertex toVertex, String edgeLabel) {
public Edge addEdge(Vertex fromVertex, Vertex toVertex, String edgeLabel) {
LOG.debug("Adding edge for {} -> label {} -> {}", fromVertex, edgeLabel, toVertex);
return titanGraph.addEdge(null, fromVertex, toVertex, edgeLabel);
}
public static Vertex findVertex(TitanGraph titanGraph, String propertyKey, Object value) {
public Vertex findVertex(String propertyKey, Object value) {
LOG.debug("Finding vertex for {}={}", propertyKey, value);
GraphQuery query = titanGraph.query().has(propertyKey, value);
......@@ -94,6 +120,20 @@ public final class GraphHelper {
return results.hasNext() ? results.next() : null;
}
public static Iterable<Edge> getOutGoingEdgesByLabel(Vertex instanceVertex, String edgeLabel) {
if(instanceVertex != null && edgeLabel != null) {
return instanceVertex.getEdges(Direction.OUT, edgeLabel);
}
return null;
}
public Edge getOutGoingEdgeById(String edgeId) {
if(edgeId != null) {
return titanGraph.getEdge(edgeId);
}
return null;
}
public static String vertexString(final Vertex vertex) {
StringBuilder properties = new StringBuilder();
for (String propertyKey : vertex.getPropertyKeys()) {
......@@ -110,7 +150,17 @@ public final class GraphHelper {
public static void setProperty(Vertex vertex, String propertyName, Object value) {
LOG.debug("Setting property {} = \"{}\" to vertex {}", propertyName, value, vertex);
vertex.setProperty(propertyName, value);
Object existValue = vertex.getProperty(propertyName);
if(value == null || (value instanceof Collection && ((Collection) value).isEmpty())) {
if(existValue != null) {
LOG.info("Removing property - {} value from vertex {}", propertyName, vertex);
vertex.removeProperty(propertyName);
}
} else {
if (!value.equals(existValue)) {
vertex.setProperty(propertyName, value);
}
}
}
public static void addProperty(Vertex vertex, String propertyName, Object value) {
......@@ -118,7 +168,108 @@ public final class GraphHelper {
((TitanVertex)vertex).addProperty(propertyName, value);
}
/*
public Edge removeRelation(String edgeId, boolean cascade) {
LOG.debug("Removing edge with id {}", edgeId);
final Edge edge = titanGraph.getEdge(edgeId);
titanGraph.removeEdge(edge);
LOG.info("Removed edge {}", edge);
if (cascade) {
Vertex referredVertex = edge.getVertex(Direction.IN);
titanGraph.removeVertex(referredVertex);
LOG.info("Removed vertex {}", referredVertex);
}
return edge;
}
public Vertex getVertexForGUID(String guid) throws EntityNotFoundException {
return getVertexForProperty(Constants.GUID_PROPERTY_KEY, guid);
}
public Vertex getVertexForProperty(String propertyKey, Object value) throws EntityNotFoundException {
Vertex instanceVertex = findVertex(propertyKey, value);
if (instanceVertex == null) {
LOG.debug("Could not find a vertex with {}={}", propertyKey, value);
throw new EntityNotFoundException("Could not find an entity in the repository with " + propertyKey + "="
+ value);
} else {
LOG.debug("Found a vertex {} with {}={}", instanceVertex, propertyKey, value);
}
return instanceVertex;
}
public static String getQualifiedFieldName(ITypedInstance typedInstance, AttributeInfo attributeInfo) throws AtlasException {
IDataType dataType = typeSystem.getDataType(IDataType.class, typedInstance.getTypeName());
return getQualifiedFieldName(dataType, attributeInfo.name);
}
public static String getQualifiedFieldName(IDataType dataType, String attributeName) throws AtlasException {
return dataType.getTypeCategory() == DataTypes.TypeCategory.STRUCT ? dataType.getName() + "." + attributeName
// else class or trait
: ((HierarchicalType) dataType).getQualifiedName(attributeName);
}
public static String getTraitLabel(String typeName, String attrName) {
return typeName + "." + attrName;
}
public static List<String> getTraitNames(Vertex entityVertex) {
ArrayList<String> traits = new ArrayList<>();
for (TitanProperty property : ((TitanVertex) entityVertex).getProperties(Constants.TRAIT_NAMES_PROPERTY_KEY)) {
traits.add((String) property.getValue());
}
return traits;
}
public static String getEdgeLabel(ITypedInstance typedInstance, AttributeInfo aInfo) throws AtlasException {
IDataType dataType = typeSystem.getDataType(IDataType.class, typedInstance.getTypeName());
return getEdgeLabel(dataType, aInfo);
}
public static String getEdgeLabel(IDataType dataType, AttributeInfo aInfo) throws AtlasException {
return GraphHelper.EDGE_LABEL_PREFIX + getQualifiedFieldName(dataType, aInfo.name);
}
public static Id getIdFromVertex(String dataTypeName, Vertex vertex) {
return new Id(vertex.<String>getProperty(Constants.GUID_PROPERTY_KEY),
vertex.<Integer>getProperty(Constants.VERSION_PROPERTY_KEY), dataTypeName);
}
public static String getTypeName(Vertex instanceVertex) {
return instanceVertex.getProperty(Constants.ENTITY_TYPE_PROPERTY_KEY);
}
/**
* For the given type, finds an unique attribute and checks if there is an existing instance with the same
* unique value
*
* @param classType
* @param instance
* @return
* @throws AtlasException
*/
public Vertex getVertexForInstanceByUniqueAttribute(ClassType classType, IReferenceableInstance instance)
throws AtlasException {
Vertex result = null;
for (AttributeInfo attributeInfo : classType.fieldMapping().fields.values()) {
if (attributeInfo.isUnique) {
String propertyKey = getQualifiedFieldName(classType, attributeInfo.name);
try {
result = getVertexForProperty(propertyKey, instance.get(attributeInfo.name));
LOG.debug("Found vertex by unique attribute : " + propertyKey + "=" + instance.get(attributeInfo.name));
} catch (EntityNotFoundException e) {
//Its ok if there is no entity with the same unique value
}
}
}
return result;
}
public static void dumpToLog(final Graph graph) {
LOG.debug("*******************Graph Dump****************************");
LOG.debug("Vertices of {}", graph);
......@@ -132,5 +283,4 @@ public final class GraphHelper {
}
LOG.debug("*******************Graph Dump****************************");
}
*/
}
\ No newline at end of file
repository/src/main/java/org/apache/atlas/repository/graph/GraphToTypedInstanceMapper.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.repository.graph;
import com.thinkaurelius.titan.core.TitanGraph;
import com.tinkerpop.blueprints.Direction;
import com.tinkerpop.blueprints.Edge;
import com.tinkerpop.blueprints.Vertex;
import org.apache.atlas.AtlasException;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.typesystem.ITypedInstance;
import org.apache.atlas.typesystem.ITypedReferenceableInstance;
import org.apache.atlas.typesystem.ITypedStruct;
import org.apache.atlas.typesystem.persistence.Id;
import org.apache.atlas.typesystem.types.AttributeInfo;
import org.apache.atlas.typesystem.types.ClassType;
import org.apache.atlas.typesystem.types.DataTypes;
import org.apache.atlas.typesystem.types.IDataType;
import org.apache.atlas.typesystem.types.Multiplicity;
import org.apache.atlas.typesystem.types.StructType;
import org.apache.atlas.typesystem.types.TraitType;
import org.apache.atlas.typesystem.types.TypeSystem;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
public final class GraphToTypedInstanceMapper {
private static final Logger LOG = LoggerFactory.getLogger(GraphToTypedInstanceMapper.class);
private static TypeSystem typeSystem = TypeSystem.getInstance();
private final TitanGraph titanGraph;
public GraphToTypedInstanceMapper(TitanGraph titanGraph) {
this.titanGraph = titanGraph;
}
public ITypedReferenceableInstance mapGraphToTypedInstance(String guid, Vertex instanceVertex)
throws AtlasException {
LOG.debug("Mapping graph root vertex {} to typed instance for guid {}", instanceVertex, guid);
String typeName = instanceVertex.getProperty(Constants.ENTITY_TYPE_PROPERTY_KEY);
List<String> traits = GraphHelper.getTraitNames(instanceVertex);
Id id = new Id(guid, instanceVertex.<Integer>getProperty(Constants.VERSION_PROPERTY_KEY), typeName);
LOG.debug("Created id {} for instance type {}", id, typeName);
ClassType classType = typeSystem.getDataType(ClassType.class, typeName);
ITypedReferenceableInstance typedInstance =
classType.createInstance(id, traits.toArray(new String[traits.size()]));
mapVertexToInstance(instanceVertex, typedInstance, classType.fieldMapping().fields);
mapVertexToInstanceTraits(instanceVertex, typedInstance, traits);
return typedInstance;
}
private void mapVertexToInstanceTraits(Vertex instanceVertex, ITypedReferenceableInstance typedInstance,
List<String> traits) throws AtlasException {
for (String traitName : traits) {
LOG.debug("mapping trait {} to instance", traitName);
TraitType traitType = typeSystem.getDataType(TraitType.class, traitName);
mapVertexToTraitInstance(instanceVertex, typedInstance, traitName, traitType);
}
}
public void mapVertexToInstance(Vertex instanceVertex, ITypedInstance typedInstance,
Map<String, AttributeInfo> fields) throws AtlasException {
LOG.debug("Mapping vertex {} to instance {} for fields", instanceVertex, typedInstance.getTypeName(),
fields);
for (AttributeInfo attributeInfo : fields.values()) {
mapVertexToAttribute(instanceVertex, typedInstance, attributeInfo);
}
}
private void mapVertexToAttribute(Vertex instanceVertex, ITypedInstance typedInstance,
AttributeInfo attributeInfo) throws AtlasException {
LOG.debug("Mapping attributeInfo {}", attributeInfo.name);
final IDataType dataType = attributeInfo.dataType();
final String vertexPropertyName = GraphHelper.getQualifiedFieldName(typedInstance, attributeInfo);
switch (dataType.getTypeCategory()) {
case PRIMITIVE:
mapVertexToPrimitive(instanceVertex, typedInstance, attributeInfo);
break; // add only if vertex has this attribute
case ENUM:
if (instanceVertex.getProperty(vertexPropertyName) == null) {
return;
}
typedInstance.set(attributeInfo.name,
dataType.convert(instanceVertex.<String>getProperty(vertexPropertyName),
Multiplicity.REQUIRED));
break;
case ARRAY:
mapVertexToArrayInstance(instanceVertex, typedInstance, attributeInfo, vertexPropertyName);
break;
case MAP:
mapVertexToMapInstance(instanceVertex, typedInstance, attributeInfo, vertexPropertyName);
break;
case STRUCT:
mapVertexToStructInstance(instanceVertex, typedInstance, attributeInfo);
break;
case TRAIT:
// do NOTHING - handled in class
break;
case CLASS:
String relationshipLabel = GraphHelper.getEdgeLabel(typedInstance, attributeInfo);
Object idOrInstance = mapVertexToClassReference(instanceVertex, attributeInfo, relationshipLabel,
attributeInfo.dataType());
if (idOrInstance != null) {
typedInstance.set(attributeInfo.name, idOrInstance);
}
break;
default:
break;
}
}
private Object mapVertexToClassReference(Vertex instanceVertex, AttributeInfo attributeInfo,
String relationshipLabel, IDataType dataType) throws AtlasException {
LOG.debug("Finding edge for {} -> label {} ", instanceVertex, relationshipLabel);
Iterator<Edge> results = instanceVertex.getEdges(Direction.OUT, relationshipLabel).iterator();
if (results.hasNext()) {
final Vertex referenceVertex = results.next().getVertex(Direction.IN);
if (referenceVertex != null) {
final String guid = referenceVertex.getProperty(Constants.GUID_PROPERTY_KEY);
LOG.debug("Found vertex {} for label {} with guid {}", referenceVertex, relationshipLabel, guid);
if (attributeInfo.isComposite) {
//Also, when you retrieve a type's instance, you get the complete object graph of the composites
LOG.debug("Found composite, mapping vertex to instance");
return mapGraphToTypedInstance(guid, referenceVertex);
} else {
Id referenceId =
new Id(guid, referenceVertex.<Integer>getProperty(Constants.VERSION_PROPERTY_KEY),
dataType.getName());
LOG.debug("Found non-composite, adding id {} ", referenceId);
return referenceId;
}
}
}
return null;
}
@SuppressWarnings("unchecked")
private void mapVertexToArrayInstance(Vertex instanceVertex, ITypedInstance typedInstance,
AttributeInfo attributeInfo, String propertyName) throws AtlasException {
LOG.debug("mapping vertex {} to array {}", instanceVertex, attributeInfo.name);
List list = instanceVertex.getProperty(propertyName);
if (list == null || list.size() == 0) {
return;
}
DataTypes.ArrayType arrayType = (DataTypes.ArrayType) attributeInfo.dataType();
final IDataType elementType = arrayType.getElemType();
String edgeLabel = GraphHelper.EDGE_LABEL_PREFIX + propertyName;
ArrayList values = new ArrayList();
for (int index = 0; index < list.size(); index++) {
values.add(mapVertexToCollectionEntry(instanceVertex, attributeInfo, elementType, list.get(index),
edgeLabel));
}
if (values.size() > 0) {
typedInstance.set(attributeInfo.name, values);
}
}
private Object mapVertexToCollectionEntry(Vertex instanceVertex, AttributeInfo attributeInfo,
IDataType elementType, Object value, String edgeLabel) throws AtlasException {
switch (elementType.getTypeCategory()) {
case PRIMITIVE:
case ENUM:
return value;
case ARRAY:
case MAP:
case TRAIT:
// do nothing
break;
case STRUCT:
return getStructInstanceFromVertex(instanceVertex, elementType, attributeInfo.name, edgeLabel,
(String) value);
case CLASS:
return mapVertexToClassReference(instanceVertex, attributeInfo, edgeLabel, elementType, (String) value);
default:
break;
}
throw new IllegalArgumentException();
}
@SuppressWarnings("unchecked")
private void mapVertexToMapInstance(Vertex instanceVertex, ITypedInstance typedInstance,
AttributeInfo attributeInfo, final String propertyName) throws AtlasException {
LOG.debug("mapping vertex {} to array {}", instanceVertex, attributeInfo.name);
List<String> keys = instanceVertex.getProperty(propertyName);
if (keys == null || keys.size() == 0) {
return;
}
DataTypes.MapType mapType = (DataTypes.MapType) attributeInfo.dataType();
final IDataType valueType = mapType.getValueType();
HashMap values = new HashMap();
for (String key : keys) {
final String keyPropertyName = propertyName + "." + key;
final String edgeLabel = GraphHelper.EDGE_LABEL_PREFIX + keyPropertyName;
final Object keyValue = instanceVertex.getProperty(keyPropertyName);
Object mapValue = mapVertexToCollectionEntry(instanceVertex, attributeInfo, valueType, keyValue, edgeLabel);
if (mapValue != null) {
values.put(key, mapValue);
}
}
if (!values.isEmpty()) {
typedInstance.set(attributeInfo.name, values);
}
}
private ITypedStruct getStructInstanceFromVertex(Vertex instanceVertex, IDataType elemType,
String attributeName, String relationshipLabel, String edgeId) throws AtlasException {
LOG.debug("Finding edge for {} -> label {} ", instanceVertex, relationshipLabel);
for (Edge edge : instanceVertex.getEdges(Direction.OUT, relationshipLabel)) {
if (edgeId.equals(String.valueOf(edge.getId()))) {
Vertex structInstanceVertex = edge.getVertex(Direction.IN);
LOG.debug("mapping vertex {} to struct {}", structInstanceVertex, attributeName);
if (structInstanceVertex != null) {
LOG.debug("Found struct instance vertex {}, mapping to instance {} ", structInstanceVertex,
elemType.getName());
StructType structType = typeSystem.getDataType(StructType.class, elemType.getName());
ITypedStruct structInstance = structType.createInstance();
mapVertexToInstance(structInstanceVertex, structInstance, structType.fieldMapping().fields);
return structInstance;
}
break;
}
}
return null;
}
private Object mapVertexToClassReference(Vertex instanceVertex, AttributeInfo attributeInfo,
String relationshipLabel, IDataType dataType, String edgeId) throws AtlasException {
LOG.debug("Finding edge for {} -> label {} ", instanceVertex, relationshipLabel);
for (Edge edge : instanceVertex.getEdges(Direction.OUT, relationshipLabel)) {
if (edgeId.equals(String.valueOf(edge.getId()))) {
final Vertex referenceVertex = edge.getVertex(Direction.IN);
if (referenceVertex != null) {
final String guid = referenceVertex.getProperty(Constants.GUID_PROPERTY_KEY);
LOG.debug("Found vertex {} for label {} with guid {}", referenceVertex, relationshipLabel,
guid);
if (attributeInfo.isComposite) {
//Also, when you retrieve a type's instance, you get the complete object graph of the composites
LOG.debug("Found composite, mapping vertex to instance");
return mapGraphToTypedInstance(guid, referenceVertex);
} else {
Id referenceId =
new Id(guid, referenceVertex.<Integer>getProperty(Constants.VERSION_PROPERTY_KEY),
dataType.getName());
LOG.debug("Found non-composite, adding id {} ", referenceId);
return referenceId;
}
}
break;
}
}
return null;
}
private void mapVertexToStructInstance(Vertex instanceVertex, ITypedInstance typedInstance,
AttributeInfo attributeInfo) throws AtlasException {
LOG.debug("mapping vertex {} to struct {}", instanceVertex, attributeInfo.name);
StructType structType = typeSystem.getDataType(StructType.class, attributeInfo.dataType().getName());
ITypedStruct structInstance = null;
String relationshipLabel = GraphHelper.getEdgeLabel(typedInstance, attributeInfo);
LOG.debug("Finding edge for {} -> label {} ", instanceVertex, relationshipLabel);
final Iterable<Edge> edges = instanceVertex.getEdges(Direction.OUT, relationshipLabel);
if (edges.iterator().hasNext()) {
structInstance = structType.createInstance();
typedInstance.set(attributeInfo.name, structInstance);
}
for (Edge edge : edges) {
final Vertex structInstanceVertex = edge.getVertex(Direction.IN);
if (structInstanceVertex != null) {
LOG.debug("Found struct instance vertex {}, mapping to instance {} ", structInstanceVertex,
structInstance.getTypeName());
mapVertexToInstance(structInstanceVertex, structInstance, structType.fieldMapping().fields);
break;
}
}
}
private void mapVertexToTraitInstance(Vertex instanceVertex, ITypedReferenceableInstance typedInstance,
String traitName, TraitType traitType) throws AtlasException {
ITypedStruct traitInstance = (ITypedStruct) typedInstance.getTrait(traitName);
mapVertexToTraitInstance(instanceVertex, typedInstance.getTypeName(), traitName, traitType, traitInstance);
}
private void mapVertexToTraitInstance(Vertex instanceVertex, String typedInstanceTypeName, String traitName,
TraitType traitType, ITypedStruct traitInstance) throws AtlasException {
String relationshipLabel = GraphHelper.getTraitLabel(typedInstanceTypeName, traitName);
LOG.debug("Finding edge for {} -> label {} ", instanceVertex, relationshipLabel);
for (Edge edge : instanceVertex.getEdges(Direction.OUT, relationshipLabel)) {
final Vertex traitInstanceVertex = edge.getVertex(Direction.IN);
if (traitInstanceVertex != null) {
LOG.debug("Found trait instance vertex {}, mapping to instance {} ", traitInstanceVertex,
traitInstance.getTypeName());
mapVertexToInstance(traitInstanceVertex, traitInstance, traitType.fieldMapping().fields);
break;
}
}
}
private void mapVertexToPrimitive(Vertex instanceVertex, ITypedInstance typedInstance,
AttributeInfo attributeInfo) throws AtlasException {
LOG.debug("Adding primitive {} from vertex {}", attributeInfo, instanceVertex);
final String vertexPropertyName = GraphHelper.getQualifiedFieldName(typedInstance, attributeInfo);
if (instanceVertex.getProperty(vertexPropertyName) == null) {
return;
}
if (attributeInfo.dataType() == DataTypes.STRING_TYPE) {
typedInstance.setString(attributeInfo.name, instanceVertex.<String>getProperty(vertexPropertyName));
} else if (attributeInfo.dataType() == DataTypes.SHORT_TYPE) {
typedInstance.setShort(attributeInfo.name, instanceVertex.<Short>getProperty(vertexPropertyName));
} else if (attributeInfo.dataType() == DataTypes.INT_TYPE) {
typedInstance.setInt(attributeInfo.name, instanceVertex.<Integer>getProperty(vertexPropertyName));
} else if (attributeInfo.dataType() == DataTypes.BIGINTEGER_TYPE) {
typedInstance.setBigInt(attributeInfo.name, instanceVertex.<BigInteger>getProperty(vertexPropertyName));
} else if (attributeInfo.dataType() == DataTypes.BOOLEAN_TYPE) {
typedInstance.setBoolean(attributeInfo.name, instanceVertex.<Boolean>getProperty(vertexPropertyName));
} else if (attributeInfo.dataType() == DataTypes.BYTE_TYPE) {
typedInstance.setByte(attributeInfo.name, instanceVertex.<Byte>getProperty(vertexPropertyName));
} else if (attributeInfo.dataType() == DataTypes.LONG_TYPE) {
typedInstance.setLong(attributeInfo.name, instanceVertex.<Long>getProperty(vertexPropertyName));
} else if (attributeInfo.dataType() == DataTypes.FLOAT_TYPE) {
typedInstance.setFloat(attributeInfo.name, instanceVertex.<Float>getProperty(vertexPropertyName));
} else if (attributeInfo.dataType() == DataTypes.DOUBLE_TYPE) {
typedInstance.setDouble(attributeInfo.name, instanceVertex.<Double>getProperty(vertexPropertyName));
} else if (attributeInfo.dataType() == DataTypes.BIGDECIMAL_TYPE) {
typedInstance
.setBigDecimal(attributeInfo.name, instanceVertex.<BigDecimal>getProperty(vertexPropertyName));
} else if (attributeInfo.dataType() == DataTypes.DATE_TYPE) {
final Long dateVal = instanceVertex.<Long>getProperty(vertexPropertyName);
typedInstance.setDate(attributeInfo.name, new Date(dateVal));
}
}
public ITypedInstance getReferredEntity(String edgeId, IDataType<?> referredType) throws AtlasException {
final Edge edge = titanGraph.getEdge(edgeId);
if (edge != null) {
final Vertex referredVertex = edge.getVertex(Direction.IN);
if (referredVertex != null) {
switch (referredType.getTypeCategory()) {
case STRUCT:
LOG.debug("Found struct instance vertex {}, mapping to instance {} ", referredVertex,
referredType.getName());
StructType structType = (StructType) referredType;
ITypedStruct instance = structType.createInstance();
Map<String, AttributeInfo> fields = structType.fieldMapping().fields;
mapVertexToInstance(referredVertex, instance, fields);
return instance;
case CLASS:
//TODO isComposite handling for class loads
final String guid = referredVertex.getProperty(Constants.GUID_PROPERTY_KEY);
Id referenceId =
new Id(guid, referredVertex.<Integer>getProperty(Constants.VERSION_PROPERTY_KEY),
referredType.getName());
return referenceId;
default:
throw new UnsupportedOperationException("Loading " + referredType.getTypeCategory() + " is not supported");
}
}
}
return null;
}
}
repository/src/main/java/org/apache/atlas/repository/graph/TypedInstanceToGraphMapper.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.repository.graph;
import com.thinkaurelius.titan.core.SchemaViolationException;
import com.tinkerpop.blueprints.Direction;
import com.tinkerpop.blueprints.Edge;
import com.tinkerpop.blueprints.Vertex;
import org.apache.atlas.AtlasException;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.RepositoryException;
import org.apache.atlas.typesystem.IReferenceableInstance;
import org.apache.atlas.typesystem.ITypedInstance;
import org.apache.atlas.typesystem.ITypedReferenceableInstance;
import org.apache.atlas.typesystem.ITypedStruct;
import org.apache.atlas.typesystem.exception.EntityExistsException;
import org.apache.atlas.typesystem.exception.EntityNotFoundException;
import org.apache.atlas.typesystem.persistence.Id;
import org.apache.atlas.typesystem.persistence.ReferenceableInstance;
import org.apache.atlas.typesystem.types.AttributeInfo;
import org.apache.atlas.typesystem.types.ClassType;
import org.apache.atlas.typesystem.types.DataTypes;
import org.apache.atlas.typesystem.types.EnumValue;
import org.apache.atlas.typesystem.types.IDataType;
import org.apache.atlas.typesystem.types.Multiplicity;
import org.apache.atlas.typesystem.types.ObjectGraphWalker;
import org.apache.atlas.typesystem.types.TraitType;
import org.apache.atlas.typesystem.types.TypeSystem;
import org.apache.atlas.utils.MD5Utils;
import org.apache.commons.lang3.tuple.Pair;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.security.MessageDigest;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
public final class TypedInstanceToGraphMapper {
private static final Logger LOG = LoggerFactory.getLogger(TypedInstanceToGraphMapper.class);
private final Map<Id, Vertex> idToVertexMap = new HashMap<>();
private final TypeSystem typeSystem = TypeSystem.getInstance();
private final GraphToTypedInstanceMapper graphToTypedInstanceMapper;
private static final GraphHelper graphHelper = GraphHelper.getInstance();
private final String SIGNATURE_HASH_PROPERTY_KEY = Constants.INTERNAL_PROPERTY_KEY_PREFIX + "signature";
public enum Operation {
CREATE,
UPDATE_PARTIAL,
UPDATE_FULL,
DELETE
}
public TypedInstanceToGraphMapper(GraphToTypedInstanceMapper graphToTypedInstanceMapper) {
this.graphToTypedInstanceMapper = graphToTypedInstanceMapper;
}
String[] mapTypedInstanceToGraph(Operation operation, ITypedReferenceableInstance... typedInstances)
throws AtlasException {
List<String> guids = new ArrayList<>();
for (ITypedReferenceableInstance typedInstance : typedInstances) {
Collection<IReferenceableInstance> newInstances = walkClassInstances(typedInstance);
Pair<List<ITypedReferenceableInstance>, List<ITypedReferenceableInstance>> instancesPair =
createVerticesAndDiscoverInstances(newInstances);
switch (operation) {
case CREATE:
addOrUpdateAttributesAndTraits(operation, instancesPair.getLeft());
addFullTextProperty(instancesPair.getLeft());
break;
case UPDATE_FULL:
case UPDATE_PARTIAL:
List<ITypedReferenceableInstance> instancesForUpdate = instancesPair.getLeft();
instancesForUpdate.addAll(instancesPair.getRight());
addOrUpdateAttributesAndTraits(operation, instancesForUpdate);
addFullTextProperty(instancesForUpdate);
break;
case DELETE:
throw new UnsupportedOperationException("Not handled - " + operation);
}
//Return guid for
addToGuids(typedInstance, guids);
}
return guids.toArray(new String[guids.size()]);
}
private Collection<IReferenceableInstance> walkClassInstances(ITypedReferenceableInstance typedInstance)
throws RepositoryException {
EntityProcessor entityProcessor = new EntityProcessor();
try {
LOG.debug("Walking the object graph for instance {}", typedInstance.getTypeName());
new ObjectGraphWalker(typeSystem, entityProcessor, typedInstance).walk();
} catch (AtlasException me) {
throw new RepositoryException("TypeSystem error when walking the ObjectGraph", me);
}
entityProcessor.addInstanceIfNotExists(typedInstance);
return entityProcessor.getInstances();
}
private void addOrUpdateAttributesAndTraits(Operation operation, List<ITypedReferenceableInstance> instances) throws AtlasException {
for (ITypedReferenceableInstance instance : instances) {
try {
//new vertex, set all the properties
addOrUpdateAttributesAndTraits(operation, instance);
} catch (SchemaViolationException e) {
throw new EntityExistsException(instance, e);
}
}
}
private void addOrUpdateAttributesAndTraits(Operation operation, ITypedReferenceableInstance typedInstance)
throws AtlasException {
LOG.debug("Adding/Updating typed instance {}", typedInstance.getTypeName());
Id id = typedInstance.getId();
if (id == null) { // oops
throw new RepositoryException("id cannot be null");
}
Vertex instanceVertex = idToVertexMap.get(id);
// add the attributes for the instance
ClassType classType = typeSystem.getDataType(ClassType.class, typedInstance.getTypeName());
final Map<String, AttributeInfo> fields = classType.fieldMapping().fields;
mapInstanceToVertex(typedInstance, instanceVertex, fields, false, operation);
if (Operation.CREATE.equals(operation)) {
//TODO - Handle Trait updates
addTraits(typedInstance, instanceVertex, classType);
}
}
private void mapInstanceToVertex(ITypedInstance typedInstance, Vertex instanceVertex,
Map<String, AttributeInfo> fields, boolean mapOnlyUniqueAttributes, Operation operation)
throws AtlasException {
LOG.debug("Mapping instance {} of {} to vertex {}", typedInstance, typedInstance.getTypeName(),
instanceVertex);
for (AttributeInfo attributeInfo : fields.values()) {
if (mapOnlyUniqueAttributes && !attributeInfo.isUnique) {
continue;
}
mapAttributesToVertex(typedInstance, instanceVertex, attributeInfo, operation);
}
}
void mapAttributesToVertex(ITypedInstance typedInstance, Vertex instanceVertex,
AttributeInfo attributeInfo, Operation operation) throws AtlasException {
Object attrValue = typedInstance.get(attributeInfo.name);
LOG.debug("mapping attribute {} = {}", attributeInfo.name, attrValue);
final String propertyName = GraphHelper.getQualifiedFieldName(typedInstance, attributeInfo);
String edgeLabel = GraphHelper.getEdgeLabel(typedInstance, attributeInfo);
if (attrValue != null || operation == Operation.UPDATE_FULL) {
switch (attributeInfo.dataType().getTypeCategory()) {
case PRIMITIVE:
case ENUM:
mapPrimitiveOrEnumToVertex(typedInstance, instanceVertex, attributeInfo);
break;
case ARRAY:
mapArrayCollectionToVertex(typedInstance, instanceVertex, attributeInfo, operation);
break;
case MAP:
mapMapCollectionToVertex(typedInstance, instanceVertex, attributeInfo, operation);
break;
case STRUCT:
case CLASS:
Iterator<Edge> outGoingEdgesIterator =
GraphHelper.getOutGoingEdgesByLabel(instanceVertex, edgeLabel).iterator();
String currentEntry =
outGoingEdgesIterator.hasNext() ? outGoingEdgesIterator.next().getId().toString() : null;
addOrUpdateCollectionEntry(instanceVertex, attributeInfo, attributeInfo.dataType(), attrValue,
currentEntry, propertyName, operation);
break;
case TRAIT:
// do NOTHING - this is taken care of earlier
break;
default:
throw new IllegalArgumentException("Unknown type category: " + attributeInfo.dataType().getTypeCategory());
}
}
}
private Pair<List<ITypedReferenceableInstance>, List<ITypedReferenceableInstance>> createVerticesAndDiscoverInstances(
Collection<IReferenceableInstance> instances) throws AtlasException {
List<ITypedReferenceableInstance> instancesToCreate = new ArrayList<>();
List<ITypedReferenceableInstance> instancesToUpdate = new ArrayList<>();
for (IReferenceableInstance instance : instances) {
Id id = instance.getId();
if (!idToVertexMap.containsKey(id)) {
Vertex instanceVertex;
if (id.isAssigned()) { // has a GUID
instanceVertex = graphHelper.getVertexForGUID(id.id);
if (!(instance instanceof ReferenceableInstance)) {
throw new IllegalStateException(
String.format("%s is not of type ITypedReferenceableInstance", instance));
}
instancesToUpdate.add((ITypedReferenceableInstance) instance);
} else {
//Check if there is already an instance with the same unique attribute value
ClassType classType = typeSystem.getDataType(ClassType.class, instance.getTypeName());
instanceVertex = graphHelper.getVertexForInstanceByUniqueAttribute(classType, instance);
//no entity with the given unique attribute, create new
if (instanceVertex == null) {
ITypedReferenceableInstance newInstance = classType.convert(instance, Multiplicity.REQUIRED);
instanceVertex = graphHelper.createVertexWithIdentity(newInstance, classType.getAllSuperTypeNames());
instancesToCreate.add(newInstance);
//Map only unique attributes for cases of circular references
mapInstanceToVertex(newInstance, instanceVertex, classType.fieldMapping().fields, true, Operation.CREATE);
} else {
if (!(instance instanceof ReferenceableInstance)) {
throw new IllegalStateException(
String.format("%s is not of type ITypedReferenceableInstance", instance));
}
instancesToUpdate.add((ITypedReferenceableInstance) instance);
}
}
idToVertexMap.put(id, instanceVertex);
}
}
return Pair.of(instancesToCreate, instancesToUpdate);
}
private void addToGuids(ITypedReferenceableInstance typedInstance, List<String> guids) {
Vertex instanceVertex = idToVertexMap.get(typedInstance.getId());
String guid = instanceVertex.getProperty(Constants.GUID_PROPERTY_KEY);
guids.add(guid);
}
private void addFullTextProperty(List<ITypedReferenceableInstance> instances) throws AtlasException {
FullTextMapper fulltextMapper = new FullTextMapper(graphToTypedInstanceMapper);
for (ITypedReferenceableInstance typedInstance : instances) { // Traverse
Vertex instanceVertex = getClassVertex(typedInstance);
String fullText = fulltextMapper.mapRecursive(instanceVertex, true);
GraphHelper.setProperty(instanceVertex, Constants.ENTITY_TEXT_PROPERTY_KEY, fullText);
}
}
private void addTraits(ITypedReferenceableInstance typedInstance, Vertex instanceVertex, ClassType classType) throws AtlasException {
for (String traitName : typedInstance.getTraits()) {
LOG.debug("mapping trait {}", traitName);
GraphHelper.addProperty(instanceVertex, Constants.TRAIT_NAMES_PROPERTY_KEY, traitName);
ITypedStruct traitInstance = (ITypedStruct) typedInstance.getTrait(traitName);
// add the attributes for the trait instance
mapTraitInstanceToVertex(traitInstance, classType, instanceVertex);
}
}
/******************************************** STRUCT **************************************************/
private Pair<Vertex, Edge> updateStructVertex(ITypedStruct structInstance, Edge relEdge, Operation operation) throws AtlasException {
//Already existing vertex. Update
Vertex structInstanceVertex = relEdge.getVertex(Direction.IN);
// Update attributes
final MessageDigest digester = MD5Utils.getDigester();
String newSignature = structInstance.getSignatureHash(digester);
String curSignature = structInstanceVertex.getProperty(SIGNATURE_HASH_PROPERTY_KEY);
if (!newSignature.equals(curSignature)) {
//Update struct vertex instance only if there is a change
LOG.debug("Updating struct {} since signature has changed {} {} ", structInstance, curSignature, newSignature);
mapInstanceToVertex(structInstance, structInstanceVertex, structInstance.fieldMapping().fields, false, operation);
GraphHelper.setProperty(structInstanceVertex, SIGNATURE_HASH_PROPERTY_KEY, String.valueOf(newSignature));
}
return Pair.of(structInstanceVertex, relEdge);
}
private Pair<Vertex, Edge> addStructVertex(ITypedStruct structInstance, Vertex instanceVertex, AttributeInfo attributeInfo, String edgeLabel) throws AtlasException {
// add a new vertex for the struct or trait instance
Vertex structInstanceVertex = graphHelper.createVertexWithoutIdentity(structInstance.getTypeName(), null,
Collections.<String>emptySet()); // no super types for struct type
LOG.debug("created vertex {} for struct {} value {}", structInstanceVertex, attributeInfo.name, structInstance);
// map all the attributes to this new vertex
mapInstanceToVertex(structInstance, structInstanceVertex, structInstance.fieldMapping().fields, false, Operation.CREATE);
// add an edge to the newly created vertex from the parent
Edge relEdge = graphHelper.addEdge(instanceVertex, structInstanceVertex, edgeLabel);
return Pair.of(structInstanceVertex, relEdge);
}
/******************************************** ARRAY **************************************************/
private void mapArrayCollectionToVertex(ITypedInstance typedInstance, Vertex instanceVertex,
AttributeInfo attributeInfo, Operation operation) throws AtlasException {
LOG.debug("Mapping instance {} to vertex {} for name {}", typedInstance.getTypeName(), instanceVertex,
attributeInfo.name);
List newElements = (List) typedInstance.get(attributeInfo.name);
boolean empty = (newElements == null || newElements.isEmpty());
if (!empty || operation == Operation.UPDATE_FULL) {
String propertyName = GraphHelper.getQualifiedFieldName(typedInstance, attributeInfo);
List<String> currentEntries = instanceVertex.getProperty(propertyName);
IDataType elementType = ((DataTypes.ArrayType) attributeInfo.dataType()).getElemType();
List<String> newEntries = new ArrayList<>();
if (newElements != null && !newElements.isEmpty()) {
int index = 0;
for (; index < newElements.size(); index++) {
String currentEntry =
(currentEntries != null && index < currentEntries.size()) ? currentEntries.get(index) : null;
String newEntry = addOrUpdateCollectionEntry(instanceVertex, attributeInfo, elementType,
newElements.get(index), currentEntry, propertyName, operation);
newEntries.add(newEntry);
}
//Remove extra entries in the list
if (currentEntries != null) {
if (index < currentEntries.size()) {
for (; index < currentEntries.size(); index++) {
removeUnusedReference(currentEntries.get(index), attributeInfo, elementType);
}
}
}
}
// for dereference on way out
GraphHelper.setProperty(instanceVertex, propertyName, newEntries);
}
}
/******************************************** MAP **************************************************/
private void mapMapCollectionToVertex(ITypedInstance typedInstance, Vertex instanceVertex,
AttributeInfo attributeInfo, Operation operation) throws AtlasException {
LOG.debug("Mapping instance {} to vertex {} for name {}", typedInstance.getTypeName(), instanceVertex,
attributeInfo.name);
@SuppressWarnings("unchecked") Map<Object, Object> collection =
(Map<Object, Object>) typedInstance.get(attributeInfo.name);
boolean empty = (collection == null || collection.isEmpty());
if (!empty || operation == Operation.UPDATE_FULL) {
String propertyName = GraphHelper.getQualifiedFieldName(typedInstance, attributeInfo);
IDataType elementType = ((DataTypes.MapType) attributeInfo.dataType()).getValueType();
if (!empty) {
for (Map.Entry entry : collection.entrySet()) {
String myPropertyName = propertyName + "." + entry.getKey().toString();
String currentEntry = instanceVertex.getProperty(myPropertyName);
String newEntry = addOrUpdateCollectionEntry(instanceVertex, attributeInfo, elementType,
entry.getValue(), currentEntry, myPropertyName, operation);
//Add/Update/Remove property value
GraphHelper.setProperty(instanceVertex, myPropertyName, newEntry);
}
//Remove unused key references
List<Object> origKeys = instanceVertex.getProperty(propertyName);
if (origKeys != null) {
if (collection != null) {
origKeys.removeAll(collection.keySet());
}
for (Object unusedKey : origKeys) {
String edgeLabel = GraphHelper.getEdgeLabel(typedInstance, attributeInfo) + "." + unusedKey;
if (instanceVertex.getEdges(Direction.OUT, edgeLabel).iterator().hasNext()) {
Edge edge = instanceVertex.getEdges(Direction.OUT, edgeLabel).iterator().next();
removeUnusedReference(edge.getId().toString(), attributeInfo,
((DataTypes.MapType) attributeInfo.dataType()).getValueType());
}
}
}
}
// for dereference on way out
GraphHelper.setProperty(instanceVertex, propertyName, collection == null ? null : new ArrayList(collection.keySet()));
}
}
/******************************************** ARRAY & MAP **************************************************/
private String addOrUpdateCollectionEntry(Vertex instanceVertex, AttributeInfo attributeInfo,
IDataType elementType, Object newVal, String curVal, String propertyName,
Operation operation)
throws AtlasException {
final String edgeLabel = GraphHelper.EDGE_LABEL_PREFIX + propertyName;
switch (elementType.getTypeCategory()) {
case PRIMITIVE:
case ENUM:
return newVal != null ? newVal.toString() : null;
case ARRAY:
case MAP:
case TRAIT:
// do nothing
return null;
case STRUCT:
return addOrUpdateStruct(instanceVertex, attributeInfo, elementType, (ITypedStruct) newVal, curVal, edgeLabel, operation);
case CLASS:
return addOrUpdateClassVertex(instanceVertex, attributeInfo, elementType,
(ITypedReferenceableInstance) newVal, curVal, edgeLabel, operation);
default:
throw new IllegalArgumentException("Unknown type category: " + elementType.getTypeCategory());
}
}
private String addOrUpdateStruct(Vertex instanceVertex, AttributeInfo attributeInfo, IDataType elementType,
ITypedStruct structAttr, String curVal,
String edgeLabel, Operation operation) throws AtlasException {
Pair<Vertex, Edge> vertexEdgePair = null;
if (curVal != null && structAttr == null) {
//remove edge
removeUnusedReference(curVal, attributeInfo, elementType);
} else if (curVal != null && structAttr != null) {
//update
Edge edge = graphHelper.getOutGoingEdgeById(curVal);
vertexEdgePair = updateStructVertex(structAttr, edge, operation);
} else if (structAttr != null) {
//add
vertexEdgePair = addStructVertex(structAttr, instanceVertex, attributeInfo, edgeLabel);
}
return (vertexEdgePair != null) ? vertexEdgePair.getRight().getId().toString() : null;
}
private String addOrUpdateClassVertex(Vertex instanceVertex, AttributeInfo attributeInfo, IDataType elementType,
ITypedReferenceableInstance newVal, String curVal,
String edgeLabel, Operation operation) throws AtlasException {
Vertex toVertex = getClassVertex(newVal);
if(toVertex == null && newVal != null) {
LOG.error("Could not find vertex for Class Reference " + newVal);
throw new EntityNotFoundException("Could not find vertex for Class Reference " + newVal);
}
Pair<Vertex, Edge> vertexEdgePair = null;
if (curVal != null && newVal == null) {
//remove edge
removeUnusedReference(curVal, attributeInfo, elementType);
} else if (curVal != null && newVal != null) {
Edge edge = graphHelper.getOutGoingEdgeById(curVal);
Id classRefId = getId(newVal);
vertexEdgePair = updateClassEdge(classRefId, newVal, instanceVertex, edge, toVertex, attributeInfo, elementType, edgeLabel, operation);
} else if (newVal != null){
vertexEdgePair = addClassEdge(instanceVertex, toVertex, edgeLabel);
}
return (vertexEdgePair != null) ? vertexEdgePair.getRight().getId().toString() : null;
}
/******************************************** CLASS **************************************************/
private Pair<Vertex, Edge> addClassEdge(Vertex instanceVertex, Vertex toVertex, String edgeLabel) throws AtlasException {
// add an edge to the class vertex from the instance
Edge edge = graphHelper.addEdge(instanceVertex, toVertex, edgeLabel);
return Pair.of(toVertex, edge);
}
private Vertex getClassVertex(ITypedReferenceableInstance typedReference) throws EntityNotFoundException {
Vertex referenceVertex = null;
Id id = null;
if (typedReference != null) {
id = typedReference instanceof Id ? (Id) typedReference : typedReference.getId();
if (id.isAssigned()) {
referenceVertex = graphHelper.getVertexForGUID(id.id);
} else {
referenceVertex = idToVertexMap.get(id);
}
}
return referenceVertex;
}
private Id getId(ITypedReferenceableInstance typedReference) throws EntityNotFoundException {
Id id = null;
if (typedReference != null) {
id = typedReference instanceof Id ? (Id) typedReference : typedReference.getId();
}
if (id.isUnassigned()) {
Vertex classVertex = idToVertexMap.get(id);
String guid = classVertex.getProperty(Constants.GUID_PROPERTY_KEY);
id = new Id(guid, 0, typedReference.getTypeName());
}
return id;
}
private Pair<Vertex, Edge> updateClassEdge(Id id, final ITypedReferenceableInstance typedInstance,
Vertex instanceVertex, Edge edge, Vertex toVertex,
AttributeInfo attributeInfo, IDataType dataType,
String edgeLabel, Operation operation) throws AtlasException {
Pair<Vertex, Edge> result = Pair.of(toVertex, edge);
Edge newEdge = edge;
// Update edge if it exists
Vertex invertex = edge.getVertex(Direction.IN);
String currentGUID = invertex.getProperty(Constants.GUID_PROPERTY_KEY);
Id currentId = new Id(currentGUID, 0, (String) invertex.getProperty(Constants.ENTITY_TYPE_PROPERTY_KEY));
if (!currentId.equals(id)) {
// add an edge to the class vertex from the instance
if(toVertex != null) {
newEdge = graphHelper.addEdge(instanceVertex, toVertex, edgeLabel);
result = Pair.of(toVertex, newEdge);
}
removeUnusedReference(edge.getId().toString(), attributeInfo, dataType);
}
if (attributeInfo.isComposite) {
//Update the attributes also if composite
if (typedInstance.fieldMapping() != null) {
//In case of Id instance, fieldMapping is null
mapInstanceToVertex(typedInstance, toVertex, typedInstance.fieldMapping().fields , false, operation);
//Update full text for the updated composite vertex
addFullTextProperty(new ArrayList<ITypedReferenceableInstance>() {{ add(typedInstance); }});
}
}
return result;
}
/******************************************** TRAITS ****************************************************/
void mapTraitInstanceToVertex(ITypedStruct traitInstance, IDataType entityType, Vertex parentInstanceVertex)
throws AtlasException {
// add a new vertex for the struct or trait instance
final String traitName = traitInstance.getTypeName();
Vertex traitInstanceVertex = graphHelper.createVertexWithoutIdentity(traitInstance.getTypeName(), null,
typeSystem.getDataType(TraitType.class, traitName).getAllSuperTypeNames());
LOG.debug("created vertex {} for trait {}", traitInstanceVertex, traitName);
// map all the attributes to this newly created vertex
mapInstanceToVertex(traitInstance, traitInstanceVertex, traitInstance.fieldMapping().fields, false, Operation.CREATE);
// add an edge to the newly created vertex from the parent
String relationshipLabel = GraphHelper.getTraitLabel(entityType.getName(), traitName);
graphHelper.addEdge(parentInstanceVertex, traitInstanceVertex, relationshipLabel);
}
/******************************************** PRIMITIVES **************************************************/
private void mapPrimitiveOrEnumToVertex(ITypedInstance typedInstance, Vertex instanceVertex,
AttributeInfo attributeInfo) throws AtlasException {
Object attrValue = typedInstance.get(attributeInfo.name);
final String vertexPropertyName = GraphHelper.getQualifiedFieldName(typedInstance, attributeInfo);
Object propertyValue = null;
if (attrValue == null ) {
propertyValue = null;
} else if (attributeInfo.dataType() == DataTypes.STRING_TYPE) {
propertyValue = typedInstance.getString(attributeInfo.name);
} else if (attributeInfo.dataType() == DataTypes.SHORT_TYPE) {
propertyValue = typedInstance.getShort(attributeInfo.name);
} else if (attributeInfo.dataType() == DataTypes.INT_TYPE) {
propertyValue = typedInstance.getInt(attributeInfo.name);
} else if (attributeInfo.dataType() == DataTypes.BIGINTEGER_TYPE) {
propertyValue = typedInstance.getBigInt(attributeInfo.name);
} else if (attributeInfo.dataType() == DataTypes.BOOLEAN_TYPE) {
propertyValue = typedInstance.getBoolean(attributeInfo.name);
} else if (attributeInfo.dataType() == DataTypes.BYTE_TYPE) {
propertyValue = typedInstance.getByte(attributeInfo.name);
} else if (attributeInfo.dataType() == DataTypes.LONG_TYPE) {
propertyValue = typedInstance.getLong(attributeInfo.name);
} else if (attributeInfo.dataType() == DataTypes.FLOAT_TYPE) {
propertyValue = typedInstance.getFloat(attributeInfo.name);
} else if (attributeInfo.dataType() == DataTypes.DOUBLE_TYPE) {
propertyValue = typedInstance.getDouble(attributeInfo.name);
} else if (attributeInfo.dataType() == DataTypes.BIGDECIMAL_TYPE) {
propertyValue = typedInstance.getBigDecimal(attributeInfo.name);
} else if (attributeInfo.dataType() == DataTypes.DATE_TYPE) {
final Date dateVal = typedInstance.getDate(attributeInfo.name);
//Convert Property value to Long while persisting
propertyValue = dateVal.getTime();
} else if (attributeInfo.dataType().getTypeCategory() == DataTypes.TypeCategory.ENUM) {
if (attrValue != null) {
propertyValue = ((EnumValue)attrValue).value;
}
}
GraphHelper.setProperty(instanceVertex, vertexPropertyName, propertyValue);
}
private Edge removeUnusedReference(String edgeId, AttributeInfo attributeInfo, IDataType<?> elementType) {
//Remove edges for property values which do not exist any more
Edge removedRelation = null;
switch (elementType.getTypeCategory()) {
case STRUCT:
removedRelation = graphHelper.removeRelation(edgeId, true);
//Remove the vertex from state so that further processing no longer uses this
break;
case CLASS:
removedRelation = graphHelper.removeRelation(edgeId, attributeInfo.isComposite);
break;
}
return removedRelation;
}
}
repository/src/main/java/org/apache/atlas/repository/memory/ReplaceIdWithInstance.java
......@@ -55,7 +55,6 @@ public class ReplaceIdWithInstance implements ObjectGraphWalker.NodeProcessor {
} else if (!nd.aInfo.isComposite || nd.value == null) {
// do nothing
} else if (nd.aInfo.dataType().getTypeCategory() == DataTypes.TypeCategory.CLASS) {
if (nd.value != null && nd.value instanceof Id) {
Id id = (Id) nd.value;
ITypedReferenceableInstance r = getInstance(id);
......
repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java
......@@ -23,8 +23,11 @@ import com.google.common.collect.ImmutableList;
import com.google.inject.Provider;
import org.apache.atlas.AtlasClient;
import org.apache.atlas.AtlasException;
import org.apache.atlas.ParamChecker;
import org.apache.atlas.TypeNotFoundException;
import org.apache.atlas.repository.RepositoryException;
import org.apache.atlas.typesystem.exception.EntityNotFoundException;
import org.apache.atlas.typesystem.exception.TypeNotFoundException;
import org.apache.atlas.typesystem.persistence.ReferenceableInstance;
import org.apache.atlas.utils.ParamChecker;
import org.apache.atlas.classification.InterfaceAudience;
import org.apache.atlas.listener.EntityChangeListener;
import org.apache.atlas.listener.TypesChangeListener;
......@@ -39,6 +42,7 @@ import org.apache.atlas.typesystem.Struct;
import org.apache.atlas.typesystem.TypesDef;
import org.apache.atlas.typesystem.json.InstanceSerialization;
import org.apache.atlas.typesystem.json.TypesSerialization;
import org.apache.atlas.typesystem.persistence.Id;
import org.apache.atlas.typesystem.types.AttributeDefinition;
import org.apache.atlas.typesystem.types.AttributeInfo;
import org.apache.atlas.typesystem.types.ClassType;
......@@ -61,6 +65,7 @@ import scala.actors.threadpool.Arrays;
import javax.inject.Inject;
import javax.inject.Singleton;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
......@@ -282,7 +287,7 @@ public class DefaultMetadataService implements MetadataService {
entitites.add(repository.getEntityDefinition(guid));
}
onEntitiesAddedToRepo(entitites);
onEntitiesAdded(entitites);
return new JSONArray(Arrays.asList(guids)).toString();
}
......@@ -325,12 +330,17 @@ public class DefaultMetadataService implements MetadataService {
return InstanceSerialization.toJson(instance, true);
}
@Override
public String getEntityDefinition(String entityType, String attribute, String value) throws AtlasException {
private ITypedReferenceableInstance getEntityDefinitionReference(String entityType, String attribute, String value)
throws AtlasException {
validateTypeExists(entityType);
validateUniqueAttribute(entityType, attribute);
final ITypedReferenceableInstance instance = repository.getEntityDefinition(entityType, attribute, value);
return repository.getEntityDefinition(entityType, attribute, value);
}
@Override
public String getEntityDefinition(String entityType, String attribute, String value) throws AtlasException {
final ITypedReferenceableInstance instance = getEntityDefinitionReference(entityType, attribute, value);
return InstanceSerialization.toJson(instance, true);
}
......@@ -361,15 +371,146 @@ public class DefaultMetadataService implements MetadataService {
return repository.getEntityList(entityType);
}
/**
* Updates an entity, instance of the type based on the guid set.
*
* @param entityInstanceDefinition json array of entity definitions
* @return guids - json array of guids
*/
@Override
public void updateEntity(String guid, String property, String value) throws AtlasException {
public String updateEntities(String entityInstanceDefinition) throws AtlasException {
ParamChecker.notEmpty(entityInstanceDefinition, "Entity instance definition cannot be empty");
ITypedReferenceableInstance[] typedInstances = deserializeClassInstances(entityInstanceDefinition);
String[] guids = repository.updateEntities(typedInstances);
onEntitiesAdded(Arrays.asList(typedInstances));
return new JSONArray(Arrays.asList(guids)).toString();
}
@Override
public void updateEntityAttributeByGuid(final String guid, String attributeName, String value) throws AtlasException {
ParamChecker.notEmpty(guid, "guid cannot be null");
ParamChecker.notEmpty(property, "property cannot be null");
ParamChecker.notEmpty(attributeName, "property cannot be null");
ParamChecker.notEmpty(value, "property value cannot be null");
repository.updateEntity(guid, property, value);
ITypedReferenceableInstance existInstance = validateEntityExists(guid);
ClassType type = typeSystem.getDataType(ClassType.class, existInstance.getTypeName());
ITypedReferenceableInstance newInstance = type.createInstance();
AttributeInfo attributeInfo = type.fieldMapping.fields.get(attributeName);
if (attributeInfo == null) {
throw new AtlasException("Invalid property " + attributeName + " for entity " + existInstance.getTypeName());
}
DataTypes.TypeCategory attrTypeCategory = attributeInfo.dataType().getTypeCategory();
switch(attrTypeCategory) {
case PRIMITIVE:
newInstance.set(attributeName, value);
break;
case CLASS:
Id id = new Id(value, 0, attributeInfo.dataType().getName());
newInstance.set(attributeName, id);
break;
default:
throw new AtlasException("Update of " + attrTypeCategory + " is not supported");
}
((ReferenceableInstance)newInstance).replaceWithNewId(new Id(guid, 0, newInstance.getTypeName()));
repository.updatePartial(newInstance);
onEntitiesUpdated(new ArrayList<ITypedReferenceableInstance>() {{
add(repository.getEntityDefinition(guid));
}});
}
private ITypedReferenceableInstance validateEntityExists(String guid)
throws EntityNotFoundException, RepositoryException {
final ITypedReferenceableInstance instance = repository.getEntityDefinition(guid);
if (instance == null) {
throw new EntityNotFoundException(String.format("Entity with guid %s not found ", guid));
}
return instance;
}
@Override
public void updateEntityPartialByGuid(final String guid, Referenceable newEntity) throws AtlasException {
ParamChecker.notEmpty(guid, "guid cannot be null");
ParamChecker.notNull(newEntity, "updatedEntity cannot be null");
ITypedReferenceableInstance existInstance = validateEntityExists(guid);
onEntityUpdated(repository.getEntityDefinition(guid));
ITypedReferenceableInstance newInstance = convertToTypedInstance(newEntity, existInstance.getTypeName());
((ReferenceableInstance)newInstance).replaceWithNewId(new Id(guid, 0, newInstance.getTypeName()));
repository.updatePartial(newInstance);
onEntitiesUpdated(new ArrayList<ITypedReferenceableInstance>() {{
add(repository.getEntityDefinition(guid));
}});
}
private ITypedReferenceableInstance convertToTypedInstance(Referenceable updatedEntity, String typeName) throws AtlasException {
ClassType type = typeSystem.getDataType(ClassType.class, typeName);
ITypedReferenceableInstance newInstance = type.createInstance();
for (String attributeName : updatedEntity.getValuesMap().keySet()) {
AttributeInfo attributeInfo = type.fieldMapping.fields.get(attributeName);
if (attributeInfo == null) {
throw new AtlasException("Invalid property " + attributeName + " for entity " + updatedEntity);
}
DataTypes.TypeCategory attrTypeCategory = attributeInfo.dataType().getTypeCategory();
Object value = updatedEntity.get(attributeName);
if (value != null) {
switch (attrTypeCategory) {
case CLASS:
if (value instanceof Referenceable) {
newInstance.set(attributeName, value);
} else {
Id id = new Id((String) value, 0, attributeInfo.dataType().getName());
newInstance.set(attributeName, id);
}
break;
case ENUM:
case PRIMITIVE:
case ARRAY:
case STRUCT:
case MAP:
newInstance.set(attributeName, value);
break;
case TRAIT:
//TODO - handle trait updates as well?
default:
throw new AtlasException("Update of " + attrTypeCategory + " is not supported");
}
}
}
return newInstance;
}
@Override
public String updateEntityByUniqueAttribute(String typeName, String uniqueAttributeName, String attrValue,
Referenceable updatedEntity) throws AtlasException {
ParamChecker.notEmpty(typeName, "typeName cannot be null");
ParamChecker.notEmpty(uniqueAttributeName, "uniqueAttributeName cannot be null");
ParamChecker.notNull(attrValue, "value cannot be null");
ParamChecker.notNull(updatedEntity, "updatedEntity cannot be null");
ITypedReferenceableInstance oldInstance = getEntityDefinitionReference(typeName, uniqueAttributeName, attrValue);
final ITypedReferenceableInstance newInstance = convertToTypedInstance(updatedEntity, typeName);
((ReferenceableInstance)newInstance).replaceWithNewId(oldInstance.getId());
repository.updatePartial(newInstance);
onEntitiesUpdated(new ArrayList<ITypedReferenceableInstance>() {{
add(newInstance);
}});
return newInstance.getId()._getId();
}
private void validateTypeExists(String entityType) throws AtlasException {
......@@ -485,8 +626,7 @@ public class DefaultMetadataService implements MetadataService {
}
}
private void onEntitiesAddedToRepo(Collection<ITypedReferenceableInstance> entities) throws AtlasException {
private void onEntitiesAdded(Collection<ITypedReferenceableInstance> entities) throws AtlasException {
for (EntityChangeListener listener : entityChangeListeners) {
listener.onEntitiesAdded(entities);
}
......@@ -509,10 +649,10 @@ public class DefaultMetadataService implements MetadataService {
}
}
private void onEntityUpdated(ITypedReferenceableInstance entity)
private void onEntitiesUpdated(Collection<ITypedReferenceableInstance> entities)
throws AtlasException {
for (EntityChangeListener listener : entityChangeListeners) {
listener.onEntityUpdated(entity);
listener.onEntitiesUpdated(entities);
}
}
......
repository/src/main/scala/org/apache/atlas/query/GraphPersistenceStrategies.scala
......@@ -26,7 +26,7 @@ import com.tinkerpop.blueprints.{Vertex, Direction}
import org.apache.atlas.AtlasException
import org.apache.atlas.query.Expressions.{ComparisonExpression, ExpressionException}
import org.apache.atlas.query.TypeUtils.FieldInfo
import org.apache.atlas.repository.graph.GraphBackedMetadataRepository
import org.apache.atlas.repository.graph.{GraphHelper, GraphBackedMetadataRepository}
import org.apache.atlas.typesystem.persistence.Id
import org.apache.atlas.typesystem.types.DataTypes._
import org.apache.atlas.typesystem.types._
......@@ -199,7 +199,7 @@ object GraphPersistenceStrategy1 extends GraphPersistenceStrategies {
def traitLabel(cls: IDataType[_], traitName: String) = s"${cls.getName}.$traitName"
def fieldNameInVertex(dataType: IDataType[_], aInfo: AttributeInfo) = GraphBackedMetadataRepository.getQualifiedName(dataType, aInfo.name)
def fieldNameInVertex(dataType: IDataType[_], aInfo: AttributeInfo) = GraphHelper.getQualifiedFieldName(dataType, aInfo.name)
def getIdFromVertex(dataTypeNm: String, v: TitanVertex): Id =
new Id(v.getId.toString, 0, dataTypeNm)
......
repository/src/test/java/org/apache/atlas/BaseHiveRepositoryTest.java
......@@ -21,10 +21,10 @@ import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import com.thinkaurelius.titan.core.TitanGraph;
import com.thinkaurelius.titan.core.util.TitanCleanup;
import org.apache.atlas.repository.graph.GraphBackedMetadataRepository;
import org.apache.atlas.repository.MetadataRepository;
import org.apache.atlas.repository.graph.GraphBackedSearchIndexer;
import org.apache.atlas.repository.graph.GraphProvider;
import org.apache.atlas.services.DefaultMetadataService;
import org.apache.atlas.services.MetadataService;
import org.apache.atlas.typesystem.ITypedReferenceableInstance;
import org.apache.atlas.typesystem.Referenceable;
import org.apache.atlas.typesystem.TypesDef;
......@@ -55,10 +55,10 @@ import java.util.List;
public class BaseHiveRepositoryTest {
@Inject
protected DefaultMetadataService metadataService;
protected MetadataService metadataService;
@Inject
protected GraphBackedMetadataRepository repository;
protected MetadataRepository repository;
@Inject
protected GraphProvider<TitanGraph> graphProvider;
......@@ -67,7 +67,7 @@ public class BaseHiveRepositoryTest {
setUpTypes();
new GraphBackedSearchIndexer(graphProvider);
setupInstances();
// TestUtils.dumpGraph(graphProvider.get());
TestUtils.dumpGraph(graphProvider.get());
}
protected void tearDown() throws Exception {
......@@ -190,17 +190,20 @@ public class BaseHiveRepositoryTest {
Id salesDB = database("Sales", "Sales Database", "John ETL", "hdfs://host:8000/apps/warehouse/sales");
Referenceable sd =
storageDescriptor("hdfs://host:8000/apps/warehouse/sales", "TextInputFormat", "TextOutputFormat", true, ImmutableList.of(column("time_id", "int", "time id")));
storageDescriptor("hdfs://host:8000/apps/warehouse/sales", "TextInputFormat", "TextOutputFormat", true, ImmutableList.of(
column("time_id", "int", "time id")));
List<Referenceable> salesFactColumns = ImmutableList
.of(column("time_id", "int", "time id"), column("product_id", "int", "product id"),
.of(column("time_id", "int", "time id"),
column("product_id", "int", "product id"),
column("customer_id", "int", "customer id", "PII"),
column("sales", "double", "product id", "Metric"));
Id salesFact = table("sales_fact", "sales fact table", salesDB, sd, "Joe", "Managed", salesFactColumns, "Fact");
List<Referenceable> timeDimColumns = ImmutableList
.of(column("time_id", "int", "time id"), column("dayOfYear", "int", "day Of Year"),
.of(column("time_id", "int", "time id"),
column("dayOfYear", "int", "day Of Year"),
column("weekDay", "int", "week Day"));
Id timeDim = table("time_dim", "time dimension table", salesDB, sd, "John Doe", "External", timeDimColumns,
......@@ -217,7 +220,8 @@ public class BaseHiveRepositoryTest {
ImmutableList.of(salesFactDaily), "create table as select ", "plan", "id", "graph", "ETL");
List<Referenceable> productDimColumns = ImmutableList
.of(column("product_id", "int", "product id"), column("product_name", "string", "product name"),
.of(column("product_id", "int", "product id"),
column("product_name", "string", "product name"),
column("brand_name", "int", "brand name"));
Id productDim =
......@@ -226,7 +230,8 @@ public class BaseHiveRepositoryTest {
view("product_dim_view", reportingDB, ImmutableList.of(productDim), "Dimension", "JdbcAccess");
List<Referenceable> customerDimColumns = ImmutableList.of(column("customer_id", "int", "customer id", "PII"),
List<Referenceable> customerDimColumns = ImmutableList.of(
column("customer_id", "int", "customer id", "PII"),
column("name", "string", "customer name", "PII"),
column("address", "string", "customer address", "PII"));
......
repository/src/test/java/org/apache/atlas/TestUtils.java
......@@ -187,6 +187,8 @@ public final class TestUtils {
public static final String DATABASE_TYPE = "hive_database";
public static final String DATABASE_NAME = "foo";
public static final String TABLE_TYPE = "hive_table";
public static final String PARTITION_TYPE = "partition_type";
public static final String SERDE_TYPE = "serdeType";
public static final String TABLE_NAME = "bar";
public static final String CLASSIFICATION = "classification";
public static final String PII = "PII";
......@@ -208,7 +210,8 @@ public final class TestUtils {
StructTypeDefinition structTypeDefinition = new StructTypeDefinition("serdeType",
new AttributeDefinition[]{createRequiredAttrDef("name", DataTypes.STRING_TYPE),
createRequiredAttrDef("serde", DataTypes.STRING_TYPE)});
createRequiredAttrDef("serde", DataTypes.STRING_TYPE),
createOptionalAttrDef("description", DataTypes.STRING_TYPE)});
EnumValue values[] = {new EnumValue("MANAGED", 1), new EnumValue("EXTERNAL", 2),};
......@@ -244,21 +247,23 @@ public final class TestUtils {
new AttributeDefinition("parametersMap",
DataTypes.mapTypeName(DataTypes.STRING_TYPE.getName(), DataTypes.STRING_TYPE.getName()),
Multiplicity.OPTIONAL, true, null),
// map of classes - todo - enable this
// new AttributeDefinition("columnsMap",
// DataTypes.mapTypeName(DataTypes.STRING_TYPE.getName(),
// "column_type"),
// Multiplicity.COLLECTION, true, null),
// map of structs todo - enable this
// new AttributeDefinition("partitionsMap",
// DataTypes.mapTypeName(DataTypes.STRING_TYPE.getName(),
// "partition_type"),
// Multiplicity.COLLECTION, true, null),
//map of classes -
new AttributeDefinition("columnsMap",
DataTypes.mapTypeName(DataTypes.STRING_TYPE.getName(),
"column_type"),
Multiplicity.COLLECTION, true, null),
//map of structs
new AttributeDefinition("partitionsMap",
DataTypes.mapTypeName(DataTypes.STRING_TYPE.getName(),
"partition_type"),
Multiplicity.COLLECTION, true, null),
// struct reference
new AttributeDefinition("serde1", "serdeType", Multiplicity.OPTIONAL, false, null),
new AttributeDefinition("serde2", "serdeType", Multiplicity.OPTIONAL, false, null),
// class reference
new AttributeDefinition("database", DATABASE_TYPE, Multiplicity.REQUIRED, true, null));
new AttributeDefinition("database", DATABASE_TYPE, Multiplicity.REQUIRED, false, null),
//class reference as composite
new AttributeDefinition("databaseComposite", DATABASE_TYPE, Multiplicity.OPTIONAL, true, null));
HierarchicalTypeDefinition<TraitType> piiTypeDefinition =
createTraitTypeDef(PII, ImmutableList.<String>of());
......
repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java
......@@ -24,7 +24,7 @@ import org.apache.atlas.BaseHiveRepositoryTest;
import org.apache.atlas.RepositoryMetadataModule;
import org.apache.atlas.TestUtils;
import org.apache.atlas.discovery.graph.GraphBackedDiscoveryService;
import org.apache.atlas.repository.graph.GraphBackedMetadataRepository;
import org.apache.atlas.repository.MetadataRepository;
import org.apache.atlas.repository.graph.GraphProvider;
import org.apache.atlas.typesystem.ITypedReferenceableInstance;
import org.apache.atlas.typesystem.Referenceable;
......@@ -55,7 +55,7 @@ public class GraphBackedDiscoveryServiceTest extends BaseHiveRepositoryTest {
private GraphProvider<TitanGraph> graphProvider;
@Inject
private GraphBackedMetadataRepository repositoryService;
private MetadataRepository repositoryService;
@Inject
private GraphBackedDiscoveryService discoveryService;
......
repository/src/test/java/org/apache/atlas/discovery/HiveLineageServiceTest.java
......@@ -20,8 +20,7 @@ package org.apache.atlas.discovery;
import org.apache.atlas.BaseHiveRepositoryTest;
import org.apache.atlas.RepositoryMetadataModule;
import org.apache.atlas.discovery.graph.GraphBackedDiscoveryService;
import org.apache.atlas.repository.EntityNotFoundException;
import org.apache.atlas.typesystem.exception.EntityNotFoundException;
import org.codehaus.jettison.json.JSONArray;
import org.codehaus.jettison.json.JSONObject;
import org.testng.Assert;
......@@ -32,7 +31,6 @@ import org.testng.annotations.Guice;
import org.testng.annotations.Test;
import javax.inject.Inject;
import java.util.List;
/**
* Unit tests for Hive LineageService.
......@@ -41,7 +39,7 @@ import java.util.List;
public class HiveLineageServiceTest extends BaseHiveRepositoryTest {
@Inject
private GraphBackedDiscoveryService discoveryService;
private DiscoveryService discoveryService;
@Inject
private HiveLineageService hiveLineageService;
......
repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryTest.java
......@@ -29,13 +29,13 @@ import org.apache.atlas.RepositoryMetadataModule;
import org.apache.atlas.TestUtils;
import org.apache.atlas.discovery.graph.GraphBackedDiscoveryService;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.EntityNotFoundException;
import org.apache.atlas.repository.RepositoryException;
import org.apache.atlas.typesystem.IStruct;
import org.apache.atlas.typesystem.ITypedReferenceableInstance;
import org.apache.atlas.typesystem.ITypedStruct;
import org.apache.atlas.typesystem.Referenceable;
import org.apache.atlas.typesystem.Struct;
import org.apache.atlas.typesystem.exception.EntityNotFoundException;
import org.apache.atlas.typesystem.persistence.Id;
import org.apache.atlas.typesystem.types.ClassType;
import org.apache.atlas.typesystem.types.DataTypes;
......@@ -131,7 +131,7 @@ public class GraphBackedMetadataRepositoryTest {
Assert.assertNotNull(entity);
}
@Test(expectedExceptions = RepositoryException.class)
@Test(expectedExceptions = EntityNotFoundException.class)
public void testGetEntityDefinitionNonExistent() throws Exception {
repositoryService.getEntityDefinition("blah");
Assert.fail();
......@@ -342,13 +342,13 @@ public class GraphBackedMetadataRepositoryTest {
}
Id expected = new Id(guid, tableVertex.<Integer>getProperty(Constants.VERSION_PROPERTY_KEY), TestUtils.TABLE_TYPE);
Assert.assertEquals(repositoryService.getIdFromVertex(TestUtils.TABLE_TYPE, tableVertex), expected);
Assert.assertEquals(GraphHelper.getIdFromVertex(TestUtils.TABLE_TYPE, tableVertex), expected);
}
@Test(dependsOnMethods = "testCreateEntity")
public void testGetTypeName() throws Exception {
Vertex tableVertex = getTableEntityVertex();
Assert.assertEquals(repositoryService.getTypeName(tableVertex), TestUtils.TABLE_TYPE);
Assert.assertEquals(GraphHelper.getTypeName(tableVertex), TestUtils.TABLE_TYPE);
}
@Test(dependsOnMethods = "testCreateEntity")
......@@ -415,6 +415,9 @@ public class GraphBackedMetadataRepositoryTest {
public void testBug37860() throws Exception {
String dslQuery = "hive_table as t where name = 'bar' "
+ "database where name = 'foo' and description = 'foo database' select t";
TestUtils.dumpGraph(graphProvider.get());
System.out.println("Executing dslQuery = " + dslQuery);
String jsonResults = discoveryService.searchByDSL(dslQuery);
Assert.assertNotNull(jsonResults);
......@@ -446,6 +449,8 @@ public class GraphBackedMetadataRepositoryTest {
//but with elasticsearch, doesn't work without sleep. why??
long sleepInterval = 1000;
TestUtils.dumpGraph(graphProvider.get());
//person in hr department whose name is john
Thread.sleep(sleepInterval);
String response = discoveryService.searchByFullText("john");
......@@ -475,31 +480,36 @@ public class GraphBackedMetadataRepositoryTest {
@Test(dependsOnMethods = "testSubmitEntity")
public void testUpdateEntity_MultiplicityOneNonCompositeReference() throws Exception {
ITypedReferenceableInstance john = repositoryService.getEntityDefinition("Person", "name", "John");
String johnGuid = john.getId()._getId();
Id johnGuid = john.getId();
ITypedReferenceableInstance max = repositoryService.getEntityDefinition("Person", "name", "Max");
String maxGuid = max.getId()._getId();
ITypedReferenceableInstance jane = repositoryService.getEntityDefinition("Person", "name", "Jane");
String janeGuid = jane.getId()._getId();
Id janeGuid = jane.getId();
// Update max's mentor reference to john.
repositoryService.updateEntity(maxGuid, "mentor", johnGuid);
ClassType personType = typeSystem.getDataType(ClassType.class, "Person");
ITypedReferenceableInstance instance = personType.createInstance(max.getId());
instance.set("mentor", johnGuid);
repositoryService.updatePartial(instance);
// Verify the update was applied correctly - john should now be max's mentor.
max = repositoryService.getEntityDefinition(maxGuid);
Object object = max.get("mentor");
Assert.assertTrue(object instanceof ITypedReferenceableInstance);
ITypedReferenceableInstance refTarget = (ITypedReferenceableInstance) object;
Assert.assertEquals(refTarget.getId()._getId(), johnGuid);
Assert.assertEquals(refTarget.getId()._getId(), johnGuid._getId());
// Update max's mentor reference to jane.
repositoryService.updateEntity(maxGuid, "mentor", janeGuid);
instance = personType.createInstance(max.getId());
instance.set("mentor", janeGuid);
repositoryService.updatePartial(instance);
// Verify the update was applied correctly - jane should now be max's mentor.
max = repositoryService.getEntityDefinition(maxGuid);
object = max.get("mentor");
Assert.assertTrue(object instanceof ITypedReferenceableInstance);
refTarget = (ITypedReferenceableInstance) object;
Assert.assertEquals(refTarget.getId()._getId(), janeGuid);
Assert.assertEquals(refTarget.getId()._getId(), janeGuid._getId());
}
private ITypedReferenceableInstance createHiveTableInstance(Referenceable databaseInstance) throws Exception {
......
repository/src/test/java/org/apache/atlas/repository/graph/GraphRepoMapperScaleTest.java
......@@ -18,15 +18,9 @@
package org.apache.atlas.repository.graph;
import com.google.inject.Inject;
import com.thinkaurelius.titan.core.TitanFactory;
import com.thinkaurelius.titan.core.TitanGraph;
import com.thinkaurelius.titan.core.TitanIndexQuery;
import com.thinkaurelius.titan.core.util.TitanCleanup;
import com.thinkaurelius.titan.diskstorage.BackendException;
import com.thinkaurelius.titan.diskstorage.configuration.ReadConfiguration;
import com.thinkaurelius.titan.diskstorage.configuration.backend.CommonsConfiguration;
import com.thinkaurelius.titan.graphdb.configuration.GraphDatabaseConfiguration;
import com.tinkerpop.blueprints.Compare;
import com.tinkerpop.blueprints.GraphQuery;
import com.tinkerpop.blueprints.Predicate;
......@@ -42,19 +36,16 @@ import org.apache.atlas.typesystem.types.ClassType;
import org.apache.atlas.typesystem.types.IDataType;
import org.apache.atlas.typesystem.types.Multiplicity;
import org.apache.atlas.typesystem.types.TypeSystem;
import org.apache.commons.io.FileUtils;
import org.testng.Assert;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Guice;
import org.testng.annotations.Test;
import java.io.File;
import java.io.IOException;
import javax.inject.Inject;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Date;
import java.util.Random;
@Test
@Guice(modules = RepositoryMetadataModule.class)
......@@ -63,15 +54,13 @@ public class GraphRepoMapperScaleTest {
private static final String DATABASE_NAME = "foo";
private static final String TABLE_NAME = "bar";
private static final String INDEX_DIR =
System.getProperty("java.io.tmpdir", "/tmp") + "/atlas-test" + new Random().nextLong();
@Inject
GraphProvider<TitanGraph> graphProvider;
@Inject
private GraphBackedMetadataRepository repositoryService;
@Inject
private GraphBackedSearchIndexer searchIndexer;
private TypeSystem typeSystem = TypeSystem.getInstance();
......@@ -81,7 +70,7 @@ public class GraphRepoMapperScaleTest {
@BeforeClass
@GraphTransaction
public void setUp() throws Exception {
searchIndexer = new GraphBackedSearchIndexer(graphProvider);
//Make sure we can cleanup the index directory
Collection<IDataType> typesAdded = TestUtils.createHiveTypes(typeSystem);
searchIndexer.onAdd(typesAdded);
}
......@@ -127,7 +116,6 @@ public class GraphRepoMapperScaleTest {
//Elasticsearch requires some time before index is updated
Thread.sleep(5000);
searchWithOutIndex(Constants.GUID_PROPERTY_KEY, dbGUID);
searchWithOutIndex(Constants.ENTITY_TYPE_PROPERTY_KEY, "column_type");
searchWithOutIndex(Constants.ENTITY_TYPE_PROPERTY_KEY, TestUtils.TABLE_TYPE);
......
repository/src/test/java/org/apache/atlas/service/DefaultMetadataServiceTest.java
......@@ -18,29 +18,40 @@
package org.apache.atlas.service;
import com.google.common.collect.ImmutableList;
import com.google.inject.Inject;
import com.thinkaurelius.titan.core.TitanGraph;
import com.thinkaurelius.titan.core.util.TitanCleanup;
import org.apache.atlas.typesystem.exception.TypeNotFoundException;
import org.apache.atlas.typesystem.exception.EntityNotFoundException;
import org.apache.atlas.utils.ParamChecker;
import org.apache.atlas.RepositoryMetadataModule;
import org.apache.atlas.TestUtils;
import org.apache.atlas.TypeNotFoundException;
import org.apache.atlas.repository.EntityNotFoundException;
import org.apache.atlas.repository.graph.GraphProvider;
import org.apache.atlas.services.MetadataService;
import org.apache.atlas.typesystem.Referenceable;
import org.apache.atlas.typesystem.Struct;
import org.apache.atlas.typesystem.TypesDef;
import org.apache.atlas.typesystem.json.InstanceSerialization;
import org.apache.atlas.typesystem.json.TypesSerialization;
import org.apache.atlas.typesystem.types.EnumType;
import org.apache.atlas.typesystem.persistence.Id;
import org.apache.atlas.typesystem.types.EnumValue;
import org.apache.atlas.typesystem.types.TypeSystem;
import org.apache.atlas.typesystem.types.ValueConversionException;
import org.apache.commons.lang.RandomStringUtils;
import org.codehaus.jettison.json.JSONArray;
import org.testng.Assert;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.AfterTest;
import org.testng.annotations.BeforeTest;
import org.testng.annotations.Guice;
import org.testng.annotations.Test;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@Guice(modules = RepositoryMetadataModule.class)
public class DefaultMetadataServiceTest {
@Inject
......@@ -48,8 +59,16 @@ public class DefaultMetadataServiceTest {
@Inject
private GraphProvider<TitanGraph> graphProvider;
private Referenceable db = createDBEntity();
private Id dbId;
private Referenceable table;
private Id tableId;
@BeforeClass
@BeforeTest
public void setUp() throws Exception {
TypesDef typesDef = TestUtils.defineHiveTypes();
try {
......@@ -57,10 +76,21 @@ public class DefaultMetadataServiceTest {
} catch (TypeNotFoundException e) {
metadataService.createType(TypesSerialization.toJson(typesDef));
}
String dbGUid = createInstance(db);
dbId = new Id(dbGUid, 0, TestUtils.DATABASE_TYPE);
table = createTableEntity(dbId);
String tableGuid = createInstance(table);
String tableDefinitionJson =
metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, "name", (String) table.get("name"));
table = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true);
tableId = new Id(tableGuid, 0, TestUtils.TABLE_TYPE);
}
@AfterClass
public void shudown() {
@AfterTest
public void shutdown() {
TypeSystem.getInstance().reset();
try {
//TODO - Fix failure during shutdown while using BDB
graphProvider.get().shutdown();
......@@ -82,6 +112,16 @@ public class DefaultMetadataServiceTest {
return new JSONArray(response).getString(0);
}
private String updateInstance(Referenceable entity) throws Exception {
ParamChecker.notNull(entity, "Entity");
ParamChecker.notNull(entity.getId(), "Entity");
String entityjson = InstanceSerialization.toJson(entity, true);
JSONArray entitiesJson = new JSONArray();
entitiesJson.put(entityjson);
String response = metadataService.updateEntities(entitiesJson.toString());
return new JSONArray(response).getString(0);
}
private Referenceable createDBEntity() {
Referenceable entity = new Referenceable(TestUtils.DATABASE_TYPE);
String dbName = RandomStringUtils.randomAlphanumeric(10);
......@@ -90,6 +130,18 @@ public class DefaultMetadataServiceTest {
return entity;
}
private Referenceable createTableEntity(Id dbId) {
Referenceable entity = new Referenceable(TestUtils.TABLE_TYPE);
String tableName = RandomStringUtils.randomAlphanumeric(10);
entity.set("name", tableName);
entity.set("description", "random table");
entity.set("type", "type");
entity.set("tableType", "MANAGED");
entity.set("database", dbId);
entity.set("created", new Date());
return entity;
}
@Test
public void testCreateEntityWithUniqueAttribute() throws Exception {
//name is the unique attribute
......@@ -116,30 +168,447 @@ public class DefaultMetadataServiceTest {
table.set("description", "random table");
table.set("type", "type");
table.set("tableType", "MANAGED");
table.set("database", db);
table.set("database", new Id(dbId, 0, TestUtils.DATABASE_TYPE));
table.set("databaseComposite", db);
createInstance(table);
//table create should re-use the db instance created earlier
String tableDefinitionJson =
metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, "name", (String) table.get("name"));
Referenceable tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true);
Referenceable actualDb = (Referenceable) tableDefinition.get("database");
Referenceable actualDb = (Referenceable) tableDefinition.get("databaseComposite");
Assert.assertEquals(actualDb.getId().id, dbId);
}
@Test
public void testCreateEntityWithEnum() throws Exception {
Referenceable dbEntity = createDBEntity();
String db = createInstance(dbEntity);
public void testUpdateEntityByUniqueAttribute() throws Exception {
final List<String> colNameList = ImmutableList.of("col1", "col2");
Referenceable tableUpdated = new Referenceable(TestUtils.TABLE_TYPE, new HashMap<String, Object>() {{
put("columnNames", colNameList);
}});
metadataService.updateEntityByUniqueAttribute(table.getTypeName(), "name", (String) table.get("name"), tableUpdated);
Referenceable table = new Referenceable(TestUtils.TABLE_TYPE);
table.set("name", TestUtils.randomString());
table.set("description", "random table");
table.set("type", "type");
table.set("tableType", "MANAGED");
table.set("database", dbEntity);
createInstance(table);
String tableDefinitionJson =
metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, "name", (String) table.get("name"));
Referenceable tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true);
List<String> actualColumns = (List) tableDefinition.get("columnNames");
Assert.assertEquals(actualColumns, colNameList);
}
@Test
public void testUpdateEntityWithMap() throws Exception {
final Map<String, Struct> partsMap = new HashMap<>();
partsMap.put("part0", new Struct("partition_type",
new HashMap<String, Object>() {{
put("name", "test");
}}));
table.set("partitionsMap", partsMap);
updateInstance(table);
String tableDefinitionJson =
metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, "name", (String) table.get("name"));
Referenceable tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true);
Assert.assertTrue(partsMap.get("part0").equalsContents(((Map<String, Struct>)tableDefinition.get("partitionsMap")).get("part0")));
//update map - add a map key
partsMap.put("part1", new Struct("partition_type",
new HashMap<String, Object>() {{
put("name", "test1");
}}));
table.set("partitionsMap", partsMap);
updateInstance(table);
tableDefinitionJson =
metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, "name", (String) table.get("name"));
tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true);
Assert.assertEquals(((Map<String, Struct>)tableDefinition.get("partitionsMap")).size(), 2);
Assert.assertTrue(partsMap.get("part1").equalsContents(((Map<String, Struct>)tableDefinition.get("partitionsMap")).get("part1")));
//update map - remove a key and add another key
partsMap.remove("part0");
partsMap.put("part2", new Struct("partition_type",
new HashMap<String, Object>() {{
put("name", "test2");
}}));
table.set("partitionsMap", partsMap);
updateInstance(table);
tableDefinitionJson =
metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, "name", (String) table.get("name"));
tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true);
Assert.assertEquals(((Map<String, Struct>)tableDefinition.get("partitionsMap")).size(), 2);
Assert.assertNull(((Map<String, Struct>)tableDefinition.get("partitionsMap")).get("part0"));
Assert.assertTrue(partsMap.get("part2").equalsContents(((Map<String, Struct>)tableDefinition.get("partitionsMap")).get("part2")));
//update struct value for existing map key
Struct partition2 = (Struct)partsMap.get("part2");
partition2.set("name", "test2Updated");
updateInstance(table);
tableDefinitionJson =
metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, "name", (String) table.get("name"));
tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true);
Assert.assertEquals(((Map<String, Struct>)tableDefinition.get("partitionsMap")).size(), 2);
Assert.assertNull(((Map<String, Struct>)tableDefinition.get("partitionsMap")).get("part0"));
Assert.assertTrue(partsMap.get("part2").equalsContents(((Map<String, Struct>)tableDefinition.get("partitionsMap")).get("part2")));
}
@Test
public void testUpdateEntityAddAndUpdateArrayAttr() throws Exception {
//Update entity, add new array attribute
//add array of primitives
final List<String> colNameList = ImmutableList.of("col1", "col2");
Referenceable tableUpdated = new Referenceable(TestUtils.TABLE_TYPE, new HashMap<String, Object>() {{
put("columnNames", colNameList);
}});
metadataService.updateEntityPartialByGuid(tableId._getId(), tableUpdated);
String tableDefinitionJson =
metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, "name", (String) table.get("name"));
Referenceable tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true);
List<String> actualColumns = (List) tableDefinition.get("columnNames");
Assert.assertEquals(actualColumns, colNameList);
//update array of primitives
final List<String> updatedColNameList = ImmutableList.of("col2", "col3");
tableUpdated = new Referenceable(TestUtils.TABLE_TYPE, new HashMap<String, Object>() {{
put("columnNames", updatedColNameList);
}});
metadataService.updateEntityPartialByGuid(tableId.getId()._getId(), tableUpdated);
tableDefinitionJson =
metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, "name", (String) table.get("name"));
tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true);
actualColumns = (List) tableDefinition.get("columnNames");
Assert.assertEquals(actualColumns, updatedColNameList);
}
@Test
public void testUpdateEntityArrayOfClass() throws Exception {
//test array of class with id
final List<Referenceable> columns = new ArrayList<>();
Map<String, Object> values = new HashMap<>();
values.put("name", "col1");
values.put("type", "type");
Referenceable ref = new Referenceable("column_type", values);
columns.add(ref);
Referenceable tableUpdated = new Referenceable(TestUtils.TABLE_TYPE, new HashMap<String, Object>() {{
put("columns", columns);
}});
metadataService.updateEntityPartialByGuid(tableId._getId(), tableUpdated);
String tableDefinitionJson =
metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, "name", (String) table.get("name"));
Referenceable tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true);
final List<Referenceable> arrClsColumns = (List) tableDefinition.get("columns");
Assert.assertTrue(arrClsColumns.get(0).equalsContents(columns.get(0)));
//Partial update. Add col5 But also update col1
Map<String, Object> valuesCol5 = new HashMap<>();
valuesCol5.put("name", "col5");
valuesCol5.put("type", "type");
ref = new Referenceable("column_type", valuesCol5);
//update col1
arrClsColumns.get(0).set("type", "type1");
//add col5
final List<Referenceable> updateColumns = new ArrayList<>(arrClsColumns);
updateColumns.add(ref);
tableUpdated = new Referenceable(TestUtils.TABLE_TYPE, new HashMap<String, Object>() {{
put("columns", updateColumns);
}});
metadataService.updateEntityPartialByGuid(tableId._getId(), tableUpdated);
tableDefinitionJson =
metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, "name", (String) table.get("name"));
tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true);
List<Referenceable> arrColumnsList = (List) tableDefinition.get("columns");
Assert.assertEquals(arrColumnsList.size(), 2);
Assert.assertTrue(arrColumnsList.get(0).equalsContents(updateColumns.get(0)));
Assert.assertTrue(arrColumnsList.get(1).equalsContents(updateColumns.get(1)));
//Complete update. Add array elements - col3,4
Map<String, Object> values1 = new HashMap<>();
values1.put("name", "col3");
values1.put("type", "type");
Referenceable ref1 = new Referenceable("column_type", values1);
columns.add(ref1);
Map<String, Object> values2 = new HashMap<>();
values2.put("name", "col4");
values2.put("type", "type");
Referenceable ref2 = new Referenceable("column_type", values2);
columns.add(ref2);
table.set("columns", columns);
updateInstance(table);
tableDefinitionJson =
metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, "name", (String) table.get("name"));
tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true);
arrColumnsList = (List) tableDefinition.get("columns");
Assert.assertEquals(arrColumnsList.size(), columns.size());
Assert.assertTrue(arrColumnsList.get(1).equalsContents(columns.get(1)));
Assert.assertTrue(arrColumnsList.get(2).equalsContents(columns.get(2)));
//Remove a class reference/Id and insert another reference
//Also covers isComposite case since columns is a composite
values.clear();
columns.clear();
values.put("name", "col2");
values.put("type", "type");
ref = new Referenceable("column_type", values);
columns.add(ref);
table.set("columns", columns);
updateInstance(table);
tableDefinitionJson =
metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, "name", (String) table.get("name"));
tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true);
arrColumnsList = (List) tableDefinition.get("columns");
Assert.assertEquals(arrColumnsList.size(), columns.size());
Assert.assertTrue(arrColumnsList.get(0).equalsContents(columns.get(0)));
//Update array column to null
table.setNull("columns");
String newtableId = updateInstance(table);
Assert.assertEquals(newtableId, tableId._getId());
tableDefinitionJson =
metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, "name", (String) table.get("name"));
tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true);
Assert.assertNull(tableDefinition.get("columns"));
}
@Test
public void testStructs() throws Exception {
Struct serdeInstance = new Struct(TestUtils.SERDE_TYPE);
serdeInstance.set("name", "serde1Name");
serdeInstance.set("serde", "test");
serdeInstance.set("description", "testDesc");
table.set("serde1", serdeInstance);
String newtableId = updateInstance(table);
Assert.assertEquals(newtableId, tableId._getId());
String tableDefinitionJson =
metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, "name", (String) table.get("name"));
Referenceable tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true);
Assert.assertNotNull(tableDefinition.get("serde1"));
Assert.assertTrue(serdeInstance.equalsContents(tableDefinition.get("serde1")));
//update struct attribute
serdeInstance.set("serde", "testUpdated");
updateInstance(table);
tableDefinitionJson =
metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, "name", (String) table.get("name"));
tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true);
Assert.assertTrue(serdeInstance.equalsContents(tableDefinition.get("serde1")));
//set to null
serdeInstance.setNull("description");
updateInstance(table);
tableDefinitionJson =
metadataService.getEntityDefinition(tableId._getId());
tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true);
Assert.assertNull(((Struct)tableDefinition.get("serde1")).get("description"));
}
@Test
public void testClassUpdate() throws Exception {
//Create new db instance
final Referenceable databaseInstance = new Referenceable(TestUtils.DATABASE_TYPE);
databaseInstance.set("name", TestUtils.randomString());
databaseInstance.set("description", "new database");
String dbId = createInstance(databaseInstance);
/*Update reference property with Id */
metadataService.updateEntityAttributeByGuid(tableId._getId(), "database", dbId);
String tableDefinitionJson =
metadataService.getEntityDefinition(tableId._getId());
Referenceable tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true);
Assert.assertEquals(dbId, (((Id)tableDefinition.get("database"))._getId()));
/* Update with referenceable - TODO - Fails . Need to fix this */
/*final String dbName = TestUtils.randomString();
final Referenceable databaseInstance2 = new Referenceable(TestUtils.DATABASE_TYPE);
databaseInstance2.set("name", dbName);
databaseInstance2.set("description", "new database 2");
Referenceable updateTable = new Referenceable(TestUtils.TABLE_TYPE, new HashMap<String, Object>() {{
put("database", databaseInstance2);
}});
metadataService.updateEntityAttributeByGuid(tableId._getId(), updateTable);
tableDefinitionJson =
metadataService.getEntityDefinition(tableId._getId());
Referenceable tableDefinitionActual = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true);
String dbDefJson = metadataService.getEntityDefinition(TestUtils.DATABASE_TYPE, "name", dbName);
Referenceable dbDef = InstanceSerialization.fromJsonReferenceable(dbDefJson, true);
Assert.assertNotEquals(dbId, (((Id) tableDefinitionActual.get("database"))._getId()));
Assert.assertEquals(dbDef.getId()._getId(), (((Id) tableDefinitionActual.get("database"))._getId())); */
}
@Test
public void testArrayOfStructs() throws Exception {
//Add array of structs
TestUtils.dumpGraph(graphProvider.get());
final Struct partition1 = new Struct(TestUtils.PARTITION_TYPE);
partition1.set("name", "part1");
final Struct partition2 = new Struct(TestUtils.PARTITION_TYPE);
partition2.set("name", "part2");
List<Struct> partitions = new ArrayList<Struct>(){{ add(partition1); add(partition2); }};
table.set("partitions", partitions);
String newtableId = updateInstance(table);
Assert.assertEquals(newtableId, tableId._getId());
String tableDefinitionJson =
metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, "name", (String) table.get("name"));
Referenceable tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true);
Assert.assertNotNull(tableDefinition.get("partitions"));
List<Struct> partitionsActual = (List<Struct>) tableDefinition.get("partitions");
Assert.assertEquals(partitionsActual.size(), 2);
Assert.assertTrue(partitions.get(0).equalsContents(partitionsActual.get(0)));
//add a new element to array of struct
final Struct partition3 = new Struct(TestUtils.PARTITION_TYPE);
partition3.set("name", "part3");
partitions.add(partition3);
table.set("partitions", partitions);
newtableId = updateInstance(table);
Assert.assertEquals(newtableId, tableId._getId());
tableDefinitionJson =
metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, "name", (String) table.get("name"));
tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true);
Assert.assertNotNull(tableDefinition.get("partitions"));
partitionsActual = (List<Struct>) tableDefinition.get("partitions");
Assert.assertEquals(partitionsActual.size(), 3);
Assert.assertTrue(partitions.get(2).equalsContents(partitionsActual.get(2)));
//remove one of the struct values
partitions.remove(1);
table.set("partitions", partitions);
newtableId = updateInstance(table);
Assert.assertEquals(newtableId, tableId._getId());
tableDefinitionJson =
metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, "name", (String) table.get("name"));
tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true);
Assert.assertNotNull(tableDefinition.get("partitions"));
partitionsActual = (List<Struct>) tableDefinition.get("partitions");
Assert.assertEquals(partitionsActual.size(), 2);
Assert.assertTrue(partitions.get(0).equalsContents(partitionsActual.get(0)));
Assert.assertTrue(partitions.get(1).equalsContents(partitionsActual.get(1)));
//Update struct value within array of struct
partition1.set("name", "part4");
newtableId = updateInstance(table);
Assert.assertEquals(newtableId, tableId._getId());
tableDefinitionJson =
metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, "name", (String) table.get("name"));
tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true);
Assert.assertNotNull(tableDefinition.get("partitions"));
partitionsActual = (List<Struct>) tableDefinition.get("partitions");
Assert.assertEquals(partitionsActual.size(), 2);
Assert.assertTrue(partitions.get(0).equalsContents(partitionsActual.get(0)));
//add a repeated element to array of struct
final Struct partition4 = new Struct(TestUtils.PARTITION_TYPE);
partition4.set("name", "part4");
partitions.add(partition4);
table.set("partitions", partitions);
newtableId = updateInstance(table);
Assert.assertEquals(newtableId, tableId._getId());
tableDefinitionJson =
metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, "name", (String) table.get("name"));
tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true);
Assert.assertNotNull(tableDefinition.get("partitions"));
partitionsActual = (List<Struct>) tableDefinition.get("partitions");
Assert.assertEquals(partitionsActual.size(), 3);
Assert.assertEquals(partitionsActual.get(2).get("name"), "part4");
Assert.assertEquals(partitionsActual.get(0).get("name"), "part4");
Assert.assertTrue(partitions.get(2).equalsContents(partitionsActual.get(2)));
// Remove all elements. Should set array attribute to null
partitions.clear();
newtableId = updateInstance(table);
Assert.assertEquals(newtableId, tableId._getId());
tableDefinitionJson =
metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, "name", (String) table.get("name"));
tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true);
Assert.assertNull(tableDefinition.get("partitions"));
}
@Test(expectedExceptions = ValueConversionException.class)
public void testUpdateRequiredAttrToNull() throws Exception {
//Update required attribute
String tableDefinitionJson =
metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, "name", (String) table.get("name"));
Referenceable tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true);
Assert.assertEquals(tableDefinition.get("description"), "random table");
table.setNull("description");
updateInstance(table);
Assert.fail("Expected exception while updating required attribute to null");
}
@Test
public void testUpdateOptionalAttrToNull() throws Exception {
String tableDefinitionJson =
metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, "name", (String) table.get("name"));
Referenceable tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true);
//Update optional Attribute
Assert.assertNotNull(tableDefinition.get("created"));
//Update optional attribute
table.setNull("created");
String newtableId = updateInstance(table);
Assert.assertEquals(newtableId, tableId._getId());
tableDefinitionJson =
metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, "name", (String) table.get("name"));
tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true);
Assert.assertNull(tableDefinition.get("created"));
}
@Test
public void testCreateEntityWithEnum() throws Exception {
String tableDefinitionJson =
metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, "name", (String) table.get("name"));
Referenceable tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true);
......
repository/src/test/scala/org/apache/atlas/query/GremlinTest.scala
......@@ -39,7 +39,6 @@ class GremlinTest extends BaseGremlinTest {
gProvider = new TitanGraphProvider();
gp = new DefaultGraphPersistenceStrategy(new GraphBackedMetadataRepository(gProvider))
g = QueryTestsUtils.setupTestGraph(gProvider)
}
@AfterClass
......
server-api/pom.xml 0 → 100644
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one
~ or more contributor license agreements. See the NOTICE file
~ distributed with this work for additional information
~ regarding copyright ownership. The ASF licenses this file
~ to you under the Apache License, Version 2.0 (the
~ "License"); you may not use this file except in compliance
~ with the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>apache-atlas</artifactId>
<groupId>org.apache.atlas</groupId>
<version>0.6-incubating-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>atlas-server-api</artifactId>
<name>Apache Atlas Server API</name>
<description>Apache Atlas Server related APIs</description>
<packaging>jar</packaging>
<dependencies>
<dependency>
<groupId>org.testng</groupId>
<artifactId>testng</artifactId>
</dependency>
<dependency>
<groupId>com.google.inject</groupId>
<artifactId>guice</artifactId>
</dependency>
<dependency>
<groupId>org.apache.atlas</groupId>
<artifactId>atlas-typesystem</artifactId>
</dependency>
</dependencies>
</project>
\ No newline at end of file
common/src/main/java/org/apache/atlas/listener/EntityChangeListener.java → server-api/src/main/java/org/apache/atlas/listener/EntityChangeListener.java
......@@ -41,11 +41,11 @@ public interface EntityChangeListener {
/**
* This is upon updating an entity.
*
* @param entity the updated entity
* @param entities the updated entities
*
* @throws AtlasException if the listener notification fails
*/
void onEntityUpdated(ITypedReferenceableInstance entity) throws AtlasException;
void onEntitiesUpdated(Collection<ITypedReferenceableInstance> entities) throws AtlasException;
/**
* This is upon adding a new trait to a typed instance.
......
repository/src/main/java/org/apache/atlas/services/MetadataService.java → server-api/src/main/java/org/apache/atlas/services/MetadataService.java 100755 → 100644
......@@ -20,6 +20,7 @@ package org.apache.atlas.services;
import org.apache.atlas.AtlasException;
import org.apache.atlas.listener.EntityChangeListener;
import org.apache.atlas.typesystem.Referenceable;
import org.apache.atlas.typesystem.types.DataTypes;
import org.codehaus.jettison.json.JSONObject;
......@@ -105,16 +106,45 @@ public interface MetadataService {
/**
* Adds the property to the given entity id(guid).
* Currently supports updates only on PRIMITIVE, CLASS attribute types
*
* @param guid entity id
* @param property property name
* @param attribute property name
* @param value property value
*/
void updateEntity(String guid, String property, String value) throws AtlasException;
void updateEntityAttributeByGuid(String guid, String attribute, String value) throws AtlasException;
/**
* Supports Partial updates of an entity. Users can update a subset of attributes for an entity identified by its guid
* Note however that it cannot be used to set attribute values to null or delete attrbute values
*
*/
void updateEntityPartialByGuid(String guid, Referenceable entity) throws AtlasException;
/**
* Batch API - Adds/Updates the given entity id(guid).
*
* @param entityJson entity json
* @return List of guids which were updated and ones which were newly created as part of the updated entity
*/
String updateEntities(String entityJson) throws AtlasException;
// Trait management functions
/**
* Updates entity identified by a qualified name
*
* @param typeName
* @param uniqueAttributeName
* @param attrValue
* @param updatedEntity
* @return Guid of updated entity
* @throws AtlasException
*/
String updateEntityByUniqueAttribute(String typeName, String uniqueAttributeName, String attrValue,
Referenceable updatedEntity) throws AtlasException;
/**
* Gets the list of trait names for a given entity represented by a guid.
*
* @param guid globally unique identifier for the entity
......
repository/src/main/java/org/apache/atlas/repository/EntityExistsException.java → server-api/src/main/java/org/apache/atlas/typesystem/exception/EntityExistsException.java
......@@ -16,7 +16,7 @@
* limitations under the License.
*/
package org.apache.atlas.repository;
package org.apache.atlas.typesystem.exception;
import org.apache.atlas.AtlasException;
import org.apache.atlas.typesystem.IReferenceableInstance;
......
repository/src/main/java/org/apache/atlas/repository/EntityNotFoundException.java → server-api/src/main/java/org/apache/atlas/typesystem/exception/EntityNotFoundException.java
......@@ -16,12 +16,14 @@
* limitations under the License.
*/
package org.apache.atlas.repository;
package org.apache.atlas.typesystem.exception;
import org.apache.atlas.AtlasException;
/**
* A simple wrapper for 404.
*/
public class EntityNotFoundException extends RepositoryException {
public class EntityNotFoundException extends AtlasException {
public EntityNotFoundException() {
}
......
typesystem/pom.xml
......@@ -110,13 +110,8 @@
</dependency>
<dependency>
<groupId>com.google.inject</groupId>
<artifactId>guice</artifactId>
</dependency>
<dependency>
<groupId>commons-configuration</groupId>
<artifactId>commons-configuration</artifactId>
<groupId>org.apache.atlas</groupId>
<artifactId>atlas-common</artifactId>
</dependency>
<dependency>
......
typesystem/src/main/java/org/apache/atlas/typesystem/IInstance.java
......@@ -33,6 +33,8 @@ public interface IInstance {
void set(String attrName, Object val) throws AtlasException;
void setNull(String attrName) throws AtlasException;
Map<String, Object> getValuesMap() throws AtlasException;
}
typesystem/src/main/java/org/apache/atlas/typesystem/ITypedInstance.java
......@@ -23,6 +23,7 @@ import org.apache.atlas.typesystem.types.FieldMapping;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.security.MessageDigest;
import java.util.Date;
/**
......@@ -37,8 +38,6 @@ public interface ITypedInstance extends IInstance {
FieldMapping fieldMapping();
void setNull(String attrName) throws AtlasException;
boolean getBoolean(String attrName) throws AtlasException;
byte getByte(String attrName) throws AtlasException;
......@@ -82,4 +81,6 @@ public interface ITypedInstance extends IInstance {
void setDate(String attrName, Date val) throws AtlasException;
void setString(String attrName, String val) throws AtlasException;
String getSignatureHash(MessageDigest digester) throws AtlasException;
}
typesystem/src/main/java/org/apache/atlas/typesystem/Referenceable.java
......@@ -24,7 +24,6 @@ import org.apache.atlas.AtlasException;
import org.apache.atlas.classification.InterfaceAudience;
import org.apache.atlas.typesystem.persistence.Id;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
......@@ -95,8 +94,10 @@ public class Referenceable extends Struct implements IReferenceableInstance {
*/
@SuppressWarnings("unused")
private Referenceable() {
this("", "", Collections.<String, Object>emptyMap(), Collections.<String>emptyList(),
Collections.<String, IStruct>emptyMap());
super(null, null);
id = null;
traitNames = ImmutableList.of();
traits = ImmutableMap.of();
}
@Override
......@@ -114,6 +115,42 @@ public class Referenceable extends Struct implements IReferenceableInstance {
return traits.get(typeName);
}
/**
* Matches traits, values associated with this Referenceable and skips the id match
* @param o The Referenceable which needs to be matched with
* @return
*/
public boolean equalsContents(Object o) {
if(this == o) {
return true;
}
if(o == null) {
return false;
}
if (o.getClass() != getClass()) {
return false;
}
if(!super.equalsContents(o)) {
return false;
}
Referenceable obj = (Referenceable)o;
if (!traitNames.equals(obj.getTraits())) {
return false;
}
return true;
}
public String toString() {
return "{" +
"Id='" + id + '\'' +
", traits=" + traitNames +
", values=" + getValuesMap() +
'}';
}
private static Map<String, IStruct> getTraits(IReferenceableInstance instance) throws AtlasException {
Map<String, IStruct> traits = new HashMap<>();
for (String traitName : instance.getTraits() ) {
......
typesystem/src/main/java/org/apache/atlas/typesystem/Struct.java
......@@ -18,6 +18,7 @@
package org.apache.atlas.typesystem;
import org.apache.atlas.AtlasException;
import org.apache.atlas.classification.InterfaceAudience;
import java.util.Collections;
......@@ -67,7 +68,52 @@ public class Struct implements IStruct {
}
@Override
public void setNull(String attrName) throws AtlasException {
values.remove(attrName);
}
@Override
public Map<String, Object> getValuesMap() {
return values;
}
@Override
public int hashCode() {
int result = typeName.hashCode();
result = 31 * result + values.hashCode();
return result;
}
/**
* equalContents instead of equals since values is a mutable attribute and could lead
* to incorrect behaviour when added to collections and mutated after that
* i.e when the attribute is mutated collections.contains(struct) returns false
* due to hashcode having changed for the struct.
* @param o
* @return
*/
public boolean equalsContents(Object o) {
if (this == o) {
return true;
}
if (o == null) {
return false;
}
if (o.getClass() != getClass()) {
return false;
}
Struct obj = (Struct)o;
if(!typeName.equals(obj.getTypeName())) {
return false;
}
if(!values.equals(obj.getValuesMap())) {
return false;
}
return true;
}
}
typesystem/src/main/java/org/apache/atlas/TypeExistsException.java → typesystem/src/main/java/org/apache/atlas/typesystem/exception/TypeExistsException.java
......@@ -16,7 +16,9 @@
* limitations under the License.
*/
package org.apache.atlas;
package org.apache.atlas.typesystem.exception;
import org.apache.atlas.AtlasException;
public class TypeExistsException extends AtlasException {
public TypeExistsException(String message) {
......
typesystem/src/main/java/org/apache/atlas/TypeNotFoundException.java → typesystem/src/main/java/org/apache/atlas/typesystem/exception/TypeNotFoundException.java
......@@ -16,7 +16,9 @@
* limitations under the License.
*/
package org.apache.atlas;
package org.apache.atlas.typesystem.exception;
import org.apache.atlas.AtlasException;
/**
* A simple wrapper for 404.
......
typesystem/src/main/java/org/apache/atlas/typesystem/persistence/DownCastStructInstance.java
......@@ -52,6 +52,11 @@ public class DownCastStructInstance implements IStruct {
fieldMapping.set(this, attrName, val);
}
@Override
public void setNull(String attrName) throws AtlasException {
throw new UnsupportedOperationException("unset on attributes are not allowed");
}
/*
* Use only for json serialization
* @nonpublic
......
typesystem/src/main/java/org/apache/atlas/typesystem/persistence/Id.java
......@@ -20,12 +20,16 @@ package org.apache.atlas.typesystem.persistence;
import com.google.common.collect.ImmutableList;
import org.apache.atlas.AtlasException;
import org.apache.atlas.utils.ParamChecker;
import org.apache.atlas.typesystem.IStruct;
import org.apache.atlas.typesystem.ITypedReferenceableInstance;
import org.apache.atlas.typesystem.types.FieldMapping;
import org.apache.atlas.utils.MD5Utils;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.nio.charset.Charset;
import java.security.MessageDigest;
import java.util.Date;
import java.util.Map;
import java.util.UUID;
......@@ -37,6 +41,8 @@ public class Id implements ITypedReferenceableInstance {
public final int version;
public Id(String id, int version, String className) {
ParamChecker.notEmpty(className, "id");
ParamChecker.notEmpty(className, "className");
this.id = id;
this.className = className;
this.version = version;
......@@ -248,4 +254,12 @@ public class Id implements ITypedReferenceableInstance {
public void setString(String attrName, String val) throws AtlasException {
throw new AtlasException("Get/Set not supported on an Id object");
}
@Override
public String getSignatureHash(MessageDigest digester) throws AtlasException {
digester.update(id.getBytes(Charset.forName("UTF-8")));
digester.update(className.getBytes(Charset.forName("UTF-8")));
byte[] digest = digester.digest();
return MD5Utils.toString(digest);
}
}
typesystem/src/main/java/org/apache/atlas/typesystem/persistence/ReferenceableInstance.java
......@@ -24,10 +24,14 @@ import org.apache.atlas.AtlasException;
import org.apache.atlas.typesystem.IStruct;
import org.apache.atlas.typesystem.ITypedReferenceableInstance;
import org.apache.atlas.typesystem.ITypedStruct;
import org.apache.atlas.typesystem.types.ClassType;
import org.apache.atlas.typesystem.types.FieldMapping;
import org.apache.atlas.typesystem.types.TypeSystem;
import org.apache.atlas.utils.MD5Utils;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.security.MessageDigest;
import java.util.Date;
/*
......@@ -75,7 +79,7 @@ public class ReferenceableInstance extends StructInstance implements ITypedRefer
* @nopub
* @param id
*/
void replaceWithNewId(Id id) {
public void replaceWithNewId(Id id) {
this.id = id;
}
......@@ -92,4 +96,12 @@ public class ReferenceableInstance extends StructInstance implements ITypedRefer
throw new RuntimeException(me);
}
}
@Override
public String getSignatureHash(MessageDigest digester) throws AtlasException {
ClassType classType = TypeSystem.getInstance().getDataType(ClassType.class, getTypeName());
classType.updateSignatureHash(digester, this);
byte[] digest = digester.digest();
return MD5Utils.toString(digest);
}
}
typesystem/src/main/java/org/apache/atlas/typesystem/persistence/StructInstance.java
......@@ -29,12 +29,15 @@ import org.apache.atlas.typesystem.types.DataTypes;
import org.apache.atlas.typesystem.types.EnumType;
import org.apache.atlas.typesystem.types.EnumValue;
import org.apache.atlas.typesystem.types.FieldMapping;
import org.apache.atlas.typesystem.types.StructType;
import org.apache.atlas.typesystem.types.TypeSystem;
import org.apache.atlas.typesystem.types.TypeUtils;
import org.apache.atlas.typesystem.types.ValueConversionException;
import org.apache.atlas.utils.MD5Utils;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.security.MessageDigest;
import java.util.Date;
import java.util.HashMap;
import java.util.Map;
......@@ -229,6 +232,30 @@ public class StructInstance implements ITypedStruct {
}
int nullPos = fieldMapping.fieldNullPos.get(attrName);
nullFlags[nullPos] = true;
int pos = fieldMapping.fieldPos.get(attrName);
if (i.dataType() == DataTypes.BIGINTEGER_TYPE) {
bigIntegers[pos] = null;
} else if (i.dataType() == DataTypes.BIGDECIMAL_TYPE) {
bigDecimals[pos] = null;
} else if (i.dataType() == DataTypes.DATE_TYPE) {
dates[pos] = null;
} else if (i.dataType() == DataTypes.STRING_TYPE) {
strings[pos] = null;
} else if (i.dataType().getTypeCategory() == DataTypes.TypeCategory.ARRAY) {
arrays[pos] = null;
} else if (i.dataType().getTypeCategory() == DataTypes.TypeCategory.MAP) {
maps[pos] = null;
} else if (i.dataType().getTypeCategory() == DataTypes.TypeCategory.STRUCT
|| i.dataType().getTypeCategory() == DataTypes.TypeCategory.TRAIT) {
structs[pos] = null;
} else if (i.dataType().getTypeCategory() == DataTypes.TypeCategory.CLASS) {
ids[pos] = null;
referenceables[pos] = null;
} else {
throw new AtlasException(String.format("Unknown datatype %s", i.dataType()));
}
}
/*
......@@ -729,4 +756,12 @@ public class StructInstance implements ITypedStruct {
throw new RuntimeException(me);
}
}
@Override
public String getSignatureHash(MessageDigest digester) throws AtlasException {
StructType structType = TypeSystem.getInstance().getDataType(StructType.class, getTypeName());
structType.updateSignatureHash(digester, this);
byte[] digest = digester.digest();
return MD5Utils.toString(digest);
}
}
typesystem/src/main/java/org/apache/atlas/typesystem/types/AttributeDefinition.java
......@@ -18,7 +18,7 @@
package org.apache.atlas.typesystem.types;
import org.apache.atlas.ParamChecker;
import org.apache.atlas.utils.ParamChecker;
public final class AttributeDefinition {
......
typesystem/src/main/java/org/apache/atlas/typesystem/types/ClassType.java
......@@ -34,6 +34,8 @@ import org.apache.atlas.typesystem.persistence.StructInstance;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.nio.charset.Charset;
import java.security.MessageDigest;
import java.util.Date;
import java.util.List;
import java.util.Map;
......@@ -123,9 +125,9 @@ public class ClassType extends HierarchicalType<ClassType, IReferenceableInstanc
r != null ? createInstanceWithTraits(id, r, r.getTraits().toArray(new String[0])) :
createInstance(id);
if (id != null && id.isAssigned()) {
return tr;
}
// if (id != null && id.isAssigned()) {
// return tr;
// }
for (Map.Entry<String, AttributeInfo> e : fieldMapping.fields.entrySet()) {
String attrKey = e.getKey();
......@@ -214,4 +216,24 @@ public class ClassType extends HierarchicalType<ClassType, IReferenceableInstanc
public List<String> getNames(AttributeInfo info) {
return infoToNameMap.get(info);
}
@Override
public void updateSignatureHash(MessageDigest digester, Object val) throws AtlasException {
if( !(val instanceof ITypedReferenceableInstance)) {
throw new IllegalArgumentException("Unexpected value type " + val.getClass().getSimpleName() + ". Expected instance of ITypedStruct");
}
digester.update(getName().getBytes(Charset.forName("UTF-8")));
if(fieldMapping.fields != null && val != null) {
IReferenceableInstance typedValue = (IReferenceableInstance) val;
if(fieldMapping.fields.values() != null) {
for (AttributeInfo aInfo : fieldMapping.fields.values()) {
Object attrVal = typedValue.get(aInfo.name);
if (attrVal != null) {
aInfo.dataType().updateSignatureHash(digester, attrVal);
}
}
}
}
}
}
\ No newline at end of file
typesystem/src/main/java/org/apache/atlas/typesystem/types/DataTypes.java
......@@ -29,10 +29,13 @@ import org.apache.commons.lang3.StringUtils;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.nio.charset.Charset;
import java.security.MessageDigest;
import java.text.ParseException;
import java.util.Collection;
import java.util.Date;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
public class DataTypes {
......@@ -95,6 +98,14 @@ public class DataTypes {
return nullValue();
}
@Override
public void updateSignatureHash(MessageDigest digester, Object val) throws AtlasException {
if ( val != null ) {
digester.update(val.toString().getBytes(Charset.forName("UTF-8")));
}
}
}
public static class BooleanType extends PrimitiveType<Boolean> {
......@@ -161,6 +172,13 @@ public class DataTypes {
public Byte nullValue() {
return 0;
}
@Override
public void updateSignatureHash(MessageDigest digester, Object val) throws AtlasException {
if ( val != null ) {
digester.update(((Byte) val).byteValue());
}
}
}
public static class ShortType extends PrimitiveType<Short> {
......@@ -508,6 +526,7 @@ public class DataTypes {
} else if (val instanceof Iterator) {
it = (Iterator) val;
}
if (it != null) {
ImmutableCollection.Builder b = m.isUnique ? ImmutableSet.builder() : ImmutableList.builder();
while (it.hasNext()) {
......@@ -557,6 +576,15 @@ public class DataTypes {
public TypeCategory getTypeCategory() {
return TypeCategory.ARRAY;
}
@Override
public void updateSignatureHash(MessageDigest digester, Object val) throws AtlasException {
IDataType elemType = getElemType();
List vals = (List) val;
for (Object listElem : vals) {
elemType.updateSignatureHash(digester, listElem);
}
}
}
public static class MapType extends AbstractDataType<ImmutableMap<?, ?>> {
......@@ -586,7 +614,7 @@ public class DataTypes {
}
protected void setValueType(IDataType valueType) {
this.keyType = valueType;
this.valueType = valueType;
}
@Override
......@@ -605,7 +633,8 @@ public class DataTypes {
Map.Entry e = it.next();
b.put(keyType.convert(e.getKey(),
TypeSystem.getInstance().allowNullsInCollections() ? Multiplicity.OPTIONAL :
Multiplicity.REQUIRED), valueType.convert(e.getValue(),
Multiplicity.REQUIRED),
valueType.convert(e.getValue(),
TypeSystem.getInstance().allowNullsInCollections() ? Multiplicity.OPTIONAL :
Multiplicity.REQUIRED));
}
......@@ -657,6 +686,17 @@ public class DataTypes {
public TypeCategory getTypeCategory() {
return TypeCategory.MAP;
}
@Override
public void updateSignatureHash(MessageDigest digester, Object val) throws AtlasException {
IDataType keyType = getKeyType();
IDataType valueType = getValueType();
Map vals = (Map) val;
for (Object key : vals.keySet()) {
keyType.updateSignatureHash(digester, key);
valueType.updateSignatureHash(digester, vals.get(key));
}
}
}
}
typesystem/src/main/java/org/apache/atlas/typesystem/types/EnumType.java
......@@ -23,6 +23,9 @@ import com.google.common.collect.ImmutableMap;
import org.apache.atlas.AtlasException;
import scala.math.BigInt;
import java.nio.charset.Charset;
import java.security.MessageDigest;
public class EnumType extends AbstractDataType<EnumValue> {
public final TypeSystem typeSystem;
......@@ -80,7 +83,7 @@ public class EnumType extends AbstractDataType<EnumValue> {
public void validateUpdate(IDataType newType) throws TypeUpdateException {
super.validateUpdate(newType);
EnumType enumType = (EnumType)newType;
EnumType enumType = (EnumType) newType;
for (EnumValue enumValue : values()) {
//The old enum value should be part of new enum definition as well
if (!enumType.valueMap.containsKey(enumValue.value)) {
......@@ -96,6 +99,12 @@ public class EnumType extends AbstractDataType<EnumValue> {
}
}
public void updateSignatureHash(MessageDigest digester, Object val) throws AtlasException {
if (val != null) {
digester.update(fromValue((String) val).toString().getBytes(Charset.forName("UTF-8")));
}
}
public EnumValue fromOrdinal(int o) {
return ordinalMap.get(o);
}
......
typesystem/src/main/java/org/apache/atlas/typesystem/types/EnumTypeDefinition.java
......@@ -18,7 +18,7 @@
package org.apache.atlas.typesystem.types;
import org.apache.atlas.ParamChecker;
import org.apache.atlas.utils.ParamChecker;
import java.util.Arrays;
......
typesystem/src/main/java/org/apache/atlas/typesystem/types/EnumValue.java
......@@ -18,7 +18,7 @@
package org.apache.atlas.typesystem.types;
import org.apache.atlas.ParamChecker;
import org.apache.atlas.utils.ParamChecker;
public class EnumValue {
......
typesystem/src/main/java/org/apache/atlas/typesystem/types/IDataType.java
......@@ -20,6 +20,8 @@ package org.apache.atlas.typesystem.types;
import org.apache.atlas.AtlasException;
import java.security.MessageDigest;
public interface IDataType<T> {
String getName();
......@@ -30,5 +32,7 @@ public interface IDataType<T> {
void output(T val, Appendable buf, String prefix) throws AtlasException;
void validateUpdate(IDataType newType) throws TypeUpdateException;
void updateSignatureHash(MessageDigest digester, Object val) throws AtlasException;
}
typesystem/src/main/java/org/apache/atlas/typesystem/types/Multiplicity.java
......@@ -25,8 +25,8 @@ public final class Multiplicity {
public static final Multiplicity OPTIONAL = new Multiplicity(0, 1, false);
public static final Multiplicity REQUIRED = new Multiplicity(1, 1, false);
public static final Multiplicity COLLECTION = new Multiplicity(1, Integer.MAX_VALUE, false);
public static final Multiplicity SET = new Multiplicity(1, Integer.MAX_VALUE, true);
public static final Multiplicity COLLECTION = new Multiplicity(0, Integer.MAX_VALUE, false);
public static final Multiplicity SET = new Multiplicity(0, Integer.MAX_VALUE, true);
public final int lower;
public final int upper;
......
typesystem/src/main/java/org/apache/atlas/typesystem/types/ObjectGraphWalker.java
......@@ -214,5 +214,11 @@ public class ObjectGraphWalker {
this.aInfo = aInfo;
this.value = value;
}
@Override
public String toString(){
StringBuilder string = new StringBuilder().append(instance).append(aInfo).append(value);
return string.toString();
}
}
}
typesystem/src/main/java/org/apache/atlas/typesystem/types/StructType.java
......@@ -22,6 +22,8 @@ import org.apache.atlas.AtlasException;
import org.apache.atlas.typesystem.IStruct;
import org.apache.atlas.typesystem.ITypedStruct;
import java.nio.charset.Charset;
import java.security.MessageDigest;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
......@@ -193,6 +195,24 @@ public class StructType extends AbstractDataType<IStruct> implements IConstructa
handler.output(s, buf, prefix);
}
@Override
public void updateSignatureHash(MessageDigest digester, Object val) throws AtlasException {
if( !(val instanceof ITypedStruct)) {
throw new IllegalArgumentException("Unexpected value type " + val.getClass().getSimpleName() + ". Expected instance of ITypedStruct");
}
digester.update(getName().getBytes(Charset.forName("UTF-8")));
if(fieldMapping.fields != null && val != null) {
IStruct typedValue = (IStruct) val;
for (AttributeInfo aInfo : fieldMapping.fields.values()) {
Object attrVal = typedValue.get(aInfo.name);
if(attrVal != null) {
aInfo.dataType().updateSignatureHash(digester, attrVal);
}
}
}
}
public List<String> getNames(AttributeInfo info) {
return infoToNameMap.get(info);
}
......
typesystem/src/main/java/org/apache/atlas/typesystem/types/StructTypeDefinition.java
......@@ -18,7 +18,7 @@
package org.apache.atlas.typesystem.types;
import org.apache.atlas.ParamChecker;
import org.apache.atlas.utils.ParamChecker;
import java.util.Arrays;
......
typesystem/src/main/java/org/apache/atlas/typesystem/types/TraitType.java
......@@ -23,6 +23,8 @@ import org.apache.atlas.AtlasException;
import org.apache.atlas.typesystem.IStruct;
import org.apache.atlas.typesystem.ITypedStruct;
import java.nio.charset.Charset;
import java.security.MessageDigest;
import java.util.List;
import java.util.Map;
......@@ -68,6 +70,24 @@ public class TraitType extends HierarchicalType<TraitType, IStruct>
}
@Override
public void updateSignatureHash(MessageDigest digester, Object val) throws AtlasException {
if( !(val instanceof ITypedStruct)) {
throw new IllegalArgumentException("Unexpected value type " + val.getClass().getSimpleName() + ". Expected instance of ITypedStruct");
}
digester.update(getName().getBytes(Charset.forName("UTF-8")));
if(fieldMapping.fields != null && val != null) {
IStruct typedValue = (IStruct) val;
for (AttributeInfo aInfo : fieldMapping.fields.values()) {
Object attrVal = typedValue.get(aInfo.name);
if(attrVal != null) {
aInfo.dataType().updateSignatureHash(digester, attrVal);
}
}
}
}
@Override
public List<String> getNames(AttributeInfo info) {
return infoToNameMap.get(info);
}
......
typesystem/src/main/java/org/apache/atlas/typesystem/types/TypeSystem.java
......@@ -22,10 +22,10 @@ import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Multimap;
import org.apache.atlas.AtlasException;
import org.apache.atlas.TypeExistsException;
import org.apache.atlas.TypeNotFoundException;
import org.apache.atlas.classification.InterfaceAudience;
import org.apache.atlas.typesystem.TypesDef;
import org.apache.atlas.typesystem.exception.TypeExistsException;
import org.apache.atlas.typesystem.exception.TypeNotFoundException;
import javax.inject.Singleton;
import java.lang.reflect.Constructor;
......
typesystem/src/main/scala/org/apache/atlas/typesystem/json/Serialization.scala
......@@ -29,18 +29,22 @@ import org.json4s.native.Serialization._
import scala.collection.JavaConversions._
import scala.collection.JavaConverters._
class BigDecimalSerializer extends CustomSerializer[java.math.BigDecimal](format => ( {
case JDecimal(e) => e.bigDecimal
}, {
case e: java.math.BigDecimal => JDecimal(new BigDecimal(e))
}
class BigDecimalSerializer extends CustomSerializer[java.math.BigDecimal](format => (
{
case JDecimal(e) => e.bigDecimal
},
{
case e: java.math.BigDecimal => JDecimal(new BigDecimal(e))
}
))
class BigIntegerSerializer extends CustomSerializer[java.math.BigInteger](format => ( {
case JInt(e) => e.bigInteger
}, {
case e: java.math.BigInteger => JInt(new BigInt(e))
}
class BigIntegerSerializer extends CustomSerializer[java.math.BigInteger](format => (
{
case JInt(e) => e.bigInteger
},
{
case e: java.math.BigInteger => JInt(new BigInt(e))
}
))
class IdSerializer extends CustomSerializer[Id](format => ( {
......@@ -292,12 +296,19 @@ object Serialization {
read[ReferenceableInstance](jsonStr)
}
def traitFromJson(jsonStr: String): ITypedInstance = {
implicit val formats = org.json4s.native.Serialization.formats(NoTypeHints) + new TypedStructSerializer +
new TypedReferenceableInstanceSerializer + new BigDecimalSerializer + new BigIntegerSerializer
def traitFromJson(jsonStr: String): ITypedInstance = {
implicit val formats = org.json4s.native.Serialization.formats(NoTypeHints) + new TypedStructSerializer +
new TypedReferenceableInstanceSerializer + new BigDecimalSerializer + new BigIntegerSerializer
read[StructInstance](jsonStr)
}
read[StructInstance](jsonStr)
}
def arrayFromJson(jsonStr: String): ITypedInstance = {
implicit val formats = org.json4s.native.Serialization.formats(NoTypeHints) + new TypedStructSerializer +
new TypedReferenceableInstanceSerializer + new BigDecimalSerializer + new BigIntegerSerializer
read[StructInstance](jsonStr)
}
}
webapp/src/main/java/org/apache/atlas/web/filters/AuditFilter.java
......@@ -86,7 +86,7 @@ public class AuditFilter implements Filter {
LOG.debug("Audit: {}/{} performed request {} {} ({}) at time {}", who, fromAddress, whatRequest, whatURL,
whatAddrs, whenISO9601);
audit(who, fromAddress, fromHost, whatURL, whatAddrs, whenISO9601);
audit(who, fromAddress, whatRequest, fromHost, whatURL, whatAddrs, whenISO9601);
}
private String getUserFromRequest(HttpServletRequest httpRequest) {
......@@ -95,9 +95,9 @@ public class AuditFilter implements Filter {
return userFromRequest == null ? "UNKNOWN" : userFromRequest;
}
private void audit(String who, String fromAddress, String fromHost, String whatURL, String whatAddrs,
private void audit(String who, String fromAddress, String whatRequest, String fromHost, String whatURL, String whatAddrs,
String whenISO9601) {
AUDIT_LOG.info("Audit: {}/{}-{} performed request {} ({}) at time {}", who, fromAddress, fromHost, whatURL,
AUDIT_LOG.info("Audit: {}/{}-{} performed request {} {} ({}) at time {}", who, fromAddress, fromHost, whatRequest, whatURL,
whatAddrs, whenISO9601);
}
......
webapp/src/main/java/org/apache/atlas/web/resources/EntityResource.java
......@@ -21,11 +21,13 @@ package org.apache.atlas.web.resources;
import com.google.common.base.Preconditions;
import org.apache.atlas.AtlasClient;
import org.apache.atlas.AtlasException;
import org.apache.atlas.ParamChecker;
import org.apache.atlas.TypeNotFoundException;
import org.apache.atlas.repository.EntityExistsException;
import org.apache.atlas.repository.EntityNotFoundException;
import org.apache.atlas.typesystem.exception.EntityExistsException;
import org.apache.atlas.typesystem.exception.EntityNotFoundException;
import org.apache.atlas.typesystem.exception.TypeNotFoundException;
import org.apache.atlas.utils.ParamChecker;
import org.apache.atlas.services.MetadataService;
import org.apache.atlas.typesystem.Referenceable;
import org.apache.atlas.typesystem.json.InstanceSerialization;
import org.apache.atlas.typesystem.types.ValueConversionException;
import org.apache.atlas.web.util.Servlets;
import org.apache.commons.lang.StringUtils;
......@@ -85,7 +87,6 @@ public class EntityResource {
this.metadataService = metadataService;
}
/**
* Submits the entity definitions (instances).
* The body contains the JSONArray of entity json. The service takes care of de-duping the entities based on any
......@@ -138,6 +139,175 @@ public class EntityResource {
}
/**
* Complete update of a set of entities - the values not specified will be replaced with null/removed
* Adds/Updates given entities identified by its GUID or unique attribute
* @return response payload as json
*/
@PUT
@Consumes(Servlets.JSON_MEDIA_TYPE)
@Produces(Servlets.JSON_MEDIA_TYPE)
public Response updateEntities(@Context HttpServletRequest request) {
try {
final String entities = Servlets.getRequestPayload(request);
LOG.debug("updating entities {} ", AtlasClient.toString(new JSONArray(entities)));
final String guids = metadataService.updateEntities(entities);
JSONObject response = new JSONObject();
response.put(AtlasClient.REQUEST_ID, Servlets.getRequestId());
response.put(AtlasClient.GUID, new JSONArray(guids));
response.put(AtlasClient.DEFINITION, metadataService.getEntityDefinition(new JSONArray(guids).getString(0)));
return Response.ok(response).build();
} catch(EntityExistsException e) {
LOG.error("Unique constraint violation", e);
throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.CONFLICT));
} catch (ValueConversionException ve) {
LOG.error("Unable to persist entity instance due to a desrialization error ", ve);
throw new WebApplicationException(Servlets.getErrorResponse(ve.getCause(), Response.Status.BAD_REQUEST));
} catch (AtlasException | IllegalArgumentException e) {
LOG.error("Unable to persist entity instance", e);
throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST));
} catch (Throwable e) {
LOG.error("Unable to persist entity instance", e);
throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR));
}
}
/**
* Adds/Updates given entity identified by its unique attribute( entityType, attributeName and value)
* Updates support only partial update of an entity - Adds/updates any new values specified
* Updates do not support removal of attribute values
*
* @param entityType the entity type
* @param attribute the unique attribute used to identify the entity
* @param value the unique attributes value
* @param request The updated entity json
* @return response payload as json
* The body contains the JSONArray of entity json. The service takes care of de-duping the entities based on any
* unique attribute for the give type.
*/
@POST
@Path("qualifiedName")
@Consumes(Servlets.JSON_MEDIA_TYPE)
@Produces(Servlets.JSON_MEDIA_TYPE)
public Response updateByUniqueAttribute(@QueryParam("type") String entityType,
@QueryParam("property") String attribute,
@QueryParam("value") String value, @Context HttpServletRequest request) {
try {
String entities = Servlets.getRequestPayload(request);
LOG.debug("Partially updating entity by unique attribute {} {} {} {} ", entityType, attribute, value, entities);
Referenceable updatedEntity =
InstanceSerialization.fromJsonReferenceable(entities, true);
final String guid = metadataService.updateEntityByUniqueAttribute(entityType, attribute, value, updatedEntity);
JSONObject response = new JSONObject();
response.put(AtlasClient.REQUEST_ID, Thread.currentThread().getName());
response.put(AtlasClient.GUID, guid);
return Response.ok(response).build();
} catch (ValueConversionException ve) {
LOG.error("Unable to persist entity instance due to a desrialization error ", ve);
throw new WebApplicationException(Servlets.getErrorResponse(ve.getCause(), Response.Status.BAD_REQUEST));
} catch(EntityExistsException e) {
LOG.error("Unique constraint violation", e);
throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.CONFLICT));
} catch (EntityNotFoundException e) {
LOG.error("An entity with type={} and qualifiedName={} does not exist", entityType, value, e);
throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.NOT_FOUND));
} catch (AtlasException | IllegalArgumentException e) {
LOG.error("Unable to create/update entity {}" + entityType + ":" + attribute + "." + value, e);
throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST));
} catch (Throwable e) {
LOG.error("Unable to update entity {}" + entityType + ":" + attribute + "." + value, e);
throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR));
}
}
/**
* Updates entity identified by its GUID
* Support Partial update of an entity - Adds/updates any new values specified
* Does not support removal of attribute values
*
* @param guid
* @param request The updated entity json
* @return
*/
@POST
@Path("{guid}")
@Consumes(Servlets.JSON_MEDIA_TYPE)
@Produces(Servlets.JSON_MEDIA_TYPE)
public Response updateEntityByGuid(@PathParam("guid") String guid, @QueryParam("property") String attribute,
@Context HttpServletRequest request) {
if (StringUtils.isEmpty(attribute)) {
return updateEntityPartialByGuid(guid, request);
} else {
return updateEntityAttributeByGuid(guid, attribute, request);
}
}
private Response updateEntityPartialByGuid(String guid, HttpServletRequest request) {
try {
ParamChecker.notEmpty(guid, "Guid property cannot be null");
final String entityJson = Servlets.getRequestPayload(request);
LOG.debug("partially updating entity for guid {} : {} ", guid, entityJson);
Referenceable updatedEntity =
InstanceSerialization.fromJsonReferenceable(entityJson, true);
metadataService.updateEntityPartialByGuid(guid, updatedEntity);
JSONObject response = new JSONObject();
response.put(AtlasClient.REQUEST_ID, Thread.currentThread().getName());
return Response.ok(response).build();
} catch (EntityNotFoundException e) {
LOG.error("An entity with GUID={} does not exist", guid, e);
throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.NOT_FOUND));
} catch (AtlasException | IllegalArgumentException e) {
LOG.error("Unable to update entity {}", guid, e);
throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST));
} catch (Throwable e) {
LOG.error("Unable to update entity {}", guid, e);
throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR));
}
}
/**
* Supports Partial updates
* Adds/Updates given entity specified by its GUID
* Supports updation of only simple primitive attributes like strings, ints, floats, enums, class references and
* does not support updation of complex types like arrays, maps
* @param guid entity id
* @param property property to add
* @postbody property's value
* @return response payload as json
*/
private Response updateEntityAttributeByGuid(String guid, String property, HttpServletRequest request) {
try {
Preconditions.checkNotNull(property, "Entity property cannot be null");
String value = Servlets.getRequestPayload(request);
Preconditions.checkNotNull(value, "Entity value cannot be null");
metadataService.updateEntityAttributeByGuid(guid, property, value);
JSONObject response = new JSONObject();
response.put(AtlasClient.REQUEST_ID, Thread.currentThread().getName());
response.put(AtlasClient.GUID, guid);
return Response.ok(response).build();
} catch (EntityNotFoundException e) {
LOG.error("An entity with GUID={} does not exist", guid, e);
throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.NOT_FOUND));
} catch (AtlasException | IllegalArgumentException e) {
LOG.error("Unable to add property {} to entity id {}", property, guid, e);
throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST));
} catch (Throwable e) {
LOG.error("Unable to add property {} to entity id {}", property, guid, e);
throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR));
}
}
/**
* Fetch the complete definition of an entity given its GUID.
*
* @param guid GUID for the entity
......@@ -265,39 +435,6 @@ public class EntityResource {
}
}
/**
* Adds property to the given entity id
* @param guid entity id
* @param property property to add
* @param value property's value
* @return response payload as json
*/
@PUT
@Path("{guid}")
@Consumes(Servlets.JSON_MEDIA_TYPE)
@Produces(Servlets.JSON_MEDIA_TYPE)
public Response update(@PathParam("guid") String guid, @QueryParam("property") String property,
@QueryParam("value") String value) {
try {
Preconditions.checkNotNull(property, "Entity property cannot be null");
Preconditions.checkNotNull(value, "Entity value cannot be null");
metadataService.updateEntity(guid, property, value);
JSONObject response = new JSONObject();
response.put(AtlasClient.REQUEST_ID, Thread.currentThread().getName());
return Response.ok(response).build();
} catch (EntityNotFoundException e) {
LOG.error("An entity with GUID={} does not exist", guid, e);
throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.NOT_FOUND));
} catch (AtlasException | IllegalArgumentException e) {
LOG.error("Unable to add property {} to entity id {}", property, guid, e);
throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST));
} catch (Throwable e) {
LOG.error("Unable to add property {} to entity id {}", property, guid, e);
throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR));
}
}
// Trait management functions
......
webapp/src/main/java/org/apache/atlas/web/resources/HiveLineageResource.java
......@@ -19,10 +19,10 @@
package org.apache.atlas.web.resources;
import org.apache.atlas.AtlasClient;
import org.apache.atlas.ParamChecker;
import org.apache.atlas.typesystem.exception.EntityNotFoundException;
import org.apache.atlas.utils.ParamChecker;
import org.apache.atlas.discovery.DiscoveryException;
import org.apache.atlas.discovery.LineageService;
import org.apache.atlas.repository.EntityNotFoundException;
import org.apache.atlas.web.util.Servlets;
import org.codehaus.jettison.json.JSONObject;
import org.slf4j.Logger;
......
webapp/src/main/java/org/apache/atlas/web/resources/MetadataDiscoveryResource.java
......@@ -20,7 +20,7 @@ package org.apache.atlas.web.resources;
import com.google.common.base.Preconditions;
import org.apache.atlas.AtlasClient;
import org.apache.atlas.ParamChecker;
import org.apache.atlas.utils.ParamChecker;
import org.apache.atlas.discovery.DiscoveryException;
import org.apache.atlas.discovery.DiscoveryService;
import org.apache.atlas.web.util.Servlets;
......
webapp/src/main/java/org/apache/atlas/web/resources/TypesResource.java
......@@ -21,8 +21,8 @@ package org.apache.atlas.web.resources;
import com.sun.jersey.api.client.ClientResponse;
import org.apache.atlas.AtlasClient;
import org.apache.atlas.AtlasException;
import org.apache.atlas.TypeExistsException;
import org.apache.atlas.services.MetadataService;
import org.apache.atlas.typesystem.exception.TypeExistsException;
import org.apache.atlas.typesystem.types.DataTypes;
import org.apache.atlas.web.util.Servlets;
import org.codehaus.jettison.json.JSONArray;
......
webapp/src/main/java/org/apache/atlas/web/util/Servlets.java
......@@ -19,7 +19,7 @@
package org.apache.atlas.web.util;
import org.apache.atlas.AtlasClient;
import org.apache.atlas.ParamChecker;
import org.apache.atlas.utils.ParamChecker;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringEscapeUtils;
import org.apache.commons.lang3.StringUtils;
......
webapp/src/test/java/org/apache/atlas/notification/EntityNotificationIT.java
......@@ -118,7 +118,7 @@ public class EntityNotificationIT extends BaseResourceIT {
final String guid = tableId._getId();
serviceClient.updateEntity(guid, property, newValue);
serviceClient.updateEntityAttribute(guid, property, newValue);
waitForNotification(MAX_WAIT_TIME);
......
webapp/src/test/java/org/apache/atlas/notification/NotificationHookConsumerIT.java
......@@ -30,7 +30,7 @@ import org.testng.annotations.Guice;
import org.testng.annotations.Test;
@Guice(modules = NotificationModule.class)
public class NotificationHookConsumerIT extends BaseResourceIT{
public class NotificationHookConsumerIT extends BaseResourceIT {
@Inject
private NotificationInterface kafka;
......
webapp/src/test/java/org/apache/atlas/web/resources/BaseResourceIT.java
......@@ -42,6 +42,7 @@ import org.apache.atlas.typesystem.types.Multiplicity;
import org.apache.atlas.typesystem.types.StructTypeDefinition;
import org.apache.atlas.typesystem.types.TraitType;
import org.apache.atlas.typesystem.types.utils.TypesUtil;
import org.apache.atlas.utils.ParamChecker;
import org.apache.atlas.web.util.Servlets;
import org.apache.commons.configuration.Configuration;
import org.apache.commons.lang.RandomStringUtils;
......@@ -83,7 +84,11 @@ public abstract class BaseResourceIT {
protected void createType(TypesDef typesDef) throws Exception {
HierarchicalTypeDefinition<ClassType> sampleType = typesDef.classTypesAsJavaList().get(0);
if (serviceClient.getType(sampleType.typeName) == null) {
try {
serviceClient.getType(sampleType.typeName);
LOG.info("Types already exist. Skipping type creation");
} catch(AtlasServiceException ase) {
//Expected if type doesnt exist
String typesAsJSON = TypesSerialization.toJson(typesDef);
createType(typesAsJSON);
}
......
webapp/src/test/java/org/apache/atlas/web/resources/EntityJerseyResourceIT.java
......@@ -52,7 +52,10 @@ import org.testng.annotations.Test;
import javax.ws.rs.HttpMethod;
import javax.ws.rs.core.Response;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.UUID;
/**
......@@ -174,7 +177,6 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
@Test
public void testSubmitEntityWithBadDateFormat() throws Exception {
try {
Referenceable tableInstance = createHiveTableInstance("db" + randomString(), "table" + randomString());
tableInstance.set("lastAccessTime", "2014-07-11");
......@@ -191,8 +193,7 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
final String guid = tableId._getId();
//add property
String description = "bar table - new desc";
ClientResponse clientResponse = addProperty(guid, "description", description);
Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode());
addProperty(guid, "description", description);
String entityRef = getEntityDefinition(getEntityDefinition(guid));
Assert.assertNotNull(entityRef);
......@@ -200,13 +201,16 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
tableInstance.set("description", description);
//invalid property for the type
clientResponse = addProperty(guid, "invalid_property", "bar table");
Assert.assertEquals(clientResponse.getStatus(), Response.Status.BAD_REQUEST.getStatusCode());
try {
addProperty(guid, "invalid_property", "bar table");
Assert.fail("Expected AtlasServiceException");
} catch (AtlasServiceException e) {
Assert.assertEquals(e.getStatus().getStatusCode(), Response.Status.BAD_REQUEST.getStatusCode());
}
//non-string property, update
String currentTime = String.valueOf(System.currentTimeMillis());
clientResponse = addProperty(guid, "createTime", currentTime);
Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode());
addProperty(guid, "createTime", currentTime);
entityRef = getEntityDefinition(getEntityDefinition(guid));
Assert.assertNotNull(entityRef);
......@@ -222,12 +226,16 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
Assert.fail();
}
@Test(dependsOnMethods = "testSubmitEntity", expectedExceptions = IllegalArgumentException.class)
@Test(dependsOnMethods = "testSubmitEntity")
public void testAddNullPropertyValue() throws Exception {
final String guid = tableId._getId();
//add property
addProperty(guid, "description", null);
Assert.fail();
try {
addProperty(guid, "description", null);
Assert.fail("Expected AtlasServiceException");
} catch(AtlasServiceException e) {
Assert.assertEquals(e.getStatus().getStatusCode(), Response.Status.BAD_REQUEST.getStatusCode());
}
}
@Test(dependsOnMethods = "testSubmitEntity")
......@@ -242,8 +250,7 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
//Add reference property
final String guid = tableId._getId();
ClientResponse clientResponse = addProperty(guid, "db", dbId);
Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode());
addProperty(guid, "db", dbId);
}
@Test(dependsOnMethods = "testSubmitEntity")
......@@ -264,11 +271,8 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
InstanceSerialization.fromJsonReferenceable(definition, true);
}
private ClientResponse addProperty(String guid, String property, String value) {
WebResource resource = service.path(ENTITIES).path(guid);
return resource.queryParam("property", property).queryParam("value", value).accept(Servlets.JSON_MEDIA_TYPE)
.type(Servlets.JSON_MEDIA_TYPE).method(HttpMethod.PUT, ClientResponse.class);
private void addProperty(String guid, String property, String value) throws AtlasServiceException {
serviceClient.updateEntityAttribute(guid, property, value);
}
private ClientResponse getEntityDefinition(String guid) {
......@@ -547,4 +551,82 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
Referenceable getReferenceable = InstanceSerialization.fromJsonReferenceable(definition, true);
Assert.assertEquals(getReferenceable.get(attrName), attrValue);
}
@Test(dependsOnMethods = "testSubmitEntity")
public void testPartialUpdate() throws Exception {
final List<Referenceable> columns = new ArrayList<>();
Map<String, Object> values = new HashMap<>();
values.put("name", "col1");
values.put("dataType", "string");
values.put("comment", "col1 comment");
Referenceable ref = new Referenceable(BaseResourceIT.COLUMN_TYPE, values);
columns.add(ref);
Referenceable tableUpdated = new Referenceable(BaseResourceIT.HIVE_TABLE_TYPE, new HashMap<String, Object>() {{
put("columns", columns);
}});
LOG.debug("Updating entity= " + tableUpdated);
serviceClient.updateEntity(tableId._getId(), tableUpdated);
ClientResponse response = getEntityDefinition(tableId._getId());
String definition = getEntityDefinition(response);
Referenceable getReferenceable = InstanceSerialization.fromJsonReferenceable(definition, true);
List<Referenceable> refs = (List<Referenceable>) getReferenceable.get("columns");
Assert.assertTrue(refs.get(0).equalsContents(columns.get(0)));
//Update by unique attribute
values.put("dataType", "int");
ref = new Referenceable(BaseResourceIT.COLUMN_TYPE, values);
columns.set(0, ref);
tableUpdated = new Referenceable(BaseResourceIT.HIVE_TABLE_TYPE, new HashMap<String, Object>() {{
put("columns", columns);
}});
LOG.debug("Updating entity= " + tableUpdated);
serviceClient.updateEntity(BaseResourceIT.HIVE_TABLE_TYPE, "name", (String) tableInstance.get("name"),
tableUpdated);
response = getEntityDefinition(tableId._getId());
definition = getEntityDefinition(response);
getReferenceable = InstanceSerialization.fromJsonReferenceable(definition, true);
refs = (List<Referenceable>) getReferenceable.get("columns");
Assert.assertTrue(refs.get(0).equalsContents(columns.get(0)));
Assert.assertEquals(refs.get(0).get("dataType"), "int");
}
@Test(dependsOnMethods = "testSubmitEntity")
public void testCompleteUpdate() throws Exception {
final List<Referenceable> columns = new ArrayList<>();
Map<String, Object> values1 = new HashMap<>();
values1.put("name", "col3");
values1.put("dataType", "string");
values1.put("comment", "col3 comment");
Map<String, Object> values2 = new HashMap<>();
values2.put("name", "col4");
values2.put("dataType", "string");
values2.put("comment", "col4 comment");
Referenceable ref1 = new Referenceable(BaseResourceIT.COLUMN_TYPE, values1);
Referenceable ref2 = new Referenceable(BaseResourceIT.COLUMN_TYPE, values2);
columns.add(ref1);
columns.add(ref2);
tableInstance.set("columns", columns);
LOG.debug("Replacing entity= " + tableInstance);
serviceClient.updateEntities(tableInstance);
ClientResponse response = getEntityDefinition(tableId._getId());
String definition = getEntityDefinition(response);
Referenceable getReferenceable = InstanceSerialization.fromJsonReferenceable(definition, true);
List<Referenceable> refs = (List<Referenceable>) getReferenceable.get("columns");
Assert.assertEquals(refs.size(), 2);
Assert.assertTrue(refs.get(0).equalsContents(columns.get(0)));
Assert.assertTrue(refs.get(1).equalsContents(columns.get(1)));
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment