Skip to content

A
atlas
Commit 51656991 by Shwetha GS
Options

ATLAS-47 Entity mutations for complex types (sumasai via shwethags)

parent 6c3f0964
Showing with 1206 additions and 236 deletions
addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java
......@@ -479,12 +479,13 @@ public class HiveMetaStoreBridge {
HiveDataModelGenerator dataModelGenerator = new HiveDataModelGenerator();
AtlasClient dgiClient = getAtlasClient();
//Register hive data model if its not already registered
if (dgiClient.getType(HiveDataTypes.HIVE_PROCESS.getName()) == null) {
try {
dgiClient.getType(HiveDataTypes.HIVE_PROCESS.getName());
LOG.info("Hive data model is already registered!");
} catch(AtlasServiceException ase) {
//Expected in case types do not exist
LOG.info("Registering Hive data model");
dgiClient.createType(dataModelGenerator.getModelAsJson());
} else {
LOG.info("Hive data model is already registered!");
}
}
......
addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java
......@@ -20,7 +20,7 @@ package org.apache.atlas.hive.hook;
import org.apache.atlas.ApplicationProperties;
import org.apache.atlas.AtlasClient;
import org.apache.atlas.ParamChecker;
import org.apache.atlas.utils.ParamChecker;
import org.apache.atlas.hive.bridge.HiveMetaStoreBridge;
import org.apache.atlas.hive.model.HiveDataModelGenerator;
import org.apache.atlas.hive.model.HiveDataTypes;
......
client/src/main/java/org/apache/atlas/AtlasClient.java
......@@ -133,40 +133,43 @@ public class AtlasClient {
enum API {
//Type operations
CREATE_TYPE(BASE_URI + TYPES, HttpMethod.POST),
UPDATE_TYPE(BASE_URI + TYPES, HttpMethod.PUT),
GET_TYPE(BASE_URI + TYPES, HttpMethod.GET),
LIST_TYPES(BASE_URI + TYPES, HttpMethod.GET),
LIST_TRAIT_TYPES(BASE_URI + TYPES + "?type=trait", HttpMethod.GET),
CREATE_TYPE(BASE_URI + TYPES, HttpMethod.POST, Response.Status.CREATED),
UPDATE_TYPE(BASE_URI + TYPES, HttpMethod.PUT, Response.Status.OK),
GET_TYPE(BASE_URI + TYPES, HttpMethod.GET, Response.Status.OK),
LIST_TYPES(BASE_URI + TYPES, HttpMethod.GET, Response.Status.OK),
LIST_TRAIT_TYPES(BASE_URI + TYPES + "?type=trait", HttpMethod.GET, Response.Status.OK),
//Entity operations
CREATE_ENTITY(BASE_URI + URI_ENTITY, HttpMethod.POST),
GET_ENTITY(BASE_URI + URI_ENTITY, HttpMethod.GET),
UPDATE_ENTITY(BASE_URI + URI_ENTITY, HttpMethod.PUT),
LIST_ENTITIES(BASE_URI + URI_ENTITY, HttpMethod.GET),
CREATE_ENTITY(BASE_URI + URI_ENTITY, HttpMethod.POST, Response.Status.CREATED),
GET_ENTITY(BASE_URI + URI_ENTITY, HttpMethod.GET, Response.Status.OK),
UPDATE_ENTITY(BASE_URI + URI_ENTITY, HttpMethod.PUT, Response.Status.OK),
UPDATE_ENTITY_PARTIAL(BASE_URI + URI_ENTITY, HttpMethod.POST, Response.Status.OK),
LIST_ENTITIES(BASE_URI + URI_ENTITY, HttpMethod.GET, Response.Status.OK),
//Trait operations
ADD_TRAITS(BASE_URI + URI_ENTITY, HttpMethod.POST),
DELETE_TRAITS(BASE_URI + URI_ENTITY, HttpMethod.DELETE),
LIST_TRAITS(BASE_URI + URI_ENTITY, HttpMethod.GET),
ADD_TRAITS(BASE_URI + URI_ENTITY, HttpMethod.POST, Response.Status.CREATED),
DELETE_TRAITS(BASE_URI + URI_ENTITY, HttpMethod.DELETE, Response.Status.OK),
LIST_TRAITS(BASE_URI + URI_ENTITY, HttpMethod.GET, Response.Status.OK),
//Search operations
SEARCH(BASE_URI + URI_SEARCH, HttpMethod.GET),
SEARCH_DSL(BASE_URI + URI_SEARCH + "/dsl", HttpMethod.GET),
SEARCH_GREMLIN(BASE_URI + URI_SEARCH + "/gremlin", HttpMethod.GET),
SEARCH_FULL_TEXT(BASE_URI + URI_SEARCH + "/fulltext", HttpMethod.GET),
SEARCH(BASE_URI + URI_SEARCH, HttpMethod.GET, Response.Status.OK),
SEARCH_DSL(BASE_URI + URI_SEARCH + "/dsl", HttpMethod.GET, Response.Status.OK),
SEARCH_GREMLIN(BASE_URI + URI_SEARCH + "/gremlin", HttpMethod.GET, Response.Status.OK),
SEARCH_FULL_TEXT(BASE_URI + URI_SEARCH + "/fulltext", HttpMethod.GET, Response.Status.OK),
//Lineage operations
LINEAGE_INPUTS_GRAPH(BASE_URI + URI_LINEAGE, HttpMethod.GET),
LINEAGE_OUTPUTS_GRAPH(BASE_URI + URI_LINEAGE, HttpMethod.GET),
LINEAGE_SCHEMA(BASE_URI + URI_LINEAGE, HttpMethod.GET);
LINEAGE_INPUTS_GRAPH(BASE_URI + URI_LINEAGE, HttpMethod.GET, Response.Status.OK),
LINEAGE_OUTPUTS_GRAPH(BASE_URI + URI_LINEAGE, HttpMethod.GET, Response.Status.OK),
LINEAGE_SCHEMA(BASE_URI + URI_LINEAGE, HttpMethod.GET, Response.Status.OK);
private final String method;
private final String path;
private final Response.Status status;
API(String path, String method) {
API(String path, String method, Response.Status status) {
this.path = path;
this.method = method;
this.status = status;
}
public String getMethod() {
......@@ -176,6 +179,8 @@ public class AtlasClient {
public String getPath() {
return path;
}
public Response.Status getExpectedStatus() { return status; }
}
/**
......@@ -231,7 +236,7 @@ public class AtlasClient {
JSONObject response = callAPIWithResource(API.GET_TYPE, resource);
return response.getString(DEFINITION);
} catch (AtlasServiceException e) {
if (e.getStatus() == ClientResponse.Status.NOT_FOUND) {
if (Response.Status.NOT_FOUND.equals(e.getStatus())) {
return null;
}
throw e;
......@@ -266,11 +271,82 @@ public class AtlasClient {
}
public JSONArray createEntity(Referenceable... entities) throws AtlasServiceException {
JSONArray entityArray = getEntitiesArray(entities);
return createEntity(entityArray);
}
private JSONArray getEntitiesArray(Referenceable[] entities) {
JSONArray entityArray = new JSONArray(entities.length);
for (Referenceable entity : entities) {
entityArray.put(InstanceSerialization.toJson(entity, true));
}
return createEntity(entityArray);
return entityArray;
}
/**
* Replaces entity definitions identified by their guid or unique attribute
* Updates properties set in the definition for the entity corresponding to guid
* @param entities entities to be updated
* @return json array of guids which were updated/created
* @throws AtlasServiceException
*/
public JSONArray updateEntities(Referenceable... entities) throws AtlasServiceException {
JSONArray entitiesArray = getEntitiesArray(entities);
JSONObject response = callAPI(API.UPDATE_ENTITY, entitiesArray.toString());
try {
return response.getJSONArray(GUID);
} catch (JSONException e) {
throw new AtlasServiceException(API.UPDATE_ENTITY, e);
}
}
/**
* Supports Partial updates
* Updates property for the entity corresponding to guid
* @param guid guid
* @param attribute property key
* @param value property value
*/
public void updateEntityAttribute(String guid, String attribute, String value) throws AtlasServiceException {
API api = API.UPDATE_ENTITY_PARTIAL;
WebResource resource = getResource(api, guid);
resource = resource.queryParam(ATTRIBUTE_NAME, attribute);
callAPIWithResource(api, resource, value);
}
/**
* Supports Partial updates
* Updates properties set in the definition for the entity corresponding to guid
* @param guid guid
* @param entity entity definition
*/
public void updateEntity(String guid, Referenceable entity) throws AtlasServiceException {
String entityJson = InstanceSerialization.toJson(entity, true);
callAPI(API.UPDATE_ENTITY_PARTIAL, entityJson, guid);
}
/**
* Supports Partial updates
* Updates properties set in the definition for the entity corresponding to guid
* @param entityType Type of the entity being updated
* @param uniqueAttributeName Attribute Name that uniquely identifies the entity
* @param uniqueAttributeValue Attribute Value that uniquely identifies the entity
* @param entity entity definition
*/
public String updateEntity(String entityType, String uniqueAttributeName, String uniqueAttributeValue,
Referenceable entity) throws AtlasServiceException {
API api = API.UPDATE_ENTITY_PARTIAL;
WebResource resource = getResource(api, "qualifiedName");
resource = resource.queryParam(TYPE, entityType);
resource = resource.queryParam(ATTRIBUTE_NAME, uniqueAttributeName);
resource = resource.queryParam(ATTRIBUTE_VALUE, uniqueAttributeValue);
String entityJson = InstanceSerialization.toJson(entity, true);
JSONObject response = callAPIWithResource(api, resource, entityJson);
try {
return response.getString(GUID);
} catch (JSONException e) {
throw new AtlasServiceException(api, e);
}
}
/**
......@@ -351,19 +427,6 @@ public class AtlasClient {
}
/**
* Updates property for the entity corresponding to guid
* @param guid guid
* @param property property key
* @param value property value
*/
public JSONObject updateEntity(String guid, String property, String value) throws AtlasServiceException {
WebResource resource = getResource(API.UPDATE_ENTITY, guid);
resource = resource.queryParam(ATTRIBUTE_NAME, property);
resource = resource.queryParam(ATTRIBUTE_VALUE, value);
return callAPIWithResource(API.UPDATE_ENTITY, resource);
}
/**
* Search using gremlin/dsl/full text
* @param searchQuery
* @return
......@@ -488,13 +551,11 @@ public class AtlasClient {
}
private JSONObject callAPIWithResource(API api, WebResource resource, Object requestObject)
throws AtlasServiceException {
throws AtlasServiceException {
ClientResponse clientResponse = resource.accept(JSON_MEDIA_TYPE).type(JSON_MEDIA_TYPE)
.method(api.getMethod(), ClientResponse.class, requestObject);
.method(api.getMethod(), ClientResponse.class, requestObject);
Response.Status expectedStatus =
HttpMethod.POST.equals(api.getMethod()) ? Response.Status.CREATED : Response.Status.OK;
if (clientResponse.getStatus() == expectedStatus.getStatusCode()) {
if (clientResponse.getStatus() == api.getExpectedStatus().getStatusCode()) {
String responseAsString = clientResponse.getEntity(String.class);
try {
return new JSONObject(responseAsString);
......
common/pom.xml
......@@ -33,13 +33,19 @@
<dependencies>
<dependency>
<groupId>org.apache.atlas</groupId>
<artifactId>atlas-typesystem</artifactId>
<groupId>org.testng</groupId>
<artifactId>testng</artifactId>
</dependency>
<dependency>
<groupId>org.testng</groupId>
<artifactId>testng</artifactId>
<groupId>com.google.inject</groupId>
<artifactId>guice</artifactId>
</dependency>
<dependency>
<groupId>commons-configuration</groupId>
<artifactId>commons-configuration</artifactId>
</dependency>
</dependencies>
</project>
common/src/main/java/org/apache/atlas/utils/MD5Utils.java 0 → 100644
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.utils;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
public class MD5Utils {
private static final ThreadLocal<MessageDigest> DIGESTER_FACTORY =
new ThreadLocal<MessageDigest>() {
@Override
protected MessageDigest initialValue() {
try {
return MessageDigest.getInstance("MD5");
} catch (NoSuchAlgorithmException e) {
throw new RuntimeException(e);
}
}
};
/**
* Create a thread local MD5 digester
*/
public static MessageDigest getDigester() {
MessageDigest digester = DIGESTER_FACTORY.get();
digester.reset();
return digester;
}
private static final char[] HEX_DIGITS =
{'0','1','2','3','4','5','6','7','8','9','a','b','c','d','e','f'};
public static String toString(byte[] digest) {
StringBuilder buf = new StringBuilder(MD5_LEN*2);
for (int i = 0; i < MD5_LEN; i++) {
int b = digest[i];
buf.append(HEX_DIGITS[(b >> 4) & 0xf]);
buf.append(HEX_DIGITS[b & 0xf]);
}
return buf.toString();
}
public static final int MD5_LEN = 16;
}
typesystem/src/main/java/org/apache/atlas/ParamChecker.java → common/src/main/java/org/apache/atlas/utils/ParamChecker.java
......@@ -15,7 +15,7 @@
* limitations under the License.
*/
package org.apache.atlas;
package org.apache.atlas.utils;
import java.util.Arrays;
import java.util.Collection;
......
docs/src/site/twiki/Configuration.twiki
......@@ -60,8 +60,9 @@ Without Ranger, HBase shell can be used to set the permissions.
</verbatim>
---++++ Graph Search Index
This section sets up the graph db - titan - to use an search indexing system. The example
configuration below setsup to use an embedded Elastic search indexing system.
configuration below sets up to use an embedded Elastic search indexing system.
<verbatim>
atlas.graph.index.search.backend=elasticsearch
......@@ -72,6 +73,7 @@ atlas.graph.index.search.elasticsearch.create.sleep=2000
</verbatim>
---++++ Graph Search Index - Solr
Please note that Solr installation in Cloud mode is a prerequisite before configuring Solr as the search indexing backend. Refer InstallationSteps section for Solr installation/configuration.
<verbatim>
atlas.graph.index.search.backend=solr5
......
docs/src/site/twiki/InstallationSteps.twiki
......@@ -124,7 +124,7 @@ export METADATA_SERVER_OPTS="-Djava.awt.headless=true -Djava.security.krb5.realm
* Hbase as the Storage Backend for the Graph Repository
By default, Atlas uses Titan as the graph repository and is the only graph repository implementation available currently.
The HBase versions currently supported are 0.98.x, 1.0.x, 1.1.x. For configuring ATLAS graph persistence on HBase, please go through the "Configuration - Graph persistence engine - HBase" section
The HBase versions currently supported are 1.1.x. For configuring ATLAS graph persistence on HBase, please go through the "Configuration - Graph persistence engine - HBase" section
for more details.
Pre-requisites for running HBase as a distributed cluster
......
notification/pom.xml
......@@ -48,6 +48,11 @@
</dependency>
<dependency>
<groupId>org.apache.atlas</groupId>
<artifactId>atlas-server-api</artifactId>
</dependency>
<dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka-clients</artifactId>
</dependency>
......
notification/src/main/java/org/apache/atlas/notification/entity/NotificationEntityChangeListener.java
......@@ -73,8 +73,8 @@ public class NotificationEntityChangeListener implements EntityChangeListener {
}
@Override
public void onEntityUpdated(ITypedReferenceableInstance entity) throws AtlasException {
notifyOfEntityEvent(Collections.singleton(entity), EntityNotification.OperationType.ENTITY_UPDATE);
public void onEntitiesUpdated(Collection<ITypedReferenceableInstance> entities) throws AtlasException {
notifyOfEntityEvent(entities, EntityNotification.OperationType.ENTITY_UPDATE);
}
@Override
......
pom.xml
......@@ -407,6 +407,7 @@
<modules>
<module>common</module>
<module>typesystem</module>
<module>server-api</module>
<module>notification</module>
<module>client</module>
<module>titan</module>
......@@ -932,6 +933,12 @@
<dependency>
<groupId>org.apache.atlas</groupId>
<artifactId>atlas-server-api</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.atlas</groupId>
<artifactId>atlas-repository</artifactId>
<version>${project.version}</version>
</dependency>
......
release-log.txt
......@@ -14,6 +14,7 @@ ATLAS-54 Rename configs in hive hook (shwethags)
ATLAS-3 Mixed Index creation fails with Date types (sumasai via shwethags)
ALL CHANGES:
ATLAS-47 Entity mutations for complex types (sumasai via shwethags)
ATLAS-345 UI: Should allow tag addition on any search result that returns a reference-able entity (darshankumar89 via shwethags)
ATLAS-279 UI not displaying results for certain successful "select" search queries (anilsg via shwethags)
ATLAS-242 The qualified name for hive entities should be backward compatible (shwethags)
......
repository/pom.xml
......@@ -41,7 +41,7 @@
<dependency>
<groupId>org.apache.atlas</groupId>
<artifactId>atlas-common</artifactId>
<artifactId>atlas-server-api</artifactId>
</dependency>
<dependency>
......
repository/src/main/java/org/apache/atlas/RepositoryMetadataModule.java
......@@ -49,11 +49,13 @@ public class RepositoryMetadataModule extends com.google.inject.AbstractModule {
@Override
protected void configure() {
// special wiring for Titan Graph
ThrowingProviderBinder.create(binder()).bind(GraphProvider.class, TitanGraph.class).to(TitanGraphProvider.class)
.asEagerSingleton();
// allow for dynamic binding of the metadata repo & graph service
// bind the MetadataRepositoryService interface to an implementation
bind(MetadataRepository.class).to(GraphBackedMetadataRepository.class).asEagerSingleton();
......
repository/src/main/java/org/apache/atlas/discovery/HiveLineageService.java
......@@ -22,14 +22,14 @@ import com.thinkaurelius.titan.core.TitanGraph;
import org.apache.atlas.ApplicationProperties;
import org.apache.atlas.AtlasException;
import org.apache.atlas.GraphTransaction;
import org.apache.atlas.ParamChecker;
import org.apache.atlas.typesystem.exception.EntityNotFoundException;
import org.apache.atlas.utils.ParamChecker;
import org.apache.atlas.discovery.graph.DefaultGraphPersistenceStrategy;
import org.apache.atlas.discovery.graph.GraphBackedDiscoveryService;
import org.apache.atlas.query.Expressions;
import org.apache.atlas.query.GremlinQueryResult;
import org.apache.atlas.query.HiveLineageQuery;
import org.apache.atlas.query.HiveWhereUsedQuery;
import org.apache.atlas.repository.EntityNotFoundException;
import org.apache.atlas.repository.MetadataRepository;
import org.apache.atlas.repository.graph.GraphProvider;
import org.apache.atlas.typesystem.persistence.ReferenceableInstance;
......
repository/src/main/java/org/apache/atlas/discovery/graph/DefaultGraphPersistenceStrategy.java
......@@ -30,6 +30,7 @@ import org.apache.atlas.query.TypeUtils;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.MetadataRepository;
import org.apache.atlas.repository.graph.GraphBackedMetadataRepository;
import org.apache.atlas.repository.graph.GraphHelper;
import org.apache.atlas.typesystem.ITypedReferenceableInstance;
import org.apache.atlas.typesystem.ITypedStruct;
import org.apache.atlas.typesystem.persistence.Id;
......@@ -43,7 +44,7 @@ import org.apache.atlas.typesystem.types.TypeSystem;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import javax.inject.Inject;
import java.util.List;
/**
......@@ -55,6 +56,7 @@ public class DefaultGraphPersistenceStrategy implements GraphPersistenceStrategi
private final GraphBackedMetadataRepository metadataRepository;
@Inject
public DefaultGraphPersistenceStrategy(MetadataRepository metadataRepository) {
this.metadataRepository = (GraphBackedMetadataRepository) metadataRepository;
}
......@@ -71,7 +73,11 @@ public class DefaultGraphPersistenceStrategy implements GraphPersistenceStrategi
@Override
public String edgeLabel(IDataType<?> dataType, AttributeInfo aInfo) {
return metadataRepository.getEdgeLabel(dataType, aInfo);
try {
return metadataRepository.getEdgeLabel(dataType, aInfo);
} catch (AtlasException e) {
throw new RuntimeException(e);
}
}
@Override
......@@ -90,7 +96,7 @@ public class DefaultGraphPersistenceStrategy implements GraphPersistenceStrategi
@Override
public List<String> traitNames(TitanVertex vertex) {
return metadataRepository.getTraitNames(vertex);
return GraphHelper.getTraitNames(vertex);
}
@Override
......@@ -100,7 +106,7 @@ public class DefaultGraphPersistenceStrategy implements GraphPersistenceStrategi
@Override
public Id getIdFromVertex(String dataTypeName, TitanVertex vertex) {
return metadataRepository.getIdFromVertex(dataTypeName, vertex);
return GraphHelper.getIdFromVertex(dataTypeName, vertex);
}
@Override
......
repository/src/main/java/org/apache/atlas/repository/MetadataRepository.java
......@@ -19,6 +19,8 @@
package org.apache.atlas.repository;
import org.apache.atlas.AtlasException;
import org.apache.atlas.typesystem.exception.EntityExistsException;
import org.apache.atlas.typesystem.exception.EntityNotFoundException;
import org.apache.atlas.typesystem.ITypedReferenceableInstance;
import org.apache.atlas.typesystem.ITypedStruct;
import org.apache.atlas.typesystem.types.AttributeInfo;
......@@ -70,7 +72,7 @@ public interface MetadataRepository {
* @param aInfo attribute info
* @return edge label for a given attribute
*/
String getEdgeLabel(IDataType<?> dataType, AttributeInfo aInfo);
String getEdgeLabel(IDataType<?> dataType, AttributeInfo aInfo) throws AtlasException;
/**
* Creates an entity definition (instance) corresponding to a given type.
......@@ -89,7 +91,7 @@ public interface MetadataRepository {
* @return entity (typed instance) definition
* @throws RepositoryException
*/
ITypedReferenceableInstance getEntityDefinition(String guid) throws RepositoryException;
ITypedReferenceableInstance getEntityDefinition(String guid) throws RepositoryException, EntityNotFoundException;
/**
* Gets the list of entities for a given entity type.
......@@ -108,20 +110,6 @@ public interface MetadataRepository {
* @throws RepositoryException
*/
// boolean deleteEntity(String guid) throws RepositoryException;
/**
* Updates an entity given its GUID with the attribute name and value.
*
* @param guid globally unique identifier for the entity
* @param attributeName name of the attribute
* @param attributeValue value of the attribute
* @return an entity instance with updated state
* @throws RepositoryException
*/
//ITypedReferenceableInstance updateEntity(String guid, String attributeName,
// String attributeValue) throws RepositoryException;
// Trait management functions
/**
......@@ -149,15 +137,19 @@ public interface MetadataRepository {
* @param traitNameToBeDeleted name of the trait
* @throws RepositoryException
*/
void deleteTrait(String guid, String traitNameToBeDeleted) throws RepositoryException;
void deleteTrait(String guid, String traitNameToBeDeleted) throws EntityNotFoundException, RepositoryException;
/**
* Adds/Updates the property to the entity that corresponds to the GUID
* Supports only primitive attribute/Class Id updations.
*/
void updatePartial(ITypedReferenceableInstance entity) throws RepositoryException;
/**
* Adds/Updates the property to/in the entity that corresponds to the GUID
* @param guid entity id
* @param property property name
* @param value property value
* Adds the property to the entity that corresponds to the GUID
* @param entitiesToBeUpdated The entities to be updated
*/
void updateEntity(String guid, String property, String value) throws RepositoryException;
String[] updateEntities(ITypedReferenceableInstance... entitiesToBeUpdated) throws RepositoryException;
/**
* Returns the entity for the given type and qualified name
......@@ -166,5 +158,5 @@ public interface MetadataRepository {
* @param value
* @return entity instance
*/
ITypedReferenceableInstance getEntityDefinition(String entityType, String attribute, String value) throws AtlasException;
ITypedReferenceableInstance getEntityDefinition(String entityType, String attribute, Object value) throws AtlasException;
}
repository/src/main/java/org/apache/atlas/repository/graph/EntityProcessor.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.repository.graph;
import org.apache.atlas.AtlasException;
import org.apache.atlas.repository.RepositoryException;
import org.apache.atlas.typesystem.IReferenceableInstance;
import org.apache.atlas.typesystem.ITypedReferenceableInstance;
import org.apache.atlas.typesystem.persistence.Id;
import org.apache.atlas.typesystem.types.DataTypes;
import org.apache.atlas.typesystem.types.ObjectGraphWalker;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.LinkedHashMap;
import java.util.Map;
public final class EntityProcessor implements ObjectGraphWalker.NodeProcessor {
private final Map<Id, IReferenceableInstance> idToInstanceMap;
public EntityProcessor() {
idToInstanceMap = new LinkedHashMap<>();
}
public Collection<IReferenceableInstance> getInstances() {
ArrayList<IReferenceableInstance> instances = new ArrayList<IReferenceableInstance>(idToInstanceMap.values());
Collections.reverse(instances);
return instances;
}
@Override
public void processNode(ObjectGraphWalker.Node nd) throws AtlasException {
IReferenceableInstance ref = null;
Id id = null;
if (nd.attributeName == null) {
ref = (IReferenceableInstance) nd.instance;
id = ref.getId();
} else if (nd.aInfo.dataType().getTypeCategory() == DataTypes.TypeCategory.CLASS) {
if (nd.value != null && (nd.value instanceof Id)) {
id = (Id) nd.value;
}
}
if (id != null) {
if (id.isUnassigned()) {
if (ref != null) {
if (idToInstanceMap.containsKey(id)) { // Oops
throw new RepositoryException(
String.format("Unexpected internal error: Id %s processed again", id));
}
idToInstanceMap.put(id, ref);
}
}
}
}
public void addInstanceIfNotExists(ITypedReferenceableInstance ref) {
if(!idToInstanceMap.containsKey(ref.getId())) {
idToInstanceMap.put(ref.getId(), ref);
}
}
}
repository/src/main/java/org/apache/atlas/repository/graph/FullTextMapper.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.repository.graph;
import com.tinkerpop.blueprints.Vertex;
import org.apache.atlas.AtlasException;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.typesystem.ITypedInstance;
import org.apache.atlas.typesystem.ITypedReferenceableInstance;
import org.apache.atlas.typesystem.types.AttributeInfo;
import org.apache.atlas.typesystem.types.DataTypes;
import org.apache.atlas.typesystem.types.EnumValue;
import org.apache.atlas.typesystem.types.IDataType;
import org.apache.commons.lang.StringUtils;
import java.util.List;
import java.util.Map;
public class FullTextMapper {
private final GraphToTypedInstanceMapper graphToTypedInstanceMapper;
private static final GraphHelper graphHelper = GraphHelper.getInstance();
private static final String FULL_TEXT_DELIMITER = " ";
FullTextMapper(GraphToTypedInstanceMapper graphToTypedInstanceMapper) {
this.graphToTypedInstanceMapper = graphToTypedInstanceMapper;
}
public String mapRecursive(Vertex instanceVertex, boolean followReferences) throws AtlasException {
String guid = instanceVertex.getProperty(Constants.GUID_PROPERTY_KEY);
ITypedReferenceableInstance typedReference =
graphToTypedInstanceMapper.mapGraphToTypedInstance(guid, instanceVertex);
String fullText = forInstance(typedReference, followReferences);
StringBuilder fullTextBuilder =
new StringBuilder(typedReference.getTypeName()).append(FULL_TEXT_DELIMITER).append(fullText);
List<String> traits = typedReference.getTraits();
for (String traitName : traits) {
String traitText = forInstance((ITypedInstance) typedReference.getTrait(traitName), false);
fullTextBuilder.append(FULL_TEXT_DELIMITER).append(traitName).append(FULL_TEXT_DELIMITER)
.append(traitText);
}
return fullTextBuilder.toString();
}
private String forAttribute(IDataType type, Object value, boolean followReferences)
throws AtlasException {
if (value == null) {
return null;
}
switch (type.getTypeCategory()) {
case PRIMITIVE:
return String.valueOf(value);
case ENUM:
return ((EnumValue) value).value;
case ARRAY:
StringBuilder fullText = new StringBuilder();
IDataType elemType = ((DataTypes.ArrayType) type).getElemType();
List list = (List) value;
for (Object element : list) {
String elemFullText = forAttribute(elemType, element, false);
if (StringUtils.isNotEmpty(elemFullText)) {
fullText = fullText.append(FULL_TEXT_DELIMITER).append(elemFullText);
}
}
return fullText.toString();
case MAP:
fullText = new StringBuilder();
IDataType keyType = ((DataTypes.MapType) type).getKeyType();
IDataType valueType = ((DataTypes.MapType) type).getValueType();
Map map = (Map) value;
for (Object entryObj : map.entrySet()) {
Map.Entry entry = (Map.Entry) entryObj;
String keyFullText = forAttribute(keyType, entry.getKey(), false);
if (StringUtils.isNotEmpty(keyFullText)) {
fullText = fullText.append(FULL_TEXT_DELIMITER).append(keyFullText);
}
String valueFullText = forAttribute(valueType, entry.getValue(), false);
if (StringUtils.isNotEmpty(valueFullText)) {
fullText = fullText.append(FULL_TEXT_DELIMITER).append(valueFullText);
}
}
return fullText.toString();
case CLASS:
if (followReferences) {
String refGuid = ((ITypedReferenceableInstance) value).getId()._getId();
Vertex refVertex = graphHelper.getVertexForGUID(refGuid);
return mapRecursive(refVertex, false);
}
break;
case STRUCT:
if (followReferences) {
return forInstance((ITypedInstance) value, true);
}
break;
default:
throw new IllegalStateException("Unhandled type category " + type.getTypeCategory());
}
return null;
}
private String forInstance(ITypedInstance typedInstance, boolean followReferences)
throws AtlasException {
StringBuilder fullText = new StringBuilder();
for (AttributeInfo attributeInfo : typedInstance.fieldMapping().fields.values()) {
Object attrValue = typedInstance.get(attributeInfo.name);
if (attrValue == null) {
continue;
}
String attrFullText = forAttribute(attributeInfo.dataType(), attrValue, followReferences);
if (StringUtils.isNotEmpty(attrFullText)) {
fullText =
fullText.append(FULL_TEXT_DELIMITER).append(attributeInfo.name).append(FULL_TEXT_DELIMITER)
.append(attrFullText);
}
}
return fullText.toString();
}
}
repository/src/main/java/org/apache/atlas/repository/memory/ReplaceIdWithInstance.java
......@@ -55,7 +55,6 @@ public class ReplaceIdWithInstance implements ObjectGraphWalker.NodeProcessor {
} else if (!nd.aInfo.isComposite || nd.value == null) {
// do nothing
} else if (nd.aInfo.dataType().getTypeCategory() == DataTypes.TypeCategory.CLASS) {
if (nd.value != null && nd.value instanceof Id) {
Id id = (Id) nd.value;
ITypedReferenceableInstance r = getInstance(id);
......
repository/src/main/scala/org/apache/atlas/query/GraphPersistenceStrategies.scala
......@@ -26,7 +26,7 @@ import com.tinkerpop.blueprints.{Vertex, Direction}
import org.apache.atlas.AtlasException
import org.apache.atlas.query.Expressions.{ComparisonExpression, ExpressionException}
import org.apache.atlas.query.TypeUtils.FieldInfo
import org.apache.atlas.repository.graph.GraphBackedMetadataRepository
import org.apache.atlas.repository.graph.{GraphHelper, GraphBackedMetadataRepository}
import org.apache.atlas.typesystem.persistence.Id
import org.apache.atlas.typesystem.types.DataTypes._
import org.apache.atlas.typesystem.types._
......@@ -199,7 +199,7 @@ object GraphPersistenceStrategy1 extends GraphPersistenceStrategies {
def traitLabel(cls: IDataType[_], traitName: String) = s"${cls.getName}.$traitName"
def fieldNameInVertex(dataType: IDataType[_], aInfo: AttributeInfo) = GraphBackedMetadataRepository.getQualifiedName(dataType, aInfo.name)
def fieldNameInVertex(dataType: IDataType[_], aInfo: AttributeInfo) = GraphHelper.getQualifiedFieldName(dataType, aInfo.name)
def getIdFromVertex(dataTypeNm: String, v: TitanVertex): Id =
new Id(v.getId.toString, 0, dataTypeNm)
......
repository/src/test/java/org/apache/atlas/BaseHiveRepositoryTest.java
......@@ -21,10 +21,10 @@ import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import com.thinkaurelius.titan.core.TitanGraph;
import com.thinkaurelius.titan.core.util.TitanCleanup;
import org.apache.atlas.repository.graph.GraphBackedMetadataRepository;
import org.apache.atlas.repository.MetadataRepository;
import org.apache.atlas.repository.graph.GraphBackedSearchIndexer;
import org.apache.atlas.repository.graph.GraphProvider;
import org.apache.atlas.services.DefaultMetadataService;
import org.apache.atlas.services.MetadataService;
import org.apache.atlas.typesystem.ITypedReferenceableInstance;
import org.apache.atlas.typesystem.Referenceable;
import org.apache.atlas.typesystem.TypesDef;
......@@ -55,10 +55,10 @@ import java.util.List;
public class BaseHiveRepositoryTest {
@Inject
protected DefaultMetadataService metadataService;
protected MetadataService metadataService;
@Inject
protected GraphBackedMetadataRepository repository;
protected MetadataRepository repository;
@Inject
protected GraphProvider<TitanGraph> graphProvider;
......@@ -67,7 +67,7 @@ public class BaseHiveRepositoryTest {
setUpTypes();
new GraphBackedSearchIndexer(graphProvider);
setupInstances();
// TestUtils.dumpGraph(graphProvider.get());
TestUtils.dumpGraph(graphProvider.get());
}
protected void tearDown() throws Exception {
......@@ -190,17 +190,20 @@ public class BaseHiveRepositoryTest {
Id salesDB = database("Sales", "Sales Database", "John ETL", "hdfs://host:8000/apps/warehouse/sales");
Referenceable sd =
storageDescriptor("hdfs://host:8000/apps/warehouse/sales", "TextInputFormat", "TextOutputFormat", true, ImmutableList.of(column("time_id", "int", "time id")));
storageDescriptor("hdfs://host:8000/apps/warehouse/sales", "TextInputFormat", "TextOutputFormat", true, ImmutableList.of(
column("time_id", "int", "time id")));
List<Referenceable> salesFactColumns = ImmutableList
.of(column("time_id", "int", "time id"), column("product_id", "int", "product id"),
.of(column("time_id", "int", "time id"),
column("product_id", "int", "product id"),
column("customer_id", "int", "customer id", "PII"),
column("sales", "double", "product id", "Metric"));
Id salesFact = table("sales_fact", "sales fact table", salesDB, sd, "Joe", "Managed", salesFactColumns, "Fact");
List<Referenceable> timeDimColumns = ImmutableList
.of(column("time_id", "int", "time id"), column("dayOfYear", "int", "day Of Year"),
.of(column("time_id", "int", "time id"),
column("dayOfYear", "int", "day Of Year"),
column("weekDay", "int", "week Day"));
Id timeDim = table("time_dim", "time dimension table", salesDB, sd, "John Doe", "External", timeDimColumns,
......@@ -217,7 +220,8 @@ public class BaseHiveRepositoryTest {
ImmutableList.of(salesFactDaily), "create table as select ", "plan", "id", "graph", "ETL");
List<Referenceable> productDimColumns = ImmutableList
.of(column("product_id", "int", "product id"), column("product_name", "string", "product name"),
.of(column("product_id", "int", "product id"),
column("product_name", "string", "product name"),
column("brand_name", "int", "brand name"));
Id productDim =
......@@ -226,7 +230,8 @@ public class BaseHiveRepositoryTest {
view("product_dim_view", reportingDB, ImmutableList.of(productDim), "Dimension", "JdbcAccess");
List<Referenceable> customerDimColumns = ImmutableList.of(column("customer_id", "int", "customer id", "PII"),
List<Referenceable> customerDimColumns = ImmutableList.of(
column("customer_id", "int", "customer id", "PII"),
column("name", "string", "customer name", "PII"),
column("address", "string", "customer address", "PII"));
......
repository/src/test/java/org/apache/atlas/TestUtils.java
......@@ -187,6 +187,8 @@ public final class TestUtils {
public static final String DATABASE_TYPE = "hive_database";
public static final String DATABASE_NAME = "foo";
public static final String TABLE_TYPE = "hive_table";
public static final String PARTITION_TYPE = "partition_type";
public static final String SERDE_TYPE = "serdeType";
public static final String TABLE_NAME = "bar";
public static final String CLASSIFICATION = "classification";
public static final String PII = "PII";
......@@ -208,7 +210,8 @@ public final class TestUtils {
StructTypeDefinition structTypeDefinition = new StructTypeDefinition("serdeType",
new AttributeDefinition[]{createRequiredAttrDef("name", DataTypes.STRING_TYPE),
createRequiredAttrDef("serde", DataTypes.STRING_TYPE)});
createRequiredAttrDef("serde", DataTypes.STRING_TYPE),
createOptionalAttrDef("description", DataTypes.STRING_TYPE)});
EnumValue values[] = {new EnumValue("MANAGED", 1), new EnumValue("EXTERNAL", 2),};
......@@ -244,21 +247,23 @@ public final class TestUtils {
new AttributeDefinition("parametersMap",
DataTypes.mapTypeName(DataTypes.STRING_TYPE.getName(), DataTypes.STRING_TYPE.getName()),
Multiplicity.OPTIONAL, true, null),
// map of classes - todo - enable this
// new AttributeDefinition("columnsMap",
// DataTypes.mapTypeName(DataTypes.STRING_TYPE.getName(),
// "column_type"),
// Multiplicity.COLLECTION, true, null),
// map of structs todo - enable this
// new AttributeDefinition("partitionsMap",
// DataTypes.mapTypeName(DataTypes.STRING_TYPE.getName(),
// "partition_type"),
// Multiplicity.COLLECTION, true, null),
//map of classes -
new AttributeDefinition("columnsMap",
DataTypes.mapTypeName(DataTypes.STRING_TYPE.getName(),
"column_type"),
Multiplicity.COLLECTION, true, null),
//map of structs
new AttributeDefinition("partitionsMap",
DataTypes.mapTypeName(DataTypes.STRING_TYPE.getName(),
"partition_type"),
Multiplicity.COLLECTION, true, null),
// struct reference
new AttributeDefinition("serde1", "serdeType", Multiplicity.OPTIONAL, false, null),
new AttributeDefinition("serde2", "serdeType", Multiplicity.OPTIONAL, false, null),
// class reference
new AttributeDefinition("database", DATABASE_TYPE, Multiplicity.REQUIRED, true, null));
new AttributeDefinition("database", DATABASE_TYPE, Multiplicity.REQUIRED, false, null),
//class reference as composite
new AttributeDefinition("databaseComposite", DATABASE_TYPE, Multiplicity.OPTIONAL, true, null));
HierarchicalTypeDefinition<TraitType> piiTypeDefinition =
createTraitTypeDef(PII, ImmutableList.<String>of());
......
repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java
......@@ -24,7 +24,7 @@ import org.apache.atlas.BaseHiveRepositoryTest;
import org.apache.atlas.RepositoryMetadataModule;
import org.apache.atlas.TestUtils;
import org.apache.atlas.discovery.graph.GraphBackedDiscoveryService;
import org.apache.atlas.repository.graph.GraphBackedMetadataRepository;
import org.apache.atlas.repository.MetadataRepository;
import org.apache.atlas.repository.graph.GraphProvider;
import org.apache.atlas.typesystem.ITypedReferenceableInstance;
import org.apache.atlas.typesystem.Referenceable;
......@@ -55,7 +55,7 @@ public class GraphBackedDiscoveryServiceTest extends BaseHiveRepositoryTest {
private GraphProvider<TitanGraph> graphProvider;
@Inject
private GraphBackedMetadataRepository repositoryService;
private MetadataRepository repositoryService;
@Inject
private GraphBackedDiscoveryService discoveryService;
......
repository/src/test/java/org/apache/atlas/discovery/HiveLineageServiceTest.java
......@@ -20,8 +20,7 @@ package org.apache.atlas.discovery;
import org.apache.atlas.BaseHiveRepositoryTest;
import org.apache.atlas.RepositoryMetadataModule;
import org.apache.atlas.discovery.graph.GraphBackedDiscoveryService;
import org.apache.atlas.repository.EntityNotFoundException;
import org.apache.atlas.typesystem.exception.EntityNotFoundException;
import org.codehaus.jettison.json.JSONArray;
import org.codehaus.jettison.json.JSONObject;
import org.testng.Assert;
......@@ -32,7 +31,6 @@ import org.testng.annotations.Guice;
import org.testng.annotations.Test;
import javax.inject.Inject;
import java.util.List;
/**
* Unit tests for Hive LineageService.
......@@ -41,7 +39,7 @@ import java.util.List;
public class HiveLineageServiceTest extends BaseHiveRepositoryTest {
@Inject
private GraphBackedDiscoveryService discoveryService;
private DiscoveryService discoveryService;
@Inject
private HiveLineageService hiveLineageService;
......
repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryTest.java
......@@ -29,13 +29,13 @@ import org.apache.atlas.RepositoryMetadataModule;
import org.apache.atlas.TestUtils;
import org.apache.atlas.discovery.graph.GraphBackedDiscoveryService;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.EntityNotFoundException;
import org.apache.atlas.repository.RepositoryException;
import org.apache.atlas.typesystem.IStruct;
import org.apache.atlas.typesystem.ITypedReferenceableInstance;
import org.apache.atlas.typesystem.ITypedStruct;
import org.apache.atlas.typesystem.Referenceable;
import org.apache.atlas.typesystem.Struct;
import org.apache.atlas.typesystem.exception.EntityNotFoundException;
import org.apache.atlas.typesystem.persistence.Id;
import org.apache.atlas.typesystem.types.ClassType;
import org.apache.atlas.typesystem.types.DataTypes;
......@@ -131,7 +131,7 @@ public class GraphBackedMetadataRepositoryTest {
Assert.assertNotNull(entity);
}
@Test(expectedExceptions = RepositoryException.class)
@Test(expectedExceptions = EntityNotFoundException.class)
public void testGetEntityDefinitionNonExistent() throws Exception {
repositoryService.getEntityDefinition("blah");
Assert.fail();
......@@ -342,13 +342,13 @@ public class GraphBackedMetadataRepositoryTest {
}
Id expected = new Id(guid, tableVertex.<Integer>getProperty(Constants.VERSION_PROPERTY_KEY), TestUtils.TABLE_TYPE);
Assert.assertEquals(repositoryService.getIdFromVertex(TestUtils.TABLE_TYPE, tableVertex), expected);
Assert.assertEquals(GraphHelper.getIdFromVertex(TestUtils.TABLE_TYPE, tableVertex), expected);
}
@Test(dependsOnMethods = "testCreateEntity")
public void testGetTypeName() throws Exception {
Vertex tableVertex = getTableEntityVertex();
Assert.assertEquals(repositoryService.getTypeName(tableVertex), TestUtils.TABLE_TYPE);
Assert.assertEquals(GraphHelper.getTypeName(tableVertex), TestUtils.TABLE_TYPE);
}
@Test(dependsOnMethods = "testCreateEntity")
......@@ -415,6 +415,9 @@ public class GraphBackedMetadataRepositoryTest {
public void testBug37860() throws Exception {
String dslQuery = "hive_table as t where name = 'bar' "
+ "database where name = 'foo' and description = 'foo database' select t";
TestUtils.dumpGraph(graphProvider.get());
System.out.println("Executing dslQuery = " + dslQuery);
String jsonResults = discoveryService.searchByDSL(dslQuery);
Assert.assertNotNull(jsonResults);
......@@ -446,6 +449,8 @@ public class GraphBackedMetadataRepositoryTest {
//but with elasticsearch, doesn't work without sleep. why??
long sleepInterval = 1000;
TestUtils.dumpGraph(graphProvider.get());
//person in hr department whose name is john
Thread.sleep(sleepInterval);
String response = discoveryService.searchByFullText("john");
......@@ -475,31 +480,36 @@ public class GraphBackedMetadataRepositoryTest {
@Test(dependsOnMethods = "testSubmitEntity")
public void testUpdateEntity_MultiplicityOneNonCompositeReference() throws Exception {
ITypedReferenceableInstance john = repositoryService.getEntityDefinition("Person", "name", "John");
String johnGuid = john.getId()._getId();
Id johnGuid = john.getId();
ITypedReferenceableInstance max = repositoryService.getEntityDefinition("Person", "name", "Max");
String maxGuid = max.getId()._getId();
ITypedReferenceableInstance jane = repositoryService.getEntityDefinition("Person", "name", "Jane");
String janeGuid = jane.getId()._getId();
Id janeGuid = jane.getId();
// Update max's mentor reference to john.
repositoryService.updateEntity(maxGuid, "mentor", johnGuid);
ClassType personType = typeSystem.getDataType(ClassType.class, "Person");
ITypedReferenceableInstance instance = personType.createInstance(max.getId());
instance.set("mentor", johnGuid);
repositoryService.updatePartial(instance);
// Verify the update was applied correctly - john should now be max's mentor.
max = repositoryService.getEntityDefinition(maxGuid);
Object object = max.get("mentor");
Assert.assertTrue(object instanceof ITypedReferenceableInstance);
ITypedReferenceableInstance refTarget = (ITypedReferenceableInstance) object;
Assert.assertEquals(refTarget.getId()._getId(), johnGuid);
Assert.assertEquals(refTarget.getId()._getId(), johnGuid._getId());
// Update max's mentor reference to jane.
repositoryService.updateEntity(maxGuid, "mentor", janeGuid);
instance = personType.createInstance(max.getId());
instance.set("mentor", janeGuid);
repositoryService.updatePartial(instance);
// Verify the update was applied correctly - jane should now be max's mentor.
max = repositoryService.getEntityDefinition(maxGuid);
object = max.get("mentor");
Assert.assertTrue(object instanceof ITypedReferenceableInstance);
refTarget = (ITypedReferenceableInstance) object;
Assert.assertEquals(refTarget.getId()._getId(), janeGuid);
Assert.assertEquals(refTarget.getId()._getId(), janeGuid._getId());
}
private ITypedReferenceableInstance createHiveTableInstance(Referenceable databaseInstance) throws Exception {
......
repository/src/test/java/org/apache/atlas/repository/graph/GraphRepoMapperScaleTest.java
......@@ -18,15 +18,9 @@
package org.apache.atlas.repository.graph;
import com.google.inject.Inject;
import com.thinkaurelius.titan.core.TitanFactory;
import com.thinkaurelius.titan.core.TitanGraph;
import com.thinkaurelius.titan.core.TitanIndexQuery;
import com.thinkaurelius.titan.core.util.TitanCleanup;
import com.thinkaurelius.titan.diskstorage.BackendException;
import com.thinkaurelius.titan.diskstorage.configuration.ReadConfiguration;
import com.thinkaurelius.titan.diskstorage.configuration.backend.CommonsConfiguration;
import com.thinkaurelius.titan.graphdb.configuration.GraphDatabaseConfiguration;
import com.tinkerpop.blueprints.Compare;
import com.tinkerpop.blueprints.GraphQuery;
import com.tinkerpop.blueprints.Predicate;
......@@ -42,19 +36,16 @@ import org.apache.atlas.typesystem.types.ClassType;
import org.apache.atlas.typesystem.types.IDataType;
import org.apache.atlas.typesystem.types.Multiplicity;
import org.apache.atlas.typesystem.types.TypeSystem;
import org.apache.commons.io.FileUtils;
import org.testng.Assert;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Guice;
import org.testng.annotations.Test;
import java.io.File;
import java.io.IOException;
import javax.inject.Inject;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Date;
import java.util.Random;
@Test
@Guice(modules = RepositoryMetadataModule.class)
......@@ -63,15 +54,13 @@ public class GraphRepoMapperScaleTest {
private static final String DATABASE_NAME = "foo";
private static final String TABLE_NAME = "bar";
private static final String INDEX_DIR =
System.getProperty("java.io.tmpdir", "/tmp") + "/atlas-test" + new Random().nextLong();
@Inject
GraphProvider<TitanGraph> graphProvider;
@Inject
private GraphBackedMetadataRepository repositoryService;
@Inject
private GraphBackedSearchIndexer searchIndexer;
private TypeSystem typeSystem = TypeSystem.getInstance();
......@@ -81,7 +70,7 @@ public class GraphRepoMapperScaleTest {
@BeforeClass
@GraphTransaction
public void setUp() throws Exception {
searchIndexer = new GraphBackedSearchIndexer(graphProvider);
//Make sure we can cleanup the index directory
Collection<IDataType> typesAdded = TestUtils.createHiveTypes(typeSystem);
searchIndexer.onAdd(typesAdded);
}
......@@ -127,7 +116,6 @@ public class GraphRepoMapperScaleTest {
//Elasticsearch requires some time before index is updated
Thread.sleep(5000);
searchWithOutIndex(Constants.GUID_PROPERTY_KEY, dbGUID);
searchWithOutIndex(Constants.ENTITY_TYPE_PROPERTY_KEY, "column_type");
searchWithOutIndex(Constants.ENTITY_TYPE_PROPERTY_KEY, TestUtils.TABLE_TYPE);
......
repository/src/test/scala/org/apache/atlas/query/GremlinTest.scala
......@@ -39,7 +39,6 @@ class GremlinTest extends BaseGremlinTest {
gProvider = new TitanGraphProvider();
gp = new DefaultGraphPersistenceStrategy(new GraphBackedMetadataRepository(gProvider))
g = QueryTestsUtils.setupTestGraph(gProvider)
}
@AfterClass
......
server-api/pom.xml 0 → 100644
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one
~ or more contributor license agreements. See the NOTICE file
~ distributed with this work for additional information
~ regarding copyright ownership. The ASF licenses this file
~ to you under the Apache License, Version 2.0 (the
~ "License"); you may not use this file except in compliance
~ with the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>apache-atlas</artifactId>
<groupId>org.apache.atlas</groupId>
<version>0.6-incubating-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>atlas-server-api</artifactId>
<name>Apache Atlas Server API</name>
<description>Apache Atlas Server related APIs</description>
<packaging>jar</packaging>
<dependencies>
<dependency>
<groupId>org.testng</groupId>
<artifactId>testng</artifactId>
</dependency>
<dependency>
<groupId>com.google.inject</groupId>
<artifactId>guice</artifactId>
</dependency>
<dependency>
<groupId>org.apache.atlas</groupId>
<artifactId>atlas-typesystem</artifactId>
</dependency>
</dependencies>
</project>
\ No newline at end of file
common/src/main/java/org/apache/atlas/listener/EntityChangeListener.java → server-api/src/main/java/org/apache/atlas/listener/EntityChangeListener.java
......@@ -41,11 +41,11 @@ public interface EntityChangeListener {
/**
* This is upon updating an entity.
*
* @param entity the updated entity
* @param entities the updated entities
*
* @throws AtlasException if the listener notification fails
*/
void onEntityUpdated(ITypedReferenceableInstance entity) throws AtlasException;
void onEntitiesUpdated(Collection<ITypedReferenceableInstance> entities) throws AtlasException;
/**
* This is upon adding a new trait to a typed instance.
......
repository/src/main/java/org/apache/atlas/services/MetadataService.java → server-api/src/main/java/org/apache/atlas/services/MetadataService.java 100755 → 100644
......@@ -20,6 +20,7 @@ package org.apache.atlas.services;
import org.apache.atlas.AtlasException;
import org.apache.atlas.listener.EntityChangeListener;
import org.apache.atlas.typesystem.Referenceable;
import org.apache.atlas.typesystem.types.DataTypes;
import org.codehaus.jettison.json.JSONObject;
......@@ -105,16 +106,45 @@ public interface MetadataService {
/**
* Adds the property to the given entity id(guid).
* Currently supports updates only on PRIMITIVE, CLASS attribute types
*
* @param guid entity id
* @param property property name
* @param attribute property name
* @param value property value
*/
void updateEntity(String guid, String property, String value) throws AtlasException;
void updateEntityAttributeByGuid(String guid, String attribute, String value) throws AtlasException;
/**
* Supports Partial updates of an entity. Users can update a subset of attributes for an entity identified by its guid
* Note however that it cannot be used to set attribute values to null or delete attrbute values
*
*/
void updateEntityPartialByGuid(String guid, Referenceable entity) throws AtlasException;
/**
* Batch API - Adds/Updates the given entity id(guid).
*
* @param entityJson entity json
* @return List of guids which were updated and ones which were newly created as part of the updated entity
*/
String updateEntities(String entityJson) throws AtlasException;
// Trait management functions
/**
* Updates entity identified by a qualified name
*
* @param typeName
* @param uniqueAttributeName
* @param attrValue
* @param updatedEntity
* @return Guid of updated entity
* @throws AtlasException
*/
String updateEntityByUniqueAttribute(String typeName, String uniqueAttributeName, String attrValue,
Referenceable updatedEntity) throws AtlasException;
/**
* Gets the list of trait names for a given entity represented by a guid.
*
* @param guid globally unique identifier for the entity
......
repository/src/main/java/org/apache/atlas/repository/EntityExistsException.java → server-api/src/main/java/org/apache/atlas/typesystem/exception/EntityExistsException.java
......@@ -16,7 +16,7 @@
* limitations under the License.
*/
package org.apache.atlas.repository;
package org.apache.atlas.typesystem.exception;
import org.apache.atlas.AtlasException;
import org.apache.atlas.typesystem.IReferenceableInstance;
......
repository/src/main/java/org/apache/atlas/repository/EntityNotFoundException.java → server-api/src/main/java/org/apache/atlas/typesystem/exception/EntityNotFoundException.java
......@@ -16,12 +16,14 @@
* limitations under the License.
*/
package org.apache.atlas.repository;
package org.apache.atlas.typesystem.exception;
import org.apache.atlas.AtlasException;
/**
* A simple wrapper for 404.
*/
public class EntityNotFoundException extends RepositoryException {
public class EntityNotFoundException extends AtlasException {
public EntityNotFoundException() {
}
......
typesystem/pom.xml
......@@ -110,13 +110,8 @@
</dependency>
<dependency>
<groupId>com.google.inject</groupId>
<artifactId>guice</artifactId>
</dependency>
<dependency>
<groupId>commons-configuration</groupId>
<artifactId>commons-configuration</artifactId>
<groupId>org.apache.atlas</groupId>
<artifactId>atlas-common</artifactId>
</dependency>
<dependency>
......
typesystem/src/main/java/org/apache/atlas/typesystem/IInstance.java
......@@ -33,6 +33,8 @@ public interface IInstance {
void set(String attrName, Object val) throws AtlasException;
void setNull(String attrName) throws AtlasException;
Map<String, Object> getValuesMap() throws AtlasException;
}
typesystem/src/main/java/org/apache/atlas/typesystem/ITypedInstance.java
......@@ -23,6 +23,7 @@ import org.apache.atlas.typesystem.types.FieldMapping;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.security.MessageDigest;
import java.util.Date;
/**
......@@ -37,8 +38,6 @@ public interface ITypedInstance extends IInstance {
FieldMapping fieldMapping();
void setNull(String attrName) throws AtlasException;
boolean getBoolean(String attrName) throws AtlasException;
byte getByte(String attrName) throws AtlasException;
......@@ -82,4 +81,6 @@ public interface ITypedInstance extends IInstance {
void setDate(String attrName, Date val) throws AtlasException;
void setString(String attrName, String val) throws AtlasException;
String getSignatureHash(MessageDigest digester) throws AtlasException;
}
typesystem/src/main/java/org/apache/atlas/typesystem/Referenceable.java
......@@ -24,7 +24,6 @@ import org.apache.atlas.AtlasException;
import org.apache.atlas.classification.InterfaceAudience;
import org.apache.atlas.typesystem.persistence.Id;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
......@@ -95,8 +94,10 @@ public class Referenceable extends Struct implements IReferenceableInstance {
*/
@SuppressWarnings("unused")
private Referenceable() {
this("", "", Collections.<String, Object>emptyMap(), Collections.<String>emptyList(),
Collections.<String, IStruct>emptyMap());
super(null, null);
id = null;
traitNames = ImmutableList.of();
traits = ImmutableMap.of();
}
@Override
......@@ -114,6 +115,42 @@ public class Referenceable extends Struct implements IReferenceableInstance {
return traits.get(typeName);
}
/**
* Matches traits, values associated with this Referenceable and skips the id match
* @param o The Referenceable which needs to be matched with
* @return
*/
public boolean equalsContents(Object o) {
if(this == o) {
return true;
}
if(o == null) {
return false;
}
if (o.getClass() != getClass()) {
return false;
}
if(!super.equalsContents(o)) {
return false;
}
Referenceable obj = (Referenceable)o;
if (!traitNames.equals(obj.getTraits())) {
return false;
}
return true;
}
public String toString() {
return "{" +
"Id='" + id + '\'' +
", traits=" + traitNames +
", values=" + getValuesMap() +
'}';
}
private static Map<String, IStruct> getTraits(IReferenceableInstance instance) throws AtlasException {
Map<String, IStruct> traits = new HashMap<>();
for (String traitName : instance.getTraits() ) {
......
typesystem/src/main/java/org/apache/atlas/typesystem/Struct.java
......@@ -18,6 +18,7 @@
package org.apache.atlas.typesystem;
import org.apache.atlas.AtlasException;
import org.apache.atlas.classification.InterfaceAudience;
import java.util.Collections;
......@@ -67,7 +68,52 @@ public class Struct implements IStruct {
}
@Override
public void setNull(String attrName) throws AtlasException {
values.remove(attrName);
}
@Override
public Map<String, Object> getValuesMap() {
return values;
}
@Override
public int hashCode() {
int result = typeName.hashCode();
result = 31 * result + values.hashCode();
return result;
}
/**
* equalContents instead of equals since values is a mutable attribute and could lead
* to incorrect behaviour when added to collections and mutated after that
* i.e when the attribute is mutated collections.contains(struct) returns false
* due to hashcode having changed for the struct.
* @param o
* @return
*/
public boolean equalsContents(Object o) {
if (this == o) {
return true;
}
if (o == null) {
return false;
}
if (o.getClass() != getClass()) {
return false;
}
Struct obj = (Struct)o;
if(!typeName.equals(obj.getTypeName())) {
return false;
}
if(!values.equals(obj.getValuesMap())) {
return false;
}
return true;
}
}
typesystem/src/main/java/org/apache/atlas/TypeExistsException.java → typesystem/src/main/java/org/apache/atlas/typesystem/exception/TypeExistsException.java
......@@ -16,7 +16,9 @@
* limitations under the License.
*/
package org.apache.atlas;
package org.apache.atlas.typesystem.exception;
import org.apache.atlas.AtlasException;
public class TypeExistsException extends AtlasException {
public TypeExistsException(String message) {
......
typesystem/src/main/java/org/apache/atlas/TypeNotFoundException.java → typesystem/src/main/java/org/apache/atlas/typesystem/exception/TypeNotFoundException.java
......@@ -16,7 +16,9 @@
* limitations under the License.
*/
package org.apache.atlas;
package org.apache.atlas.typesystem.exception;
import org.apache.atlas.AtlasException;
/**
* A simple wrapper for 404.
......
typesystem/src/main/java/org/apache/atlas/typesystem/persistence/DownCastStructInstance.java
......@@ -52,6 +52,11 @@ public class DownCastStructInstance implements IStruct {
fieldMapping.set(this, attrName, val);
}
@Override
public void setNull(String attrName) throws AtlasException {
throw new UnsupportedOperationException("unset on attributes are not allowed");
}
/*
* Use only for json serialization
* @nonpublic
......
typesystem/src/main/java/org/apache/atlas/typesystem/persistence/Id.java
......@@ -20,12 +20,16 @@ package org.apache.atlas.typesystem.persistence;
import com.google.common.collect.ImmutableList;
import org.apache.atlas.AtlasException;
import org.apache.atlas.utils.ParamChecker;
import org.apache.atlas.typesystem.IStruct;
import org.apache.atlas.typesystem.ITypedReferenceableInstance;
import org.apache.atlas.typesystem.types.FieldMapping;
import org.apache.atlas.utils.MD5Utils;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.nio.charset.Charset;
import java.security.MessageDigest;
import java.util.Date;
import java.util.Map;
import java.util.UUID;
......@@ -37,6 +41,8 @@ public class Id implements ITypedReferenceableInstance {
public final int version;
public Id(String id, int version, String className) {
ParamChecker.notEmpty(className, "id");
ParamChecker.notEmpty(className, "className");
this.id = id;
this.className = className;
this.version = version;
......@@ -248,4 +254,12 @@ public class Id implements ITypedReferenceableInstance {
public void setString(String attrName, String val) throws AtlasException {
throw new AtlasException("Get/Set not supported on an Id object");
}
@Override
public String getSignatureHash(MessageDigest digester) throws AtlasException {
digester.update(id.getBytes(Charset.forName("UTF-8")));
digester.update(className.getBytes(Charset.forName("UTF-8")));
byte[] digest = digester.digest();
return MD5Utils.toString(digest);
}
}
typesystem/src/main/java/org/apache/atlas/typesystem/persistence/ReferenceableInstance.java
......@@ -24,10 +24,14 @@ import org.apache.atlas.AtlasException;
import org.apache.atlas.typesystem.IStruct;
import org.apache.atlas.typesystem.ITypedReferenceableInstance;
import org.apache.atlas.typesystem.ITypedStruct;
import org.apache.atlas.typesystem.types.ClassType;
import org.apache.atlas.typesystem.types.FieldMapping;
import org.apache.atlas.typesystem.types.TypeSystem;
import org.apache.atlas.utils.MD5Utils;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.security.MessageDigest;
import java.util.Date;
/*
......@@ -75,7 +79,7 @@ public class ReferenceableInstance extends StructInstance implements ITypedRefer
* @nopub
* @param id
*/
void replaceWithNewId(Id id) {
public void replaceWithNewId(Id id) {
this.id = id;
}
......@@ -92,4 +96,12 @@ public class ReferenceableInstance extends StructInstance implements ITypedRefer
throw new RuntimeException(me);
}
}
@Override
public String getSignatureHash(MessageDigest digester) throws AtlasException {
ClassType classType = TypeSystem.getInstance().getDataType(ClassType.class, getTypeName());
classType.updateSignatureHash(digester, this);
byte[] digest = digester.digest();
return MD5Utils.toString(digest);
}
}
typesystem/src/main/java/org/apache/atlas/typesystem/persistence/StructInstance.java
......@@ -29,12 +29,15 @@ import org.apache.atlas.typesystem.types.DataTypes;
import org.apache.atlas.typesystem.types.EnumType;
import org.apache.atlas.typesystem.types.EnumValue;
import org.apache.atlas.typesystem.types.FieldMapping;
import org.apache.atlas.typesystem.types.StructType;
import org.apache.atlas.typesystem.types.TypeSystem;
import org.apache.atlas.typesystem.types.TypeUtils;
import org.apache.atlas.typesystem.types.ValueConversionException;
import org.apache.atlas.utils.MD5Utils;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.security.MessageDigest;
import java.util.Date;
import java.util.HashMap;
import java.util.Map;
......@@ -229,6 +232,30 @@ public class StructInstance implements ITypedStruct {
}
int nullPos = fieldMapping.fieldNullPos.get(attrName);
nullFlags[nullPos] = true;
int pos = fieldMapping.fieldPos.get(attrName);
if (i.dataType() == DataTypes.BIGINTEGER_TYPE) {
bigIntegers[pos] = null;
} else if (i.dataType() == DataTypes.BIGDECIMAL_TYPE) {
bigDecimals[pos] = null;
} else if (i.dataType() == DataTypes.DATE_TYPE) {
dates[pos] = null;
} else if (i.dataType() == DataTypes.STRING_TYPE) {
strings[pos] = null;
} else if (i.dataType().getTypeCategory() == DataTypes.TypeCategory.ARRAY) {
arrays[pos] = null;
} else if (i.dataType().getTypeCategory() == DataTypes.TypeCategory.MAP) {
maps[pos] = null;
} else if (i.dataType().getTypeCategory() == DataTypes.TypeCategory.STRUCT
|| i.dataType().getTypeCategory() == DataTypes.TypeCategory.TRAIT) {
structs[pos] = null;
} else if (i.dataType().getTypeCategory() == DataTypes.TypeCategory.CLASS) {
ids[pos] = null;
referenceables[pos] = null;
} else {
throw new AtlasException(String.format("Unknown datatype %s", i.dataType()));
}
}
/*
......@@ -729,4 +756,12 @@ public class StructInstance implements ITypedStruct {
throw new RuntimeException(me);
}
}
@Override
public String getSignatureHash(MessageDigest digester) throws AtlasException {
StructType structType = TypeSystem.getInstance().getDataType(StructType.class, getTypeName());
structType.updateSignatureHash(digester, this);
byte[] digest = digester.digest();
return MD5Utils.toString(digest);
}
}
typesystem/src/main/java/org/apache/atlas/typesystem/types/AttributeDefinition.java
......@@ -18,7 +18,7 @@
package org.apache.atlas.typesystem.types;
import org.apache.atlas.ParamChecker;
import org.apache.atlas.utils.ParamChecker;
public final class AttributeDefinition {
......
typesystem/src/main/java/org/apache/atlas/typesystem/types/ClassType.java
......@@ -34,6 +34,8 @@ import org.apache.atlas.typesystem.persistence.StructInstance;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.nio.charset.Charset;
import java.security.MessageDigest;
import java.util.Date;
import java.util.List;
import java.util.Map;
......@@ -123,9 +125,9 @@ public class ClassType extends HierarchicalType<ClassType, IReferenceableInstanc
r != null ? createInstanceWithTraits(id, r, r.getTraits().toArray(new String[0])) :
createInstance(id);
if (id != null && id.isAssigned()) {
return tr;
}
// if (id != null && id.isAssigned()) {
// return tr;
// }
for (Map.Entry<String, AttributeInfo> e : fieldMapping.fields.entrySet()) {
String attrKey = e.getKey();
......@@ -214,4 +216,24 @@ public class ClassType extends HierarchicalType<ClassType, IReferenceableInstanc
public List<String> getNames(AttributeInfo info) {
return infoToNameMap.get(info);
}
@Override
public void updateSignatureHash(MessageDigest digester, Object val) throws AtlasException {
if( !(val instanceof ITypedReferenceableInstance)) {
throw new IllegalArgumentException("Unexpected value type " + val.getClass().getSimpleName() + ". Expected instance of ITypedStruct");
}
digester.update(getName().getBytes(Charset.forName("UTF-8")));
if(fieldMapping.fields != null && val != null) {
IReferenceableInstance typedValue = (IReferenceableInstance) val;
if(fieldMapping.fields.values() != null) {
for (AttributeInfo aInfo : fieldMapping.fields.values()) {
Object attrVal = typedValue.get(aInfo.name);
if (attrVal != null) {
aInfo.dataType().updateSignatureHash(digester, attrVal);
}
}
}
}
}
}
\ No newline at end of file
typesystem/src/main/java/org/apache/atlas/typesystem/types/DataTypes.java
......@@ -29,10 +29,13 @@ import org.apache.commons.lang3.StringUtils;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.nio.charset.Charset;
import java.security.MessageDigest;
import java.text.ParseException;
import java.util.Collection;
import java.util.Date;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
public class DataTypes {
......@@ -95,6 +98,14 @@ public class DataTypes {
return nullValue();
}
@Override
public void updateSignatureHash(MessageDigest digester, Object val) throws AtlasException {
if ( val != null ) {
digester.update(val.toString().getBytes(Charset.forName("UTF-8")));
}
}
}
public static class BooleanType extends PrimitiveType<Boolean> {
......@@ -161,6 +172,13 @@ public class DataTypes {
public Byte nullValue() {
return 0;
}
@Override
public void updateSignatureHash(MessageDigest digester, Object val) throws AtlasException {
if ( val != null ) {
digester.update(((Byte) val).byteValue());
}
}
}
public static class ShortType extends PrimitiveType<Short> {
......@@ -508,6 +526,7 @@ public class DataTypes {
} else if (val instanceof Iterator) {
it = (Iterator) val;
}
if (it != null) {
ImmutableCollection.Builder b = m.isUnique ? ImmutableSet.builder() : ImmutableList.builder();
while (it.hasNext()) {
......@@ -557,6 +576,15 @@ public class DataTypes {
public TypeCategory getTypeCategory() {
return TypeCategory.ARRAY;
}
@Override
public void updateSignatureHash(MessageDigest digester, Object val) throws AtlasException {
IDataType elemType = getElemType();
List vals = (List) val;
for (Object listElem : vals) {
elemType.updateSignatureHash(digester, listElem);
}
}
}
public static class MapType extends AbstractDataType<ImmutableMap<?, ?>> {
......@@ -586,7 +614,7 @@ public class DataTypes {
}
protected void setValueType(IDataType valueType) {
this.keyType = valueType;
this.valueType = valueType;
}
@Override
......@@ -605,7 +633,8 @@ public class DataTypes {
Map.Entry e = it.next();
b.put(keyType.convert(e.getKey(),
TypeSystem.getInstance().allowNullsInCollections() ? Multiplicity.OPTIONAL :
Multiplicity.REQUIRED), valueType.convert(e.getValue(),
Multiplicity.REQUIRED),
valueType.convert(e.getValue(),
TypeSystem.getInstance().allowNullsInCollections() ? Multiplicity.OPTIONAL :
Multiplicity.REQUIRED));
}
......@@ -657,6 +686,17 @@ public class DataTypes {
public TypeCategory getTypeCategory() {
return TypeCategory.MAP;
}
@Override
public void updateSignatureHash(MessageDigest digester, Object val) throws AtlasException {
IDataType keyType = getKeyType();
IDataType valueType = getValueType();
Map vals = (Map) val;
for (Object key : vals.keySet()) {
keyType.updateSignatureHash(digester, key);
valueType.updateSignatureHash(digester, vals.get(key));
}
}
}
}
typesystem/src/main/java/org/apache/atlas/typesystem/types/EnumType.java
......@@ -23,6 +23,9 @@ import com.google.common.collect.ImmutableMap;
import org.apache.atlas.AtlasException;
import scala.math.BigInt;
import java.nio.charset.Charset;
import java.security.MessageDigest;
public class EnumType extends AbstractDataType<EnumValue> {
public final TypeSystem typeSystem;
......@@ -80,7 +83,7 @@ public class EnumType extends AbstractDataType<EnumValue> {
public void validateUpdate(IDataType newType) throws TypeUpdateException {
super.validateUpdate(newType);
EnumType enumType = (EnumType)newType;
EnumType enumType = (EnumType) newType;
for (EnumValue enumValue : values()) {
//The old enum value should be part of new enum definition as well
if (!enumType.valueMap.containsKey(enumValue.value)) {
......@@ -96,6 +99,12 @@ public class EnumType extends AbstractDataType<EnumValue> {
}
}
public void updateSignatureHash(MessageDigest digester, Object val) throws AtlasException {
if (val != null) {
digester.update(fromValue((String) val).toString().getBytes(Charset.forName("UTF-8")));
}
}
public EnumValue fromOrdinal(int o) {
return ordinalMap.get(o);
}
......
typesystem/src/main/java/org/apache/atlas/typesystem/types/EnumTypeDefinition.java
......@@ -18,7 +18,7 @@
package org.apache.atlas.typesystem.types;
import org.apache.atlas.ParamChecker;
import org.apache.atlas.utils.ParamChecker;
import java.util.Arrays;
......
typesystem/src/main/java/org/apache/atlas/typesystem/types/EnumValue.java
......@@ -18,7 +18,7 @@
package org.apache.atlas.typesystem.types;
import org.apache.atlas.ParamChecker;
import org.apache.atlas.utils.ParamChecker;
public class EnumValue {
......
typesystem/src/main/java/org/apache/atlas/typesystem/types/IDataType.java
......@@ -20,6 +20,8 @@ package org.apache.atlas.typesystem.types;
import org.apache.atlas.AtlasException;
import java.security.MessageDigest;
public interface IDataType<T> {
String getName();
......@@ -30,5 +32,7 @@ public interface IDataType<T> {
void output(T val, Appendable buf, String prefix) throws AtlasException;
void validateUpdate(IDataType newType) throws TypeUpdateException;
void updateSignatureHash(MessageDigest digester, Object val) throws AtlasException;
}
typesystem/src/main/java/org/apache/atlas/typesystem/types/Multiplicity.java
......@@ -25,8 +25,8 @@ public final class Multiplicity {
public static final Multiplicity OPTIONAL = new Multiplicity(0, 1, false);
public static final Multiplicity REQUIRED = new Multiplicity(1, 1, false);
public static final Multiplicity COLLECTION = new Multiplicity(1, Integer.MAX_VALUE, false);
public static final Multiplicity SET = new Multiplicity(1, Integer.MAX_VALUE, true);
public static final Multiplicity COLLECTION = new Multiplicity(0, Integer.MAX_VALUE, false);
public static final Multiplicity SET = new Multiplicity(0, Integer.MAX_VALUE, true);
public final int lower;
public final int upper;
......
typesystem/src/main/java/org/apache/atlas/typesystem/types/ObjectGraphWalker.java
......@@ -214,5 +214,11 @@ public class ObjectGraphWalker {
this.aInfo = aInfo;
this.value = value;
}
@Override
public String toString(){
StringBuilder string = new StringBuilder().append(instance).append(aInfo).append(value);
return string.toString();
}
}
}
typesystem/src/main/java/org/apache/atlas/typesystem/types/StructType.java
......@@ -22,6 +22,8 @@ import org.apache.atlas.AtlasException;
import org.apache.atlas.typesystem.IStruct;
import org.apache.atlas.typesystem.ITypedStruct;
import java.nio.charset.Charset;
import java.security.MessageDigest;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
......@@ -193,6 +195,24 @@ public class StructType extends AbstractDataType<IStruct> implements IConstructa
handler.output(s, buf, prefix);
}
@Override
public void updateSignatureHash(MessageDigest digester, Object val) throws AtlasException {
if( !(val instanceof ITypedStruct)) {
throw new IllegalArgumentException("Unexpected value type " + val.getClass().getSimpleName() + ". Expected instance of ITypedStruct");
}
digester.update(getName().getBytes(Charset.forName("UTF-8")));
if(fieldMapping.fields != null && val != null) {
IStruct typedValue = (IStruct) val;
for (AttributeInfo aInfo : fieldMapping.fields.values()) {
Object attrVal = typedValue.get(aInfo.name);
if(attrVal != null) {
aInfo.dataType().updateSignatureHash(digester, attrVal);
}
}
}
}
public List<String> getNames(AttributeInfo info) {
return infoToNameMap.get(info);
}
......
typesystem/src/main/java/org/apache/atlas/typesystem/types/StructTypeDefinition.java
......@@ -18,7 +18,7 @@
package org.apache.atlas.typesystem.types;
import org.apache.atlas.ParamChecker;
import org.apache.atlas.utils.ParamChecker;
import java.util.Arrays;
......
typesystem/src/main/java/org/apache/atlas/typesystem/types/TraitType.java
......@@ -23,6 +23,8 @@ import org.apache.atlas.AtlasException;
import org.apache.atlas.typesystem.IStruct;
import org.apache.atlas.typesystem.ITypedStruct;
import java.nio.charset.Charset;
import java.security.MessageDigest;
import java.util.List;
import java.util.Map;
......@@ -68,6 +70,24 @@ public class TraitType extends HierarchicalType<TraitType, IStruct>
}
@Override
public void updateSignatureHash(MessageDigest digester, Object val) throws AtlasException {
if( !(val instanceof ITypedStruct)) {
throw new IllegalArgumentException("Unexpected value type " + val.getClass().getSimpleName() + ". Expected instance of ITypedStruct");
}
digester.update(getName().getBytes(Charset.forName("UTF-8")));
if(fieldMapping.fields != null && val != null) {
IStruct typedValue = (IStruct) val;
for (AttributeInfo aInfo : fieldMapping.fields.values()) {
Object attrVal = typedValue.get(aInfo.name);
if(attrVal != null) {
aInfo.dataType().updateSignatureHash(digester, attrVal);
}
}
}
}
@Override
public List<String> getNames(AttributeInfo info) {
return infoToNameMap.get(info);
}
......
typesystem/src/main/java/org/apache/atlas/typesystem/types/TypeSystem.java
......@@ -22,10 +22,10 @@ import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Multimap;
import org.apache.atlas.AtlasException;
import org.apache.atlas.TypeExistsException;
import org.apache.atlas.TypeNotFoundException;
import org.apache.atlas.classification.InterfaceAudience;
import org.apache.atlas.typesystem.TypesDef;
import org.apache.atlas.typesystem.exception.TypeExistsException;
import org.apache.atlas.typesystem.exception.TypeNotFoundException;
import javax.inject.Singleton;
import java.lang.reflect.Constructor;
......
typesystem/src/main/scala/org/apache/atlas/typesystem/json/Serialization.scala
......@@ -29,18 +29,22 @@ import org.json4s.native.Serialization._
import scala.collection.JavaConversions._
import scala.collection.JavaConverters._
class BigDecimalSerializer extends CustomSerializer[java.math.BigDecimal](format => ( {
case JDecimal(e) => e.bigDecimal
}, {
case e: java.math.BigDecimal => JDecimal(new BigDecimal(e))
}
class BigDecimalSerializer extends CustomSerializer[java.math.BigDecimal](format => (
{
case JDecimal(e) => e.bigDecimal
},
{
case e: java.math.BigDecimal => JDecimal(new BigDecimal(e))
}
))
class BigIntegerSerializer extends CustomSerializer[java.math.BigInteger](format => ( {
case JInt(e) => e.bigInteger
}, {
case e: java.math.BigInteger => JInt(new BigInt(e))
}
class BigIntegerSerializer extends CustomSerializer[java.math.BigInteger](format => (
{
case JInt(e) => e.bigInteger
},
{
case e: java.math.BigInteger => JInt(new BigInt(e))
}
))
class IdSerializer extends CustomSerializer[Id](format => ( {
......@@ -292,12 +296,19 @@ object Serialization {
read[ReferenceableInstance](jsonStr)
}
def traitFromJson(jsonStr: String): ITypedInstance = {
implicit val formats = org.json4s.native.Serialization.formats(NoTypeHints) + new TypedStructSerializer +
new TypedReferenceableInstanceSerializer + new BigDecimalSerializer + new BigIntegerSerializer
def traitFromJson(jsonStr: String): ITypedInstance = {
implicit val formats = org.json4s.native.Serialization.formats(NoTypeHints) + new TypedStructSerializer +
new TypedReferenceableInstanceSerializer + new BigDecimalSerializer + new BigIntegerSerializer
read[StructInstance](jsonStr)
}
read[StructInstance](jsonStr)
}
def arrayFromJson(jsonStr: String): ITypedInstance = {
implicit val formats = org.json4s.native.Serialization.formats(NoTypeHints) + new TypedStructSerializer +
new TypedReferenceableInstanceSerializer + new BigDecimalSerializer + new BigIntegerSerializer
read[StructInstance](jsonStr)
}
}
webapp/src/main/java/org/apache/atlas/web/filters/AuditFilter.java
......@@ -86,7 +86,7 @@ public class AuditFilter implements Filter {
LOG.debug("Audit: {}/{} performed request {} {} ({}) at time {}", who, fromAddress, whatRequest, whatURL,
whatAddrs, whenISO9601);
audit(who, fromAddress, fromHost, whatURL, whatAddrs, whenISO9601);
audit(who, fromAddress, whatRequest, fromHost, whatURL, whatAddrs, whenISO9601);
}
private String getUserFromRequest(HttpServletRequest httpRequest) {
......@@ -95,9 +95,9 @@ public class AuditFilter implements Filter {
return userFromRequest == null ? "UNKNOWN" : userFromRequest;
}
private void audit(String who, String fromAddress, String fromHost, String whatURL, String whatAddrs,
private void audit(String who, String fromAddress, String whatRequest, String fromHost, String whatURL, String whatAddrs,
String whenISO9601) {
AUDIT_LOG.info("Audit: {}/{}-{} performed request {} ({}) at time {}", who, fromAddress, fromHost, whatURL,
AUDIT_LOG.info("Audit: {}/{}-{} performed request {} {} ({}) at time {}", who, fromAddress, fromHost, whatRequest, whatURL,
whatAddrs, whenISO9601);
}
......
webapp/src/main/java/org/apache/atlas/web/resources/HiveLineageResource.java
......@@ -19,10 +19,10 @@
package org.apache.atlas.web.resources;
import org.apache.atlas.AtlasClient;
import org.apache.atlas.ParamChecker;
import org.apache.atlas.typesystem.exception.EntityNotFoundException;
import org.apache.atlas.utils.ParamChecker;
import org.apache.atlas.discovery.DiscoveryException;
import org.apache.atlas.discovery.LineageService;
import org.apache.atlas.repository.EntityNotFoundException;
import org.apache.atlas.web.util.Servlets;
import org.codehaus.jettison.json.JSONObject;
import org.slf4j.Logger;
......
webapp/src/main/java/org/apache/atlas/web/resources/MetadataDiscoveryResource.java
......@@ -20,7 +20,7 @@ package org.apache.atlas.web.resources;
import com.google.common.base.Preconditions;
import org.apache.atlas.AtlasClient;
import org.apache.atlas.ParamChecker;
import org.apache.atlas.utils.ParamChecker;
import org.apache.atlas.discovery.DiscoveryException;
import org.apache.atlas.discovery.DiscoveryService;
import org.apache.atlas.web.util.Servlets;
......
webapp/src/main/java/org/apache/atlas/web/resources/TypesResource.java
......@@ -21,8 +21,8 @@ package org.apache.atlas.web.resources;
import com.sun.jersey.api.client.ClientResponse;
import org.apache.atlas.AtlasClient;
import org.apache.atlas.AtlasException;
import org.apache.atlas.TypeExistsException;
import org.apache.atlas.services.MetadataService;
import org.apache.atlas.typesystem.exception.TypeExistsException;
import org.apache.atlas.typesystem.types.DataTypes;
import org.apache.atlas.web.util.Servlets;
import org.codehaus.jettison.json.JSONArray;
......
webapp/src/main/java/org/apache/atlas/web/util/Servlets.java
......@@ -19,7 +19,7 @@
package org.apache.atlas.web.util;
import org.apache.atlas.AtlasClient;
import org.apache.atlas.ParamChecker;
import org.apache.atlas.utils.ParamChecker;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringEscapeUtils;
import org.apache.commons.lang3.StringUtils;
......
webapp/src/test/java/org/apache/atlas/notification/EntityNotificationIT.java
......@@ -118,7 +118,7 @@ public class EntityNotificationIT extends BaseResourceIT {
final String guid = tableId._getId();
serviceClient.updateEntity(guid, property, newValue);
serviceClient.updateEntityAttribute(guid, property, newValue);
waitForNotification(MAX_WAIT_TIME);
......
webapp/src/test/java/org/apache/atlas/notification/NotificationHookConsumerIT.java
......@@ -30,7 +30,7 @@ import org.testng.annotations.Guice;
import org.testng.annotations.Test;
@Guice(modules = NotificationModule.class)
public class NotificationHookConsumerIT extends BaseResourceIT{
public class NotificationHookConsumerIT extends BaseResourceIT {
@Inject
private NotificationInterface kafka;
......
webapp/src/test/java/org/apache/atlas/web/resources/BaseResourceIT.java
......@@ -42,6 +42,7 @@ import org.apache.atlas.typesystem.types.Multiplicity;
import org.apache.atlas.typesystem.types.StructTypeDefinition;
import org.apache.atlas.typesystem.types.TraitType;
import org.apache.atlas.typesystem.types.utils.TypesUtil;
import org.apache.atlas.utils.ParamChecker;
import org.apache.atlas.web.util.Servlets;
import org.apache.commons.configuration.Configuration;
import org.apache.commons.lang.RandomStringUtils;
......@@ -83,7 +84,11 @@ public abstract class BaseResourceIT {
protected void createType(TypesDef typesDef) throws Exception {
HierarchicalTypeDefinition<ClassType> sampleType = typesDef.classTypesAsJavaList().get(0);
if (serviceClient.getType(sampleType.typeName) == null) {
try {
serviceClient.getType(sampleType.typeName);
LOG.info("Types already exist. Skipping type creation");
} catch(AtlasServiceException ase) {
//Expected if type doesnt exist
String typesAsJSON = TypesSerialization.toJson(typesDef);
createType(typesAsJSON);
}
......
webapp/src/test/java/org/apache/atlas/web/resources/EntityJerseyResourceIT.java
......@@ -52,7 +52,10 @@ import org.testng.annotations.Test;
import javax.ws.rs.HttpMethod;
import javax.ws.rs.core.Response;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.UUID;
/**
......@@ -174,7 +177,6 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
@Test
public void testSubmitEntityWithBadDateFormat() throws Exception {
try {
Referenceable tableInstance = createHiveTableInstance("db" + randomString(), "table" + randomString());
tableInstance.set("lastAccessTime", "2014-07-11");
......@@ -191,8 +193,7 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
final String guid = tableId._getId();
//add property
String description = "bar table - new desc";
ClientResponse clientResponse = addProperty(guid, "description", description);
Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode());
addProperty(guid, "description", description);
String entityRef = getEntityDefinition(getEntityDefinition(guid));
Assert.assertNotNull(entityRef);
......@@ -200,13 +201,16 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
tableInstance.set("description", description);
//invalid property for the type
clientResponse = addProperty(guid, "invalid_property", "bar table");
Assert.assertEquals(clientResponse.getStatus(), Response.Status.BAD_REQUEST.getStatusCode());
try {
addProperty(guid, "invalid_property", "bar table");
Assert.fail("Expected AtlasServiceException");
} catch (AtlasServiceException e) {
Assert.assertEquals(e.getStatus().getStatusCode(), Response.Status.BAD_REQUEST.getStatusCode());
}
//non-string property, update
String currentTime = String.valueOf(System.currentTimeMillis());
clientResponse = addProperty(guid, "createTime", currentTime);
Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode());
addProperty(guid, "createTime", currentTime);
entityRef = getEntityDefinition(getEntityDefinition(guid));
Assert.assertNotNull(entityRef);
......@@ -222,12 +226,16 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
Assert.fail();
}
@Test(dependsOnMethods = "testSubmitEntity", expectedExceptions = IllegalArgumentException.class)
@Test(dependsOnMethods = "testSubmitEntity")
public void testAddNullPropertyValue() throws Exception {
final String guid = tableId._getId();
//add property
addProperty(guid, "description", null);
Assert.fail();
try {
addProperty(guid, "description", null);
Assert.fail("Expected AtlasServiceException");
} catch(AtlasServiceException e) {
Assert.assertEquals(e.getStatus().getStatusCode(), Response.Status.BAD_REQUEST.getStatusCode());
}
}
@Test(dependsOnMethods = "testSubmitEntity")
......@@ -242,8 +250,7 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
//Add reference property
final String guid = tableId._getId();
ClientResponse clientResponse = addProperty(guid, "db", dbId);
Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode());
addProperty(guid, "db", dbId);
}
@Test(dependsOnMethods = "testSubmitEntity")
......@@ -264,11 +271,8 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
InstanceSerialization.fromJsonReferenceable(definition, true);
}
private ClientResponse addProperty(String guid, String property, String value) {
WebResource resource = service.path(ENTITIES).path(guid);
return resource.queryParam("property", property).queryParam("value", value).accept(Servlets.JSON_MEDIA_TYPE)
.type(Servlets.JSON_MEDIA_TYPE).method(HttpMethod.PUT, ClientResponse.class);
private void addProperty(String guid, String property, String value) throws AtlasServiceException {
serviceClient.updateEntityAttribute(guid, property, value);
}
private ClientResponse getEntityDefinition(String guid) {
......@@ -547,4 +551,82 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
Referenceable getReferenceable = InstanceSerialization.fromJsonReferenceable(definition, true);
Assert.assertEquals(getReferenceable.get(attrName), attrValue);
}
@Test(dependsOnMethods = "testSubmitEntity")
public void testPartialUpdate() throws Exception {
final List<Referenceable> columns = new ArrayList<>();
Map<String, Object> values = new HashMap<>();
values.put("name", "col1");
values.put("dataType", "string");
values.put("comment", "col1 comment");
Referenceable ref = new Referenceable(BaseResourceIT.COLUMN_TYPE, values);
columns.add(ref);
Referenceable tableUpdated = new Referenceable(BaseResourceIT.HIVE_TABLE_TYPE, new HashMap<String, Object>() {{
put("columns", columns);
}});
LOG.debug("Updating entity= " + tableUpdated);
serviceClient.updateEntity(tableId._getId(), tableUpdated);
ClientResponse response = getEntityDefinition(tableId._getId());
String definition = getEntityDefinition(response);
Referenceable getReferenceable = InstanceSerialization.fromJsonReferenceable(definition, true);
List<Referenceable> refs = (List<Referenceable>) getReferenceable.get("columns");
Assert.assertTrue(refs.get(0).equalsContents(columns.get(0)));
//Update by unique attribute
values.put("dataType", "int");
ref = new Referenceable(BaseResourceIT.COLUMN_TYPE, values);
columns.set(0, ref);
tableUpdated = new Referenceable(BaseResourceIT.HIVE_TABLE_TYPE, new HashMap<String, Object>() {{
put("columns", columns);
}});
LOG.debug("Updating entity= " + tableUpdated);
serviceClient.updateEntity(BaseResourceIT.HIVE_TABLE_TYPE, "name", (String) tableInstance.get("name"),
tableUpdated);
response = getEntityDefinition(tableId._getId());
definition = getEntityDefinition(response);
getReferenceable = InstanceSerialization.fromJsonReferenceable(definition, true);
refs = (List<Referenceable>) getReferenceable.get("columns");
Assert.assertTrue(refs.get(0).equalsContents(columns.get(0)));
Assert.assertEquals(refs.get(0).get("dataType"), "int");
}
@Test(dependsOnMethods = "testSubmitEntity")
public void testCompleteUpdate() throws Exception {
final List<Referenceable> columns = new ArrayList<>();
Map<String, Object> values1 = new HashMap<>();
values1.put("name", "col3");
values1.put("dataType", "string");
values1.put("comment", "col3 comment");
Map<String, Object> values2 = new HashMap<>();
values2.put("name", "col4");
values2.put("dataType", "string");
values2.put("comment", "col4 comment");
Referenceable ref1 = new Referenceable(BaseResourceIT.COLUMN_TYPE, values1);
Referenceable ref2 = new Referenceable(BaseResourceIT.COLUMN_TYPE, values2);
columns.add(ref1);
columns.add(ref2);
tableInstance.set("columns", columns);
LOG.debug("Replacing entity= " + tableInstance);
serviceClient.updateEntities(tableInstance);
ClientResponse response = getEntityDefinition(tableId._getId());
String definition = getEntityDefinition(response);
Referenceable getReferenceable = InstanceSerialization.fromJsonReferenceable(definition, true);
List<Referenceable> refs = (List<Referenceable>) getReferenceable.get("columns");
Assert.assertEquals(refs.size(), 2);
Assert.assertTrue(refs.get(0).equalsContents(columns.get(0)));
Assert.assertTrue(refs.get(1).equalsContents(columns.get(1)));
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment