Skip to content

A
atlas
Commit 5bb2dcbe by David Radley Committed by Madhan Neethiraj
Options

ATLAS-1891: added RelationshipDef validation during typesystem updates

parent 4f16db45
Showing with 89 additions and 236 deletions
intg/src/main/java/org/apache/atlas/AtlasErrorCode.java
......@@ -67,7 +67,7 @@ public enum AtlasErrorCode {
INVALID_STRUCT_VALUE(400, "ATLAS-400-00-025", "not a valid struct value {0}"),
INSTANCE_LINEAGE_INVALID_PARAMS(400, "ATLAS-400-00-026", "Invalid lineage query parameters passed {0}: {1}"),
ATTRIBUTE_UPDATE_NOT_SUPPORTED(400, "ATLAS-400-00-027", "{0}.{1} : attribute update not supported"),
INVALID_VALUE(400, "ATLAS-400-00-028", "invalid value: {0}"),
INVALID_VALUE(400, "ATLAS-400-00-028", "invalid value: {0}"),
BAD_REQUEST(400, "ATLAS-400-00-029", "{0}"),
PARAMETER_PARSING_FAILED(400, "ATLAS-400-00-02A", "Parameter parsing failed at: {0}"),
MISSING_MANDATORY_ATTRIBUTE(400, "ATLAS-400-00-02B", "Mandatory field {0}.{1} has empty/null value"),
......@@ -80,7 +80,13 @@ public enum AtlasErrorCode {
RELATIONSHIPDEF_COMPOSITION_SET_CONTAINER(400, "ATLAS-400-00-033", "COMPOSITION relationshipDef {0} cannot have a SET cardinality and be a container"),
RELATIONSHIPDEF_LIST_ON_END(400, "ATLAS-400-00-034", "relationshipDef {0} cannot have a LIST cardinality on an end"),
RELATIONSHIPDEF_INVALID_END_TYPE(400, "ATLAS-400-00-035", "relationshipDef {0} has invalid end type {1}"),
INVALID_RELATIONSHIP_END_TYPE(400, "ATLAS-400-00-036", "invalid end type for relationship {0}: expected {1}, found {2}"),
INVALID_RELATIONSHIP_END_TYPE(400, "ATLAS-400-00-036", "invalid update for relationshipDef {0}: new end type {1}, existing end type {2}"),
RELATIONSHIPDEF_INVALID_END1_UPDATE(400, "ATLAS-400-00-037", "invalid update for relationshipDef {0}: new end1 {1}, existing end1 {2}"),
RELATIONSHIPDEF_INVALID_END2_UPDATE(400, "ATLAS-400-00-038", "invalid update for relationshipDef {0}: new end2 {1}, existing end2 {2}"),
RELATIONSHIPDEF_INVALID_CATEGORY_UPDATE(400, "ATLAS-400-00-039", "invalid update for relationship {0}: new relationshipDef category {1}, existing relationshipDef category {2}"),
RELATIONSHIPDEF_INVALID_NAME_UPDATE(400, "ATLAS-400-00-040", "invalid relationshipDef rename for relationship guid {0}: new name {1}, existing name {2}"),
RELATIONSHIPDEF_END1_NAME_INVALID(400, "ATLAS-400-00-020", "{0}: invalid end1 name. Name must not contain query keywords"),
RELATIONSHIPDEF_END2_NAME_INVALID(400, "ATLAS-400-00-020", "{0}: invalid end2 name. Name must not contain query keywords"),
// All Not found enums go here
TYPE_NAME_NOT_FOUND(404, "ATLAS-404-00-001", "Given typename {0} was invalid"),
TYPE_GUID_NOT_FOUND(404, "ATLAS-404-00-002", "Given type guid {0} was invalid"),
......@@ -94,14 +100,14 @@ public enum AtlasErrorCode {
INSTANCE_NOT_FOUND(404, "ATLAS-404-00-00B", "Given instance is invalid/not found: {0}"),
RELATIONSHIP_GUID_NOT_FOUND(404, "ATLAS-404-00-00C", "Given relationship guid {0} is invalid/not found"),
RELATIONSHIP_CRUD_INVALID_PARAMS(404, "ATLAS-404-00-00D", "Invalid relationship creation/updation parameters passed : {0}"),
// All data conflict errors go here
// All data conflict errors go here
TYPE_ALREADY_EXISTS(409, "ATLAS-409-00-001", "Given type {0} already exists"),
TYPE_HAS_REFERENCES(409, "ATLAS-409-00-002", "Given type {0} has references"),
INSTANCE_ALREADY_EXISTS(409, "ATLAS-409-00-003", "failed to update entity: {0}"),
RELATIONSHIP_ALREADY_EXISTS(409, "ATLAS-409-00-004", "relationship {0} already exists between entities {1} and {2}"),
TYPE_HAS_RELATIONSHIPS(409, "ATLAS-409-00-005", "Given type {0} has associated relationshipDefs"),
// All internal errors go here
// All internal errors go here
INTERNAL_ERROR(500, "ATLAS-500-00-001", "Internal server error {0}"),
INDEX_CREATION_FAILED(500, "ATLAS-500-00-002", "Index creation failed for {0}"),
INDEX_ROLLBACK_FAILED(500, "ATLAS-500-00-003", "Index rollback failed for {0}"),
......@@ -155,4 +161,4 @@ public enum AtlasErrorCode {
public String getErrorCode() {
return errorCode;
}
}
}
\ No newline at end of file
intg/src/main/java/org/apache/atlas/type/AtlasTypeUtil.java
......@@ -23,6 +23,8 @@ import org.apache.atlas.model.instance.AtlasEntityHeader;
import org.apache.atlas.model.instance.AtlasObjectId;
import org.apache.atlas.model.typedef.*;
import org.apache.atlas.model.typedef.AtlasEnumDef.AtlasEnumElementDef;
import org.apache.atlas.model.typedef.AtlasRelationshipDef.PropagateTags;
import org.apache.atlas.model.typedef.AtlasRelationshipDef.RelationshipCategory;
import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef;
import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef.Cardinality;
import org.apache.atlas.model.typedef.AtlasStructDef.AtlasConstraintDef;
......@@ -241,6 +243,18 @@ public class AtlasTypeUtil {
return new AtlasEntityDef(name, description, version, Arrays.asList(attrDefs), superTypes);
}
public static AtlasRelationshipDef createRelationshipTypeDef(String name,
String description,
String version,
RelationshipCategory relationshipCategory,
PropagateTags propagateTags,
AtlasRelationshipEndDef endDef1,
AtlasRelationshipEndDef endDef2,
AtlasAttributeDef... attrDefs) {
return new AtlasRelationshipDef(name, description, version, relationshipCategory, propagateTags,
endDef1, endDef2, Arrays.asList(attrDefs));
}
public static AtlasTypesDef getTypesDef(List<AtlasEnumDef> enums,
List<AtlasStructDef> structs,
List<AtlasClassificationDef> traits,
......
repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasTypeDefGraphStore.java
......@@ -412,11 +412,12 @@ public abstract class AtlasTypeDefGraphStore implements AtlasTypeDefStore, Activ
AtlasTypesDef ret = updateGraphStore(typesDef, ttr);
if (LOG.isDebugEnabled()) {
LOG.debug("<== AtlasTypeDefGraphStore.updateTypesDef(enums={}, structs={}, classfications={}, entities={})",
LOG.debug("<== AtlasTypeDefGraphStore.updateTypesDef(enums={}, structs={}, classfications={}, entities={}, relationships={})",
CollectionUtils.size(typesDef.getEnumDefs()),
CollectionUtils.size(typesDef.getStructDefs()),
CollectionUtils.size(typesDef.getClassificationDefs()),
CollectionUtils.size(typesDef.getEntityDefs()));
CollectionUtils.size(typesDef.getEntityDefs()),
CollectionUtils.size(typesDef.getRelationshipDefs()));
}
return ret;
......@@ -857,6 +858,7 @@ public abstract class AtlasTypeDefGraphStore implements AtlasTypeDefStore, Activ
AtlasStructDefStore structDefStore = getStructDefStore(ttr);
AtlasClassificationDefStore classifiDefStore = getClassificationDefStore(ttr);
AtlasEntityDefStore entityDefStore = getEntityDefStore(ttr);
AtlasRelationshipDefStore relationDefStore = getRelationshipDefStore(ttr);
if (CollectionUtils.isNotEmpty(typesDef.getEnumDefs())) {
for (AtlasEnumDef enumDef : typesDef.getEnumDefs()) {
......@@ -882,6 +884,12 @@ public abstract class AtlasTypeDefGraphStore implements AtlasTypeDefStore, Activ
}
}
if (CollectionUtils.isNotEmpty(typesDef.getRelationshipDefs())) {
for (AtlasRelationshipDef relationshipDef : typesDef.getRelationshipDefs()) {
ret.getRelationshipDefs().add(relationDefStore.update(relationshipDef));
}
}
return ret;
}
......
repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityDefStoreV1.java
......@@ -265,6 +265,11 @@ public class AtlasEntityDefStoreV1 extends AtlasAbstractDefStoreV1 implements At
throw new AtlasBaseException(AtlasErrorCode.TYPE_NAME_NOT_FOUND, name);
}
// error if we are trying to delete an entityDef that has a relationshipDef
if (typeDefStore.hasIncomingEdgesWithLabel(ret, AtlasGraphUtilsV1.RELATIONSHIPTYPE_EDGE_LABEL)){
throw new AtlasBaseException(AtlasErrorCode.TYPE_HAS_RELATIONSHIPS, name);
}
typeDefStore.deleteTypeVertexOutEdges(ret);
if (LOG.isDebugEnabled()) {
......@@ -313,6 +318,11 @@ public class AtlasEntityDefStoreV1 extends AtlasAbstractDefStoreV1 implements At
throw new AtlasBaseException(AtlasErrorCode.TYPE_GUID_NOT_FOUND, guid);
}
// error if we are trying to delete an entityDef that has a relationshipDef
if (typeDefStore.hasIncomingEdgesWithLabel(ret, AtlasGraphUtilsV1.RELATIONSHIPTYPE_EDGE_LABEL)){
throw new AtlasBaseException(AtlasErrorCode.TYPE_HAS_RELATIONSHIPS, typeName);
}
typeDefStore.deleteTypeVertexOutEdges(ret);
if (LOG.isDebugEnabled()) {
......
repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasGraphUtilsV1.java
......@@ -50,6 +50,7 @@ public class AtlasGraphUtilsV1 {
public static final String PROPERTY_PREFIX = Constants.INTERNAL_PROPERTY_KEY_PREFIX + "type.";
public static final String SUPERTYPE_EDGE_LABEL = PROPERTY_PREFIX + ".supertype";
public static final String VERTEX_TYPE = "typeSystem";
public static final String RELATIONSHIPTYPE_EDGE_LABEL = PROPERTY_PREFIX + ".relationshipType";
public static String getTypeDefPropertyKey(AtlasBaseTypeDef typeDef) {
......@@ -289,6 +290,22 @@ public class AtlasGraphUtilsV1 {
return vertex;
}
public static boolean relationshipTypeHasInstanceEdges(String typeName) throws AtlasBaseException {
AtlasGraphQuery query = AtlasGraphProvider.getGraphInstance()
.query()
.has(Constants.TYPE_NAME_PROPERTY_KEY, AtlasGraphQuery.ComparisionOperator.EQUAL, typeName);
Iterator<AtlasEdge> results = query.edges().iterator();
boolean hasInstanceEdges = results != null && results.hasNext();
if (LOG.isDebugEnabled()) {
LOG.debug("relationshipType {} has instance edges {}", typeName, hasInstanceEdges);
}
return hasInstanceEdges;
}
private static String toString(AtlasElement element) {
if (element instanceof AtlasVertex) {
return toString((AtlasVertex) element);
......
repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasTypeDefGraphStoreV1.java
......@@ -242,6 +242,28 @@ public class AtlasTypeDefGraphStoreV1 extends AtlasTypeDefGraphStore {
}
}
/**
* Look to see if there are any IN edges with the supplied label
* @param vertex
* @param label
* @return
* @throws AtlasBaseException
*/
boolean hasIncomingEdgesWithLabel(AtlasVertex vertex, String label) throws AtlasBaseException {
boolean foundEdges = false;
Iterator<AtlasEdge> inEdges = vertex.getEdges(AtlasEdgeDirection.IN).iterator();
while (inEdges.hasNext()) {
AtlasEdge edge = inEdges.next();
if (label.equals(edge.getLabel())) {
foundEdges = true;
break;
}
}
return foundEdges;
}
void deleteTypeVertex(AtlasVertex vertex) throws AtlasBaseException {
Iterator<AtlasEdge> inEdges = vertex.getEdges(AtlasEdgeDirection.IN).iterator();
......
repository/src/test/java/org/apache/atlas/TestModules.java
......@@ -47,16 +47,9 @@ import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.repository.impexp.ExportService;
import org.apache.atlas.repository.store.graph.AtlasEntityDefStore;
import org.apache.atlas.repository.store.graph.AtlasEntityStore;
import org.apache.atlas.repository.store.graph.AtlasRelationshipDefStore;
import org.apache.atlas.repository.store.graph.AtlasRelationshipStore;
import org.apache.atlas.repository.store.graph.v1.AtlasEntityChangeNotifier;
import org.apache.atlas.repository.store.graph.v1.AtlasEntityDefStoreV1;
import org.apache.atlas.repository.store.graph.v1.AtlasEntityStoreV1;
import org.apache.atlas.repository.store.graph.v1.AtlasRelationshipStoreV1;
import org.apache.atlas.repository.store.graph.v1.AtlasTypeDefGraphStoreV1;
import org.apache.atlas.repository.store.graph.v1.DeleteHandlerV1;
import org.apache.atlas.repository.store.graph.v1.EntityGraphMapper;
import org.apache.atlas.repository.store.graph.v1.HardDeleteHandlerV1;
import org.apache.atlas.repository.store.graph.v1.SoftDeleteHandlerV1;
import org.apache.atlas.repository.store.graph.v1.*;
import org.apache.atlas.repository.typestore.GraphBackedTypeStore;
import org.apache.atlas.repository.typestore.ITypeStore;
import org.apache.atlas.repository.typestore.StoreBackedTypeCache;
......@@ -136,6 +129,7 @@ public class TestModules {
//For testing
bind(AtlasEntityDefStore.class).to(AtlasEntityDefStoreV1.class).asEagerSingleton();
bind(AtlasRelationshipDefStore.class).to(AtlasRelationshipDefStoreV1.class).asEagerSingleton();
bind(AtlasTypeRegistry.class).asEagerSingleton();
bind(EntityGraphMapper.class).asEagerSingleton();
bind(ExportService.class).asEagerSingleton();
......
webapp/src/main/java/org/apache/atlas/examples/CreateTypesFromJsonFileUtil.java deleted 100644 → 0
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.examples;
import java.io.Console;
import java.io.File;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import org.apache.atlas.ApplicationProperties;
import org.apache.atlas.AtlasBaseClient;
import org.apache.atlas.AtlasClientV2;
import org.apache.atlas.AtlasException;
import org.apache.atlas.model.typedef.AtlasTypesDef;
import org.apache.atlas.type.AtlasType;
import org.apache.atlas.utils.AuthenticationUtil;
import org.apache.commons.configuration.Configuration;
import com.google.common.annotations.VisibleForTesting;
/**
* A driver that sets up types supplied in a file.
*/
public class CreateTypesFromJsonFileUtil extends AtlasBaseClient{
public static final String ATLAS_REST_ADDRESS = "atlas.rest.address";
public static void main(String[] args) throws Exception {
Console console = System.console();
if (console == null) {
System.err.println("No console.");
System.exit(1);
}
String[] basicAuthUsernamePassword = null;
if (!AuthenticationUtil.isKerberosAuthenticationEnabled()) {
basicAuthUsernamePassword = AuthenticationUtil.getBasicAuthenticationInput();
}
AtlasClientV2 atlasClientV2 = getAtlasClientV2(args, basicAuthUsernamePassword);
String createFileName = console.readLine("Enter fileName containing TypeDefs for create:- ");
File createFile = new File(createFileName);
String createJsonStr = new String( Files.readAllBytes(createFile.toPath()), StandardCharsets.UTF_8);
System.err.println("create json is :\n" + createJsonStr);
runTypeCreation(createJsonStr,atlasClientV2);
// String updateFileName = console.readLine("Enter fileName containing TypeDefs for update:- ");
// File updateFile = new File(updateFileName);
// String updateJsonStr = new String( Files.readAllBytes(updateFile.toPath()), StandardCharsets.UTF_8);
// System.err.println("update json is :\n" + updateJsonStr);
// runTypeUpdate(updateJsonStr,atlasClientV2);
}
@VisibleForTesting
static void runTypeCreation(String jsonStr,AtlasClientV2 atlasClientV2) throws Exception {
AtlasTypesDef typesDef = AtlasType.fromJson(jsonStr, AtlasTypesDef.class);
atlasClientV2.createAtlasTypeDefs(typesDef);
}
@VisibleForTesting
static void runTypeUpdate(String jsonStr,AtlasClientV2 atlasClientV2) throws Exception {
AtlasTypesDef typesDef = AtlasType.fromJson(jsonStr, AtlasTypesDef.class);
atlasClientV2.updateAtlasTypeDefs(typesDef);
}
private static AtlasClientV2 getAtlasClientV2(String[] args, String[] basicAuthUsernamePassword) throws AtlasException {
String[] urls = getServerUrl(args);
AtlasClientV2 atlasClientV2;
if (!AuthenticationUtil.isKerberosAuthenticationEnabled()) {
atlasClientV2 =new AtlasClientV2(urls,basicAuthUsernamePassword);
} else {
atlasClientV2 = new AtlasClientV2(urls);
}
return atlasClientV2;
}
static String[] getServerUrl(String[] args) throws AtlasException {
if (args.length > 0) {
return args[0].split(",");
}
Configuration configuration = ApplicationProperties.get();
String[] urls = configuration.getStringArray(ATLAS_REST_ADDRESS);
if (urls == null || urls.length == 0) {
System.out.println("Usage: quick_start.py <atlas endpoint of format <http/https>://<atlas-fqdn>:<atlas port> like http://localhost:21000>");
System.exit(-1);
}
return urls;
}
}
\ No newline at end of file
webapp/src/main/java/org/apache/atlas/examples/UpdateTypesFromJsonFileUtil.java deleted 100644 → 0
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.examples;
import com.google.common.annotations.VisibleForTesting;
import org.apache.atlas.ApplicationProperties;
import org.apache.atlas.AtlasBaseClient;
import org.apache.atlas.AtlasClientV2;
import org.apache.atlas.AtlasException;
import org.apache.atlas.model.typedef.AtlasTypesDef;
import org.apache.atlas.type.AtlasType;
import org.apache.atlas.utils.AuthenticationUtil;
import org.apache.commons.configuration.Configuration;
import java.io.Console;
import java.io.File;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
/**
* A driver that sets up types supplied in a file.
*/
public class UpdateTypesFromJsonFileUtil extends AtlasBaseClient{
public static final String ATLAS_REST_ADDRESS = "atlas.rest.address";
public static void main(String[] args) throws Exception {
Console console = System.console();
if (console == null) {
System.err.println("No console.");
System.exit(1);
}
String[] basicAuthUsernamePassword = null;
if (!AuthenticationUtil.isKerberosAuthenticationEnabled()) {
basicAuthUsernamePassword = AuthenticationUtil.getBasicAuthenticationInput();
}
AtlasClientV2 atlasClientV2 = getAtlasClientV2(args, basicAuthUsernamePassword);
String createFileName = console.readLine("Enter fileName containing TypeDefs for create:- ");
File createFile = new File(createFileName);
String createJsonStr = new String( Files.readAllBytes(createFile.toPath()), StandardCharsets.UTF_8);
System.err.println("create json is :\n" + createJsonStr);
runTypeCreation(createJsonStr,atlasClientV2);
// String updateFileName = console.readLine("Enter fileName containing TypeDefs for update:- ");
// File updateFile = new File(updateFileName);
// String updateJsonStr = new String( Files.readAllBytes(updateFile.toPath()), StandardCharsets.UTF_8);
// System.err.println("update json is :\n" + updateJsonStr);
// runTypeUpdate(updateJsonStr,atlasClientV2);
}
@VisibleForTesting
static void runTypeCreation(String jsonStr,AtlasClientV2 atlasClientV2) throws Exception {
AtlasTypesDef typesDef = AtlasType.fromJson(jsonStr, AtlasTypesDef.class);
atlasClientV2.createAtlasTypeDefs(typesDef);
}
@VisibleForTesting
static void runTypeUpdate(String jsonStr,AtlasClientV2 atlasClientV2) throws Exception {
AtlasTypesDef typesDef = AtlasType.fromJson(jsonStr, AtlasTypesDef.class);
atlasClientV2.updateAtlasTypeDefs(typesDef);
}
private static AtlasClientV2 getAtlasClientV2(String[] args, String[] basicAuthUsernamePassword) throws AtlasException {
String[] urls = getServerUrl(args);
AtlasClientV2 atlasClientV2;
if (!AuthenticationUtil.isKerberosAuthenticationEnabled()) {
atlasClientV2 =new AtlasClientV2(urls,basicAuthUsernamePassword);
} else {
atlasClientV2 = new AtlasClientV2(urls);
}
return atlasClientV2;
}
static String[] getServerUrl(String[] args) throws AtlasException {
if (args.length > 0) {
return args[0].split(",");
}
Configuration configuration = ApplicationProperties.get();
String[] urls = configuration.getStringArray(ATLAS_REST_ADDRESS);
if (urls == null || urls.length == 0) {
System.out.println("Usage: quick_start.py <atlas endpoint of format <http/https>://<atlas-fqdn>:<atlas port> like http://localhost:21000>");
System.exit(-1);
}
return urls;
}
}
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment