ATLAS-1657 Taxonomy term disassociation with assets fail

catalog/src/main/java/org/apache/atlas/catalog/DefaultTypeSystem.java

@@ -20,33 +20,35 @@ package org.apache.atlas.catalog;
 
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
+import org.apache.atlas.AtlasErrorCode;
 import org.apache.atlas.AtlasException;
 import org.apache.atlas.catalog.definition.ResourceDefinition;
 import org.apache.atlas.catalog.exception.CatalogRuntimeException;
 import org.apache.atlas.catalog.exception.ResourceAlreadyExistsException;
 import org.apache.atlas.catalog.exception.ResourceNotFoundException;
 import org.apache.atlas.classification.InterfaceAudience;
+import org.apache.atlas.exception.AtlasBaseException;
+import org.apache.atlas.model.typedef.AtlasClassificationDef;
+import org.apache.atlas.model.typedef.AtlasEntityDef;
+import org.apache.atlas.model.typedef.AtlasEnumDef;
+import org.apache.atlas.model.typedef.AtlasStructDef;
+import org.apache.atlas.model.typedef.AtlasTypesDef;
+import org.apache.atlas.repository.converters.TypeConverterUtil;
 import org.apache.atlas.services.MetadataService;
+import org.apache.atlas.store.AtlasTypeDefStore;
+import org.apache.atlas.type.AtlasTypeUtil;
 import org.apache.atlas.typesystem.ITypedReferenceableInstance;
 import org.apache.atlas.typesystem.Referenceable;
 import org.apache.atlas.typesystem.Struct;
-import org.apache.atlas.typesystem.TypesDef;
 import org.apache.atlas.typesystem.exception.EntityExistsException;
 import org.apache.atlas.typesystem.exception.EntityNotFoundException;
 import org.apache.atlas.typesystem.exception.TraitNotFoundException;
-import org.apache.atlas.typesystem.exception.TypeExistsException;
-import org.apache.atlas.typesystem.exception.TypeNotFoundException;
-import org.apache.atlas.typesystem.json.TypesSerialization;
 import org.apache.atlas.typesystem.types.AttributeDefinition;
 import org.apache.atlas.typesystem.types.ClassType;
-import org.apache.atlas.typesystem.types.DataTypes;
-import org.apache.atlas.typesystem.types.EnumTypeDefinition;
 import org.apache.atlas.typesystem.types.HierarchicalType;
-import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition;
-import org.apache.atlas.typesystem.types.StructTypeDefinition;
 import org.apache.atlas.typesystem.types.TraitType;
-import org.apache.atlas.typesystem.types.utils.TypesUtil;
 
+import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
@@ -58,36 +60,43 @@ import java.util.Map;
 public class DefaultTypeSystem implements AtlasTypeSystem {
     private final MetadataService metadataService;
 
+    private final AtlasTypeDefStore typeDefStore;
+
     /**
      * Constructor.
      *
      * @param metadataService  atlas metadata service
      */
-    public DefaultTypeSystem(MetadataService metadataService) throws AtlasException {
+    public DefaultTypeSystem(MetadataService metadataService, AtlasTypeDefStore typeDefStore) throws AtlasBaseException {
         this.metadataService = metadataService;
+        this.typeDefStore = typeDefStore;
         //Create namespace
         createSuperTypes();
     }
 
     @InterfaceAudience.Private
-    private void createSuperTypes() throws AtlasException {
-        HierarchicalTypeDefinition<TraitType> termType = TypesUtil
-            .createTraitTypeDef(TaxonomyResourceProvider.TAXONOMY_TERM_TYPE, ImmutableSet.<String>of(),
-                TypesUtil.createOptionalAttrDef(TaxonomyResourceProvider.NAMESPACE_ATTRIBUTE_NAME,
-                    DataTypes.STRING_TYPE));
+    private void createSuperTypes() throws AtlasBaseException {
+
+        AtlasClassificationDef termClassification = AtlasTypeUtil.createTraitTypeDef(TaxonomyResourceProvider.TAXONOMY_TERM_TYPE, TaxonomyResourceProvider.TAXONOMY_TERM_TYPE,
+            ImmutableSet.<String>of(), AtlasTypeUtil.createOptionalAttrDef(TaxonomyResourceProvider.NAMESPACE_ATTRIBUTE_NAME, "string"));
 
-        createTraitType(termType);
+        createTraitType(termClassification);
     }
 
-    private void createTraitType(HierarchicalTypeDefinition<TraitType> type) throws AtlasException {
+    private void createTraitType(AtlasClassificationDef classificationDef) throws AtlasBaseException {
         try {
-            metadataService.getTypeDefinition(type.typeName);
-        } catch(TypeNotFoundException tne) {
+            typeDefStore.getClassificationDefByName(classificationDef.getName());
+        } catch (AtlasBaseException tne) {
             //Type not found . Create
-            TypesDef typesDef = TypesUtil.getTypesDef(ImmutableList.<EnumTypeDefinition>of(), ImmutableList.<StructTypeDefinition>of(),
-                ImmutableList.of(type),
-                ImmutableList.<HierarchicalTypeDefinition<ClassType>>of());
-            metadataService.createType(TypesSerialization.toJson(typesDef));
+            if (tne.getAtlasErrorCode() == AtlasErrorCode.TYPE_NAME_NOT_FOUND) {
+                AtlasTypesDef typesDef = new AtlasTypesDef(ImmutableList.<AtlasEnumDef>of(), ImmutableList.<AtlasStructDef>of(),
+                    ImmutableList.of(classificationDef),
+                    ImmutableList.<AtlasEntityDef>of());
+
+                typeDefStore.createTypesDef(typesDef);
+            } else {
+                throw tne;
+            }
         }
     }
 
@@ -194,24 +203,37 @@ public class DefaultTypeSystem implements AtlasTypeSystem {
                                                          String name,
                                                          String description,
                                                          boolean isTrait)
-                                                         throws ResourceAlreadyExistsException {
+        throws ResourceAlreadyExistsException {
 
         try {
-            HierarchicalTypeDefinition<T> definition = null;
+            List<AtlasStructDef.AtlasAttributeDef> attrDefs  = new ArrayList<>();
+            for (AttributeDefinition attrDefinition : attributes) {
+                attrDefs.add(TypeConverterUtil.toAtlasAttributeDef(attrDefinition));
+            }
             if ( isTrait) {
-                definition = new HierarchicalTypeDefinition<>(type, name, description,
-                        ImmutableSet.of(TaxonomyResourceProvider.TAXONOMY_TERM_TYPE), attributes.toArray(new AttributeDefinition[attributes.size()]));
+                AtlasClassificationDef classificationDef = new AtlasClassificationDef(name, description, "1.0", attrDefs, ImmutableSet.of(TaxonomyResourceProvider.TAXONOMY_TERM_TYPE));
+                AtlasTypesDef typesDef = new AtlasTypesDef(ImmutableList.<AtlasEnumDef>of(), ImmutableList.<AtlasStructDef>of(),
+                    ImmutableList.of(classificationDef),
+                    ImmutableList.<AtlasEntityDef>of());
+
+                typeDefStore.createTypesDef(typesDef);
+
             } else {
-                definition = new HierarchicalTypeDefinition<>(type, name, description,
-                        ImmutableSet.<String>of(), attributes.toArray(new AttributeDefinition[attributes.size()]));
+                AtlasEntityDef entityDef = new AtlasEntityDef(name, description, "1.0", attrDefs);
+                AtlasTypesDef typesDef = new AtlasTypesDef(ImmutableList.<AtlasEnumDef>of(), ImmutableList.<AtlasStructDef>of(),
+                    ImmutableList.<AtlasClassificationDef>of(),
+                    ImmutableList.of(entityDef));
+
+                typeDefStore.createTypesDef(typesDef);
             }
 
-            metadataService.createType(TypesSerialization.toJson(definition, isTrait));
-        } catch (TypeExistsException e) {
-            throw new ResourceAlreadyExistsException(String.format("Type '%s' already exists", name));
-        } catch (AtlasException e) {
-            throw new CatalogRuntimeException(String.format(
+        } catch (AtlasBaseException e) {
+            if ( e.getAtlasErrorCode() == AtlasErrorCode.TYPE_ALREADY_EXISTS) {
+                throw new ResourceAlreadyExistsException(String.format("Type '%s' already exists", name));
+            } else {
+                throw new CatalogRuntimeException(String.format(
                     "Unable to create type '%s' in type system: %s", name, e), e);
+            }
         }
     }
 }

intg/src/main/java/org/apache/atlas/type/AtlasTypeUtil.java

@@ -121,10 +121,11 @@ public class AtlasTypeUtil {
     }
 
     public static void validateType(AtlasBaseTypeDef typeDef) throws AtlasBaseException {
-        String typeName = typeDef.getName();
+        boolean isValidName = (typeDef instanceof AtlasClassificationDef) ? isValidTraitTypeName(typeDef.getName())
+            : isValidTypeName(typeDef.getName());
 
-        if (!isValidTypeName(typeName)) {
-            throw new AtlasBaseException(AtlasErrorCode.TYPE_NAME_INVALID_FORMAT, typeName, typeDef.getCategory().name());
+        if (!isValidName) {
+            throw new AtlasBaseException(AtlasErrorCode.TYPE_NAME_INVALID_FORMAT, typeDef.getName(), typeDef.getCategory().name());
         }
     }
 

webapp/src/main/java/org/apache/atlas/util/RestUtils.java → repository/src/main/java/org/apache/atlas/repository/converters/TypeConverterUtil.java

@@ -16,7 +16,7 @@
  * limitations under the License.
  */
 
-package org.apache.atlas.util;
+package org.apache.atlas.repository.converters;
 
 import static org.apache.atlas.AtlasErrorCode.INVALID_TYPE_DEFINITION;
 import static org.apache.atlas.model.typedef.AtlasStructDef.AtlasConstraintDef.CONSTRAINT_TYPE_OWNED_REF;
@@ -30,7 +30,6 @@ import java.util.List;
 import java.util.Set;
 
 import org.apache.atlas.exception.AtlasBaseException;
-import org.apache.atlas.model.TypeCategory;
 import org.apache.atlas.model.typedef.AtlasClassificationDef;
 import org.apache.atlas.model.typedef.AtlasEntityDef;
 import org.apache.atlas.model.typedef.AtlasEnumDef;
@@ -63,36 +62,28 @@ import org.apache.atlas.typesystem.types.TraitType;
 import org.apache.atlas.typesystem.types.utils.TypesUtil;
 import org.apache.commons.collections.CollectionUtils;
 import org.apache.commons.lang.StringUtils;
-import org.codehaus.jackson.map.ObjectMapper;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Set;
-
-import static org.apache.atlas.AtlasErrorCode.INVALID_TYPE_DEFINITION;
-import static org.apache.atlas.type.AtlasTypeUtil.isArrayType;
 
 
-public final class RestUtils {
-    private RestUtils() {}
-    private static final Logger LOG = LoggerFactory.getLogger(RestUtils.class);
+public final class TypeConverterUtil {
+    private TypeConverterUtil() {}
+    private static final Logger LOG = LoggerFactory.getLogger(TypeConverterUtil.class);
 
     public static TypesDef toTypesDef(AtlasType type, AtlasTypeRegistry typeRegistry) throws AtlasBaseException {
         final TypesDef ret;
 
         if (type instanceof AtlasEnumType) {
-            ret = RestUtils.enumToTypesDef((AtlasEnumType)type);
+            ret = TypeConverterUtil.enumToTypesDef((AtlasEnumType) type);
         } else if (type instanceof AtlasEntityType) {
-            ret = RestUtils.entityToTypesDef((AtlasEntityType)type, typeRegistry);
+            ret = TypeConverterUtil.entityToTypesDef((AtlasEntityType) type, typeRegistry);
         } else if (type instanceof AtlasClassificationType) {
-            ret = RestUtils.classificationToTypesDef((AtlasClassificationType)type, typeRegistry);
+            ret = TypeConverterUtil.classificationToTypesDef((AtlasClassificationType) type, typeRegistry);
         } else if (type instanceof AtlasStructType) {
-            ret = RestUtils.structToTypesDef((AtlasStructType)type, typeRegistry);
+            ret = TypeConverterUtil.structToTypesDef((AtlasStructType) type, typeRegistry);
         } else {
             ret = new TypesDef();
         }
@@ -356,7 +347,7 @@ public final class RestUtils {
         return ret.toArray(new EnumValue[ret.size()]);
     }
 
-    private static AtlasAttributeDef toAtlasAttributeDef(final AttributeDefinition attrDefinition) {
+    public static AtlasAttributeDef toAtlasAttributeDef(final AttributeDefinition attrDefinition) {
         AtlasAttributeDef ret = new AtlasAttributeDef();
 
         ret.setName(attrDefinition.name);

webapp/src/main/java/org/apache/atlas/web/resources/EntityService.java

@@ -27,7 +27,9 @@ import org.apache.atlas.catalog.EntityTagResourceProvider;
 import org.apache.atlas.catalog.InstanceRequest;
 import org.apache.atlas.catalog.Result;
 import org.apache.atlas.catalog.exception.CatalogException;
+import org.apache.atlas.exception.AtlasBaseException;
 import org.apache.atlas.services.MetadataService;
+import org.apache.atlas.store.AtlasTypeDefStore;
 import org.apache.atlas.utils.AtlasPerfTracer;
 import org.apache.atlas.web.util.Servlets;
 import org.slf4j.Logger;
@@ -63,8 +65,8 @@ public class EntityService extends BaseService {
     private final EntityTagResourceProvider entityTagResourceProvider;
 
     @Inject
-    public EntityService(MetadataService metadataService) throws AtlasException {
-        DefaultTypeSystem typeSystem = new DefaultTypeSystem(metadataService);
+    public EntityService(MetadataService metadataService, AtlasTypeDefStore typeDefStore) throws AtlasBaseException {
+        DefaultTypeSystem typeSystem = new DefaultTypeSystem(metadataService, typeDefStore);
         entityResourceProvider = new EntityResourceProvider(typeSystem);
         entityTagResourceProvider = new EntityTagResourceProvider(typeSystem);
     }

webapp/src/main/java/org/apache/atlas/web/resources/TaxonomyService.java

@@ -22,7 +22,9 @@ import org.apache.atlas.AtlasException;
 import org.apache.atlas.catalog.*;
 import org.apache.atlas.catalog.exception.CatalogException;
 import org.apache.atlas.catalog.exception.InvalidPayloadException;
+import org.apache.atlas.exception.AtlasBaseException;
 import org.apache.atlas.services.MetadataService;
+import org.apache.atlas.store.AtlasTypeDefStore;
 import org.apache.atlas.utils.AtlasPerfTracer;
 import org.apache.atlas.web.util.Servlets;
 import org.slf4j.Logger;
@@ -58,8 +60,8 @@ public class TaxonomyService extends BaseService {
     private ResourceProvider termResourceProvider;
 
     @Inject
-    public void setMetadataService(MetadataService metadataService) throws AtlasException {
-        DefaultTypeSystem typeSystem = new DefaultTypeSystem(metadataService);
+    public void setMetadataService(MetadataService metadataService, AtlasTypeDefStore typeDefStore) throws AtlasBaseException {
+        DefaultTypeSystem typeSystem = new DefaultTypeSystem(metadataService, typeDefStore);
         taxonomyResourceProvider = createTaxonomyResourceProvider(typeSystem);
         termResourceProvider = createTermResourceProvider(typeSystem);
     }

webapp/src/main/java/org/apache/atlas/web/resources/TypesResource.java

@@ -26,7 +26,7 @@ import org.apache.atlas.model.typedef.AtlasTypesDef;
 import org.apache.atlas.type.AtlasTypeRegistry;
 import org.apache.atlas.typesystem.TypesDef;
 import org.apache.atlas.typesystem.json.TypesSerialization;
-import org.apache.atlas.util.RestUtils;
+import org.apache.atlas.repository.converters.TypeConverterUtil;
 import org.apache.atlas.utils.AtlasPerfTracer;
 import org.apache.atlas.web.rest.TypesREST;
 import org.apache.atlas.web.util.Servlets;
@@ -105,9 +105,9 @@ public class TypesResource {
                 LOG.debug("Creating type with definition {} ", typeDefinition);
             }
 
-            AtlasTypesDef createTypesDef  = RestUtils.toAtlasTypesDef(typeDefinition, typeRegistry);
+            AtlasTypesDef createTypesDef  = TypeConverterUtil.toAtlasTypesDef(typeDefinition, typeRegistry);
             AtlasTypesDef createdTypesDef = typesRest.createAtlasTypeDefs(createTypesDef);
-            List<String>  typeNames       = RestUtils.getTypeNames(createdTypesDef);
+            List<String>  typeNames       = TypeConverterUtil.getTypeNames(createdTypesDef);
 
             for (int i = 0; i < typeNames.size(); i++) {
                 final String name = typeNames.get(i);
@@ -173,9 +173,9 @@ public class TypesResource {
                 LOG.debug("Updating type with definition {} ", typeDefinition);
             }
 
-            AtlasTypesDef updateTypesDef  = RestUtils.toAtlasTypesDef(typeDefinition, typeRegistry);
+            AtlasTypesDef updateTypesDef  = TypeConverterUtil.toAtlasTypesDef(typeDefinition, typeRegistry);
             AtlasTypesDef updatedTypesDef = typesRest.updateAtlasTypeDefs(updateTypesDef);
-            List<String>  typeNames       = RestUtils.getTypeNames(updatedTypesDef);
+            List<String>  typeNames       = TypeConverterUtil.getTypeNames(updatedTypesDef);
 
             for (int i = 0; i < typeNames.size(); i++) {
                 final String name = typeNames.get(i);
@@ -231,7 +231,7 @@ public class TypesResource {
         JSONObject response = new JSONObject();
 
         try {
-            TypesDef typesDef       = RestUtils.toTypesDef(typeRegistry.getType(typeName), typeRegistry);;
+            TypesDef typesDef       = TypeConverterUtil.toTypesDef(typeRegistry.getType(typeName), typeRegistry);;
             String   typeDefinition = TypesSerialization.toJson(typesDef);
 
             response.put(AtlasClient.TYPENAME, typeName);
@@ -290,7 +290,7 @@ public class TypesResource {
         TypesREST typesRest  = resourceContext.getResource(TypesREST.class);
         JSONObject response  = new JSONObject();
         try {
-            List<String> result = RestUtils.getTypeNames(typesRest.getTypeDefHeaders(request));
+            List<String> result = TypeConverterUtil.getTypeNames(typesRest.getTypeDefHeaders(request));
 
             response.put(AtlasClient.RESULTS, new JSONArray(result));
             response.put(AtlasClient.COUNT, result.size());

webapp/src/test/java/org/apache/atlas/util/RestUtilsTest.java

@@ -28,12 +28,12 @@ import java.util.List;
 import java.util.Map;
 
 import org.apache.atlas.exception.AtlasBaseException;
-import org.apache.atlas.model.instance.AtlasEntity;
 import org.apache.atlas.model.typedef.AtlasEntityDef;
 import org.apache.atlas.model.typedef.AtlasStructDef;
 import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef;
 import org.apache.atlas.model.typedef.AtlasTypesDef;
 import org.apache.atlas.repository.Constants;
+import org.apache.atlas.repository.converters.TypeConverterUtil;
 import org.apache.atlas.repository.graphdb.AtlasVertex;
 import org.apache.atlas.repository.store.graph.v1.AtlasGraphUtilsV1;
 import org.apache.atlas.repository.store.graph.v1.AtlasStructDefStoreV1;
@@ -178,7 +178,7 @@ public class RestUtilsTest {
         for (int i = 0; i < toConvert.size(); i++) {
             AtlasEntityDef entityDef = toConvert.get(i);
             AtlasEntityType entity = reg.getEntityTypeByName(entityDef.getName());
-            HierarchicalTypeDefinition<ClassType> converted = RestUtils.toTypesDef(entity, reg)
+            HierarchicalTypeDefinition<ClassType> converted = TypeConverterUtil.toTypesDef(entity, reg)
                     .classTypesAsJavaList().get(0);
             result.add(converted);
         }
@@ -205,7 +205,7 @@ public class RestUtilsTest {
 
         String json = TypesSerialization.toJson(toConvert);
         AtlasTypeRegistry emptyRegistry = new AtlasTypeRegistry();
-        AtlasTypesDef converted = RestUtils.toAtlasTypesDef(json, emptyRegistry);
+        AtlasTypesDef converted = TypeConverterUtil.toAtlasTypesDef(json, emptyRegistry);
         List<AtlasEntityDef> convertedEntityDefs = converted.getEntityDefs();
         return convertedEntityDefs;
     }