From 6d94f39cf3b6207bc095310eebbe3292071be68b Mon Sep 17 00:00:00 2001 From: Aadarsh Jajodia <aadarshjajodia@gmail.com> Date: Fri, 10 Jan 2020 11:43:01 -0800 Subject: [PATCH] ATLAS-3486: Introduce Atlas NamespaceDef Signed-off-by: Sarath Subramanian <sarath@apache.org> --- intg/src/main/java/org/apache/atlas/AtlasErrorCode.java | 5 +++++ intg/src/main/java/org/apache/atlas/model/TypeCategory.java | 2 +- intg/src/main/java/org/apache/atlas/model/typedef/AtlasNamespaceDef.java | 88 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ intg/src/main/java/org/apache/atlas/model/typedef/AtlasStructDef.java | 33 ++++++++++++++++++++++++++++++--- intg/src/main/java/org/apache/atlas/model/typedef/AtlasTypesDef.java | 35 +++++++++++++++++++++++++++++++++-- intg/src/main/java/org/apache/atlas/store/AtlasTypeDefStore.java | 6 ++++++ intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java | 58 ++++++++++++++++++++++++++++++++++++++++++---------------- intg/src/main/java/org/apache/atlas/type/AtlasNamespaceType.java | 184 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ intg/src/main/java/org/apache/atlas/type/AtlasStructType.java | 45 +++++++++++++++++++++++++++++++++++++++++++++ intg/src/main/java/org/apache/atlas/type/AtlasTypeRegistry.java | 52 ++++++++++++++++++++++++++++++++++++++++++++++++++-- intg/src/main/java/org/apache/atlas/type/AtlasTypeUtil.java | 10 ++++++++++ intg/src/main/java/org/apache/atlas/typesystem/types/DataTypes.java | 3 ++- intg/src/test/java/org/apache/atlas/TestRelationshipUtilsV2.java | 21 +++++++++++++++++++-- intg/src/test/java/org/apache/atlas/model/typedef/TestAtlasNamespaceDef.java | 84 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java | 1 + repository/src/main/java/org/apache/atlas/repository/impexp/ExportService.java | 1 + repository/src/main/java/org/apache/atlas/repository/impexp/ExportTypeProcessor.java | 15 +++++++++++++-- repository/src/main/java/org/apache/atlas/repository/store/bootstrap/AtlasTypeDefStoreInitializer.java | 25 +++++++++++++++++++++++-- repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasTypeDefGraphStore.java | 117 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++------------------- repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasNamespaceDefStoreV2.java | 347 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasTypeDefGraphStoreV2.java | 36 +++++++++++++++++++++++------------- repository/src/test/java/org/apache/atlas/repository/store/graph/v2/AtlasNamespaceDefStoreV2Test.java | 319 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ webapp/src/main/java/org/apache/atlas/examples/QuickStartV2.java | 28 ++++++++++++++++++++++++---- webapp/src/main/java/org/apache/atlas/web/rest/TypesREST.java | 38 ++++++++++++++++++++++++++++++++++++++ 24 files changed, 1486 insertions(+), 67 deletions(-) create mode 100644 intg/src/main/java/org/apache/atlas/model/typedef/AtlasNamespaceDef.java create mode 100644 intg/src/main/java/org/apache/atlas/type/AtlasNamespaceType.java create mode 100644 intg/src/test/java/org/apache/atlas/model/typedef/TestAtlasNamespaceDef.java create mode 100644 repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasNamespaceDefStoreV2.java create mode 100644 repository/src/test/java/org/apache/atlas/repository/store/graph/v2/AtlasNamespaceDefStoreV2Test.java diff --git a/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java b/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java index 3ebd70d..2054513 100644 --- a/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java +++ b/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java @@ -161,6 +161,11 @@ public enum AtlasErrorCode { INVALID_LABEL_LENGTH(400, "ATLAS-400-00-9B", "Invalid label: {0}, label size should not be greater than {1}"), INVALID_LABEL_CHARACTERS(400, "ATLAS-400-00-9C", "Invalid label: {0}, label should contain alphanumeric characters, '_' or '-'"), INVALID_PROPAGATION_TYPE(400, "ATLAS-400-00-9D", "Invalid propagation {0} for relationship-type={1}. Default value is {2}"), + DUPLICATE_NAMESPACE_ATTRIBUTE(400, "ATLAS-400-00-094", "Duplicate Namespace Attributes: {0} not allowed within the same namespace: {1}"), + APPLICABLE_ENTITY_TYPES_DELETION_NOT_SUPPORTED(400, "ATLAS-400-00-095", "Cannot remove applicableEntityTypes in Attribute Def: {1}, defined in namespace: {2}"), + NAMESPACE_DEF_MANDATORY_ATTRIBUTE_NOT_ALLOWED(400, "ATLAS-400-00-096", "{0}.{1} : namespaces can not have mandatory attribute"), + NAMESPACE_DEF_UNIQUE_ATTRIBUTE_NOT_ALLOWED(400, "ATLAS-400-00-097", "{0}.{1} : namespaces can not have unique attribute"), + NAMESPACE_DEF_ATTRIBUTE_TYPE_INVALID(400, "ATLAS-400-00-098", "{0}.{1}: invalid attribute type. Namespace attribute cannot be of type entity/classification/struct/namespace"), UNAUTHORIZED_ACCESS(403, "ATLAS-403-00-001", "{0} is not authorized to perform {1}"), diff --git a/intg/src/main/java/org/apache/atlas/model/TypeCategory.java b/intg/src/main/java/org/apache/atlas/model/TypeCategory.java index f06f64f..cbcd0a3 100644 --- a/intg/src/main/java/org/apache/atlas/model/TypeCategory.java +++ b/intg/src/main/java/org/apache/atlas/model/TypeCategory.java @@ -18,5 +18,5 @@ package org.apache.atlas.model; public enum TypeCategory { - PRIMITIVE, OBJECT_ID_TYPE, ENUM, STRUCT, CLASSIFICATION, ENTITY, ARRAY, MAP, RELATIONSHIP + PRIMITIVE, OBJECT_ID_TYPE, ENUM, STRUCT, CLASSIFICATION, ENTITY, ARRAY, MAP, RELATIONSHIP, NAMESPACE } diff --git a/intg/src/main/java/org/apache/atlas/model/typedef/AtlasNamespaceDef.java b/intg/src/main/java/org/apache/atlas/model/typedef/AtlasNamespaceDef.java new file mode 100644 index 0000000..713a2c2 --- /dev/null +++ b/intg/src/main/java/org/apache/atlas/model/typedef/AtlasNamespaceDef.java @@ -0,0 +1,88 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.model.typedef; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import org.apache.atlas.model.TypeCategory; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; +import java.io.Serializable; +import java.util.List; +import java.util.Map; + +import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.NONE; +import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.PUBLIC_ONLY; + +@JsonAutoDetect(getterVisibility=PUBLIC_ONLY, setterVisibility=PUBLIC_ONLY, fieldVisibility=NONE) +@JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL) +@JsonIgnoreProperties(ignoreUnknown=true) +@XmlRootElement +@XmlAccessorType(XmlAccessType.PROPERTY) +public class AtlasNamespaceDef extends AtlasStructDef implements Serializable { + private static final long serialVersionUID = 1L; + + public static final String ATTR_OPTION_APPLICABLE_ENTITY_TYPES = "applicableEntityTypes"; + public static final String ATTR_MAX_STRING_LENGTH = "maxStrLength"; + public static final String ATTR_VALID_PATTERN = "validPattern"; + + public AtlasNamespaceDef() { + this(null, null, null, null); + } + + public AtlasNamespaceDef(String name, String description) { + this(name, description, null, null, null); + } + + public AtlasNamespaceDef(String name, String description, String typeVersion) { + this(name, description, typeVersion, null, null); + } + + public AtlasNamespaceDef(String name, String description, String typeVersion, List<AtlasAttributeDef> attributeDefs) { + this(name, description, typeVersion, attributeDefs, null); + } + + public AtlasNamespaceDef(String name, String description, String typeVersion, List<AtlasAttributeDef> attributeDefs, Map<String, String> options) { + super(TypeCategory.NAMESPACE, name, description, typeVersion, attributeDefs, options); + } + + public AtlasNamespaceDef(AtlasNamespaceDef other) { + super(other); + } + + @Override + public String toString() { + return toString(new StringBuilder()).toString(); + } + + @Override + public StringBuilder toString(StringBuilder sb) { + if (sb == null) { + sb = new StringBuilder(); + } + + sb.append("AtlasNamespaceDef{"); + super.toString(sb); + sb.append('}'); + + return sb; + } +} \ No newline at end of file diff --git a/intg/src/main/java/org/apache/atlas/model/typedef/AtlasStructDef.java b/intg/src/main/java/org/apache/atlas/model/typedef/AtlasStructDef.java index bb7ead0..1d4e37b 100644 --- a/intg/src/main/java/org/apache/atlas/model/typedef/AtlasStructDef.java +++ b/intg/src/main/java/org/apache/atlas/model/typedef/AtlasStructDef.java @@ -290,9 +290,9 @@ public class AtlasStructDef extends AtlasBaseTypeDef implements Serializable { private String description; private int searchWeight = DEFAULT_SEARCHWEIGHT; private IndexType indexType = null; - private List<AtlasConstraintDef> constraints; private Map<String, String> options; + private String displayName; public AtlasAttributeDef() { this(null, null); } @@ -373,9 +373,18 @@ public class AtlasStructDef extends AtlasBaseTypeDef implements Serializable { setDescription((other.getDescription())); setSearchWeight(other.getSearchWeight()); setIndexType(other.getIndexType()); + setDisplayName(other.getDisplayName()); } } + public void setDisplayName(String displayName) { + this.displayName = displayName; + } + + public String getDisplayName() { + return displayName; + } + public int getSearchWeight() { return searchWeight; } @@ -510,6 +519,22 @@ public class AtlasStructDef extends AtlasBaseTypeDef implements Serializable { getOptions().get(AtlasAttributeDef.ATTRDEF_OPTION_SOFT_REFERENCE).equals(STRING_TRUE); } + @JsonIgnore + public void setOption(String name, String value) { + if (this.options == null) { + this.options = new HashMap<>(); + } + + this.options.put(name, value); + } + + @JsonIgnore + public String getOption(String name) { + Map<String, String> option = this.options; + + return option != null ? option.get(name) : null; + } + public String getDescription() { return description; } @@ -538,6 +563,7 @@ public class AtlasStructDef extends AtlasBaseTypeDef implements Serializable { sb.append(", options='").append(options).append('\''); sb.append(", searchWeight='").append(searchWeight).append('\''); sb.append(", indexType='").append(indexType).append('\''); + sb.append(", displayName='").append(displayName).append('\''); sb.append(", constraints=["); if (CollectionUtils.isNotEmpty(constraints)) { int i = 0; @@ -574,12 +600,13 @@ public class AtlasStructDef extends AtlasBaseTypeDef implements Serializable { Objects.equals(constraints, that.constraints) && Objects.equals(options, that.options) && Objects.equals(searchWeight, that.searchWeight) && - Objects.equals(indexType, that.indexType); + Objects.equals(indexType, that.indexType) && + Objects.equals(displayName, that.displayName); } @Override public int hashCode() { - return Objects.hash(name, typeName, isOptional, cardinality, valuesMinCount, valuesMaxCount, isUnique, isIndexable, includeInNotification, defaultValue, constraints, options, description, searchWeight, indexType); + return Objects.hash(name, typeName, isOptional, cardinality, valuesMinCount, valuesMaxCount, isUnique, isIndexable, includeInNotification, defaultValue, constraints, options, description, searchWeight, indexType, displayName); } @Override diff --git a/intg/src/main/java/org/apache/atlas/model/typedef/AtlasTypesDef.java b/intg/src/main/java/org/apache/atlas/model/typedef/AtlasTypesDef.java index 3634fdf..81ea946 100644 --- a/intg/src/main/java/org/apache/atlas/model/typedef/AtlasTypesDef.java +++ b/intg/src/main/java/org/apache/atlas/model/typedef/AtlasTypesDef.java @@ -47,6 +47,7 @@ public class AtlasTypesDef { private List<AtlasClassificationDef> classificationDefs; private List<AtlasEntityDef> entityDefs; private List<AtlasRelationshipDef> relationshipDefs; + private List<AtlasNamespaceDef> namespaceDefs; public AtlasTypesDef() { enumDefs = new ArrayList<>(); @@ -54,6 +55,7 @@ public class AtlasTypesDef { classificationDefs = new ArrayList<>(); entityDefs = new ArrayList<>(); relationshipDefs = new ArrayList<>(); + namespaceDefs = new ArrayList<>(); } /** @@ -66,7 +68,7 @@ public class AtlasTypesDef { */ public AtlasTypesDef(List<AtlasEnumDef> enumDefs, List<AtlasStructDef> structDefs, List<AtlasClassificationDef> classificationDefs, List<AtlasEntityDef> entityDefs) { - this(enumDefs, structDefs, classificationDefs, entityDefs,new ArrayList<AtlasRelationshipDef>()); + this(enumDefs, structDefs, classificationDefs, entityDefs, new ArrayList<>(), new ArrayList<>()); } /** * Create the TypesDef. This created definitions for each of the types. @@ -81,12 +83,23 @@ public class AtlasTypesDef { List<AtlasClassificationDef> classificationDefs, List<AtlasEntityDef> entityDefs, List<AtlasRelationshipDef> relationshipDefs) { + this(enumDefs, structDefs, classificationDefs, entityDefs, relationshipDefs, new ArrayList<>()); + } + + public AtlasTypesDef(List<AtlasEnumDef> enumDefs, + List<AtlasStructDef> structDefs, + List<AtlasClassificationDef> classificationDefs, + List<AtlasEntityDef> entityDefs, + List<AtlasRelationshipDef> relationshipDefs, + List<AtlasNamespaceDef> namespaceDefs) { this.enumDefs = enumDefs; this.structDefs = structDefs; this.classificationDefs = classificationDefs; this.entityDefs = entityDefs; this.relationshipDefs = relationshipDefs; + this.namespaceDefs = namespaceDefs; } + public List<AtlasEnumDef> getEnumDefs() { return enumDefs; } @@ -125,6 +138,14 @@ public class AtlasTypesDef { this.relationshipDefs = relationshipDefs; } + public void setNamespaceDefs(List<AtlasNamespaceDef> namespaceDefs) { + this.namespaceDefs = namespaceDefs; + } + + public List<AtlasNamespaceDef> getNamespaceDefs() { + return namespaceDefs; + } + public boolean hasClassificationDef(String name) { return hasTypeDef(classificationDefs, name); } @@ -144,6 +165,9 @@ public class AtlasTypesDef { return hasTypeDef(relationshipDefs, name); } + public boolean hasNamespaceDef(String name) { + return hasTypeDef(namespaceDefs, name); + } private <T extends AtlasBaseTypeDef> boolean hasTypeDef(Collection<T> typeDefs, String name) { if (CollectionUtils.isNotEmpty(typeDefs)) { @@ -163,7 +187,8 @@ public class AtlasTypesDef { CollectionUtils.isEmpty(structDefs) && CollectionUtils.isEmpty(classificationDefs) && CollectionUtils.isEmpty(entityDefs) && - CollectionUtils.isEmpty(relationshipDefs); + CollectionUtils.isEmpty(relationshipDefs) && + CollectionUtils.isEmpty(namespaceDefs); } public void clear() { @@ -185,6 +210,10 @@ public class AtlasTypesDef { if (relationshipDefs != null) { relationshipDefs.clear(); } + + if (namespaceDefs != null) { + namespaceDefs.clear(); + } } public StringBuilder toString(StringBuilder sb) { if (sb == null) { @@ -206,6 +235,8 @@ public class AtlasTypesDef { sb.append("}"); sb.append("relationshipDefs={"); AtlasBaseTypeDef.dumpObjects(relationshipDefs, sb); + sb.append("namespaceDefs={"); + AtlasBaseTypeDef.dumpObjects(namespaceDefs, sb); sb.append("}"); return sb; diff --git a/intg/src/main/java/org/apache/atlas/store/AtlasTypeDefStore.java b/intg/src/main/java/org/apache/atlas/store/AtlasTypeDefStore.java index 4ee68a9..b08ace4 100644 --- a/intg/src/main/java/org/apache/atlas/store/AtlasTypeDefStore.java +++ b/intg/src/main/java/org/apache/atlas/store/AtlasTypeDefStore.java @@ -23,6 +23,7 @@ import org.apache.atlas.model.typedef.AtlasBaseTypeDef; import org.apache.atlas.model.typedef.AtlasClassificationDef; import org.apache.atlas.model.typedef.AtlasEntityDef; import org.apache.atlas.model.typedef.AtlasEnumDef; +import org.apache.atlas.model.typedef.AtlasNamespaceDef; import org.apache.atlas.model.typedef.AtlasRelationshipDef; import org.apache.atlas.model.typedef.AtlasStructDef; import org.apache.atlas.model.typedef.AtlasTypesDef; @@ -85,6 +86,11 @@ public interface AtlasTypeDefStore { AtlasRelationshipDef updateRelationshipDefByGuid(String guid, AtlasRelationshipDef structDef) throws AtlasBaseException; + /* Namespace Def operations */ + AtlasNamespaceDef getNamespaceDefByName(String name) throws AtlasBaseException; + + AtlasNamespaceDef getNamespaceDefByGuid(String guid) throws AtlasBaseException; + /* Bulk Operations */ AtlasTypesDef createTypesDef(AtlasTypesDef atlasTypesDef) throws AtlasBaseException; diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java b/intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java index 4742d1c..02d6d58 100644 --- a/intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java +++ b/intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java @@ -31,6 +31,7 @@ import org.apache.atlas.model.typedef.AtlasEntityDef.AtlasRelationshipAttributeD import org.apache.atlas.model.typedef.AtlasRelationshipDef.PropagateTags; import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef; import org.apache.atlas.type.AtlasBuiltInTypes.AtlasObjectIdType; +import org.apache.atlas.type.AtlasNamespaceType.AtlasNamespaceAttribute; import org.apache.atlas.utils.AtlasEntityUtil; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.MapUtils; @@ -47,6 +48,7 @@ import java.util.List; import java.util.Map; import java.util.Set; + /** * class that implements behaviour of an entity-type. */ @@ -72,22 +74,23 @@ public class AtlasEntityType extends AtlasStructType { private final AtlasEntityDef entityDef; private final String typeQryStr; - private List<AtlasEntityType> superTypes = Collections.emptyList(); - private Set<String> allSuperTypes = Collections.emptySet(); - private Set<String> subTypes = Collections.emptySet(); - private Set<String> allSubTypes = Collections.emptySet(); - private Set<String> typeAndAllSubTypes = Collections.emptySet(); - private Set<String> typeAndAllSuperTypes = Collections.emptySet(); - private Map<String, Map<String, AtlasAttribute>> relationshipAttributes = Collections.emptyMap(); - private List<AtlasAttribute> ownedRefAttributes = Collections.emptyList(); - private String typeAndAllSubTypesQryStr = ""; - private boolean isInternalType = false; - private Map<String, AtlasAttribute> headerAttributes = Collections.emptyMap(); - private Map<String, AtlasAttribute> minInfoAttributes = Collections.emptyMap(); - private List<AtlasAttribute> dynAttributes = Collections.emptyList(); - private List<AtlasAttribute> dynEvalTriggerAttributes = Collections.emptyList(); - private Map<String,List<TemplateToken>> parsedTemplates = Collections.emptyMap(); - private Set<String> tagPropagationEdges = Collections.emptySet(); + private List<AtlasEntityType> superTypes = Collections.emptyList(); + private Set<String> allSuperTypes = Collections.emptySet(); + private Set<String> subTypes = Collections.emptySet(); + private Set<String> allSubTypes = Collections.emptySet(); + private Set<String> typeAndAllSubTypes = Collections.emptySet(); + private Set<String> typeAndAllSuperTypes = Collections.emptySet(); + private Map<String, Map<String, AtlasAttribute>> relationshipAttributes = Collections.emptyMap(); + private List<AtlasAttribute> ownedRefAttributes = Collections.emptyList(); + private String typeAndAllSubTypesQryStr = ""; + private boolean isInternalType = false; + private Map<String, AtlasAttribute> headerAttributes = Collections.emptyMap(); + private Map<String, AtlasAttribute> minInfoAttributes = Collections.emptyMap(); + private List<AtlasAttribute> dynAttributes = Collections.emptyList(); + private List<AtlasAttribute> dynEvalTriggerAttributes = Collections.emptyList(); + private Map<String,List<TemplateToken>> parsedTemplates = Collections.emptyMap(); + private Set<String> tagPropagationEdges = Collections.emptySet(); + private Map<String, List<AtlasNamespaceAttribute>> namespaceAttributes = Collections.emptyMap(); public AtlasEntityType(AtlasEntityDef entityDef) { super(entityDef); @@ -140,6 +143,7 @@ public class AtlasEntityType extends AtlasStructType { this.typeAndAllSubTypes = new HashSet<>(); // this will be populated in resolveReferencesPhase2() this.relationshipAttributes = new HashMap<>(); // this will be populated in resolveReferencesPhase3() this.tagPropagationEdges = new HashSet<>(); // this will be populated in resolveReferencesPhase2() + this.namespaceAttributes = new HashMap<>(); // this will be populated in resolveReferences(), from AtlasNamespaceType this.typeAndAllSubTypes.add(this.getTypeName()); @@ -259,6 +263,7 @@ public class AtlasEntityType extends AtlasStructType { relationshipAttributes = Collections.unmodifiableMap(relationshipAttributes); ownedRefAttributes = Collections.unmodifiableList(ownedRefAttributes); tagPropagationEdges = Collections.unmodifiableSet(tagPropagationEdges); + namespaceAttributes = Collections.unmodifiableMap(namespaceAttributes); entityDef.setSubTypes(subTypes); @@ -361,6 +366,14 @@ public class AtlasEntityType extends AtlasStructType { public String[] getTagPropagationEdgesArray() { return CollectionUtils.isNotEmpty(tagPropagationEdges) ? tagPropagationEdges.toArray(new String[tagPropagationEdges.size()]) : null; } + + public Map<String, List<AtlasNamespaceAttribute>> getNamespaceAttributes() { + return namespaceAttributes; + } + + public List<AtlasNamespaceAttribute> getNamespaceAttributes(String nsName) { + return namespaceAttributes.get(nsName); + } public Map<String,List<TemplateToken>> getParsedTemplates() { return parsedTemplates; } @@ -452,6 +465,19 @@ public class AtlasEntityType extends AtlasStructType { return relationshipAttributes.containsKey(attributeName); } + public void addNamespaceAttribute(AtlasNamespaceAttribute attribute) { + String nsName = attribute.getDefinedInType().getTypeName(); + List<AtlasNamespaceAttribute> attributes = namespaceAttributes.get(nsName); + + if (attributes == null) { + attributes = new ArrayList<>(); + + namespaceAttributes.put(nsName, attributes); + } + + attributes.add(attribute); + } + public String getQualifiedAttributeName(String attrName) throws AtlasBaseException { AtlasAttribute ret = getAttribute(attrName); diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasNamespaceType.java b/intg/src/main/java/org/apache/atlas/type/AtlasNamespaceType.java new file mode 100644 index 0000000..a141d4a --- /dev/null +++ b/intg/src/main/java/org/apache/atlas/type/AtlasNamespaceType.java @@ -0,0 +1,184 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.type; + +import org.apache.atlas.AtlasErrorCode; +import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.model.instance.AtlasStruct; +import org.apache.atlas.model.typedef.AtlasNamespaceDef; +import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef; +import org.apache.commons.collections.CollectionUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.*; + +import static org.apache.atlas.model.typedef.AtlasNamespaceDef.*; + + +public class AtlasNamespaceType extends AtlasStructType { + private static final Logger LOG = LoggerFactory.getLogger(AtlasNamespaceType.class); + + private final AtlasNamespaceDef namespaceDef; + + + public AtlasNamespaceType(AtlasNamespaceDef namespaceDef) { + super(namespaceDef); + + this.namespaceDef = namespaceDef; + } + + @Override + public boolean isValidValue(Object o) { + return true; // there is no runtime instance for Namespaces, so return true + } + + @Override + public AtlasStruct createDefaultValue() { + return null; // there is no runtime instance for Namespaces, so return null + } + + @Override + public Object getNormalizedValue(Object a) { + return null; // there is no runtime instance for Namespaces, so return null + } + + public AtlasNamespaceDef getNamespaceDef() { + return namespaceDef; + } + + @Override + void resolveReferences(AtlasTypeRegistry typeRegistry) throws AtlasBaseException { + super.resolveReferences(typeRegistry); + + Map<String, AtlasNamespaceAttribute> a = new HashMap<>(); + + for (AtlasAttribute attribute : super.allAttributes.values()) { + AtlasAttributeDef attributeDef = attribute.getAttributeDef(); + String attrName = attribute.getName(); + AtlasType attrType = attribute.getAttributeType(); + + if (attrType instanceof AtlasArrayType) { + attrType = ((AtlasArrayType) attrType).getElementType(); + } else if (attrType instanceof AtlasMapType) { + attrType = ((AtlasMapType) attrType).getValueType(); + } + + // check if attribute type is not struct/classification/entity/namespace + if (attrType instanceof AtlasStructType) { + throw new AtlasBaseException(AtlasErrorCode.NAMESPACE_DEF_ATTRIBUTE_TYPE_INVALID, getTypeName(), attrName); + } + + if (!attributeDef.getIsOptional()) { + throw new AtlasBaseException(AtlasErrorCode.NAMESPACE_DEF_MANDATORY_ATTRIBUTE_NOT_ALLOWED, getTypeName(), attrName); + } + + if (attributeDef.getIsUnique()) { + throw new AtlasBaseException(AtlasErrorCode.NAMESPACE_DEF_UNIQUE_ATTRIBUTE_NOT_ALLOWED, getTypeName(), attrName); + } + + Set<String> entityTypeNames = attribute.getOptionSet(ATTR_OPTION_APPLICABLE_ENTITY_TYPES); + Set<AtlasEntityType> entityTypes = new HashSet<>(); + + if (CollectionUtils.isNotEmpty(entityTypeNames)) { + for (String entityTypeName : entityTypeNames) { + AtlasEntityType entityType = typeRegistry.getEntityTypeByName(entityTypeName); + + if (entityType == null) { + throw new AtlasBaseException(AtlasErrorCode.TYPE_NAME_NOT_FOUND, entityTypeName); + } + + entityTypes.add(entityType); + } + } else { + throw new AtlasBaseException(AtlasErrorCode.MISSING_MANDATORY_ATTRIBUTE, attributeDef.getName(), "options." + ATTR_OPTION_APPLICABLE_ENTITY_TYPES); + } + + AtlasNamespaceAttribute nsAttribute; + if (attribute.getAttributeType() instanceof AtlasBuiltInTypes.AtlasStringType) { + Integer maxStringLength = attribute.getOptionInt(ATTR_MAX_STRING_LENGTH); + if (maxStringLength == null) { + throw new AtlasBaseException(AtlasErrorCode.MISSING_MANDATORY_ATTRIBUTE, attributeDef.getName(), "options." + ATTR_MAX_STRING_LENGTH); + } + + String validPattern = attribute.getOptionString(ATTR_VALID_PATTERN); + nsAttribute = new AtlasNamespaceAttribute(attribute, entityTypes, maxStringLength, validPattern); + } else { + nsAttribute = new AtlasNamespaceAttribute(attribute, entityTypes); + } + + a.put(attrName, nsAttribute); + } + + super.allAttributes = Collections.unmodifiableMap(a); + } + + @Override + void resolveReferencesPhase2(AtlasTypeRegistry typeRegistry) throws AtlasBaseException { + super.resolveReferencesPhase2(typeRegistry); + + for (AtlasAttribute attribute : super.allAttributes.values()) { + AtlasNamespaceAttribute nsAttribute = (AtlasNamespaceAttribute) attribute; + Set<AtlasEntityType> entityTypes = nsAttribute.getApplicableEntityTypes(); + + if (CollectionUtils.isNotEmpty(entityTypes)) { + for (AtlasEntityType entityType : entityTypes) { + entityType.addNamespaceAttribute(nsAttribute); + } + } + } + } + + public static class AtlasNamespaceAttribute extends AtlasAttribute { + private final Set<AtlasEntityType> applicableEntityTypes; + private final int maxStringLength; + private final String validPattern; + + public AtlasNamespaceAttribute(AtlasAttribute attribute, Set<AtlasEntityType> applicableEntityTypes) { + super(attribute); + this.maxStringLength = 0; + this.validPattern = null; + this.applicableEntityTypes = applicableEntityTypes; + } + + public AtlasNamespaceAttribute(AtlasAttribute attribute, Set<AtlasEntityType> applicableEntityTypes, + int maxStringLength, String validPattern) { + super(attribute); + this.maxStringLength = maxStringLength; + this.validPattern = validPattern; + this.applicableEntityTypes = applicableEntityTypes; + } + + @Override + public AtlasNamespaceType getDefinedInType() { + return (AtlasNamespaceType) super.getDefinedInType(); + } + + public Set<AtlasEntityType> getApplicableEntityTypes() { + return applicableEntityTypes; + } + + public String getValidPattern() { + return validPattern; + } + + public int getMaxStringLength() { + return maxStringLength; + } + } +} diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java b/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java index 3475ce6..6bd3a11 100644 --- a/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java +++ b/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java @@ -112,6 +112,10 @@ public class AtlasStructType extends AtlasType { throw new AtlasBaseException(AtlasErrorCode.ATTRIBUTE_TYPE_INVALID, getTypeName(), attributeDef.getName()); } + if (attrType instanceof AtlasNamespaceType) { + throw new AtlasBaseException(AtlasErrorCode.ATTRIBUTE_TYPE_INVALID, getTypeName(), attributeDef.getName()); + } + a.put(attributeDef.getName(), attribute); } @@ -794,6 +798,26 @@ public class AtlasStructType extends AtlasType { this(definedInType, attrDef, attributeType, null, null); } + public AtlasAttribute(AtlasAttribute other) { + this.definedInType = other.definedInType; + this.attributeType = other.attributeType; + this.attributeDef = other.attributeDef; + this.qualifiedName = other.qualifiedName; + this.vertexPropertyName = other.vertexPropertyName; + this.vertexUniquePropertyName = other.vertexUniquePropertyName; + this.isOwnedRef = other.isOwnedRef; + this.isObjectRef = other.isObjectRef; + this.inverseRefAttributeName = other.inverseRefAttributeName; + this.inverseRefAttribute = other.inverseRefAttribute; + this.relationshipName = other.relationshipName; + this.relationshipEdgeLabel = other.relationshipEdgeLabel; + this.relationshipEdgeDirection = other.relationshipEdgeDirection; + this.isLegacyAttribute = other.isLegacyAttribute; + this.indexFieldName = other.indexFieldName; + this.isDynAttribute = false; + this.isDynAttributeEvalTrigger = false; + } + public AtlasStructType getDefinedInType() { return definedInType; } public AtlasStructDef getDefinedInDef() { return definedInType.getStructDef(); } @@ -860,6 +884,27 @@ public class AtlasStructType extends AtlasType { public void setIsDynAttributeEvalTrigger(boolean isDynAttributeEvalTrigger) { this.isDynAttributeEvalTrigger = isDynAttributeEvalTrigger; } + public Set<String> getOptionSet(String optionName) { + String strValue = attributeDef.getOption(optionName); + Set<String> ret = StringUtils.isBlank(strValue) ? null : AtlasType.fromJson(strValue, Set.class); + + return ret; + } + + public Integer getOptionInt(String optionName) { + String strValue = attributeDef.getOption(optionName); + Integer ret = StringUtils.isBlank(strValue) ? null : Integer.parseInt(strValue); + + return ret; + } + + public String getOptionString(String optionName) { + String strValue = attributeDef.getOption(optionName); + String ret = StringUtils.isBlank(strValue) ? null : strValue; + + return ret; + } + public static String getEdgeLabel(String property) { return "__" + property; } diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasTypeRegistry.java b/intg/src/main/java/org/apache/atlas/type/AtlasTypeRegistry.java index b071dc9..97e27d0 100644 --- a/intg/src/main/java/org/apache/atlas/type/AtlasTypeRegistry.java +++ b/intg/src/main/java/org/apache/atlas/type/AtlasTypeRegistry.java @@ -23,6 +23,7 @@ import org.apache.atlas.model.typedef.AtlasBaseTypeDef; import org.apache.atlas.model.typedef.AtlasClassificationDef; import org.apache.atlas.model.typedef.AtlasEntityDef; import org.apache.atlas.model.typedef.AtlasEnumDef; +import org.apache.atlas.model.typedef.AtlasNamespaceDef; import org.apache.atlas.model.typedef.AtlasRelationshipDef; import org.apache.atlas.model.typedef.AtlasStructDef; import org.apache.atlas.model.typedef.AtlasTypesDef; @@ -192,6 +193,15 @@ public class AtlasTypeRegistry { return registryData.classificationDefs.getTypeByName(name); } + public Collection<AtlasNamespaceType> getAllNamespaceTypes() { + return registryData.namespaceDefs.getAllTypes(); + } + + public Collection<AtlasNamespaceDef> getAllNamespaceDefs() { + return registryData.namespaceDefs.getAll(); + } + + public Collection<AtlasRelationshipDef> getAllRelationshipDefs() { return registryData.relationshipDefs.getAll(); } public Collection<AtlasEntityDef> getAllEntityDefs() { return registryData.entityDefs.getAll(); } @@ -217,6 +227,15 @@ public class AtlasTypeRegistry { public AtlasRelationshipDef getRelationshipDefByName(String name) { return registryData.relationshipDefs.getTypeDefByName(name); } + + public AtlasNamespaceDef getNamespaceDefByGuid(String guid) { + return registryData.namespaceDefs.getTypeDefByGuid(guid); + } + + public AtlasNamespaceDef getNamespaceDefByName(String name) { + return registryData.namespaceDefs.getTypeDefByName(name); + } + public AtlasRelationshipType getRelationshipTypeByName(String name) { return registryData.relationshipDefs.getTypeByName(name); } public AtlasTransientTypeRegistry lockTypeRegistryForUpdate() throws AtlasBaseException { @@ -271,6 +290,7 @@ public class AtlasTypeRegistry { final TypeDefCache<AtlasClassificationDef, AtlasClassificationType> classificationDefs; final TypeDefCache<AtlasEntityDef, AtlasEntityType> entityDefs; final TypeDefCache<AtlasRelationshipDef, AtlasRelationshipType> relationshipDefs; + final TypeDefCache<AtlasNamespaceDef, AtlasNamespaceType> namespaceDefs; final TypeDefCache<? extends AtlasBaseTypeDef, ? extends AtlasType>[] allDefCaches; RegistryData() { @@ -280,7 +300,9 @@ public class AtlasTypeRegistry { classificationDefs = new TypeDefCache<>(allTypes); entityDefs = new TypeDefCache<>(allTypes); relationshipDefs = new TypeDefCache<>(allTypes); - allDefCaches = new TypeDefCache[] { enumDefs, structDefs, classificationDefs, entityDefs, relationshipDefs }; + namespaceDefs = new TypeDefCache<>(allTypes); + allDefCaches = new TypeDefCache[] { enumDefs, structDefs, classificationDefs, entityDefs, + relationshipDefs, namespaceDefs }; init(); } @@ -339,6 +361,7 @@ public class AtlasTypeRegistry { classificationDefs.updateGuid(typeName, guid); entityDefs.updateGuid(typeName, guid); relationshipDefs.updateGuid(typeName, guid); + namespaceDefs.updateGuid(typeName, guid); } } @@ -349,6 +372,7 @@ public class AtlasTypeRegistry { classificationDefs.removeTypeDefByGuid(guid); entityDefs.removeTypeDefByGuid(guid); relationshipDefs.removeTypeDefByGuid(guid); + namespaceDefs.removeTypeDefByGuid(guid); } } @@ -359,6 +383,7 @@ public class AtlasTypeRegistry { classificationDefs.removeTypeDefByName(typeName); entityDefs.removeTypeDefByName(typeName); relationshipDefs.removeTypeDefByName(typeName); + namespaceDefs.removeTypeDefByName(typeName); } } @@ -369,7 +394,7 @@ public class AtlasTypeRegistry { classificationDefs.clear(); entityDefs.clear(); relationshipDefs.clear(); - + namespaceDefs.clear(); init(); } } @@ -388,6 +413,7 @@ public class AtlasTypeRegistry { addTypesWithNoRefResolve(parent.getAllClassificationDefs()); addTypesWithNoRefResolve(parent.getAllEntityDefs()); addTypesWithNoRefResolve(parent.getAllRelationshipDefs()); + addTypesWithNoRefResolve(parent.getAllNamespaceDefs()); addedTypes.clear(); updatedTypes.clear(); @@ -467,6 +493,7 @@ public class AtlasTypeRegistry { addTypesWithNoRefResolve(typesDef.getClassificationDefs()); addTypesWithNoRefResolve(typesDef.getEntityDefs()); addTypesWithNoRefResolve(typesDef.getRelationshipDefs()); + addTypesWithNoRefResolve(typesDef.getNamespaceDefs()); } resolveReferences(); @@ -567,6 +594,7 @@ public class AtlasTypeRegistry { updateTypesWithNoRefResolve(typesDef.getClassificationDefs()); updateTypesWithNoRefResolve(typesDef.getEntityDefs()); updateTypesWithNoRefResolve(typesDef.getRelationshipDefs()); + updateTypesWithNoRefResolve(typesDef.getNamespaceDefs()); } if (LOG.isDebugEnabled()) { @@ -581,6 +609,7 @@ public class AtlasTypeRegistry { removeTypesWithNoRefResolve(typesDef.getClassificationDefs()); removeTypesWithNoRefResolve(typesDef.getEntityDefs()); removeTypesWithNoRefResolve(typesDef.getRelationshipDefs()); + removeTypesWithNoRefResolve(typesDef.getNamespaceDefs()); } resolveReferences(); @@ -615,6 +644,9 @@ public class AtlasTypeRegistry { case RELATIONSHIP: registryData.relationshipDefs.removeTypeDefByName(typeDef.getName()); break; + case NAMESPACE: + registryData.namespaceDefs.removeTypeDefByName(typeDef.getName()); + break; } deletedTypes.add(typeDef); } @@ -636,6 +668,9 @@ public class AtlasTypeRegistry { case RELATIONSHIP: registryData.relationshipDefs.removeTypeDefByGuid(typeDef.getGuid()); break; + case NAMESPACE: + registryData.namespaceDefs.removeTypeDefByGuid(typeDef.getGuid()); + break; } deletedTypes.add(typeDef); } @@ -722,6 +757,9 @@ public class AtlasTypeRegistry { AtlasRelationshipDef relationshipDef = (AtlasRelationshipDef) typeDef; registryData.relationshipDefs.addType(relationshipDef, new AtlasRelationshipType(relationshipDef)); + } else if (typeDef.getClass().equals(AtlasNamespaceDef.class)) { + AtlasNamespaceDef namespaceDef = (AtlasNamespaceDef) typeDef; + registryData.namespaceDefs.addType(namespaceDef, new AtlasNamespaceType(namespaceDef)); } addedTypes.add(typeDef); @@ -801,6 +839,11 @@ public class AtlasTypeRegistry { registryData.relationshipDefs.removeTypeDefByGuid(guid); registryData.relationshipDefs.addType(relationshipDef, new AtlasRelationshipType(relationshipDef)); + } else if (typeDef.getClass().equals(AtlasNamespaceDef.class)) { + AtlasNamespaceDef namespaceDef = (AtlasNamespaceDef) typeDef; + + registryData.namespaceDefs.removeTypeDefByGuid(guid); + registryData.namespaceDefs.addType(namespaceDef, new AtlasNamespaceType(namespaceDef)); } updatedTypes.add(typeDef); @@ -843,6 +886,11 @@ public class AtlasTypeRegistry { registryData.relationshipDefs.removeTypeDefByName(name); registryData.relationshipDefs.addType(relationshipDef, new AtlasRelationshipType(relationshipDef)); + } else if (typeDef.getClass().equals(AtlasNamespaceDef.class)) { + AtlasNamespaceDef namespaceDef = (AtlasNamespaceDef) typeDef; + + registryData.namespaceDefs.removeTypeDefByName(name); + registryData.namespaceDefs.addType(namespaceDef, new AtlasNamespaceType(namespaceDef)); } updatedTypes.add(typeDef); diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasTypeUtil.java b/intg/src/main/java/org/apache/atlas/type/AtlasTypeUtil.java index 0883d54..5b115b5 100644 --- a/intg/src/main/java/org/apache/atlas/type/AtlasTypeUtil.java +++ b/intg/src/main/java/org/apache/atlas/type/AtlasTypeUtil.java @@ -28,6 +28,7 @@ import org.apache.atlas.model.typedef.AtlasClassificationDef; import org.apache.atlas.model.typedef.AtlasEntityDef; import org.apache.atlas.model.typedef.AtlasEnumDef; import org.apache.atlas.model.typedef.AtlasEnumDef.AtlasEnumElementDef; +import org.apache.atlas.model.typedef.AtlasNamespaceDef; import org.apache.atlas.model.typedef.AtlasRelationshipDef; import org.apache.atlas.model.typedef.AtlasRelationshipDef.PropagateTags; import org.apache.atlas.model.typedef.AtlasRelationshipDef.RelationshipCategory; @@ -315,6 +316,15 @@ public class AtlasTypeUtil { return new AtlasTypesDef(enums, structs, traits, classes, relations); } + public static AtlasTypesDef getTypesDef(List<AtlasEnumDef> enums, + List<AtlasStructDef> structs, + List<AtlasClassificationDef> traits, + List<AtlasEntityDef> classes, + List<AtlasRelationshipDef> relations, + List<AtlasNamespaceDef> namespaces) { + return new AtlasTypesDef(enums, structs, traits, classes, relations, namespaces); + } + public static List<AtlasTypeDefHeader> toTypeDefHeader(AtlasTypesDef typesDef) { List<AtlasTypeDefHeader> headerList = new LinkedList<>(); if (CollectionUtils.isNotEmpty(typesDef.getEnumDefs())) { diff --git a/intg/src/main/java/org/apache/atlas/typesystem/types/DataTypes.java b/intg/src/main/java/org/apache/atlas/typesystem/types/DataTypes.java index dba2d88..d57a484 100644 --- a/intg/src/main/java/org/apache/atlas/typesystem/types/DataTypes.java +++ b/intg/src/main/java/org/apache/atlas/typesystem/types/DataTypes.java @@ -30,6 +30,7 @@ public class DataTypes { STRUCT, TRAIT, CLASS, - RELATIONSHIP + RELATIONSHIP, + NAMESPACE } } \ No newline at end of file diff --git a/intg/src/test/java/org/apache/atlas/TestRelationshipUtilsV2.java b/intg/src/test/java/org/apache/atlas/TestRelationshipUtilsV2.java index 02613b5..32ed6ee 100755 --- a/intg/src/test/java/org/apache/atlas/TestRelationshipUtilsV2.java +++ b/intg/src/test/java/org/apache/atlas/TestRelationshipUtilsV2.java @@ -27,10 +27,12 @@ import org.apache.atlas.model.typedef.AtlasClassificationDef; import org.apache.atlas.model.typedef.AtlasEntityDef; import org.apache.atlas.model.typedef.AtlasEnumDef; import org.apache.atlas.model.typedef.AtlasEnumDef.AtlasEnumElementDef; +import org.apache.atlas.model.typedef.AtlasNamespaceDef; import org.apache.atlas.model.typedef.AtlasRelationshipDef; import org.apache.atlas.model.typedef.AtlasRelationshipEndDef; import org.apache.atlas.model.typedef.AtlasStructDef; import org.apache.atlas.model.typedef.AtlasTypesDef; +import org.apache.atlas.type.AtlasType; import org.apache.commons.lang.ArrayUtils; import org.apache.commons.lang.StringUtils; @@ -38,6 +40,7 @@ import java.math.BigDecimal; import java.math.BigInteger; import java.util.*; +import static org.apache.atlas.model.typedef.AtlasNamespaceDef.ATTR_OPTION_APPLICABLE_ENTITY_TYPES; import static org.apache.atlas.model.typedef.AtlasRelationshipDef.PropagateTags.BOTH; import static org.apache.atlas.model.typedef.AtlasRelationshipDef.PropagateTags.ONE_TO_TWO; import static org.apache.atlas.model.typedef.AtlasRelationshipDef.RelationshipCategory.AGGREGATION; @@ -141,11 +144,25 @@ public final class TestRelationshipUtilsV2 { new AtlasRelationshipEndDef(PERSON_TYPE, "sibling", SINGLE), new AtlasRelationshipEndDef(PERSON_TYPE, "sibling", SINGLE)); + AtlasStructDef.AtlasAttributeDef nsAttr1 = new AtlasStructDef.AtlasAttributeDef("attr1", "int"); + AtlasStructDef.AtlasAttributeDef nsAttr2 = new AtlasStructDef.AtlasAttributeDef("attr2", "int"); + + nsAttr1.setOption(ATTR_OPTION_APPLICABLE_ENTITY_TYPES, AtlasType.toJson(Collections.singleton(DEPARTMENT_TYPE))); + nsAttr1.setIsOptional(true); + nsAttr1.setIsUnique(false); + + nsAttr2.setOption(ATTR_OPTION_APPLICABLE_ENTITY_TYPES, AtlasType.toJson(Collections.singleton(DEPARTMENT_TYPE))); + nsAttr2.setIsOptional(true); + nsAttr2.setIsUnique(false); + + AtlasNamespaceDef namespaceDef = new AtlasNamespaceDef("test_namespace", "test_description", DEFAULT_VERSION, Arrays.asList(nsAttr1, nsAttr2)); + return new AtlasTypesDef(Collections.singletonList(orgLevelType), Collections.singletonList(addressType), Collections.singletonList(securityClearanceType), Arrays.asList(personType, employeeType, departmentType, managerType), - Arrays.asList(employeeDepartmentType, employeeManagerType, employeeMentorsType, employeeFriendsType, personSiblingType)); + Arrays.asList(employeeDepartmentType, employeeManagerType, employeeMentorsType, employeeFriendsType, personSiblingType), + Collections.singletonList(namespaceDef)); } public static AtlasEntitiesWithExtInfo getDepartmentEmployeeInstances() { @@ -278,7 +295,7 @@ public final class TestRelationshipUtilsV2 { new AtlasRelationshipEndDef(TYPE_A, "mapToB", SET)); return new AtlasTypesDef(Collections.<AtlasEnumDef>emptyList(), Collections.<AtlasStructDef>emptyList(), Collections.<AtlasClassificationDef>emptyList(), Arrays.asList(aType, bType), - Arrays.asList(relationshipType1, relationshipType2, relationshipType3, relationshipType4)); + Arrays.asList(relationshipType1, relationshipType2, relationshipType3, relationshipType4), Collections.<AtlasNamespaceDef>emptyList()); } private static List<AtlasEnumElementDef> getOrgLevelElements() { diff --git a/intg/src/test/java/org/apache/atlas/model/typedef/TestAtlasNamespaceDef.java b/intg/src/test/java/org/apache/atlas/model/typedef/TestAtlasNamespaceDef.java new file mode 100644 index 0000000..8867774 --- /dev/null +++ b/intg/src/test/java/org/apache/atlas/model/typedef/TestAtlasNamespaceDef.java @@ -0,0 +1,84 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.model.typedef; + +import org.apache.atlas.type.AtlasType; +import org.testng.annotations.Test; + +import java.util.Arrays; +import java.util.Collections; + +import static org.apache.atlas.model.typedef.AtlasNamespaceDef.ATTR_OPTION_APPLICABLE_ENTITY_TYPES; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotEquals; + +public class TestAtlasNamespaceDef { + + @Test + public void namespaceDefSerDes() { + AtlasNamespaceDef namespaceDef = new AtlasNamespaceDef("test_namespace", "test_description", null); + String jsonString = AtlasType.toJson(namespaceDef); + + AtlasNamespaceDef namespaceDef1 = AtlasType.fromJson(jsonString, AtlasNamespaceDef.class); + assertEquals(namespaceDef, namespaceDef1, + "Incorrect serialization/deserialization of AtlasNamespaceDef"); + } + + @Test + public void namespaceDefEquality() { + AtlasNamespaceDef namespaceDef1 = new AtlasNamespaceDef("test_namespace", "test_description", null); + AtlasNamespaceDef namespaceDef2 = new AtlasNamespaceDef("test_namespace", "test_description", null); + assertEquals(namespaceDef1, namespaceDef2, "Namespaces should be equal because the name of the" + + "namespace is same"); + } + + @Test + public void namespaceDefUnequality() { + AtlasNamespaceDef namespaceDef1 = new AtlasNamespaceDef("test_namespace", "test_description", null); + AtlasNamespaceDef namespaceDef2 = new AtlasNamespaceDef("test_namespace1", "test_description", null); + assertNotEquals(namespaceDef1, namespaceDef2, "Namespaces should not be equal since they have a" + + "different name"); + } + + @Test + public void namespaceDefWithNamespaceAttributes() { + AtlasNamespaceDef namespaceDef1 = new AtlasNamespaceDef("test_namespace", "test_description", null); + AtlasStructDef.AtlasAttributeDef nsAttr1 = new AtlasStructDef.AtlasAttributeDef("attr1", "int"); + AtlasStructDef.AtlasAttributeDef nsAttr2 = new AtlasStructDef.AtlasAttributeDef("attr2", "int"); + + nsAttr1.setOption(ATTR_OPTION_APPLICABLE_ENTITY_TYPES, AtlasType.toJson(Collections.singleton("hive_table"))); + nsAttr2.setOption(ATTR_OPTION_APPLICABLE_ENTITY_TYPES, AtlasType.toJson(Collections.singleton("hive_table"))); + + namespaceDef1.setAttributeDefs(Arrays.asList(nsAttr1, nsAttr2)); + assertEquals(namespaceDef1.getAttributeDefs().size(), 2); + } + + @Test + public void namespaceDefWithNamespaceAttributesHavingCardinality() { + AtlasNamespaceDef namespaceDef1 = new AtlasNamespaceDef("test_namespace", "test_description", null); + AtlasStructDef.AtlasAttributeDef nsAttr1 = new AtlasStructDef.AtlasAttributeDef("attr1", "int"); + AtlasStructDef.AtlasAttributeDef nsAttr2 = new AtlasStructDef.AtlasAttributeDef("attr2", "int"); + + nsAttr1.setOption(ATTR_OPTION_APPLICABLE_ENTITY_TYPES, AtlasType.toJson(Collections.singleton("hive_table"))); + nsAttr2.setOption(ATTR_OPTION_APPLICABLE_ENTITY_TYPES, AtlasType.toJson(Collections.singleton("hive_table"))); + nsAttr2.setCardinality(AtlasStructDef.AtlasAttributeDef.Cardinality.SET); + + namespaceDef1.setAttributeDefs(Arrays.asList(nsAttr1, nsAttr2)); + assertEquals(namespaceDef1.getAttributeDefs().size(), 2); + } +} \ No newline at end of file diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java index 7c55130..a3a570b 100755 --- a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java +++ b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java @@ -33,6 +33,7 @@ import org.apache.atlas.model.TypeCategory; import org.apache.atlas.model.typedef.AtlasBaseTypeDef; import org.apache.atlas.model.typedef.AtlasEntityDef; import org.apache.atlas.model.typedef.AtlasEnumDef; +import org.apache.atlas.model.typedef.AtlasNamespaceDef; import org.apache.atlas.model.typedef.AtlasStructDef; import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef; import org.apache.atlas.repository.Constants; diff --git a/repository/src/main/java/org/apache/atlas/repository/impexp/ExportService.java b/repository/src/main/java/org/apache/atlas/repository/impexp/ExportService.java index 6016723..82a2d31 100644 --- a/repository/src/main/java/org/apache/atlas/repository/impexp/ExportService.java +++ b/repository/src/main/java/org/apache/atlas/repository/impexp/ExportService.java @@ -347,6 +347,7 @@ public class ExportService { final Set<String> structTypes = new HashSet<>(); final Set<String> enumTypes = new HashSet<>(); final Set<String> relationshipTypes = new HashSet<>(); + final Set<String> namespaceTypes = new HashSet<>(); final AtlasExportResult result; private final ZipSink sink; diff --git a/repository/src/main/java/org/apache/atlas/repository/impexp/ExportTypeProcessor.java b/repository/src/main/java/org/apache/atlas/repository/impexp/ExportTypeProcessor.java index 5bad615..b553352 100644 --- a/repository/src/main/java/org/apache/atlas/repository/impexp/ExportTypeProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/impexp/ExportTypeProcessor.java @@ -28,6 +28,7 @@ import org.apache.atlas.type.AtlasClassificationType; import org.apache.atlas.type.AtlasEntityType; import org.apache.atlas.type.AtlasEnumType; import org.apache.atlas.type.AtlasMapType; +import org.apache.atlas.type.AtlasNamespaceType; import org.apache.atlas.type.AtlasRelationshipType; import org.apache.atlas.type.AtlasStructType; import org.apache.atlas.type.AtlasType; @@ -103,12 +104,14 @@ class ExportTypeProcessor { addEntityType((AtlasEntityType)type, context); } else if (type instanceof AtlasClassificationType) { addClassificationType((AtlasClassificationType)type, context); + } else if (type instanceof AtlasRelationshipType) { + addRelationshipType(type.getTypeName(), context); + } else if (type instanceof AtlasNamespaceType) { + addNamespaceType((AtlasNamespaceType) type, context); } else if (type instanceof AtlasStructType) { addStructType((AtlasStructType)type, context); } else if (type instanceof AtlasEnumType) { addEnumType((AtlasEnumType)type, context); - } else if (type instanceof AtlasRelationshipType) { - addRelationshipType(type.getTypeName(), context); } } @@ -169,6 +172,14 @@ class ExportTypeProcessor { } } + private void addNamespaceType(AtlasNamespaceType namespaceType, ExportService.ExportContext context) { + if (!context.namespaceTypes.contains(namespaceType.getTypeName())) { + context.namespaceTypes.add(namespaceType.getTypeName()); + + addAttributeTypes(namespaceType, context); + } + } + private void addAttributeTypes(AtlasStructType structType, ExportService.ExportContext context) { for (AtlasStructDef.AtlasAttributeDef attributeDef : structType.getStructDef().getAttributeDefs()) { addType(attributeDef.getTypeName(), context); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/bootstrap/AtlasTypeDefStoreInitializer.java b/repository/src/main/java/org/apache/atlas/repository/store/bootstrap/AtlasTypeDefStoreInitializer.java index 08b00e7..2a602c8 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/bootstrap/AtlasTypeDefStoreInitializer.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/bootstrap/AtlasTypeDefStoreInitializer.java @@ -34,13 +34,13 @@ import org.apache.atlas.model.typedef.AtlasClassificationDef; import org.apache.atlas.model.typedef.AtlasEntityDef; import org.apache.atlas.model.typedef.AtlasEnumDef; import org.apache.atlas.model.typedef.AtlasEnumDef.AtlasEnumElementDef; +import org.apache.atlas.model.typedef.AtlasNamespaceDef; import org.apache.atlas.model.typedef.AtlasRelationshipDef; import org.apache.atlas.model.typedef.AtlasRelationshipDef.RelationshipCategory; import org.apache.atlas.model.typedef.AtlasRelationshipEndDef; import org.apache.atlas.model.typedef.AtlasStructDef; import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef; import org.apache.atlas.model.typedef.AtlasTypesDef; -import org.apache.atlas.repository.Constants; import org.apache.atlas.repository.graph.GraphBackedSearchIndexer; import org.apache.atlas.repository.graphdb.AtlasGraph; import org.apache.atlas.repository.patches.AtlasPatchRegistry; @@ -55,7 +55,6 @@ import org.apache.commons.collections.MapUtils; import org.apache.commons.configuration.Configuration; import org.apache.commons.lang.ObjectUtils; import org.apache.commons.lang3.StringUtils; -import org.apache.jute.Index; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.core.annotation.Order; @@ -253,6 +252,14 @@ public class AtlasTypeDefStoreInitializer implements ActiveStateChangeHandler { } } + if (CollectionUtils.isNotEmpty(typesDef.getNamespaceDefs())) { + for (AtlasNamespaceDef namespaceDef : typesDef.getNamespaceDefs()) { + if (!typeRegistry.isRegisteredType(namespaceDef.getName())) { + typesToCreate.getNamespaceDefs().add(namespaceDef); + } + } + } + return typesToCreate; } @@ -337,6 +344,20 @@ public class AtlasTypeDefStoreInitializer implements ActiveStateChangeHandler { } } + if (CollectionUtils.isNotEmpty(typesDef.getNamespaceDefs())) { + for (AtlasNamespaceDef namespaceDef : typesDef.getNamespaceDefs()) { + AtlasNamespaceDef oldNamespaceDef = typeRegistry.getNamespaceDefByName(namespaceDef.getName()); + + if (oldNamespaceDef == null) { + continue; + } + + if (updateTypeAttributes(oldNamespaceDef, namespaceDef, checkTypeVersion)) { + typesToUpdate.getNamespaceDefs().add(namespaceDef); + } + } + } + return typesToUpdate; } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasTypeDefGraphStore.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasTypeDefGraphStore.java index b04f188..e1ef849 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasTypeDefGraphStore.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasTypeDefGraphStore.java @@ -79,6 +79,8 @@ public abstract class AtlasTypeDefGraphStore implements AtlasTypeDefStore { protected abstract AtlasDefStore<AtlasRelationshipDef> getRelationshipDefStore(AtlasTypeRegistry typeRegistry); + protected abstract AtlasDefStore<AtlasNamespaceDef> getNamespaceDefStore(AtlasTypeRegistry typeRegistry); + public AtlasTypeRegistry getTypeRegistry() { return typeRegistry; } @Override @@ -97,7 +99,8 @@ public abstract class AtlasTypeDefGraphStore implements AtlasTypeDefStore { getStructDefStore(ttr).getAll(), getClassificationDefStore(ttr).getAll(), getEntityDefStore(ttr).getAll(), - getRelationshipDefStore(ttr).getAll()); + getRelationshipDefStore(ttr).getAll(), + getNamespaceDefStore(ttr).getAll()); rectifyTypeErrorsIfAny(typesDef); @@ -194,6 +197,28 @@ public abstract class AtlasTypeDefGraphStore implements AtlasTypeDefStore { } @Override + public AtlasNamespaceDef getNamespaceDefByName(String name) throws AtlasBaseException { + AtlasNamespaceDef ret = typeRegistry.getNamespaceDefByName(name); + + if (ret == null) { + throw new AtlasBaseException(AtlasErrorCode.TYPE_NAME_NOT_FOUND, name); + } + + return ret; + } + + @Override + public AtlasNamespaceDef getNamespaceDefByGuid(String guid) throws AtlasBaseException { + AtlasNamespaceDef ret = typeRegistry.getNamespaceDefByGuid(guid); + + if (ret == null) { + throw new AtlasBaseException(AtlasErrorCode.TYPE_GUID_NOT_FOUND, guid); + } + + return ret; + } + + @Override @GraphTransaction public AtlasStructDef updateStructDefByName(String name, AtlasStructDef structDef) throws AtlasBaseException { AtlasTransientTypeRegistry ttr = lockTypeRegistryAndReleasePostCommit(); @@ -325,12 +350,13 @@ public abstract class AtlasTypeDefGraphStore implements AtlasTypeDefStore { @GraphTransaction public AtlasTypesDef createTypesDef(AtlasTypesDef typesDef) throws AtlasBaseException { if (LOG.isDebugEnabled()) { - LOG.debug("==> AtlasTypeDefGraphStore.createTypesDef(enums={}, structs={}, classifications={}, entities={}, relationships={})", + LOG.debug("==> AtlasTypeDefGraphStore.createTypesDef(enums={}, structs={}, classifications={}, entities={}, relationships={}, namespaces={})", CollectionUtils.size(typesDef.getEnumDefs()), CollectionUtils.size(typesDef.getStructDefs()), CollectionUtils.size(typesDef.getClassificationDefs()), CollectionUtils.size(typesDef.getEntityDefs()), - CollectionUtils.size(typesDef.getRelationshipDefs())); + CollectionUtils.size(typesDef.getRelationshipDefs()), + CollectionUtils.size(typesDef.getNamespaceDefs())); } AtlasTransientTypeRegistry ttr = lockTypeRegistryAndReleasePostCommit(); @@ -346,12 +372,13 @@ public abstract class AtlasTypeDefGraphStore implements AtlasTypeDefStore { } if (LOG.isDebugEnabled()) { - LOG.debug("<== AtlasTypeDefGraphStore.createTypesDef(enums={}, structs={}, classfications={}, entities={}, relationships={})", + LOG.debug("<== AtlasTypeDefGraphStore.createTypesDef(enums={}, structs={}, classfications={}, entities={}, relationships={}, namespaces={})", CollectionUtils.size(typesDef.getEnumDefs()), CollectionUtils.size(typesDef.getStructDefs()), CollectionUtils.size(typesDef.getClassificationDefs()), CollectionUtils.size(typesDef.getEntityDefs()), - CollectionUtils.size(typesDef.getRelationshipDefs())); + CollectionUtils.size(typesDef.getRelationshipDefs()), + CollectionUtils.size(typesDef.getNamespaceDefs())); } return ret; @@ -429,12 +456,13 @@ public abstract class AtlasTypeDefGraphStore implements AtlasTypeDefStore { @GraphTransaction public AtlasTypesDef updateTypesDef(AtlasTypesDef typesDef) throws AtlasBaseException { if (LOG.isDebugEnabled()) { - LOG.debug("==> AtlasTypeDefGraphStore.updateTypesDef(enums={}, structs={}, classfications={}, entities={}, relationships{})", + LOG.debug("==> AtlasTypeDefGraphStore.updateTypesDef(enums={}, structs={}, classfications={}, entities={}, relationships{}, namespaceDefs={})", CollectionUtils.size(typesDef.getEnumDefs()), CollectionUtils.size(typesDef.getStructDefs()), CollectionUtils.size(typesDef.getClassificationDefs()), CollectionUtils.size(typesDef.getEntityDefs()), - CollectionUtils.size(typesDef.getRelationshipDefs())); + CollectionUtils.size(typesDef.getRelationshipDefs()), + CollectionUtils.size(typesDef.getNamespaceDefs())); } AtlasTransientTypeRegistry ttr = lockTypeRegistryAndReleasePostCommit(); @@ -459,12 +487,13 @@ public abstract class AtlasTypeDefGraphStore implements AtlasTypeDefStore { } if (LOG.isDebugEnabled()) { - LOG.debug("<== AtlasTypeDefGraphStore.updateTypesDef(enums={}, structs={}, classfications={}, entities={}, relationships={})", + LOG.debug("<== AtlasTypeDefGraphStore.updateTypesDef(enums={}, structs={}, classfications={}, entities={}, relationships={}, namespaceDefs={})", CollectionUtils.size(typesDef.getEnumDefs()), CollectionUtils.size(typesDef.getStructDefs()), CollectionUtils.size(typesDef.getClassificationDefs()), CollectionUtils.size(typesDef.getEntityDefs()), - CollectionUtils.size(typesDef.getRelationshipDefs())); + CollectionUtils.size(typesDef.getRelationshipDefs()), + CollectionUtils.size(typesDef.getNamespaceDefs())); } return ret; @@ -475,12 +504,13 @@ public abstract class AtlasTypeDefGraphStore implements AtlasTypeDefStore { @GraphTransaction public void deleteTypesDef(AtlasTypesDef typesDef) throws AtlasBaseException { if (LOG.isDebugEnabled()) { - LOG.debug("==> AtlasTypeDefGraphStore.deleteTypesDef(enums={}, structs={}, classfications={}, entities={}, relationships={})", + LOG.debug("==> AtlasTypeDefGraphStore.deleteTypesDef(enums={}, structs={}, classfications={}, entities={}, relationships={}, namespaceDefs={})", CollectionUtils.size(typesDef.getEnumDefs()), CollectionUtils.size(typesDef.getStructDefs()), CollectionUtils.size(typesDef.getClassificationDefs()), CollectionUtils.size(typesDef.getEntityDefs()), - CollectionUtils.size(typesDef.getRelationshipDefs())); + CollectionUtils.size(typesDef.getRelationshipDefs()), + CollectionUtils.size(typesDef.getNamespaceDefs())); } AtlasTransientTypeRegistry ttr = lockTypeRegistryAndReleasePostCommit(); @@ -490,6 +520,7 @@ public abstract class AtlasTypeDefGraphStore implements AtlasTypeDefStore { AtlasDefStore<AtlasClassificationDef> classifiDefStore = getClassificationDefStore(ttr); AtlasDefStore<AtlasEntityDef> entityDefStore = getEntityDefStore(ttr); AtlasDefStore<AtlasRelationshipDef> relationshipDefStore = getRelationshipDefStore(ttr); + AtlasDefStore<AtlasNamespaceDef> namespaceDefStore = getNamespaceDefStore(ttr); List<AtlasVertex> preDeleteStructDefs = new ArrayList<>(); List<AtlasVertex> preDeleteClassifiDefs = new ArrayList<>(); @@ -599,6 +630,16 @@ public abstract class AtlasTypeDefGraphStore implements AtlasTypeDefStore { } } + if (CollectionUtils.isNotEmpty(typesDef.getNamespaceDefs())) { + for (AtlasNamespaceDef namespaceDef : typesDef.getNamespaceDefs()) { + if (StringUtils.isNotBlank(namespaceDef.getGuid())) { + namespaceDefStore.deleteByGuid(namespaceDef.getGuid(), null); + } else { + namespaceDefStore.deleteByName(namespaceDef.getName(), null); + } + } + } + // Remove all from ttr.removeTypesDef(typesDef); @@ -631,6 +672,8 @@ public abstract class AtlasTypeDefGraphStore implements AtlasTypeDefStore { typesDef.setEnumDefs(Collections.singletonList((AtlasEnumDef) baseTypeDef)); } else if (baseTypeDef instanceof AtlasRelationshipDef) { typesDef.setRelationshipDefs(Collections.singletonList((AtlasRelationshipDef) baseTypeDef)); + } else if (baseTypeDef instanceof AtlasNamespaceDef) { + typesDef.setNamespaceDefs(Collections.singletonList((AtlasNamespaceDef) baseTypeDef)); } else if (baseTypeDef instanceof AtlasStructDef) { typesDef.setStructDefs(Collections.singletonList((AtlasStructDef) baseTypeDef)); } @@ -673,6 +716,12 @@ public abstract class AtlasTypeDefGraphStore implements AtlasTypeDefStore { } } + for(AtlasNamespaceType namespaceType : typeRegistry.getAllNamespaceTypes()) { + if (searchPredicates.evaluate(namespaceType)) { + typesDef.getNamespaceDefs().add(namespaceType.getNamespaceDef()); + } + } + return typesDef; } @@ -712,6 +761,9 @@ public abstract class AtlasTypeDefGraphStore implements AtlasTypeDefStore { case RELATIONSHIP: ret = ((AtlasRelationshipType) type).getRelationshipDef(); break; + case NAMESPACE: + ret = ((AtlasNamespaceType) type).getNamespaceDef(); + break; case PRIMITIVE: case OBJECT_ID_TYPE: case ARRAY: @@ -838,11 +890,13 @@ public abstract class AtlasTypeDefGraphStore implements AtlasTypeDefStore { AtlasDefStore<AtlasClassificationDef> classifiDefStore = getClassificationDefStore(ttr); AtlasDefStore<AtlasEntityDef> entityDefStore = getEntityDefStore(ttr); AtlasDefStore<AtlasRelationshipDef> relationshipDefStore = getRelationshipDefStore(ttr); + AtlasDefStore<AtlasNamespaceDef> nameSpaceDefStore = getNamespaceDefStore(ttr); - List<AtlasVertex> preCreateStructDefs = new ArrayList<>(); - List<AtlasVertex> preCreateClassifiDefs = new ArrayList<>(); - List<AtlasVertex> preCreateEntityDefs = new ArrayList<>(); + List<AtlasVertex> preCreateStructDefs = new ArrayList<>(); + List<AtlasVertex> preCreateClassifiDefs = new ArrayList<>(); + List<AtlasVertex> preCreateEntityDefs = new ArrayList<>(); List<AtlasVertex> preCreateRelationshipDefs = new ArrayList<>(); + List<AtlasVertex> preCreateNamespaceDefs = new ArrayList<>(); // for enumerations run the create if (CollectionUtils.isNotEmpty(typesDef.getEnumDefs())) { @@ -880,6 +934,12 @@ public abstract class AtlasTypeDefGraphStore implements AtlasTypeDefStore { } } + if (CollectionUtils.isNotEmpty(typesDef.getNamespaceDefs())) { + for (AtlasNamespaceDef namespaceDef : typesDef.getNamespaceDefs()) { + preCreateNamespaceDefs.add(nameSpaceDefStore.preCreate(namespaceDef)); + } + } + if (CollectionUtils.isNotEmpty(typesDef.getStructDefs())) { int i = 0; for (AtlasStructDef structDef : typesDef.getStructDefs()) { @@ -927,17 +987,30 @@ public abstract class AtlasTypeDefGraphStore implements AtlasTypeDefStore { } } + if (CollectionUtils.isNotEmpty(typesDef.getNamespaceDefs())) { + int i = 0; + for (AtlasNamespaceDef namespaceDef : typesDef.getNamespaceDefs()) { + AtlasNamespaceDef createdDef = nameSpaceDefStore.create(namespaceDef, preCreateNamespaceDefs.get(i)); + + ttr.updateGuid(createdDef.getName(), createdDef.getGuid()); + + ret.getNamespaceDefs().add(createdDef); + i++; + } + } + return ret; } private AtlasTypesDef updateGraphStore(AtlasTypesDef typesDef, AtlasTransientTypeRegistry ttr) throws AtlasBaseException { AtlasTypesDef ret = new AtlasTypesDef(); - AtlasDefStore<AtlasEnumDef> enumDefStore = getEnumDefStore(ttr); - AtlasDefStore<AtlasStructDef> structDefStore = getStructDefStore(ttr); - AtlasDefStore<AtlasClassificationDef> classifiDefStore = getClassificationDefStore(ttr); - AtlasDefStore<AtlasEntityDef> entityDefStore = getEntityDefStore(ttr); - AtlasDefStore<AtlasRelationshipDef> relationDefStore = getRelationshipDefStore(ttr); + AtlasDefStore<AtlasEnumDef> enumDefStore = getEnumDefStore(ttr); + AtlasDefStore<AtlasStructDef> structDefStore = getStructDefStore(ttr); + AtlasDefStore<AtlasClassificationDef> classifiDefStore = getClassificationDefStore(ttr); + AtlasDefStore<AtlasEntityDef> entityDefStore = getEntityDefStore(ttr); + AtlasDefStore<AtlasRelationshipDef> relationDefStore = getRelationshipDefStore(ttr); + AtlasDefStore<AtlasNamespaceDef> nameSpaceDefStore = getNamespaceDefStore(ttr); if (CollectionUtils.isNotEmpty(typesDef.getEnumDefs())) { for (AtlasEnumDef enumDef : typesDef.getEnumDefs()) { @@ -969,6 +1042,12 @@ public abstract class AtlasTypeDefGraphStore implements AtlasTypeDefStore { } } + if (CollectionUtils.isNotEmpty(typesDef.getNamespaceDefs())) { + for (AtlasNamespaceDef namespaceDef : typesDef.getNamespaceDefs()) { + ret.getNamespaceDefs().add(nameSpaceDefStore.update(namespaceDef)); + } + } + return ret; } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasNamespaceDefStoreV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasNamespaceDefStoreV2.java new file mode 100644 index 0000000..eaaf6bb --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasNamespaceDefStoreV2.java @@ -0,0 +1,347 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.repository.store.graph.v2; + +import org.apache.atlas.AtlasErrorCode; +import org.apache.atlas.authorize.AtlasAuthorizationUtils; +import org.apache.atlas.authorize.AtlasPrivilege; +import org.apache.atlas.authorize.AtlasTypeAccessRequest; +import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.model.TypeCategory; +import org.apache.atlas.model.typedef.AtlasNamespaceDef; +import org.apache.atlas.model.typedef.AtlasStructDef; +import org.apache.atlas.repository.Constants; +import org.apache.atlas.repository.graphdb.AtlasVertex; +import org.apache.atlas.type.AtlasNamespaceType; +import org.apache.atlas.type.AtlasType; +import org.apache.atlas.type.AtlasTypeRegistry; +import org.apache.atlas.typesystem.types.DataTypes; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.inject.Inject; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Set; + +import static org.apache.atlas.model.typedef.AtlasNamespaceDef.ATTR_OPTION_APPLICABLE_ENTITY_TYPES; + +public class AtlasNamespaceDefStoreV2 extends AtlasAbstractDefStoreV2<AtlasNamespaceDef> { + private static final Logger LOG = LoggerFactory.getLogger(AtlasNamespaceDefStoreV2.class); + + @Inject + public AtlasNamespaceDefStoreV2(AtlasTypeDefGraphStoreV2 typeDefStore, AtlasTypeRegistry typeRegistry) { + super(typeDefStore, typeRegistry); + } + + @Override + public AtlasVertex preCreate(AtlasNamespaceDef namespaceDef) throws AtlasBaseException { + if (LOG.isDebugEnabled()) { + LOG.debug("==> AtlasNamespaceDefStoreV2.preCreate({})", namespaceDef); + } + + validateType(namespaceDef); + + AtlasType type = typeRegistry.getType(namespaceDef.getName()); + + if (type.getTypeCategory() != TypeCategory.NAMESPACE) { + throw new AtlasBaseException(AtlasErrorCode.TYPE_MATCH_FAILED, namespaceDef.getName(), + DataTypes.TypeCategory.NAMESPACE.name()); + } + + AtlasVertex ret = typeDefStore.findTypeVertexByName(namespaceDef.getName()); + + if (ret != null) { + throw new AtlasBaseException(AtlasErrorCode.TYPE_ALREADY_EXISTS, namespaceDef.getName()); + } + + ret = typeDefStore.createTypeVertex(namespaceDef); + + updateVertexPreCreate(namespaceDef, (AtlasNamespaceType) type, ret); + + if (LOG.isDebugEnabled()) { + LOG.debug("<== AtlasNamespaceDefStoreV2.preCreate({}): {}", namespaceDef, ret); + } + + return ret; + } + + @Override + public AtlasNamespaceDef create(AtlasNamespaceDef namespaceDef, AtlasVertex preCreateResult) throws AtlasBaseException { + if (LOG.isDebugEnabled()) { + LOG.debug("==> AtlasNamespaceDefStoreV2.create({}, {})", namespaceDef, preCreateResult); + } + + AtlasAuthorizationUtils.verifyAccess(new AtlasTypeAccessRequest(AtlasPrivilege.TYPE_CREATE, namespaceDef), "create namespace-def ", namespaceDef.getName()); + + AtlasVertex vertex = (preCreateResult == null) ? preCreate(namespaceDef) : preCreateResult; + + AtlasNamespaceDef ret = toNamespaceDef(vertex); + + if (LOG.isDebugEnabled()) { + LOG.debug("<== AtlasNamespaceDefStoreV2.create({}, {}): {}", namespaceDef, preCreateResult, ret); + } + + return ret; + } + + @Override + public List<AtlasNamespaceDef> getAll() throws AtlasBaseException { + if (LOG.isDebugEnabled()) { + LOG.debug("==> AtlasNamespaceDef.getAll()"); + } + + List<AtlasNamespaceDef> ret = new ArrayList<>(); + + Iterator<AtlasVertex> vertices = typeDefStore.findTypeVerticesByCategory(DataTypes.TypeCategory.NAMESPACE); + while (vertices.hasNext()) { + ret.add(toNamespaceDef(vertices.next())); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== AtlasNamespaceDefStoreV2.getAll(): count={}", ret.size()); + } + return ret; + } + + @Override + public AtlasNamespaceDef getByName(String name) throws AtlasBaseException { + if (LOG.isDebugEnabled()) { + LOG.debug("==> AtlasNamespaceDefStoreV2.getByName({})", name); + } + + AtlasVertex vertex = typeDefStore.findTypeVertexByNameAndCategory(name, DataTypes.TypeCategory.NAMESPACE); + + if (vertex == null) { + throw new AtlasBaseException(AtlasErrorCode.TYPE_NAME_NOT_FOUND, name); + } + + vertex.getProperty(Constants.TYPE_CATEGORY_PROPERTY_KEY, String.class); + + AtlasNamespaceDef ret = toNamespaceDef(vertex); + + if (LOG.isDebugEnabled()) { + LOG.debug("<== AtlasNamespaceDefStoreV2.getByName({}): {}", name, ret); + } + + return ret; + } + + @Override + public AtlasNamespaceDef getByGuid(String guid) throws AtlasBaseException { + if (LOG.isDebugEnabled()) { + LOG.debug("==> AtlasNamespaceDefStoreV2.getByGuid({})", guid); + } + + AtlasVertex vertex = typeDefStore.findTypeVertexByGuidAndCategory(guid, DataTypes.TypeCategory.NAMESPACE); + + if (vertex == null) { + throw new AtlasBaseException(AtlasErrorCode.TYPE_GUID_NOT_FOUND, guid); + } + + AtlasNamespaceDef ret = toNamespaceDef(vertex); + + if (LOG.isDebugEnabled()) { + LOG.debug("<== AtlasNamespaceDefStoreV2.getByGuid({}): {}", guid, ret); + } + + return ret; + } + + @Override + public AtlasNamespaceDef update(AtlasNamespaceDef typeDef) throws AtlasBaseException { + if (LOG.isDebugEnabled()) { + LOG.debug("==> AtlasNamespaceDefStoreV2.update({})", typeDef); + } + + validateType(typeDef); + + AtlasNamespaceDef ret = StringUtils.isNotBlank(typeDef.getGuid()) ? updateByGuid(typeDef.getGuid(), typeDef) + : updateByName(typeDef.getName(), typeDef); + + if (LOG.isDebugEnabled()) { + LOG.debug("<== AtlasNamespaceDefStoreV2.update({}): {}", typeDef, ret); + } + + return ret; + } + + @Override + public AtlasNamespaceDef updateByName(String name, AtlasNamespaceDef typeDef) throws AtlasBaseException { + if (LOG.isDebugEnabled()) { + LOG.debug("==> AtlasNamespaceDefStoreV2.updateByName({}, {})", name, typeDef); + } + + AtlasNamespaceDef existingDef = typeRegistry.getNamespaceDefByName(name); + + AtlasAuthorizationUtils.verifyAccess(new AtlasTypeAccessRequest(AtlasPrivilege.TYPE_UPDATE, existingDef), "update namespace-def ", name); + + validateType(typeDef); + + AtlasType type = typeRegistry.getType(typeDef.getName()); + + if (type.getTypeCategory() != TypeCategory.NAMESPACE) { + throw new AtlasBaseException(AtlasErrorCode.TYPE_MATCH_FAILED, typeDef.getName(), DataTypes.TypeCategory.NAMESPACE.name()); + } + + AtlasVertex vertex = typeDefStore.findTypeVertexByNameAndCategory(name, DataTypes.TypeCategory.NAMESPACE); + + if (vertex == null) { + throw new AtlasBaseException(AtlasErrorCode.TYPE_NAME_NOT_FOUND, name); + } + + + updateVertexPreUpdate(typeDef, (AtlasNamespaceType)type, vertex); + + AtlasNamespaceDef ret = toNamespaceDef(vertex); + + if (LOG.isDebugEnabled()) { + LOG.debug("<== AtlasNamespaceDefStoreV2.updateByName({}, {}): {}", name, typeDef, ret); + } + + return ret; + } + + public AtlasNamespaceDef updateByGuid(String guid, AtlasNamespaceDef typeDef) throws AtlasBaseException { + if (LOG.isDebugEnabled()) { + LOG.debug("==> AtlasNamespaceDefStoreV2.updateByGuid({})", guid); + } + + AtlasNamespaceDef existingDef = typeRegistry.getNamespaceDefByGuid(guid); + + AtlasAuthorizationUtils.verifyAccess(new AtlasTypeAccessRequest(AtlasPrivilege.TYPE_UPDATE, existingDef), "update namespace-def ", (existingDef != null ? existingDef.getName() : guid)); + + validateType(typeDef); + + AtlasType type = typeRegistry.getTypeByGuid(guid); + + if (type.getTypeCategory() != org.apache.atlas.model.TypeCategory.NAMESPACE) { + throw new AtlasBaseException(AtlasErrorCode.TYPE_MATCH_FAILED, typeDef.getName(), DataTypes.TypeCategory.NAMESPACE.name()); + } + + AtlasVertex vertex = typeDefStore.findTypeVertexByGuidAndCategory(guid, DataTypes.TypeCategory.NAMESPACE); + + if (vertex == null) { + throw new AtlasBaseException(AtlasErrorCode.TYPE_GUID_NOT_FOUND, guid); + } + + updateVertexPreUpdate(typeDef, (AtlasNamespaceType)type, vertex); + + AtlasNamespaceDef ret = toNamespaceDef(vertex); + + if (LOG.isDebugEnabled()) { + LOG.debug("<== AtlasNamespaceDefStoreV2.updateByGuid({}): {}", guid, ret); + } + + return ret; + } + + public AtlasVertex preDeleteByName(String name) throws AtlasBaseException { + if (LOG.isDebugEnabled()) { + LOG.debug("==> AtlasNamespaceDefStoreV2.preDeleteByName({})", name); + } + + AtlasNamespaceDef existingDef = typeRegistry.getNamespaceDefByName(name); + + AtlasAuthorizationUtils.verifyAccess(new AtlasTypeAccessRequest(AtlasPrivilege.TYPE_DELETE, existingDef), "delete namespace-def ", name); + + AtlasVertex ret = typeDefStore.findTypeVertexByNameAndCategory(name, DataTypes.TypeCategory.NAMESPACE); + + if (ret == null) { + throw new AtlasBaseException(AtlasErrorCode.TYPE_NAME_NOT_FOUND, name); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== AtlasNamespaceDefStoreV2.preDeleteByName({}): {}", name, ret); + } + + return ret; + } + + public AtlasVertex preDeleteByGuid(String guid) throws AtlasBaseException { + if (LOG.isDebugEnabled()) { + LOG.debug("==> AtlasNamespaceDefStoreV2.preDeleteByGuid({})", guid); + } + + AtlasNamespaceDef existingDef = typeRegistry.getNamespaceDefByGuid(guid); + + AtlasAuthorizationUtils.verifyAccess(new AtlasTypeAccessRequest(AtlasPrivilege.TYPE_DELETE, existingDef), "delete namespace-def ", (existingDef != null ? existingDef.getName() : guid)); + + AtlasVertex ret = typeDefStore.findTypeVertexByGuidAndCategory(guid, DataTypes.TypeCategory.NAMESPACE); + + if (ret == null) { + throw new AtlasBaseException(AtlasErrorCode.TYPE_GUID_NOT_FOUND, guid); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== AtlasNamespaceDefStoreV2.preDeleteByGuid({}): ret={}", guid, ret); + } + + return ret; + } + + private void updateVertexPreCreate(AtlasNamespaceDef namespaceDef, AtlasNamespaceType namespaceType, + AtlasVertex vertex) throws AtlasBaseException { + AtlasStructDefStoreV2.updateVertexPreCreate(namespaceDef, namespaceType, vertex, typeDefStore); + } + + private void updateVertexPreUpdate(AtlasNamespaceDef namespaceDef, AtlasNamespaceType namespaceType, + AtlasVertex vertex) throws AtlasBaseException { + // Load up current struct definition for matching attributes + AtlasNamespaceDef currentNamespaceDef = toNamespaceDef(vertex); + + // Check to verify that in an update call we only allow addition of new entity types, not deletion of existing + // entity types + if (CollectionUtils.isNotEmpty(namespaceDef.getAttributeDefs())) { + for (AtlasStructDef.AtlasAttributeDef attributeDef : namespaceDef.getAttributeDefs()) { + String updatedApplicableEntityTypesString = attributeDef.getOption(ATTR_OPTION_APPLICABLE_ENTITY_TYPES); + Set<String> updatedApplicableEntityTypes = StringUtils.isBlank(updatedApplicableEntityTypesString) ? null : AtlasType.fromJson(updatedApplicableEntityTypesString, Set.class); + + AtlasStructDef.AtlasAttributeDef existingAttribute = currentNamespaceDef.getAttribute(attributeDef.getName()); + if (existingAttribute != null) { + String existingApplicableEntityTypesString = existingAttribute.getOption(ATTR_OPTION_APPLICABLE_ENTITY_TYPES); + Set<String> existingApplicableEntityTypes = StringUtils.isBlank(existingApplicableEntityTypesString) ? null : AtlasType.fromJson(existingApplicableEntityTypesString, Set.class); + + if (existingApplicableEntityTypes != null) { + if (!updatedApplicableEntityTypes.containsAll(existingApplicableEntityTypes)) { + throw new AtlasBaseException(AtlasErrorCode.APPLICABLE_ENTITY_TYPES_DELETION_NOT_SUPPORTED, + attributeDef.getName(), namespaceDef.getName()); + } + } + } + } + } + + AtlasStructDefStoreV2.updateVertexPreUpdate(namespaceDef, namespaceType, vertex, typeDefStore); + } + + private AtlasNamespaceDef toNamespaceDef(AtlasVertex vertex) throws AtlasBaseException { + AtlasNamespaceDef ret = null; + + if (vertex != null && typeDefStore.isTypeVertex(vertex, DataTypes.TypeCategory.NAMESPACE)) { + ret = new AtlasNamespaceDef(); + + AtlasStructDefStoreV2.toStructDef(vertex, ret, typeDefStore); + } + + return ret; + } +} diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasTypeDefGraphStoreV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasTypeDefGraphStoreV2.java index a5ccfb5..afdfba9 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasTypeDefGraphStoreV2.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasTypeDefGraphStoreV2.java @@ -19,17 +19,6 @@ package org.apache.atlas.repository.store.graph.v2; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; - -import static org.apache.atlas.repository.Constants.TYPE_CATEGORY_PROPERTY_KEY; -import static org.apache.atlas.repository.Constants.VERTEX_TYPE_PROPERTY_KEY; -import static org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2.VERTEX_TYPE; - -import java.util.Date; -import java.util.HashSet; -import java.util.Iterator; -import java.util.Map; -import java.util.Set; -import java.util.UUID; import org.apache.atlas.AtlasErrorCode; import org.apache.atlas.RequestContext; import org.apache.atlas.annotation.GraphTransaction; @@ -37,8 +26,12 @@ import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.listener.TypeDefChangeListener; import org.apache.atlas.model.typedef.*; import org.apache.atlas.repository.Constants; -import org.apache.atlas.repository.graphdb.*; -import org.apache.atlas.repository.store.graph.*; +import org.apache.atlas.repository.graphdb.AtlasEdge; +import org.apache.atlas.repository.graphdb.AtlasEdgeDirection; +import org.apache.atlas.repository.graphdb.AtlasGraph; +import org.apache.atlas.repository.graphdb.AtlasVertex; +import org.apache.atlas.repository.store.graph.AtlasDefStore; +import org.apache.atlas.repository.store.graph.AtlasTypeDefGraphStore; import org.apache.atlas.type.AtlasType; import org.apache.atlas.type.AtlasTypeRegistry; import org.apache.atlas.typesystem.types.DataTypes.TypeCategory; @@ -50,6 +43,16 @@ import org.springframework.stereotype.Component; import javax.inject.Inject; import javax.inject.Singleton; +import java.util.Date; +import java.util.HashSet; +import java.util.Iterator; +import java.util.Map; +import java.util.Set; +import java.util.UUID; + +import static org.apache.atlas.repository.Constants.TYPE_CATEGORY_PROPERTY_KEY; +import static org.apache.atlas.repository.Constants.VERTEX_TYPE_PROPERTY_KEY; +import static org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2.VERTEX_TYPE; /** @@ -97,6 +100,10 @@ public class AtlasTypeDefGraphStoreV2 extends AtlasTypeDefGraphStore { return new AtlasRelationshipDefStoreV2(this, typeRegistry); } + @Override + protected AtlasDefStore<AtlasNamespaceDef> getNamespaceDefStore(AtlasTypeRegistry typeRegistry) { + return new AtlasNamespaceDefStoreV2(this, typeRegistry); + } @Override @GraphTransaction @@ -490,6 +497,9 @@ public class AtlasTypeDefGraphStoreV2 extends AtlasTypeDefGraphStore { case RELATIONSHIP: return TypeCategory.RELATIONSHIP; + + case NAMESPACE: + return TypeCategory.NAMESPACE; } return null; diff --git a/repository/src/test/java/org/apache/atlas/repository/store/graph/v2/AtlasNamespaceDefStoreV2Test.java b/repository/src/test/java/org/apache/atlas/repository/store/graph/v2/AtlasNamespaceDefStoreV2Test.java new file mode 100644 index 0000000..1d6534e --- /dev/null +++ b/repository/src/test/java/org/apache/atlas/repository/store/graph/v2/AtlasNamespaceDefStoreV2Test.java @@ -0,0 +1,319 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.repository.store.graph.v2; + +import com.google.inject.Inject; +import org.apache.atlas.AtlasErrorCode; +import org.apache.atlas.TestModules; +import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.model.typedef.AtlasNamespaceDef; +import org.apache.atlas.model.typedef.AtlasStructDef; +import org.apache.atlas.model.typedef.AtlasTypesDef; +import org.apache.atlas.type.AtlasEntityType; +import org.apache.atlas.type.AtlasNamespaceType; +import org.apache.atlas.type.AtlasType; +import org.apache.atlas.type.AtlasTypeRegistry; +import org.testng.Assert; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Guice; +import org.testng.annotations.Test; + +import java.io.IOException; +import java.util.*; + +import static org.apache.atlas.model.typedef.AtlasNamespaceDef.ATTR_OPTION_APPLICABLE_ENTITY_TYPES; +import static org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.*; + +/* Please note that for these tests, since the typeRegistry can be injected only once, + * any new tests should make sure that they flush the type registry at the end of the test. + * testNG does not provide a way to execute a method after each test has completed the run, hence + * we have to manually make sure that the flushTypeRegistry method is invoked. + */ +@Guice(modules = TestModules.TestOnlyModule.class) +public class AtlasNamespaceDefStoreV2Test { + + @Inject + AtlasTypeRegistry typeRegistry; + + @Inject + private AtlasTypeDefGraphStoreV2 typeDefStore; + + private AtlasTypesDef typesDefs; + + private static int randomCount; + private static final String TEST_NAMESPACE = "test_namespace"; + private String namespaceName; + @BeforeClass + public void setup() throws IOException, AtlasBaseException { + loadBaseModel(typeDefStore, typeRegistry); + loadFsModel(typeDefStore, typeRegistry); + loadHiveModel(typeDefStore, typeRegistry); + + typesDefs = new AtlasTypesDef(Collections.emptyList(), Collections.emptyList(), Collections.emptyList(), Collections.emptyList(), Collections.emptyList(), Collections.emptyList()); + + namespaceName = TEST_NAMESPACE; + + randomCount = 1; + } + + @BeforeMethod + public void setTypeDefs() { + typesDefs = new AtlasTypesDef(Collections.emptyList(), Collections.emptyList(), Collections.emptyList(), Collections.emptyList(), Collections.emptyList(), Collections.emptyList()); + randomCount++; + namespaceName = TEST_NAMESPACE + randomCount; + } + + @Test + public void createNamespaceTypeDef() throws AtlasBaseException { + createNamespaceTypes(namespaceName); + Assert.assertEquals(typeRegistry.getAllNamespaceDefs().size(), 1); + AtlasEntityType entityType = typeRegistry.getEntityTypeByName("hive_table"); + Map<String, List<AtlasNamespaceType.AtlasNamespaceAttribute>> m1 = entityType.getNamespaceAttributes(); + Assert.assertEquals(m1.get(namespaceName).size(), 2); + } + + @Test + public void deleteNamespaceTypeDefs() throws AtlasBaseException { + createNamespaceTypes(namespaceName); + for (AtlasNamespaceDef atlasNamespaceDef : typesDefs.getNamespaceDefs()) { + if (atlasNamespaceDef.getName().equals(namespaceName)) { + typesDefs = new AtlasTypesDef(Collections.emptyList(), Collections.emptyList(), Collections.emptyList(), Collections.emptyList(), Collections.emptyList(), + Collections.emptyList()); + typesDefs.setNamespaceDefs(Arrays.asList(atlasNamespaceDef)); + typeDefStore.deleteTypesDef(typesDefs); + } + } + + for (AtlasNamespaceDef namespaceDef : typeRegistry.getAllNamespaceDefs()) { + Assert.assertNotEquals(namespaceDef.getName(), namespaceName); + } + } + + @Test + public void updateNamespaceTypeDefs() throws AtlasBaseException { + createNamespaceTypes(namespaceName); + AtlasNamespaceDef namespaceDef = findNamespaceDef(namespaceName); + Assert.assertNotNull(namespaceDef); + + addNamespaceAttribute(namespaceDef, "test_namespace_attribute3", Collections.singleton("hive_table"), + String.format("array<%s>", "string"), AtlasStructDef.AtlasAttributeDef.Cardinality.LIST); + + updateNamespaceTypeDefs(namespaceDef); + typeDefStore.updateTypesDef(typesDefs); + AtlasEntityType entityType = typeRegistry.getEntityTypeByName("hive_table"); + Map<String, List<AtlasNamespaceType.AtlasNamespaceAttribute>> m1 = entityType.getNamespaceAttributes(); + Assert.assertEquals(m1.get(namespaceName).size(), 3); + } + + /** + * Test to verify that we cannot delete attribute defs from a namespace definition + * @throws AtlasBaseException + */ + @Test + public void updateTypeDefsWithoutApplicableEntityTypes() throws AtlasBaseException { + createNamespaceTypes(namespaceName); + AtlasNamespaceDef namespaceDef = findNamespaceDef(namespaceName); + Assert.assertNotNull(namespaceDef); + + AtlasStructDef.AtlasAttributeDef namespaceAttributeDef = namespaceDef.getAttributeDefs().iterator().next(); + namespaceDef.setAttributeDefs(Arrays.asList(namespaceAttributeDef)); + + AtlasTypesDef existingTypeDefs = typesDefs; + + try { + typesDefs.setNamespaceDefs(Arrays.asList(namespaceDef)); + typeDefStore.updateTypesDef(typesDefs); + } catch (AtlasBaseException e) { + Assert.assertEquals(e.getAtlasErrorCode(), AtlasErrorCode.ATTRIBUTE_DELETION_NOT_SUPPORTED); + } finally { + typesDefs = existingTypeDefs; + } + } + + @Test + public void updateTypeDefsDeleteApplicableEntityTypes() throws AtlasBaseException { + createNamespaceTypes(namespaceName); + AtlasNamespaceDef namespaceDef = findNamespaceDef(namespaceName); + Assert.assertNotNull(namespaceDef); + + Iterator<AtlasStructDef.AtlasAttributeDef> it = namespaceDef.getAttributeDefs().iterator(); + AtlasStructDef.AtlasAttributeDef namespaceAttributeDef = it.next(); + AtlasStructDef.AtlasAttributeDef namespaceAttributeDef2 = it.next(); + + namespaceAttributeDef.setOption(ATTR_OPTION_APPLICABLE_ENTITY_TYPES, AtlasType.toJson(Collections.emptySet())); + + namespaceDef.setAttributeDefs(Arrays.asList(namespaceAttributeDef, namespaceAttributeDef2)); + + AtlasTypesDef existingTypeDefs = typesDefs; + + try { + typesDefs.setNamespaceDefs(Arrays.asList(namespaceDef)); + typeDefStore.updateTypesDef(typesDefs); + } catch (AtlasBaseException e) { + Assert.assertEquals(e.getAtlasErrorCode(), AtlasErrorCode.MISSING_MANDATORY_ATTRIBUTE); + } finally { + typesDefs = existingTypeDefs; + } + } + + /** + * Test to verify that we cannot have an empty applicable entity types in an attribute definition + * @throws AtlasBaseException + */ + @Test + public void createNsAttrDefWithoutApplicableEntityTypes() { + AtlasTypesDef existingTypeDefs = typesDefs; + + try { + typesDefs.setNamespaceDefs(Arrays.asList(createNamespaceDef2(namespaceName))); + typeDefStore.updateTypesDef(typesDefs); + } catch (AtlasBaseException e) { + Assert.assertEquals(e.getAtlasErrorCode(), AtlasErrorCode.MISSING_MANDATORY_ATTRIBUTE); + } finally { + typesDefs = existingTypeDefs; + } + } + + @Test + public void updateNsAttrDefDeleteApplicableEntityTypes() throws AtlasBaseException { + createNamespaceTypes(namespaceName); + + AtlasNamespaceDef namespaceDef = findNamespaceDef(namespaceName); + Assert.assertNotNull(namespaceDef); + + Iterator<AtlasStructDef.AtlasAttributeDef> it = namespaceDef.getAttributeDefs().iterator(); + AtlasStructDef.AtlasAttributeDef namespaceAttributeDef = it.next(); + AtlasStructDef.AtlasAttributeDef namespaceAttributeDef2 = it.next(); + + namespaceAttributeDef.setOption(ATTR_OPTION_APPLICABLE_ENTITY_TYPES, AtlasType.toJson(Collections.singleton("hive_table"))); + + namespaceDef.setAttributeDefs(Arrays.asList(namespaceAttributeDef, namespaceAttributeDef2)); + + AtlasTypesDef existingTypeDefs = typesDefs; + + try { + typesDefs.setNamespaceDefs(Arrays.asList(namespaceDef)); + typeDefStore.updateTypesDef(typesDefs); + } catch (AtlasBaseException e) { + Assert.assertEquals(e.getAtlasErrorCode(), AtlasErrorCode.APPLICABLE_ENTITY_TYPES_DELETION_NOT_SUPPORTED); + } finally { + typesDefs = existingTypeDefs; + } + } + + @Test + public void updateNsAttrDefAddApplicableEntityTypes() throws AtlasBaseException { + createNamespaceTypes(namespaceName); + + AtlasNamespaceDef namespaceDef = findNamespaceDef(namespaceName); + AtlasStructDef.AtlasAttributeDef namespaceAttributeDef1 = namespaceDef.getAttributeDefs().get(0); + AtlasStructDef.AtlasAttributeDef namespaceAttributeDef2 = namespaceDef.getAttributeDefs().get(1); + Set<String> applicableEntityTypes = AtlasType.fromJson(namespaceAttributeDef1.getOption(ATTR_OPTION_APPLICABLE_ENTITY_TYPES), Set.class); + + if (applicableEntityTypes == null) { + applicableEntityTypes = new HashSet<>(); + } + + applicableEntityTypes.add("hive_column"); + namespaceAttributeDef1.setOption(ATTR_OPTION_APPLICABLE_ENTITY_TYPES, AtlasType.toJson(applicableEntityTypes)); + namespaceDef.setAttributeDefs(Arrays.asList(namespaceAttributeDef1, namespaceAttributeDef2)); + + updateNamespaceTypeDefs(namespaceDef); + + typeDefStore.updateTypesDef(typesDefs); + + namespaceDef = findNamespaceDef(namespaceName); + namespaceAttributeDef1 = namespaceDef.getAttributeDefs().get(0); + + applicableEntityTypes = AtlasType.fromJson(namespaceAttributeDef1.getOption(ATTR_OPTION_APPLICABLE_ENTITY_TYPES), Set.class); + + Assert.assertEquals(applicableEntityTypes == null ? 0 : applicableEntityTypes.size(), 3); + } + + @Test + public void validateMaxStringLengthForStringTypes() throws AtlasBaseException { + AtlasTypesDef existingTypeDefs = typesDefs; + AtlasNamespaceDef namespaceDef1 = new AtlasNamespaceDef(namespaceName, "test_description", null); + addNamespaceAttribute(namespaceDef1, "test_namespace_attribute1", new HashSet<>(Arrays.asList("hive_table", "fs_path")), "string", + AtlasStructDef.AtlasAttributeDef.Cardinality.SINGLE); + typesDefs.setNamespaceDefs(Arrays.asList(namespaceDef1)); + try { + typeDefStore.createTypesDef(typesDefs); + } catch (AtlasBaseException exception) { + Assert.assertEquals(exception.getAtlasErrorCode(), AtlasErrorCode.MISSING_MANDATORY_ATTRIBUTE); + } finally { + typesDefs = existingTypeDefs; + } + } + + private AtlasNamespaceDef createNamespaceDef(String namespaceName) { + AtlasNamespaceDef namespaceDef1 = new AtlasNamespaceDef(namespaceName, "test_description", null); + addNamespaceAttribute(namespaceDef1, "test_namespace_attribute1", new HashSet<>(Arrays.asList("hive_table", "fs_path")), "int", + AtlasStructDef.AtlasAttributeDef.Cardinality.SINGLE); + addNamespaceAttribute(namespaceDef1, "test_namespace_attribute2", Collections.singleton("hive_table"), "int", + AtlasStructDef.AtlasAttributeDef.Cardinality.SINGLE); + return namespaceDef1; + } + + private AtlasNamespaceDef createNamespaceDef2(String namespaceName) { + AtlasNamespaceDef namespaceDef1 = new AtlasNamespaceDef(namespaceName, "test_description", null); + addNamespaceAttribute(namespaceDef1, "test_namespace_attribute1", Collections.emptySet(), "int", + AtlasStructDef.AtlasAttributeDef.Cardinality.SINGLE); + addNamespaceAttribute(namespaceDef1, "test_namespace_attribute2", Collections.singleton("hive_table"), "int", + AtlasStructDef.AtlasAttributeDef.Cardinality.SINGLE); + return namespaceDef1; + } + + private void createNamespaceTypes(String namespaceName) throws AtlasBaseException { + List<AtlasNamespaceDef> namespaceDefs = new ArrayList(typesDefs.getNamespaceDefs()); + namespaceDefs.add(createNamespaceDef(namespaceName)); + typesDefs.setNamespaceDefs(namespaceDefs); + typeDefStore.createTypesDef(typesDefs); + } + + private void addNamespaceAttribute(AtlasNamespaceDef namespaceDef, String name, Set<String> applicableEntityTypes, + String typeName, AtlasStructDef.AtlasAttributeDef.Cardinality cardinality) { + AtlasStructDef.AtlasAttributeDef attributeDef = new AtlasStructDef.AtlasAttributeDef(name, typeName); + + attributeDef.setCardinality(cardinality); + attributeDef.setOption(ATTR_OPTION_APPLICABLE_ENTITY_TYPES, AtlasType.toJson(applicableEntityTypes)); + attributeDef.setIsOptional(true); + attributeDef.setIsUnique(false); + + namespaceDef.addAttribute(attributeDef); + } + + private AtlasNamespaceDef findNamespaceDef(String namespaceName) { + for (AtlasNamespaceDef atlasNamespaceDef : typesDefs.getNamespaceDefs()) { + if (atlasNamespaceDef.getName().equals(namespaceName)) { + return atlasNamespaceDef; + } + } + + return null; + } + + private void updateNamespaceTypeDefs(AtlasNamespaceDef atlasNamespaceDef) { + for (int i = 0; i < typesDefs.getNamespaceDefs().size(); i++) { + if (typesDefs.getNamespaceDefs().get(i).getName().equals(namespaceName)) { + typesDefs.getNamespaceDefs().set(i, atlasNamespaceDef); + } + } + } +} \ No newline at end of file diff --git a/webapp/src/main/java/org/apache/atlas/examples/QuickStartV2.java b/webapp/src/main/java/org/apache/atlas/examples/QuickStartV2.java index 6cd0ee3..72f5bef 100755 --- a/webapp/src/main/java/org/apache/atlas/examples/QuickStartV2.java +++ b/webapp/src/main/java/org/apache/atlas/examples/QuickStartV2.java @@ -19,7 +19,6 @@ package org.apache.atlas.examples; import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.ImmutableMap; import com.sun.jersey.core.util.MultivaluedMapImpl; import org.apache.atlas.ApplicationProperties; import org.apache.atlas.AtlasClientV2; @@ -40,10 +39,12 @@ import org.apache.atlas.model.lineage.AtlasLineageInfo.LineageDirection; import org.apache.atlas.model.lineage.AtlasLineageInfo.LineageRelation; import org.apache.atlas.model.typedef.AtlasClassificationDef; import org.apache.atlas.model.typedef.AtlasEntityDef; +import org.apache.atlas.model.typedef.AtlasNamespaceDef; import org.apache.atlas.model.typedef.AtlasRelationshipDef; -import org.apache.atlas.model.typedef.AtlasRelationshipDef.PropagateTags; +import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef; import org.apache.atlas.model.typedef.AtlasTypesDef; -import org.apache.atlas.repository.Constants; +import org.apache.atlas.model.typedef.AtlasRelationshipDef.PropagateTags; +import org.apache.atlas.type.AtlasType; import org.apache.atlas.type.AtlasTypeUtil; import org.apache.atlas.utils.AuthenticationUtil; import org.apache.commons.collections.CollectionUtils; @@ -55,6 +56,7 @@ import java.util.*; import static java.util.Arrays.asList; import static org.apache.atlas.AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME; +import static org.apache.atlas.model.typedef.AtlasNamespaceDef.ATTR_OPTION_APPLICABLE_ENTITY_TYPES; import static org.apache.atlas.model.typedef.AtlasRelationshipDef.RelationshipCategory.AGGREGATION; import static org.apache.atlas.model.typedef.AtlasRelationshipDef.RelationshipCategory.COMPOSITION; import static org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef.Cardinality.SET; @@ -316,7 +318,25 @@ public class QuickStartV2 { List<AtlasRelationshipDef> relationshipDefs = asList(tableDatabaseTypeDef, viewDatabaseTypeDef, viewTablesTypeDef, tableColumnsTypeDef, tableStorageDescTypeDef, processProcessExecutionTypeDef); List<AtlasClassificationDef> classificationDefs = asList(dimClassifDef, factClassifDef, piiClassifDef, metricClassifDef, etlClassifDef, jdbcClassifDef, logClassifDef); - return new AtlasTypesDef(Collections.emptyList(), Collections.emptyList(), classificationDefs, entityDefs, relationshipDefs); + // Namespace definitions + AtlasAttributeDef nsAttrDef1 = new AtlasAttributeDef("attr1", "int"); + AtlasAttributeDef nsAttrDef2 = new AtlasAttributeDef("attr2", "int"); + + nsAttrDef1.setOption(ATTR_OPTION_APPLICABLE_ENTITY_TYPES, AtlasType.toJson(Collections.singleton(TABLE_TYPE))); + nsAttrDef1.setIsOptional(true); + nsAttrDef1.setIsUnique(false); + + nsAttrDef2.setOption(ATTR_OPTION_APPLICABLE_ENTITY_TYPES, AtlasType.toJson(Collections.singleton(TABLE_TYPE))); + nsAttrDef2.setIsOptional(true); + nsAttrDef2.setIsUnique(false); + + AtlasNamespaceDef testNamespaceDef = new AtlasNamespaceDef("test_namespace", "test_description", VERSION_1); + + testNamespaceDef.setAttributeDefs(Arrays.asList(nsAttrDef1, nsAttrDef2)); + + List<AtlasNamespaceDef> namespaceDefs = asList(testNamespaceDef); + + return new AtlasTypesDef(Collections.emptyList(), Collections.emptyList(), classificationDefs, entityDefs, relationshipDefs, namespaceDefs); } void createEntities() throws Exception { diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/TypesREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/TypesREST.java index fb56fad..e7cf62d 100644 --- a/webapp/src/main/java/org/apache/atlas/web/rest/TypesREST.java +++ b/webapp/src/main/java/org/apache/atlas/web/rest/TypesREST.java @@ -23,6 +23,7 @@ import org.apache.atlas.model.typedef.AtlasBaseTypeDef; import org.apache.atlas.model.typedef.AtlasClassificationDef; import org.apache.atlas.model.typedef.AtlasEntityDef; import org.apache.atlas.model.typedef.AtlasEnumDef; +import org.apache.atlas.model.typedef.AtlasNamespaceDef; import org.apache.atlas.model.typedef.AtlasRelationshipDef; import org.apache.atlas.model.typedef.AtlasStructDef; import org.apache.atlas.model.typedef.AtlasTypeDefHeader; @@ -317,6 +318,43 @@ public class TypesREST { return ret; } + + /** + * Get the namespace definition for the given guid + * @param guid namespace guid + * @return namespace definition + * @throws AtlasBaseException + * @HTTP 200 On successful lookup of the the namespace definition by it's guid + * @HTTP 404 On Failed lookup for the given guid + */ + @GET + @Path("/namespacedef/guid/{guid}") + public AtlasNamespaceDef getNamespaceDefByGuid(@PathParam("guid") String guid) throws AtlasBaseException { + Servlets.validateQueryParamLength("guid", guid); + + AtlasNamespaceDef ret = typeDefStore.getNamespaceDefByGuid(guid); + + return ret; + } + + /** + * Get the namespace definition by it's name (unique) + * @param name namespace name + * @return namespace definition + * @throws AtlasBaseException + * @HTTP 200 On successful lookup of the the namespace definition by it's name + * @HTTP 404 On Failed lookup for the given name + */ + @GET + @Path("/namespacedef/name/{name}") + public AtlasNamespaceDef getNamespaceDefByName(@PathParam("name") String name) throws AtlasBaseException { + Servlets.validateQueryParamLength("name", name); + + AtlasNamespaceDef ret = typeDefStore.getNamespaceDefByName(name); + + return ret; + } + /* Bulk API operation */ /** -- libgit2 0.27.1