Skip to content

A
atlas
Commit eb6e656b by Madhan Neethiraj
Options

ATLAS-1230: updated AtlasTypeRegistry to support batch, atomic type updates

Signed-off-by: 's avatarSuma Shivaprasad <sumasai.shivaprasad@gmail.com>
parent 4d9cf456
Showing with 871 additions and 669 deletions
intg/src/main/java/org/apache/atlas/type/AtlasArrayType.java
......@@ -103,16 +103,6 @@ public class AtlasArrayType extends AtlasType {
@Override
public void resolveReferences(AtlasTypeRegistry typeRegistry) throws AtlasBaseException {
elementType = typeRegistry.getType(elementTypeName);
if (elementType == null) {
String msg = elementTypeName + ": unknown element-type for array";
LOG.error(msg);
throw new AtlasBaseException(msg);
}
elementType.resolveReferences(typeRegistry);
}
@Override
......
intg/src/main/java/org/apache/atlas/type/AtlasClassificationType.java
......@@ -70,7 +70,7 @@ public class AtlasClassificationType extends AtlasStructType {
for (String superTypeName : classificationDef.getSuperTypes()) {
AtlasType superType = typeRegistry.getType(superTypeName);
if (superType != null && superType instanceof AtlasClassificationType) {
if (superType instanceof AtlasClassificationType) {
AtlasClassificationType superClassificationType = (AtlasClassificationType)superType;
superClassificationType.resolveReferences(typeRegistry);
......@@ -82,13 +82,8 @@ public class AtlasClassificationType extends AtlasStructType {
allS.addAll(superClassificationType.getAllSuperTypes());
}
} else {
String msg = superTypeName + ((superType == null) ? ": unknown" : ": incompatible");
msg += (" supertype in classification " + classificationDef.getName());
LOG.error(msg);
throw new AtlasBaseException(msg);
throw new AtlasBaseException(superTypeName + ": incompatible supertype in classification "
+ classificationDef.getName());
}
}
......
intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java
......@@ -68,7 +68,7 @@ public class AtlasEntityType extends AtlasStructType {
for (String superTypeName : entityDef.getSuperTypes()) {
AtlasType superType = typeRegistry.getType(superTypeName);
if (superType != null && superType instanceof AtlasEntityType) {
if (superType instanceof AtlasEntityType) {
AtlasEntityType superEntityType = (AtlasEntityType)superType;
superEntityType.resolveReferences(typeRegistry);
......@@ -80,13 +80,8 @@ public class AtlasEntityType extends AtlasStructType {
allS.addAll(superEntityType.getAllSuperTypes());
}
} else {
String msg = superTypeName + ((superType == null) ? ": unknown" : ": incompatible");
msg += (" supertype in entity " + entityDef.getName());
LOG.error(msg);
throw new AtlasBaseException(msg);
throw new AtlasBaseException(superTypeName + ": incompatible supertype in entity "
+ entityDef.getName());
}
}
......
intg/src/main/java/org/apache/atlas/type/AtlasMapType.java
......@@ -88,25 +88,6 @@ public class AtlasMapType extends AtlasType {
public void resolveReferences(AtlasTypeRegistry typeRegistry) throws AtlasBaseException {
this.keyType = typeRegistry.getType(keyTypeName);
this.valueType = typeRegistry.getType(valueTypeName);
if (keyType == null) {
String msg = keyTypeName + ": unknown key-type for map";
LOG.error(msg);
throw new AtlasBaseException(msg);
}
if (valueType == null) {
String msg = valueTypeName + ": unknown value-type for map";
LOG.error(msg);
throw new AtlasBaseException(msg);
}
keyType.resolveReferences(typeRegistry);
valueType.resolveReferences(typeRegistry);
}
@Override
......
intg/src/main/java/org/apache/atlas/type/AtlasStructType.java
......@@ -103,11 +103,6 @@ public class AtlasStructType extends AtlasType {
for (AtlasAttributeDef attributeDef : structDef.getAttributeDefs()) {
AtlasType attrType = typeRegistry.getType(attributeDef.getTypeName());
if (attrType == null) {
throw new AtlasBaseException(attributeDef.getTypeName() + ": unknown type for attribute "
+ structDef.getName() + "." + attributeDef.getName());
}
resolveConstraints(attributeDef, attrType);
Cardinality cardinality = attributeDef.getCardinality();
......@@ -407,8 +402,8 @@ public class AtlasStructType extends AtlasType {
if (StringUtils.isBlank(refAttribName)) {
throw new AtlasBaseException(getTypeName() + "." + attribDef.getName() + ": "
+ CONSTRAINT_TYPE_MAPPED_FROM_REF + " invalid constraint. missing parameter "
+ CONSTRAINT_PARAM_REF_ATTRIBUTE);
+ " invalid constraint. missing parameter " + CONSTRAINT_PARAM_REF_ATTRIBUTE
+ " in " + CONSTRAINT_TYPE_MAPPED_FROM_REF + ". params=" + constraintDef.getParams());
}
AtlasStructType structType = (AtlasStructType)attribType;
......@@ -421,8 +416,8 @@ public class AtlasStructType extends AtlasType {
}
if (!StringUtils.equals(getTypeName(), refAttrib.getTypeName())) {
throw new AtlasBaseException(getTypeName() + "." + attribDef.getName() + ": invalid constraint. Datatype of"
+ CONSTRAINT_PARAM_REF_ATTRIBUTE + " " + structType.getTypeName() + "." + refAttribName
throw new AtlasBaseException(getTypeName() + "." + attribDef.getName() + ": invalid constraint. Datatype"
+ " of " + CONSTRAINT_PARAM_REF_ATTRIBUTE + " " + structType.getTypeName() + "." + refAttribName
+ " should be " + getTypeName() + ", but found " + refAttrib.getTypeName());
}
......
intg/src/main/java/org/apache/atlas/type/AtlasTypeRegistry.java
......@@ -17,12 +17,15 @@
*/
package org.apache.atlas.type;
import com.sun.jersey.spi.resource.Singleton;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.typedef.AtlasBaseTypeDef;
import org.apache.atlas.model.typedef.AtlasClassificationDef;
import org.apache.atlas.model.typedef.AtlasEnumDef;
import org.apache.atlas.model.typedef.AtlasEntityDef;
import org.apache.atlas.model.typedef.AtlasEnumDef;
import org.apache.atlas.model.typedef.AtlasStructDef;
import org.apache.atlas.model.typedef.AtlasTypesDef;
import static org.apache.atlas.model.typedef.AtlasBaseTypeDef.ATLAS_TYPE_ARRAY_PREFIX;
import static org.apache.atlas.model.typedef.AtlasBaseTypeDef.ATLAS_TYPE_ARRAY_SUFFIX;
import static org.apache.atlas.model.typedef.AtlasBaseTypeDef.ATLAS_TYPE_MAP_PREFIX;
......@@ -37,52 +40,36 @@ import org.slf4j.LoggerFactory;
import java.util.Collection;
import java.util.Collections;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
/**
* registry for all types defined in Atlas.
*/
@Singleton
public class AtlasTypeRegistry {
private static final Logger LOG = LoggerFactory.getLogger(AtlasStructType.class);
private final Map<String, AtlasType> allTypes;
private final TypeDefCache<AtlasEnumDef> enumDefs;
private final TypeDefCache<AtlasStructDef> structDefs;
private final TypeDefCache<AtlasClassificationDef> classificationDefs;
private final TypeDefCache<AtlasEntityDef> entityDefs;
protected RegistryData registryData;
public AtlasTypeRegistry() {
allTypes = new ConcurrentHashMap<>();
enumDefs = new TypeDefCache<>(this);
structDefs = new TypeDefCache<>(this);
classificationDefs = new TypeDefCache<>(this);
entityDefs = new TypeDefCache<>(this);
registerType(new AtlasBuiltInTypes.AtlasBooleanType());
registerType(new AtlasBuiltInTypes.AtlasByteType());
registerType(new AtlasBuiltInTypes.AtlasShortType());
registerType(new AtlasBuiltInTypes.AtlasIntType());
registerType(new AtlasBuiltInTypes.AtlasLongType());
registerType(new AtlasBuiltInTypes.AtlasFloatType());
registerType(new AtlasBuiltInTypes.AtlasDoubleType());
registerType(new AtlasBuiltInTypes.AtlasBigIntegerType());
registerType(new AtlasBuiltInTypes.AtlasBigDecimalType());
registerType(new AtlasBuiltInTypes.AtlasDateType());
registerType(new AtlasBuiltInTypes.AtlasStringType());
registerType(new AtlasBuiltInTypes.AtlasObjectIdType());
}
public Collection<String> getAllTypeNames() { return Collections.unmodifiableSet(allTypes.keySet()); }
public AtlasType getType(String typeName) {
registryData = new RegistryData();
}
protected AtlasTypeRegistry(AtlasTypeRegistry other) {
registryData = new RegistryData(other.registryData);
}
public Collection<String> getAllTypeNames() { return registryData.allTypes.getAllTypeNames(); }
public AtlasType getType(String typeName) throws AtlasBaseException {
if (LOG.isDebugEnabled()) {
LOG.debug("==> AtlasTypeRegistry.getType({})", typeName);
}
AtlasType ret = allTypes.get(typeName);
AtlasType ret = registryData.allTypes.getTypeByName(typeName);
if (ret == null) {
try {
if (typeName.startsWith(ATLAS_TYPE_ARRAY_PREFIX) && typeName.endsWith(ATLAS_TYPE_ARRAY_SUFFIX)) {
int startIdx = ATLAS_TYPE_ARRAY_PREFIX.length();
int endIdx = typeName.length() - ATLAS_TYPE_ARRAY_SUFFIX.length();
......@@ -97,109 +84,193 @@ public class AtlasTypeRegistry {
String valueTypeName = keyValueTypes.length > 1 ? keyValueTypes[1] : null;
ret = new AtlasMapType(keyTypeName, valueTypeName, this);
}
} catch(AtlasBaseException excp) {
LOG.warn("failed to instantiate type for " + typeName, excp);
} else {
throw new AtlasBaseException(typeName + ": unknown typename");
}
}
if (LOG.isDebugEnabled()) {
LOG.debug("<== AtlasTypeRegistry.getType({})", typeName);
LOG.debug("<== AtlasTypeRegistry.getType({}): {}", typeName, ret);
}
return ret;
}
public AtlasType getTypeByGuid(String guid) {
if (LOG.isDebugEnabled()) {
LOG.debug("==> AtlasTypeRegistry.getTypeByGuid({})", guid);
}
AtlasType ret = registryData.allTypes.getTypeByGuid(guid);
public void addType(AtlasBaseTypeDef typeDef) throws AtlasBaseException {
if (LOG.isDebugEnabled()) {
LOG.debug("==> AtlasTypeRegistry.addType({})", typeDef);
LOG.debug("<== AtlasTypeRegistry.getTypeByGuid({}): {}", guid, ret);
}
if (typeDef == null) {
// ignore
} else if (typeDef.getClass().equals(AtlasEnumDef.class)) {
AtlasEnumDef enumDef = (AtlasEnumDef)typeDef;
return ret;
}
enumDefs.addType(enumDef, new AtlasEnumType(enumDef));
} else if (typeDef.getClass().equals(AtlasStructDef.class)) {
AtlasStructDef structDef = (AtlasStructDef)typeDef;
structDefs.addType(structDef, new AtlasStructType(structDef, this));
} else if (typeDef.getClass().equals(AtlasClassificationDef.class)) {
AtlasClassificationDef classificationDef = (AtlasClassificationDef)typeDef;
public Collection<AtlasEnumDef> getAllEnumDefs() { return registryData.enumDefs.getAll(); }
classificationDefs.addType(classificationDef, new AtlasClassificationType(classificationDef, this));
} else if (typeDef.getClass().equals(AtlasEntityDef.class)) {
AtlasEntityDef entityDef = (AtlasEntityDef)typeDef;
public AtlasEnumDef getEnumDefByGuid(String guid) {
return registryData.enumDefs.getTypeDefByGuid(guid);
}
entityDefs.addType(entityDef, new AtlasEntityType(entityDef, this));
public AtlasEnumDef getEnumDefByName(String name) {
return registryData.enumDefs.getTypeDefByName(name);
}
if (LOG.isDebugEnabled()) {
LOG.debug("<== AtlasTypeRegistry.addType({})", typeDef);
public Collection<AtlasStructDef> getAllStructDefs() { return registryData.structDefs.getAll(); }
public AtlasStructDef getStructDefByGuid(String guid) {
return registryData.structDefs.getTypeDefByGuid(guid);
}
public AtlasStructDef getStructDefByName(String name) { return registryData.structDefs.getTypeDefByName(name); }
public Collection<AtlasClassificationDef> getAllClassificationDefs() {
return registryData.classificationDefs.getAll();
}
public void addTypeWithNoRefResolve(AtlasBaseTypeDef typeDef) {
public AtlasClassificationDef getClassificationDefByGuid(String guid) {
return registryData.classificationDefs.getTypeDefByGuid(guid);
}
public AtlasClassificationDef getClassificationDefByName(String name) {
return registryData.classificationDefs.getTypeDefByName(name);
}
public Collection<AtlasEntityDef> getAllEntityDefs() { return registryData.entityDefs.getAll(); }
public AtlasEntityDef getEntityDefByGuid(String guid) {
return registryData.entityDefs.getTypeDefByGuid(guid);
}
public AtlasEntityDef getEntityDefByName(String name) {
return registryData.entityDefs.getTypeDefByName(name);
}
public AtlasTransientTypeRegistry createTransientTypeRegistry() {
return new AtlasTransientTypeRegistry(this);
}
public void commitTransientTypeRegistry(AtlasTransientTypeRegistry transientTypeRegistry) {
this.registryData = transientTypeRegistry.registryData;
}
static class RegistryData {
final TypeCache allTypes;
final TypeDefCache<AtlasEnumDef> enumDefs;
final TypeDefCache<AtlasStructDef> structDefs;
final TypeDefCache<AtlasClassificationDef> classificationDefs;
final TypeDefCache<AtlasEntityDef> entityDefs;
RegistryData() {
allTypes = new TypeCache();
enumDefs = new TypeDefCache<>(allTypes);
structDefs = new TypeDefCache<>(allTypes);
classificationDefs = new TypeDefCache<>(allTypes);
entityDefs = new TypeDefCache<>(allTypes);
allTypes.addType(new AtlasBuiltInTypes.AtlasBooleanType());
allTypes.addType(new AtlasBuiltInTypes.AtlasByteType());
allTypes.addType(new AtlasBuiltInTypes.AtlasShortType());
allTypes.addType(new AtlasBuiltInTypes.AtlasIntType());
allTypes.addType(new AtlasBuiltInTypes.AtlasLongType());
allTypes.addType(new AtlasBuiltInTypes.AtlasFloatType());
allTypes.addType(new AtlasBuiltInTypes.AtlasDoubleType());
allTypes.addType(new AtlasBuiltInTypes.AtlasBigIntegerType());
allTypes.addType(new AtlasBuiltInTypes.AtlasBigDecimalType());
allTypes.addType(new AtlasBuiltInTypes.AtlasDateType());
allTypes.addType(new AtlasBuiltInTypes.AtlasStringType());
allTypes.addType(new AtlasBuiltInTypes.AtlasObjectIdType());
}
RegistryData(RegistryData other) {
allTypes = new TypeCache(other.allTypes);
enumDefs = new TypeDefCache<>(other.enumDefs, allTypes);
structDefs = new TypeDefCache<>(other.structDefs, allTypes);
classificationDefs = new TypeDefCache<>(other.classificationDefs, allTypes);
entityDefs = new TypeDefCache<>(other.entityDefs, allTypes);
}
}
public static class AtlasTransientTypeRegistry extends AtlasTypeRegistry {
private AtlasTransientTypeRegistry(AtlasTypeRegistry parent) {
super(parent);
}
private void resolveReferences() throws AtlasBaseException {
for (AtlasType type : registryData.allTypes.getAllTypes()) {
type.resolveReferences(this);
}
}
public void addType(AtlasBaseTypeDef typeDef) throws AtlasBaseException {
if (LOG.isDebugEnabled()) {
LOG.debug("==> AtlasTypeRegistry.addTypeWithNoRefResolve({})", typeDef);
LOG.debug("==> AtlasTypeRegistry.addType({})", typeDef);
}
if (typeDef == null) {
// ignore
} else if (typeDef.getClass().equals(AtlasEnumDef.class)) {
AtlasEnumDef enumDef = (AtlasEnumDef)typeDef;
if (typeDef != null) {
addTypeWithNoRefResolve(typeDef);
enumDefs.addType(enumDef, new AtlasEnumType(enumDef));
} else if (typeDef.getClass().equals(AtlasStructDef.class)) {
AtlasStructDef structDef = (AtlasStructDef)typeDef;
resolveReferences();
}
structDefs.addType(structDef, new AtlasStructType(structDef));
} else if (typeDef.getClass().equals(AtlasClassificationDef.class)) {
AtlasClassificationDef classificationDef = (AtlasClassificationDef)typeDef;
if (LOG.isDebugEnabled()) {
LOG.debug("<== AtlasTypeRegistry.addType({})", typeDef);
}
}
classificationDefs.addType(classificationDef, new AtlasClassificationType(classificationDef));
} else if (typeDef.getClass().equals(AtlasEntityDef.class)) {
AtlasEntityDef entityDef = (AtlasEntityDef)typeDef;
public void addTypes(Collection<? extends AtlasBaseTypeDef> typeDefs) throws AtlasBaseException {
if (LOG.isDebugEnabled()) {
LOG.debug("==> AtlasTypeRegistry.addTypes(length={})", (typeDefs == null ? 0 : typeDefs.size()));
}
if (CollectionUtils.isNotEmpty(typeDefs)) {
addTypesWithNoRefResolve(typeDefs);
entityDefs.addType(entityDef, new AtlasEntityType(entityDef));
resolveReferences();
}
if (LOG.isDebugEnabled()) {
LOG.debug("<== AtlasTypeRegistry.addTypeWithNoRefResolve({})", typeDef);
LOG.debug("<== AtlasTypeRegistry.addTypes(length={})", (typeDefs == null ? 0 : typeDefs.size()));
}
}
public void updateType(AtlasBaseTypeDef typeDef) throws AtlasBaseException {
public void addTypes(AtlasTypesDef typesDef) throws AtlasBaseException {
if (LOG.isDebugEnabled()) {
LOG.debug("==> AtlasTypeRegistry.updateType({})", typeDef);
LOG.debug("==> AtlasTypeRegistry.addTypes({})", typesDef);
}
if (typeDef == null) {
// ignore
} else if (StringUtils.isNotBlank(typeDef.getGuid())) {
updateTypeByGuid(typeDef.getGuid(), typeDef);
} else if (StringUtils.isNotBlank(typeDef.getName())) {
updateTypeByName(typeDef.getName(), typeDef);
if (typesDef != null) {
addTypesWithNoRefResolve(typesDef.getEnumDefs());
addTypesWithNoRefResolve(typesDef.getStructDefs());
addTypesWithNoRefResolve(typesDef.getClassificationDefs());
addTypesWithNoRefResolve(typesDef.getEntityDefs());
resolveReferences();
}
if (LOG.isDebugEnabled()) {
LOG.debug("<== AtlasTypeRegistry.updateType({})", typeDef);
LOG.debug("<== AtlasTypeRegistry.addTypes({})", typesDef);
}
}
public void updateTypeWithNoRefResolve(AtlasBaseTypeDef typeDef) {
public void updateType(AtlasBaseTypeDef typeDef) throws AtlasBaseException {
if (LOG.isDebugEnabled()) {
LOG.debug("==> AtlasTypeRegistry.updateType({})", typeDef);
}
if (typeDef == null) {
// ignore
} else if (StringUtils.isNotBlank(typeDef.getGuid())) {
updateTypeByGuidWithNoRefResolve(typeDef.getGuid(), typeDef);
} else if (StringUtils.isNotBlank(typeDef.getName())) {
updateTypeByNameWithNoRefResolve(typeDef.getName(), typeDef);
if (typeDef != null) {
updateTypeWithNoRefResolve(typeDef);
resolveReferences();
}
if (LOG.isDebugEnabled()) {
......@@ -212,28 +283,10 @@ public class AtlasTypeRegistry {
LOG.debug("==> AtlasTypeRegistry.updateTypeByGuid({})", guid);
}
if (guid == null || typeDef == null) {
// ignore
} else if (typeDef.getClass().equals(AtlasEnumDef.class)) {
AtlasEnumDef enumDef = (AtlasEnumDef)typeDef;
enumDefs.removeTypeDefByGuid(guid);
enumDefs.addType(enumDef, new AtlasEnumType(enumDef));
} else if (typeDef.getClass().equals(AtlasStructDef.class)) {
AtlasStructDef structDef = (AtlasStructDef)typeDef;
if (guid != null && typeDef != null) {
updateTypeByGuidWithNoRefResolve(guid, typeDef);
structDefs.removeTypeDefByGuid(guid);
structDefs.addType(structDef, new AtlasStructType(structDef, this));
} else if (typeDef.getClass().equals(AtlasClassificationDef.class)) {
AtlasClassificationDef classificationDef = (AtlasClassificationDef)typeDef;
classificationDefs.removeTypeDefByGuid(guid);
classificationDefs.addType(classificationDef, new AtlasClassificationType(classificationDef, this));
} else if (typeDef.getClass().equals(AtlasEntityDef.class)) {
AtlasEntityDef entityDef = (AtlasEntityDef)typeDef;
entityDefs.removeTypeDefByGuid(guid);
entityDefs.addType(entityDef, new AtlasEntityType(entityDef, this));
resolveReferences();
}
if (LOG.isDebugEnabled()) {
......@@ -246,28 +299,10 @@ public class AtlasTypeRegistry {
LOG.debug("==> AtlasTypeRegistry.updateEnumDefByName({})", name);
}
if (name == null || typeDef == null) {
// ignore
} else if (typeDef.getClass().equals(AtlasEnumDef.class)) {
AtlasEnumDef enumDef = (AtlasEnumDef)typeDef;
enumDefs.removeTypeDefByName(name);
enumDefs.addType(enumDef, new AtlasEnumType(enumDef));
} else if (typeDef.getClass().equals(AtlasStructDef.class)) {
AtlasStructDef structDef = (AtlasStructDef)typeDef;
if (name != null && typeDef != null) {
updateTypeByNameWithNoRefResolve(name, typeDef);
structDefs.removeTypeDefByName(name);
structDefs.addType(structDef, new AtlasStructType(structDef, this));
} else if (typeDef.getClass().equals(AtlasClassificationDef.class)) {
AtlasClassificationDef classificationDef = (AtlasClassificationDef)typeDef;
classificationDefs.removeTypeDefByName(name);
classificationDefs.addType(classificationDef, new AtlasClassificationType(classificationDef, this));
} else if (typeDef.getClass().equals(AtlasEntityDef.class)) {
AtlasEntityDef entityDef = (AtlasEntityDef)typeDef;
entityDefs.removeTypeDefByName(name);
entityDefs.addType(entityDef, new AtlasEntityType(entityDef, this));
resolveReferences();
}
if (LOG.isDebugEnabled()) {
......@@ -275,84 +310,53 @@ public class AtlasTypeRegistry {
}
}
public void updateTypeByGuidWithNoRefResolve(String guid, AtlasBaseTypeDef typeDef) {
public void updateTypes(Collection<? extends AtlasBaseTypeDef> typeDefs) throws AtlasBaseException {
if (LOG.isDebugEnabled()) {
LOG.debug("==> AtlasTypeRegistry.updateTypeByGuidWithNoRefResolve({})", guid);
LOG.debug("==> AtlasTypeRegistry.updateTypes(length={})", (typeDefs == null ? 0 : typeDefs.size()));
}
if (guid == null || typeDef == null) {
// ignore
} else if (typeDef.getClass().equals(AtlasEnumDef.class)) {
AtlasEnumDef enumDef = (AtlasEnumDef)typeDef;
enumDefs.removeTypeDefByGuid(guid);
enumDefs.addType(enumDef, new AtlasEnumType(enumDef));
} else if (typeDef.getClass().equals(AtlasStructDef.class)) {
AtlasStructDef structDef = (AtlasStructDef)typeDef;
structDefs.removeTypeDefByGuid(guid);
structDefs.addType(structDef, new AtlasStructType(structDef));
} else if (typeDef.getClass().equals(AtlasClassificationDef.class)) {
AtlasClassificationDef classificationDef = (AtlasClassificationDef)typeDef;
classificationDefs.removeTypeDefByGuid(guid);
classificationDefs.addType(classificationDef, new AtlasClassificationType(classificationDef));
} else if (typeDef.getClass().equals(AtlasEntityDef.class)) {
AtlasEntityDef entityDef = (AtlasEntityDef)typeDef;
if (CollectionUtils.isNotEmpty(typeDefs)) {
updateTypesWithNoRefResolve(typeDefs);
entityDefs.removeTypeDefByGuid(guid);
entityDefs.addType(entityDef, new AtlasEntityType(entityDef));
resolveReferences();
}
if (LOG.isDebugEnabled()) {
LOG.debug("<== AtlasTypeRegistry.updateTypeByGuidWithNoRefResolve({})", guid);
LOG.debug("<== AtlasTypeRegistry.updateTypes(length={})", (typeDefs == null ? 0 : typeDefs.size()));
}
}
public void updateTypeByNameWithNoRefResolve(String name, AtlasBaseTypeDef typeDef) {
public void updateTypes(AtlasTypesDef typesDef) throws AtlasBaseException {
if (LOG.isDebugEnabled()) {
LOG.debug("==> AtlasTypeRegistry.updateTypeByNameWithNoRefResolve({})", name);
LOG.debug("==> AtlasTypeRegistry.updateTypes({})", typesDef);
}
if (name == null || typeDef == null) {
// ignore
} else if (typeDef.getClass().equals(AtlasEnumDef.class)) {
AtlasEnumDef enumDef = (AtlasEnumDef)typeDef;
enumDefs.removeTypeDefByName(name);
enumDefs.addType(enumDef, new AtlasEnumType(enumDef));
} else if (typeDef.getClass().equals(AtlasStructDef.class)) {
AtlasStructDef structDef = (AtlasStructDef)typeDef;
structDefs.removeTypeDefByName(name);
structDefs.addType(structDef, new AtlasStructType(structDef));
} else if (typeDef.getClass().equals(AtlasClassificationDef.class)) {
AtlasClassificationDef classificationDef = (AtlasClassificationDef)typeDef;
classificationDefs.removeTypeDefByName(name);
classificationDefs.addType(classificationDef, new AtlasClassificationType(classificationDef));
} else if (typeDef.getClass().equals(AtlasEntityDef.class)) {
AtlasEntityDef entityDef = (AtlasEntityDef)typeDef;
if (typesDef != null) {
updateTypesWithNoRefResolve(typesDef.getEnumDefs());
updateTypesWithNoRefResolve(typesDef.getStructDefs());
updateTypesWithNoRefResolve(typesDef.getClassificationDefs());
updateTypesWithNoRefResolve(typesDef.getEntityDefs());
entityDefs.removeTypeDefByName(name);
entityDefs.addType(entityDef, new AtlasEntityType(entityDef));
resolveReferences();
}
if (LOG.isDebugEnabled()) {
LOG.debug("<== AtlasTypeRegistry.updateTypeByNameWithNoRefResolve({})", name);
LOG.debug("<== AtlasTypeRegistry.updateTypes({})", typesDef);
}
}
public void removeTypeByGuid(String guid) {
public void removeTypeByGuid(String guid) throws AtlasBaseException {
if (LOG.isDebugEnabled()) {
LOG.debug("==> AtlasTypeRegistry.removeTypeByGuid({})", guid);
}
if (guid != null) {
enumDefs.removeTypeDefByGuid(guid);
structDefs.removeTypeDefByGuid(guid);
classificationDefs.removeTypeDefByGuid(guid);
entityDefs.removeTypeDefByGuid(guid);
registryData.enumDefs.removeTypeDefByGuid(guid);
registryData.structDefs.removeTypeDefByGuid(guid);
registryData.classificationDefs.removeTypeDefByGuid(guid);
registryData.entityDefs.removeTypeDefByGuid(guid);
resolveReferences();
}
if (LOG.isDebugEnabled()) {
......@@ -360,16 +364,18 @@ public class AtlasTypeRegistry {
}
}
public void removeTypeByName(String name) {
public void removeTypeByName(String name) throws AtlasBaseException {
if (LOG.isDebugEnabled()) {
LOG.debug("==> AtlasTypeRegistry.removeTypeByName({})", name);
}
if (name != null) {
enumDefs.removeTypeDefByName(name);
structDefs.removeTypeDefByName(name);
classificationDefs.removeTypeDefByName(name);
entityDefs.removeTypeDefByName(name);
registryData.enumDefs.removeTypeDefByName(name);
registryData.structDefs.removeTypeDefByName(name);
registryData.classificationDefs.removeTypeDefByName(name);
registryData.entityDefs.removeTypeDefByName(name);
resolveReferences();
}
if (LOG.isDebugEnabled()) {
......@@ -377,25 +383,46 @@ public class AtlasTypeRegistry {
}
}
public void addTypes(Collection<? extends AtlasBaseTypeDef> typeDefs) throws AtlasBaseException {
private void addTypeWithNoRefResolve(AtlasBaseTypeDef typeDef) {
if (LOG.isDebugEnabled()) {
LOG.debug("==> AtlasTypeRegistry.addTypes(length={})", (typeDefs == null ? 0 : typeDefs.size()));
LOG.debug("==> AtlasTypeRegistry.addTypeWithNoRefResolve({})", typeDef);
}
if (CollectionUtils.isNotEmpty(typeDefs)) {
for (AtlasBaseTypeDef typeDef : typeDefs) {
addType(typeDef);
if (typeDef != null) {
if (StringUtils.isBlank(typeDef.getGuid())) {
typeDef.setGuid(UUID.randomUUID().toString());
}
if (typeDef.getClass().equals(AtlasEnumDef.class)) {
AtlasEnumDef enumDef = (AtlasEnumDef) typeDef;
registryData.enumDefs.addType(enumDef, new AtlasEnumType(enumDef));
} else if (typeDef.getClass().equals(AtlasStructDef.class)) {
AtlasStructDef structDef = (AtlasStructDef) typeDef;
registryData.structDefs.addType(structDef, new AtlasStructType(structDef));
} else if (typeDef.getClass().equals(AtlasClassificationDef.class)) {
AtlasClassificationDef classificationDef = (AtlasClassificationDef) typeDef;
registryData.classificationDefs.addType(classificationDef,
new AtlasClassificationType(classificationDef));
} else if (typeDef.getClass().equals(AtlasEntityDef.class)) {
AtlasEntityDef entityDef = (AtlasEntityDef) typeDef;
registryData.entityDefs.addType(entityDef, new AtlasEntityType(entityDef));
}
}
if (LOG.isDebugEnabled()) {
LOG.debug("<== AtlasTypeRegistry.addTypes(length={})", (typeDefs == null ? 0 : typeDefs.size()));
LOG.debug("<== AtlasTypeRegistry.addTypeWithNoRefResolve({})", typeDef);
}
}
public void addTypesWithNoRefResolve(Collection<? extends AtlasBaseTypeDef> typeDefs) {
private void addTypesWithNoRefResolve(Collection<? extends AtlasBaseTypeDef> typeDefs) {
if (LOG.isDebugEnabled()) {
LOG.debug("==> AtlasTypeRegistry.addTypes(length={})", (typeDefs == null ? 0 : typeDefs.size()));
LOG.debug("==> AtlasTypeRegistry.addTypesWithNoRefResolve(length={})",
(typeDefs == null ? 0 : typeDefs.size()));
}
if (CollectionUtils.isNotEmpty(typeDefs)) {
......@@ -405,116 +432,205 @@ public class AtlasTypeRegistry {
}
if (LOG.isDebugEnabled()) {
LOG.debug("<== AtlasTypeRegistry.addTypes(length={})", (typeDefs == null ? 0 : typeDefs.size()));
LOG.debug("<== AtlasTypeRegistry.addTypesWithNoRefResolve(length={})",
(typeDefs == null ? 0 : typeDefs.size()));
}
}
public void updateTypes(Collection<? extends AtlasBaseTypeDef> typeDefs) throws AtlasBaseException {
private void updateTypeWithNoRefResolve(AtlasBaseTypeDef typeDef) {
if (LOG.isDebugEnabled()) {
LOG.debug("==> AtlasTypeRegistry.updateTypes(length={})", (typeDefs == null ? 0 : typeDefs.size()));
LOG.debug("==> AtlasTypeRegistry.updateType({})", typeDef);
}
if (CollectionUtils.isNotEmpty(typeDefs)) {
for (AtlasBaseTypeDef typeDef : typeDefs) {
updateType(typeDef);
}
if (typeDef == null) {
// ignore
} else if (StringUtils.isNotBlank(typeDef.getGuid())) {
updateTypeByGuidWithNoRefResolve(typeDef.getGuid(), typeDef);
} else if (StringUtils.isNotBlank(typeDef.getName())) {
updateTypeByNameWithNoRefResolve(typeDef.getName(), typeDef);
}
if (LOG.isDebugEnabled()) {
LOG.debug("<== AtlasTypeRegistry.updateTypes(length={})", (typeDefs == null ? 0 : typeDefs.size()));
LOG.debug("<== AtlasTypeRegistry.updateType({})", typeDef);
}
}
public void updateTypesWithNoRefResolve(Collection<? extends AtlasBaseTypeDef> typeDefs) {
private void updateTypeByGuidWithNoRefResolve(String guid, AtlasBaseTypeDef typeDef) {
if (LOG.isDebugEnabled()) {
LOG.debug("==> AtlasTypeRegistry.updateTypesWithNoRefResolve(length={})", (typeDefs == null ? 0 : typeDefs.size()));
LOG.debug("==> AtlasTypeRegistry.updateTypeByGuidWithNoRefResolve({})", guid);
}
if (CollectionUtils.isNotEmpty(typeDefs)) {
for (AtlasBaseTypeDef typeDef : typeDefs) {
updateTypeWithNoRefResolve(typeDef);
}
if (guid == null || typeDef == null) {
// ignore
} else if (typeDef.getClass().equals(AtlasEnumDef.class)) {
AtlasEnumDef enumDef = (AtlasEnumDef)typeDef;
registryData.enumDefs.removeTypeDefByGuid(guid);
registryData.enumDefs.addType(enumDef, new AtlasEnumType(enumDef));
} else if (typeDef.getClass().equals(AtlasStructDef.class)) {
AtlasStructDef structDef = (AtlasStructDef)typeDef;
registryData.structDefs.removeTypeDefByGuid(guid);
registryData.structDefs.addType(structDef, new AtlasStructType(structDef));
} else if (typeDef.getClass().equals(AtlasClassificationDef.class)) {
AtlasClassificationDef classificationDef = (AtlasClassificationDef)typeDef;
registryData.classificationDefs.removeTypeDefByGuid(guid);
registryData.classificationDefs.addType(classificationDef,
new AtlasClassificationType(classificationDef));
} else if (typeDef.getClass().equals(AtlasEntityDef.class)) {
AtlasEntityDef entityDef = (AtlasEntityDef)typeDef;
registryData.entityDefs.removeTypeDefByGuid(guid);
registryData.entityDefs.addType(entityDef, new AtlasEntityType(entityDef));
}
if (LOG.isDebugEnabled()) {
LOG.debug("<== AtlasTypeRegistry.updateTypesWithNoRefResolve(length={})", (typeDefs == null ? 0 : typeDefs.size()));
LOG.debug("<== AtlasTypeRegistry.updateTypeByGuidWithNoRefResolve({})", guid);
}
}
private void updateTypeByNameWithNoRefResolve(String name, AtlasBaseTypeDef typeDef) {
if (LOG.isDebugEnabled()) {
LOG.debug("==> AtlasTypeRegistry.updateTypeByNameWithNoRefResolve({})", name);
}
public Collection<AtlasEnumDef> getAllEnumDefs() { return enumDefs.getAll(); }
if (name == null || typeDef == null) {
// ignore
} else if (typeDef.getClass().equals(AtlasEnumDef.class)) {
AtlasEnumDef enumDef = (AtlasEnumDef)typeDef;
public AtlasEnumDef getEnumDefByGuid(String guid) {
return enumDefs.getTypeDefByGuid(guid);
}
registryData.enumDefs.removeTypeDefByName(name);
registryData.enumDefs.addType(enumDef, new AtlasEnumType(enumDef));
} else if (typeDef.getClass().equals(AtlasStructDef.class)) {
AtlasStructDef structDef = (AtlasStructDef)typeDef;
public AtlasEnumDef getEnumDefByName(String name) {
return enumDefs.getTypeDefByName(name);
registryData.structDefs.removeTypeDefByName(name);
registryData.structDefs.addType(structDef, new AtlasStructType(structDef));
} else if (typeDef.getClass().equals(AtlasClassificationDef.class)) {
AtlasClassificationDef classificationDef = (AtlasClassificationDef)typeDef;
registryData.classificationDefs.removeTypeDefByName(name);
registryData.classificationDefs.addType(classificationDef,
new AtlasClassificationType(classificationDef));
} else if (typeDef.getClass().equals(AtlasEntityDef.class)) {
AtlasEntityDef entityDef = (AtlasEntityDef)typeDef;
registryData.entityDefs.removeTypeDefByName(name);
registryData.entityDefs.addType(entityDef, new AtlasEntityType(entityDef));
}
if (LOG.isDebugEnabled()) {
LOG.debug("<== AtlasTypeRegistry.updateTypeByNameWithNoRefResolve({})", name);
}
}
public Collection<AtlasStructDef> getAllStructDefs() { return structDefs.getAll(); }
private void updateTypesWithNoRefResolve(Collection<? extends AtlasBaseTypeDef> typeDefs) {
if (LOG.isDebugEnabled()) {
LOG.debug("==> AtlasTypeRegistry.updateTypesWithNoRefResolve(length={})",
(typeDefs == null ? 0 : typeDefs.size()));
}
public AtlasStructDef getStructDefByGuid(String guid) {
return structDefs.getTypeDefByGuid(guid);
if (CollectionUtils.isNotEmpty(typeDefs)) {
for (AtlasBaseTypeDef typeDef : typeDefs) {
updateTypeWithNoRefResolve(typeDef);
}
}
public AtlasStructDef getStructDefByName(String name) { return structDefs.getTypeDefByName(name); }
if (LOG.isDebugEnabled()) {
LOG.debug("<== AtlasTypeRegistry.updateTypesWithNoRefResolve(length={})",
(typeDefs == null ? 0 : typeDefs.size()));
}
}
}
}
class TypeCache {
private final Map<String, AtlasType> typeGuidMap;
private final Map<String, AtlasType> typeNameMap;
public Collection<AtlasClassificationDef> getAllClassificationDefs() { return classificationDefs.getAll(); }
public TypeCache() {
typeGuidMap = new ConcurrentHashMap<>();
typeNameMap = new ConcurrentHashMap<>();
}
public AtlasClassificationDef getClassificationDefByGuid(String guid) {
return classificationDefs.getTypeDefByGuid(guid);
public TypeCache(TypeCache other) {
typeGuidMap = new ConcurrentHashMap<>(other.typeGuidMap);
typeNameMap = new ConcurrentHashMap<>(other.typeNameMap);
}
public AtlasClassificationDef getClassificationDefByName(String name) {
return classificationDefs.getTypeDefByName(name);
public void addType(AtlasType type) {
if (type != null) {
if (StringUtils.isNotEmpty(type.getTypeName())) {
typeNameMap.put(type.getTypeName(), type);
}
}
}
public void addType(AtlasBaseTypeDef typeDef, AtlasType type) {
if (typeDef != null && type != null) {
if (StringUtils.isNotEmpty(typeDef.getGuid())) {
typeGuidMap.put(typeDef.getGuid(), type);
}
public Collection<AtlasEntityDef> getAllEntityDefs() { return entityDefs.getAll(); }
if (StringUtils.isNotEmpty(typeDef.getName())) {
typeNameMap.put(typeDef.getName(), type);
}
}
}
public AtlasEntityDef getEntityDefByGuid(String guid) {
return entityDefs.getTypeDefByGuid(guid);
public Collection<String> getAllTypeNames() {
return Collections.unmodifiableCollection(typeNameMap.keySet());
}
public AtlasEntityDef getEntityDefByName(String name) {
return entityDefs.getTypeDefByName(name);
public Collection<AtlasType> getAllTypes() {
return Collections.unmodifiableCollection(typeNameMap.values());
}
public AtlasType getTypeByGuid(String guid) {
AtlasType ret = guid != null ? typeGuidMap.get(guid) : null;
public void resolveReferences() throws AtlasBaseException {
for (Map.Entry<String, AtlasType> e : allTypes.entrySet()) {
e.getValue().resolveReferences(this);
}
return ret;
}
public AtlasType getTypeByName(String name) {
AtlasType ret = name != null ? typeNameMap.get(name) : null;
private void registerType(AtlasType dataType) {
allTypes.put(dataType.getTypeName(), dataType);
return ret;
}
private void unregisterType(AtlasType dataType) {
allTypes.remove(dataType.getTypeName());
public void removeTypeByGuid(String guid) {
if (guid != null) {
typeGuidMap.remove(guid);
}
}
private void unregisterTypeByName(String typeName) {
allTypes.remove(typeName);
public void removeTypeByName(String name) {
if (name != null) {
typeNameMap.get(name);
}
}
}
class TypeDefCache<T extends AtlasBaseTypeDef> {
private final TypeCache typeCache;
private final Map<String, T> typeDefGuidMap;
private final Map<String, T> typeDefNameMap;
class TypeDefCache<T extends AtlasBaseTypeDef> {
private final AtlasTypeRegistry typeRegistry;
private final Map<String, T> typeDefGuidMap = new ConcurrentHashMap<String, T>();
private final Map<String, T> typeDefNameMap = new ConcurrentHashMap<String, T>();
public TypeDefCache(TypeCache typeCache) {
this.typeCache = typeCache;
this.typeDefGuidMap = new ConcurrentHashMap<>();
this.typeDefNameMap = new ConcurrentHashMap<>();
}
public TypeDefCache(AtlasTypeRegistry typeRegistry) {
this.typeRegistry = typeRegistry;
public TypeDefCache(TypeDefCache other, TypeCache typeCache) {
this.typeCache = typeCache;
this.typeDefGuidMap = new ConcurrentHashMap<>(other.typeDefGuidMap);
this.typeDefNameMap = new ConcurrentHashMap<>(other.typeDefNameMap);
}
public void addType(T typeDef, AtlasType type) {
if (type != null) {
if (typeDef != null && type != null) {
if (StringUtils.isNotEmpty(typeDef.getGuid())) {
typeDefGuidMap.put(typeDef.getGuid(), typeDef);
}
......@@ -523,7 +639,7 @@ public class AtlasTypeRegistry {
typeDefNameMap.put(typeDef.getName(), typeDef);
}
typeRegistry.registerType(type);
typeCache.addType(typeDef, type);
}
}
......@@ -544,25 +660,32 @@ public class AtlasTypeRegistry {
}
public void removeTypeDefByGuid(String guid) {
T typeDef = guid != null ? typeDefGuidMap.remove(guid) : null;
if (guid != null) {
T typeDef = typeDefGuidMap.remove(guid);
typeCache.removeTypeByGuid(guid);
String name = typeDef != null ? typeDef.getName() : null;
if (name != null) {
typeDefNameMap.remove(name);
typeRegistry.unregisterTypeByName(name);
typeCache.removeTypeByName(name);
}
}
}
public void removeTypeDefByName(String name) {
T typeDef = name != null ? typeDefNameMap.get(name) : null;
if (name != null) {
T typeDef = typeDefNameMap.remove(name);
typeCache.removeTypeByName(name);
String guid = typeDef != null ? typeDef.getGuid() : null;
if (guid != null) {
typeDefGuidMap.remove(guid);
}
if (name != null) {
typeRegistry.unregisterTypeByName(name);
typeCache.removeTypeByGuid(guid);
}
}
}
......
intg/src/test/java/org/apache/atlas/model/ModelTestUtil.java
......@@ -41,6 +41,7 @@ import org.apache.atlas.type.AtlasClassificationType;
import org.apache.atlas.type.AtlasEntityType;
import org.apache.atlas.type.AtlasStructType;
import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.atlas.type.AtlasTypeRegistry.AtlasTransientTypeRegistry;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
......@@ -158,7 +159,11 @@ public final class ModelTestUtil {
}
try {
typesRegistry.addType(ret);
AtlasTransientTypeRegistry ttr = typesRegistry.createTransientTypeRegistry();
ttr.addType(ret);
typesRegistry.commitTransientTypeRegistry(ttr);
} catch (AtlasBaseException excp) {
LOG.error("failed to create enum-def", excp);
......@@ -182,7 +187,11 @@ public final class ModelTestUtil {
ret.setAttributeDefs(newAttributeDefsWithAllBuiltInTypes(PREFIX_ATTRIBUTE_NAME));
try {
typesRegistry.addType(ret);
AtlasTransientTypeRegistry ttr = typesRegistry.createTransientTypeRegistry();
ttr.addType(ret);
typesRegistry.commitTransientTypeRegistry(ttr);
} catch (AtlasBaseException excp) {
LOG.error("failed to create struct-def", excp);
......@@ -220,7 +229,11 @@ public final class ModelTestUtil {
}
try {
typesRegistry.addType(ret);
AtlasTransientTypeRegistry ttr = typesRegistry.createTransientTypeRegistry();
ttr.addType(ret);
typesRegistry.commitTransientTypeRegistry(ttr);
} catch (AtlasBaseException excp) {
LOG.error("failed to create entity-def", excp);
......@@ -267,7 +280,11 @@ public final class ModelTestUtil {
}
try {
typesRegistry.addType(ret);
AtlasTransientTypeRegistry ttr = typesRegistry.createTransientTypeRegistry();
ttr.addType(ret);
typesRegistry.commitTransientTypeRegistry(ttr);
} catch (AtlasBaseException excp) {
LOG.error("failed to create classification-def", excp);
......@@ -285,10 +302,14 @@ public final class ModelTestUtil {
public static AtlasEntity newEntity(AtlasEntityDef entityDef, AtlasTypeRegistry typesRegistry) {
AtlasEntity ret = null;
try {
AtlasType dataType = typesRegistry.getType(entityDef.getName());
if (dataType != null && dataType instanceof AtlasEntityType) {
ret = ((AtlasEntityType)dataType).createDefaultValue();
if (dataType instanceof AtlasEntityType) {
ret = ((AtlasEntityType) dataType).createDefaultValue();
}
} catch (AtlasBaseException excp) {
LOG.error("failed to get entity-type " + entityDef.getName(), excp);
}
return ret;
......@@ -301,11 +322,15 @@ public final class ModelTestUtil {
public static AtlasStruct newStruct(AtlasStructDef structDef, AtlasTypeRegistry typesRegistry) {
AtlasStruct ret = null;
try {
AtlasType dataType = typesRegistry.getType(structDef.getName());
if (dataType != null && dataType instanceof AtlasStructType) {
if (dataType instanceof AtlasStructType) {
ret = ((AtlasStructType)dataType).createDefaultValue();
}
} catch (AtlasBaseException excp) {
LOG.error("failed to get struct-type " + structDef.getName(), excp);
}
return ret;
}
......@@ -318,11 +343,15 @@ public final class ModelTestUtil {
AtlasTypeRegistry typesRegistry) {
AtlasClassification ret = null;
try {
AtlasType dataType = typesRegistry.getType(classificationDef.getName());
if (dataType != null && dataType instanceof AtlasClassificationType) {
if (dataType instanceof AtlasClassificationType) {
ret = ((AtlasClassificationType)dataType).createDefaultValue();
}
} catch (AtlasBaseException excp) {
LOG.error("failed to get classification-type " + classificationDef.getName(), excp);
}
return ret;
}
......
intg/src/test/java/org/apache/atlas/model/instance/TestAtlasClassification.java
......@@ -17,6 +17,7 @@
*/
package org.apache.atlas.model.instance;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.ModelTestUtil;
import org.apache.atlas.model.typedef.AtlasClassificationDef;
import org.apache.atlas.type.AtlasType;
......@@ -31,7 +32,7 @@ import static org.testng.Assert.assertTrue;
public class TestAtlasClassification {
@Test
public void testClassificationSerDe() {
public void testClassificationSerDe() throws AtlasBaseException {
AtlasClassificationDef classificationDef = ModelTestUtil.getClassificationDef();
AtlasTypeRegistry typeRegistry = ModelTestUtil.getTypesRegistry();
AtlasType dataType = typeRegistry.getType(classificationDef.getName());
......@@ -50,7 +51,7 @@ public class TestAtlasClassification {
}
@Test
public void testClassificationSerDeWithSuperType() {
public void testClassificationSerDeWithSuperType() throws AtlasBaseException {
AtlasClassificationDef classificationDef = ModelTestUtil.getClassificationDefWithSuperType();
AtlasTypeRegistry typeRegistry = ModelTestUtil.getTypesRegistry();
AtlasType dataType = typeRegistry.getType(classificationDef.getName());
......@@ -69,7 +70,7 @@ public class TestAtlasClassification {
}
@Test
public void testClassificationSerDeWithSuperTypes() {
public void testClassificationSerDeWithSuperTypes() throws AtlasBaseException {
AtlasClassificationDef classificationDef = ModelTestUtil.getClassificationDefWithSuperTypes();
AtlasTypeRegistry typeRegistry = ModelTestUtil.getTypesRegistry();
AtlasType dataType = typeRegistry.getType(classificationDef.getName());
......
intg/src/test/java/org/apache/atlas/model/instance/TestAtlasEntity.java
......@@ -17,6 +17,7 @@
*/
package org.apache.atlas.model.instance;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.ModelTestUtil;
import org.apache.atlas.model.typedef.AtlasEntityDef;
import org.apache.atlas.type.AtlasType;
......@@ -31,7 +32,7 @@ import static org.testng.Assert.assertTrue;
public class TestAtlasEntity {
@Test
public void testEntitySerDe() {
public void testEntitySerDe() throws AtlasBaseException {
AtlasEntityDef entityDef = ModelTestUtil.getEntityDef();
AtlasTypeRegistry typeRegistry = ModelTestUtil.getTypesRegistry();
AtlasType dataType = typeRegistry.getType(entityDef.getName());
......@@ -50,7 +51,7 @@ public class TestAtlasEntity {
}
@Test
public void testEntitySerDeWithSuperType() {
public void testEntitySerDeWithSuperType() throws AtlasBaseException {
AtlasEntityDef entityDef = ModelTestUtil.getEntityDefWithSuperType();
AtlasTypeRegistry typeRegistry = ModelTestUtil.getTypesRegistry();
AtlasType dataType = typeRegistry.getType(entityDef.getName());
......@@ -69,7 +70,7 @@ public class TestAtlasEntity {
}
@Test
public void testEntitySerDeWithSuperTypes() {
public void testEntitySerDeWithSuperTypes() throws AtlasBaseException {
AtlasEntityDef entityDef = ModelTestUtil.getEntityDefWithSuperTypes();
AtlasTypeRegistry typeRegistry = ModelTestUtil.getTypesRegistry();
AtlasType dataType = typeRegistry.getType(entityDef.getName());
......
intg/src/test/java/org/apache/atlas/model/typedef/TestAtlasEntityDef.java
......@@ -84,10 +84,12 @@ public class TestAtlasEntityDef {
entityDef.addSuperType(newSuperType);
assertTrue(entityDef.hasSuperType(newSuperType));
entityDef.removeSuperType(newSuperType);
}
@Test
public void testEntityDefDefRemoveElement() {
public void testEntityDefRemoveElement() {
AtlasEntityDef entityDef = ModelTestUtil.newEntityDefWithSuperTypes();
for (String superType : entityDef.getSuperTypes()) {
......@@ -114,6 +116,9 @@ public class TestAtlasEntityDef {
for (String superType : newSuperTypes) {
assertTrue(entityDef.hasSuperType(superType));
}
// restore old sypertypes
entityDef.setSuperTypes(oldSuperTypes);
}
@Test
......
intg/src/test/java/org/apache/atlas/type/TestAtlasEntityType.java
......@@ -17,7 +17,11 @@
*/
package org.apache.atlas.type;
import java.util.*;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.ModelTestUtil;
......@@ -26,6 +30,7 @@ import org.apache.atlas.model.typedef.AtlasBaseTypeDef;
import org.apache.atlas.model.typedef.AtlasEntityDef;
import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef;
import org.apache.atlas.model.typedef.AtlasStructDef.AtlasConstraintDef;
import org.apache.atlas.type.AtlasTypeRegistry.AtlasTransientTypeRegistry;
import org.testng.annotations.Test;
import static org.testng.Assert.*;
......@@ -127,8 +132,11 @@ public class TestAtlasEntityType {
entityDefs.add(createColumnEntityDef());
try {
typeRegistry.addTypesWithNoRefResolve(entityDefs);
typeRegistry.resolveReferences();
AtlasTransientTypeRegistry ttr = typeRegistry.createTransientTypeRegistry();
ttr.addTypes(entityDefs);
typeRegistry.commitTransientTypeRegistry(ttr);
} catch (AtlasBaseException excp) {
failureMsg = excp.getMessage();
}
......@@ -144,7 +152,11 @@ public class TestAtlasEntityType {
entityDefs.add(createTableEntityDef());
try {
typeRegistry.addTypes(entityDefs);
AtlasTransientTypeRegistry ttr = typeRegistry.createTransientTypeRegistry();
ttr.addTypes(entityDefs);
typeRegistry.commitTransientTypeRegistry(ttr);
} catch (AtlasBaseException excp) {
failureMsg = excp.getMessage();
}
......@@ -161,8 +173,11 @@ public class TestAtlasEntityType {
entityDefs.add(createColumnEntityDef());
try {
typeRegistry.addTypesWithNoRefResolve(entityDefs);
typeRegistry.resolveReferences();
AtlasTransientTypeRegistry ttr = typeRegistry.createTransientTypeRegistry();
ttr.addTypes(entityDefs);
typeRegistry.commitTransientTypeRegistry(ttr);
} catch (AtlasBaseException excp) {
failureMsg = excp.getMessage();
}
......@@ -178,7 +193,11 @@ public class TestAtlasEntityType {
entityDefs.add(createColumnEntityDef());
try {
typeRegistry.addTypes(entityDefs);
AtlasTransientTypeRegistry ttr = typeRegistry.createTransientTypeRegistry();
ttr.addTypes(entityDefs);
typeRegistry.commitTransientTypeRegistry(ttr);
} catch (AtlasBaseException excp) {
failureMsg = excp.getMessage();
}
......
release-log.txt
......@@ -9,6 +9,7 @@ ATLAS-1060 Add composite indexes for exact match performance improvements for al
ATLAS-1127 Modify creation and modification timestamps to Date instead of Long(sumasai)
ALL CHANGES:
ATLAS-1230 updated AtlasTypeRegistry to support batch, atomic type updates (mneethiraj)
ATLAS-1229 Add TypeCategory and methods to access attribute definitiions in AtlasTypes (sumasai)
ATLAS-1227 Added support for attribute constraints in the API (mneethiraj)
ATLAS-1225 Optimize AtlasTypeDefGraphStore to use typeRegistry (mneethiraj via sumasai)
......
repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasTypeDefGraphStore.java
......@@ -20,7 +20,6 @@ package org.apache.atlas.repository.store.graph;
import org.apache.atlas.GraphTransaction;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.SearchFilter;
import org.apache.atlas.model.typedef.AtlasBaseTypeDef;
import org.apache.atlas.model.typedef.AtlasClassificationDef;
import org.apache.atlas.model.typedef.AtlasClassificationDef.AtlasClassificationDefs;
import org.apache.atlas.model.typedef.AtlasEntityDef;
......@@ -33,7 +32,7 @@ import org.apache.atlas.model.typedef.AtlasTypesDef;
import org.apache.atlas.repository.util.FilterUtil;
import org.apache.atlas.store.AtlasTypeDefStore;
import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.atlas.util.TypeDefSorter;
import org.apache.atlas.type.AtlasTypeRegistry.AtlasTransientTypeRegistry;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.Predicate;
import org.slf4j.Logger;
......@@ -41,7 +40,6 @@ import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
......@@ -52,60 +50,58 @@ public abstract class AtlasTypeDefGraphStore implements AtlasTypeDefStore {
private static final Logger LOG = LoggerFactory.getLogger(AtlasTypeDefGraphStore.class);
private AtlasTypeRegistry typeRegistry;
private AtlasEnumDefStore enumDefStore;
private AtlasStructDefStore structDefStore;
private AtlasClassificationDefStore classificationDefStore;
private AtlasEntityDefStore entityDefStore;
private final AtlasTypeRegistry typeRegistry;
protected AtlasTypeDefGraphStore() {
protected AtlasTypeDefGraphStore(AtlasTypeRegistry typeRegistry) {
this.typeRegistry = typeRegistry;
}
protected void init(AtlasEnumDefStore enumDefStore,
AtlasStructDefStore structDefStore,
AtlasClassificationDefStore classificationDefStore,
AtlasEntityDefStore entityDefStore) throws AtlasBaseException {
AtlasTypeRegistry typeRegistry = new AtlasTypeRegistry();
typeRegistry.addTypesWithNoRefResolve(enumDefStore.getAll());
typeRegistry.addTypesWithNoRefResolve(structDefStore.getAll());
typeRegistry.addTypesWithNoRefResolve(classificationDefStore.getAll());
typeRegistry.addTypesWithNoRefResolve(entityDefStore.getAll());
protected abstract AtlasEnumDefStore getEnumDefStore(AtlasTypeRegistry typeRegistry);
typeRegistry.resolveReferences();
protected abstract AtlasStructDefStore getStructDefStore(AtlasTypeRegistry typeRegistry);
this.enumDefStore = enumDefStore;
this.structDefStore = structDefStore;
this.classificationDefStore = classificationDefStore;
this.entityDefStore = entityDefStore;
this.typeRegistry = typeRegistry;
}
protected abstract AtlasClassificationDefStore getClassificationDefStore(AtlasTypeRegistry typeRegistry);
public AtlasTypeRegistry getTypeRegistry() {
return typeRegistry;
}
protected abstract AtlasEntityDefStore getEntityDefStore(AtlasTypeRegistry typeRegistry);
@Override
public void init() throws AtlasBaseException {
AtlasTransientTypeRegistry ttr = typeRegistry.createTransientTypeRegistry();
AtlasTypesDef typesDef = new AtlasTypesDef(getEnumDefStore(ttr).getAll(),
getStructDefStore(ttr).getAll(),
getClassificationDefStore(ttr).getAll(),
getEntityDefStore(ttr).getAll());
ttr.addTypes(typesDef);
typeRegistry.commitTransientTypeRegistry(ttr);
}
@Override
@GraphTransaction
public AtlasEnumDef createEnumDef(AtlasEnumDef enumDef) throws AtlasBaseException {
AtlasEnumDef ret = enumDefStore.create(enumDef);
AtlasTransientTypeRegistry ttr = typeRegistry.createTransientTypeRegistry();
ttr.addType(enumDef);
typeRegistry.addType(ret);
AtlasEnumDef ret = getEnumDefStore(ttr).create(enumDef);
typeRegistry.commitTransientTypeRegistry(ttr);
return ret;
}
@Override
@GraphTransaction
public List<AtlasEnumDef> createEnumDefs(List<AtlasEnumDef> atlasEnumDefs) throws AtlasBaseException {
List<AtlasEnumDef> ret = enumDefStore.create(atlasEnumDefs);
public List<AtlasEnumDef> createEnumDefs(List<AtlasEnumDef> enumDefs) throws AtlasBaseException {
AtlasTransientTypeRegistry ttr = typeRegistry.createTransientTypeRegistry();
ttr.addTypes(enumDefs);
typeRegistry.addTypes(ret);
List<AtlasEnumDef> ret = getEnumDefStore(ttr).create(enumDefs);
typeRegistry.commitTransientTypeRegistry(ttr);
return ret;
}
......@@ -143,9 +139,13 @@ public abstract class AtlasTypeDefGraphStore implements AtlasTypeDefStore {
@Override
@GraphTransaction
public AtlasEnumDef updateEnumDefByName(String name, AtlasEnumDef enumDef) throws AtlasBaseException {
AtlasEnumDef ret = enumDefStore.updateByName(name, enumDef);
AtlasTransientTypeRegistry ttr = typeRegistry.createTransientTypeRegistry();
ttr.updateTypeByName(name, enumDef);
typeRegistry.updateTypeByName(name, ret);
AtlasEnumDef ret = getEnumDefStore(ttr).updateByName(name, enumDef);
typeRegistry.commitTransientTypeRegistry(ttr);
return ret;
}
......@@ -153,9 +153,13 @@ public abstract class AtlasTypeDefGraphStore implements AtlasTypeDefStore {
@Override
@GraphTransaction
public AtlasEnumDef updateEnumDefByGuid(String guid, AtlasEnumDef enumDef) throws AtlasBaseException {
AtlasEnumDef ret = enumDefStore.updateByGuid(guid, enumDef);
AtlasTransientTypeRegistry ttr = typeRegistry.createTransientTypeRegistry();
ttr.updateTypeByGuid(guid, enumDef);
AtlasEnumDef ret = getEnumDefStore(ttr).updateByGuid(guid, enumDef);
typeRegistry.updateTypeByGuid(guid, ret);
typeRegistry.commitTransientTypeRegistry(ttr);
return ret;
}
......@@ -163,31 +167,43 @@ public abstract class AtlasTypeDefGraphStore implements AtlasTypeDefStore {
@Override
@GraphTransaction
public void deleteEnumDefByName(String name) throws AtlasBaseException {
enumDefStore.deleteByName(name);
AtlasTransientTypeRegistry ttr = typeRegistry.createTransientTypeRegistry();
typeRegistry.removeTypeByName(name);
ttr.removeTypeByName(name);
getEnumDefStore(ttr).deleteByName(name);
typeRegistry.commitTransientTypeRegistry(ttr);
}
@Override
@GraphTransaction
public void deleteEnumDefByGuid(String guid) throws AtlasBaseException {
enumDefStore.deleteByGuid(guid);
AtlasTransientTypeRegistry ttr = typeRegistry.createTransientTypeRegistry();
ttr.removeTypeByGuid(guid);
getEnumDefStore(ttr).deleteByGuid(guid);
typeRegistry.removeTypeByGuid(guid);
typeRegistry.commitTransientTypeRegistry(ttr);
}
@Override
@GraphTransaction
public AtlasEnumDefs searchEnumDefs(SearchFilter filter) throws AtlasBaseException {
return enumDefStore.search(filter);
return getEnumDefStore(typeRegistry).search(filter);
}
@Override
@GraphTransaction
public AtlasStructDef createStructDef(AtlasStructDef structDef) throws AtlasBaseException {
AtlasStructDef ret = structDefStore.create(structDef);
AtlasTransientTypeRegistry ttr = typeRegistry.createTransientTypeRegistry();
typeRegistry.addType(structDef);
ttr.addType(structDef);
AtlasStructDef ret = getStructDefStore(ttr).create(structDef);
typeRegistry.commitTransientTypeRegistry(ttr);
return ret;
}
......@@ -195,9 +211,13 @@ public abstract class AtlasTypeDefGraphStore implements AtlasTypeDefStore {
@Override
@GraphTransaction
public List<AtlasStructDef> createStructDefs(List<AtlasStructDef> structDefs) throws AtlasBaseException {
List<AtlasStructDef> ret = structDefStore.create(structDefs);
AtlasTransientTypeRegistry ttr = typeRegistry.createTransientTypeRegistry();
ttr.addTypes(structDefs);
typeRegistry.addTypes(ret);
List<AtlasStructDef> ret = getStructDefStore(ttr).create(structDefs);
typeRegistry.commitTransientTypeRegistry(ttr);
return ret;
}
......@@ -235,9 +255,13 @@ public abstract class AtlasTypeDefGraphStore implements AtlasTypeDefStore {
@Override
@GraphTransaction
public AtlasStructDef updateStructDefByName(String name, AtlasStructDef structDef) throws AtlasBaseException {
AtlasStructDef ret = structDefStore.updateByName(name, structDef);
AtlasTransientTypeRegistry ttr = typeRegistry.createTransientTypeRegistry();
ttr.updateTypeByName(name, structDef);
typeRegistry.updateTypeByName(name, ret);
AtlasStructDef ret = getStructDefStore(ttr).updateByName(name, structDef);
typeRegistry.commitTransientTypeRegistry(ttr);
return ret;
}
......@@ -245,9 +269,13 @@ public abstract class AtlasTypeDefGraphStore implements AtlasTypeDefStore {
@Override
@GraphTransaction
public AtlasStructDef updateStructDefByGuid(String guid, AtlasStructDef structDef) throws AtlasBaseException {
AtlasStructDef ret = structDefStore.updateByGuid(guid, structDef);
AtlasTransientTypeRegistry ttr = typeRegistry.createTransientTypeRegistry();
ttr.updateTypeByGuid(guid, structDef);
typeRegistry.updateTypeByGuid(guid, ret);
AtlasStructDef ret = getStructDefStore(ttr).updateByGuid(guid, structDef);
typeRegistry.commitTransientTypeRegistry(ttr);
return ret;
}
......@@ -255,32 +283,44 @@ public abstract class AtlasTypeDefGraphStore implements AtlasTypeDefStore {
@Override
@GraphTransaction
public void deleteStructDefByName(String name) throws AtlasBaseException {
structDefStore.deleteByName(name);
AtlasTransientTypeRegistry ttr = typeRegistry.createTransientTypeRegistry();
ttr.removeTypeByName(name);
getStructDefStore(ttr).deleteByName(name);
typeRegistry.removeTypeByName(name);
typeRegistry.commitTransientTypeRegistry(ttr);
}
@Override
@GraphTransaction
public void deleteStructDefByGuid(String guid) throws AtlasBaseException {
structDefStore.deleteByGuid(guid);
AtlasTransientTypeRegistry ttr = typeRegistry.createTransientTypeRegistry();
typeRegistry.removeTypeByGuid(guid);
ttr.removeTypeByGuid(guid);
getStructDefStore(ttr).deleteByGuid(guid);
typeRegistry.commitTransientTypeRegistry(ttr);
}
@Override
@GraphTransaction
public AtlasStructDefs searchStructDefs(SearchFilter filter) throws AtlasBaseException {
return structDefStore.search(filter);
return getStructDefStore(typeRegistry).search(filter);
}
@Override
@GraphTransaction
public AtlasClassificationDef createClassificationDef(AtlasClassificationDef classificationDef)
throws AtlasBaseException {
AtlasClassificationDef ret = classificationDefStore.create(classificationDef);
AtlasTransientTypeRegistry ttr = typeRegistry.createTransientTypeRegistry();
ttr.addType(classificationDef);
AtlasClassificationDef ret = getClassificationDefStore(ttr).create(classificationDef);
typeRegistry.addType(classificationDef);
typeRegistry.commitTransientTypeRegistry(ttr);
return ret;
}
......@@ -289,9 +329,13 @@ public abstract class AtlasTypeDefGraphStore implements AtlasTypeDefStore {
@GraphTransaction
public List<AtlasClassificationDef> createClassificationDefs(List<AtlasClassificationDef> classificationDefs)
throws AtlasBaseException {
List<AtlasClassificationDef> ret = classificationDefStore.create(classificationDefs);
AtlasTransientTypeRegistry ttr = typeRegistry.createTransientTypeRegistry();
typeRegistry.addTypes(ret);
ttr.addTypes(classificationDefs);
List<AtlasClassificationDef> ret = getClassificationDefStore(ttr).create(classificationDefs);
typeRegistry.commitTransientTypeRegistry(ttr);
return ret;
}
......@@ -330,9 +374,13 @@ public abstract class AtlasTypeDefGraphStore implements AtlasTypeDefStore {
@GraphTransaction
public AtlasClassificationDef updateClassificationDefByName(String name, AtlasClassificationDef classificationDef)
throws AtlasBaseException {
AtlasClassificationDef ret = classificationDefStore.updateByName(name, classificationDef);
AtlasTransientTypeRegistry ttr = typeRegistry.createTransientTypeRegistry();
ttr.updateTypeByName(name, classificationDef);
typeRegistry.updateTypeByName(name, ret);
AtlasClassificationDef ret = getClassificationDefStore(ttr).updateByName(name, classificationDef);
typeRegistry.commitTransientTypeRegistry(ttr);
return ret;
}
......@@ -341,9 +389,13 @@ public abstract class AtlasTypeDefGraphStore implements AtlasTypeDefStore {
@GraphTransaction
public AtlasClassificationDef updateClassificationDefByGuid(String guid, AtlasClassificationDef classificationDef)
throws AtlasBaseException {
AtlasClassificationDef ret = classificationDefStore.updateByGuid(guid, classificationDef);
AtlasTransientTypeRegistry ttr = typeRegistry.createTransientTypeRegistry();
ttr.updateTypeByGuid(guid, classificationDef);
typeRegistry.updateTypeByGuid(guid, ret);
AtlasClassificationDef ret = getClassificationDefStore(ttr).updateByGuid(guid, classificationDef);
typeRegistry.commitTransientTypeRegistry(ttr);
return ret;
}
......@@ -351,31 +403,43 @@ public abstract class AtlasTypeDefGraphStore implements AtlasTypeDefStore {
@Override
@GraphTransaction
public void deleteClassificationDefByName(String name) throws AtlasBaseException {
classificationDefStore.deleteByName(name);
AtlasTransientTypeRegistry ttr = typeRegistry.createTransientTypeRegistry();
ttr.removeTypeByName(name);
typeRegistry.removeTypeByName(name);
getClassificationDefStore(ttr).deleteByName(name);
typeRegistry.commitTransientTypeRegistry(ttr);
}
@Override
@GraphTransaction
public void deleteClassificationDefByGuid(String guid) throws AtlasBaseException {
classificationDefStore.deleteByGuid(guid);
AtlasTransientTypeRegistry ttr = typeRegistry.createTransientTypeRegistry();
ttr.removeTypeByGuid(guid);
getClassificationDefStore(ttr).deleteByGuid(guid);
typeRegistry.removeTypeByGuid(guid);
typeRegistry.commitTransientTypeRegistry(ttr);
}
@Override
@GraphTransaction
public AtlasClassificationDefs searchClassificationDefs(SearchFilter filter) throws AtlasBaseException {
return classificationDefStore.search(filter);
return getClassificationDefStore(typeRegistry).search(filter);
}
@Override
@GraphTransaction
public AtlasEntityDef createEntityDef(AtlasEntityDef entityDef) throws AtlasBaseException {
AtlasEntityDef ret = entityDefStore.create(entityDef);
AtlasTransientTypeRegistry ttr = typeRegistry.createTransientTypeRegistry();
typeRegistry.addType(ret);
ttr.addType(entityDef);
AtlasEntityDef ret = getEntityDefStore(ttr).create(entityDef);
typeRegistry.commitTransientTypeRegistry(ttr);
return ret;
}
......@@ -383,9 +447,13 @@ public abstract class AtlasTypeDefGraphStore implements AtlasTypeDefStore {
@Override
@GraphTransaction
public List<AtlasEntityDef> createEntityDefs(List<AtlasEntityDef> entityDefs) throws AtlasBaseException {
List<AtlasEntityDef> ret = entityDefStore.create(entityDefs);
AtlasTransientTypeRegistry ttr = typeRegistry.createTransientTypeRegistry();
ttr.addTypes(entityDefs);
List<AtlasEntityDef> ret = getEntityDefStore(ttr).create(entityDefs);
typeRegistry.addTypes(ret);
typeRegistry.commitTransientTypeRegistry(ttr);
return ret;
}
......@@ -423,9 +491,13 @@ public abstract class AtlasTypeDefGraphStore implements AtlasTypeDefStore {
@Override
@GraphTransaction
public AtlasEntityDef updateEntityDefByName(String name, AtlasEntityDef entityDef) throws AtlasBaseException {
AtlasEntityDef ret = entityDefStore.updateByName(name, entityDef);
AtlasTransientTypeRegistry ttr = typeRegistry.createTransientTypeRegistry();
typeRegistry.updateTypeByName(name, ret);
ttr.updateTypeByName(name, entityDef);
AtlasEntityDef ret = getEntityDefStore(ttr).updateByName(name, entityDef);
typeRegistry.commitTransientTypeRegistry(ttr);
return ret;
}
......@@ -433,9 +505,13 @@ public abstract class AtlasTypeDefGraphStore implements AtlasTypeDefStore {
@Override
@GraphTransaction
public AtlasEntityDef updateEntityDefByGuid(String guid, AtlasEntityDef entityDef) throws AtlasBaseException {
AtlasEntityDef ret = entityDefStore.updateByGuid(guid, entityDef);
AtlasTransientTypeRegistry ttr = typeRegistry.createTransientTypeRegistry();
ttr.updateTypeByGuid(guid, entityDef);
typeRegistry.updateTypeByGuid(guid, ret);
AtlasEntityDef ret = getEntityDefStore(ttr).updateByGuid(guid, entityDef);
typeRegistry.commitTransientTypeRegistry(ttr);
return ret;
}
......@@ -443,44 +519,50 @@ public abstract class AtlasTypeDefGraphStore implements AtlasTypeDefStore {
@Override
@GraphTransaction
public void deleteEntityDefByName(String name) throws AtlasBaseException {
entityDefStore.deleteByName(name);
AtlasTransientTypeRegistry ttr = typeRegistry.createTransientTypeRegistry();
ttr.removeTypeByName(name);
typeRegistry.removeTypeByName(name);
getEntityDefStore(ttr).deleteByName(name);
typeRegistry.commitTransientTypeRegistry(ttr);
}
@Override
@GraphTransaction
public void deleteEntityDefByGuid(String guid) throws AtlasBaseException {
entityDefStore.deleteByGuid(guid);
AtlasTransientTypeRegistry ttr = typeRegistry.createTransientTypeRegistry();
ttr.removeTypeByGuid(guid);
typeRegistry.removeTypeByGuid(guid);
getEntityDefStore(ttr).deleteByGuid(guid);
typeRegistry.commitTransientTypeRegistry(ttr);
}
@Override
@GraphTransaction
public AtlasEntityDefs searchEntityDefs(SearchFilter filter) throws AtlasBaseException {
return entityDefStore.search(filter);
return getEntityDefStore(typeRegistry).search(filter);
}
@Override
@GraphTransaction
public AtlasTypesDef createTypesDef(AtlasTypesDef typesDef) throws AtlasBaseException {
LOG.info("Creating EnumDefs");
List<AtlasEnumDef> enumDefs = enumDefStore.create(typesDef.getEnumDefs());
List<AtlasStructDef> structDefs = structDefStore.create(typesDef.getStructDefs());
List<AtlasClassificationDef> classifiDefs = classificationDefStore.create(typesDef.getClassificationDefs());
List<AtlasEntityDef> entityDefs = entityDefStore.create(typesDef.getEntityDefs());
AtlasTransientTypeRegistry ttr = typeRegistry.createTransientTypeRegistry();
// typeRegistry should be updated only after resovleReferences() returns success; until then use a temp registry
typeRegistry.addTypes(enumDefs);
typeRegistry.addTypes(structDefs);
typeRegistry.addTypes(classifiDefs);
typeRegistry.addTypes(entityDefs);
ttr.addTypes(typesDef);
typeRegistry.resolveReferences();
List<AtlasEnumDef> enumDefs = getEnumDefStore(ttr).create(typesDef.getEnumDefs());
List<AtlasStructDef> structDefs = getStructDefStore(ttr).create(typesDef.getStructDefs());
List<AtlasClassificationDef> classifiDefs = getClassificationDefStore(ttr).create(typesDef.getClassificationDefs());
List<AtlasEntityDef> entityDefs = getEntityDefStore(ttr).create(typesDef.getEntityDefs());
AtlasTypesDef ret = new AtlasTypesDef(enumDefs, structDefs, classifiDefs, entityDefs);
typeRegistry.commitTransientTypeRegistry(ttr);
return ret;
}
......@@ -488,18 +570,16 @@ public abstract class AtlasTypeDefGraphStore implements AtlasTypeDefStore {
@GraphTransaction
public AtlasTypesDef updateTypesDef(AtlasTypesDef typesDef) throws AtlasBaseException {
LOG.info("Updating EnumDefs");
List<AtlasEnumDef> enumDefs = enumDefStore.update(typesDef.getEnumDefs());
List<AtlasStructDef> structDefs = structDefStore.update(typesDef.getStructDefs());
List<AtlasClassificationDef> classifiDefs = classificationDefStore.update(typesDef.getClassificationDefs());
List<AtlasEntityDef> entityDefs = entityDefStore.update(typesDef.getEntityDefs());
AtlasTransientTypeRegistry ttr = typeRegistry.createTransientTypeRegistry();
ttr.updateTypes(typesDef);
// typeRegistry should be updated only after resovleReferences() returns success; until then use a temp registry
typeRegistry.updateTypes(enumDefs);
typeRegistry.updateTypes(structDefs);
typeRegistry.updateTypes(classifiDefs);
typeRegistry.updateTypes(entityDefs);
List<AtlasEnumDef> enumDefs = getEnumDefStore(ttr).update(typesDef.getEnumDefs());
List<AtlasStructDef> structDefs = getStructDefStore(ttr).update(typesDef.getStructDefs());
List<AtlasClassificationDef> classifiDefs = getClassificationDefStore(ttr).update(typesDef.getClassificationDefs());
List<AtlasEntityDef> entityDefs = getEntityDefStore(ttr).update(typesDef.getEntityDefs());
typeRegistry.resolveReferences();
typeRegistry.commitTransientTypeRegistry(ttr);
AtlasTypesDef ret = new AtlasTypesDef(enumDefs, structDefs, classifiDefs, entityDefs);
......@@ -513,7 +593,7 @@ public abstract class AtlasTypeDefGraphStore implements AtlasTypeDefStore {
AtlasTypesDef typesDef = new AtlasTypesDef();
Predicate searchPredicates = FilterUtil.getPredicateFromSearchFilter(searchFilter);
try {
List<AtlasEnumDef> enumDefs = enumDefStore.getAll();
List<AtlasEnumDef> enumDefs = getEnumDefStore(typeRegistry).getAll();
CollectionUtils.filter(enumDefs, searchPredicates);
typesDef.setEnumDefs(enumDefs);
} catch (AtlasBaseException ex) {
......@@ -521,7 +601,7 @@ public abstract class AtlasTypeDefGraphStore implements AtlasTypeDefStore {
}
try {
List<AtlasStructDef> structDefs = structDefStore.getAll();
List<AtlasStructDef> structDefs = getStructDefStore(typeRegistry).getAll();
CollectionUtils.filter(structDefs, searchPredicates);
typesDef.setStructDefs(structDefs);
} catch (AtlasBaseException ex) {
......@@ -529,7 +609,7 @@ public abstract class AtlasTypeDefGraphStore implements AtlasTypeDefStore {
}
try {
List<AtlasClassificationDef> classificationDefs = classificationDefStore.getAll();
List<AtlasClassificationDef> classificationDefs = getClassificationDefStore(typeRegistry).getAll();
CollectionUtils.filter(classificationDefs, searchPredicates);
typesDef.setClassificationDefs(classificationDefs);
} catch (AtlasBaseException ex) {
......@@ -537,7 +617,7 @@ public abstract class AtlasTypeDefGraphStore implements AtlasTypeDefStore {
}
try {
List<AtlasEntityDef> entityDefs = entityDefStore.getAll();
List<AtlasEntityDef> entityDefs = getEntityDefStore(typeRegistry).getAll();
CollectionUtils.filter(entityDefs, searchPredicates);
typesDef.setEntityDefs(entityDefs);
} catch (AtlasBaseException ex) {
......
repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasAbstractDefStoreV1.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.repository.store.graph.v1;
import org.apache.atlas.type.AtlasTypeRegistry;
/**
* Abstract typedef-store for v1 format.
*/
public abstract class AtlasAbstractDefStoreV1 {
protected final AtlasTypeDefGraphStoreV1 typeDefStore;
protected final AtlasTypeRegistry typeRegistry;
public AtlasAbstractDefStoreV1(AtlasTypeDefGraphStoreV1 typeDefStore, AtlasTypeRegistry typeRegistry) {
this.typeDefStore = typeDefStore;
this.typeRegistry = typeRegistry;
}
}
repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasClassificationDefStoreV1.java
......@@ -26,6 +26,9 @@ import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.repository.store.graph.AtlasClassificationDefStore;
import org.apache.atlas.repository.util.FilterUtil;
import org.apache.atlas.type.AtlasClassificationType;
import org.apache.atlas.type.AtlasType;
import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.atlas.typesystem.types.DataTypes.TypeCategory;
import org.apache.commons.collections.CollectionUtils;
import org.slf4j.Logger;
......@@ -33,21 +36,16 @@ import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
/**
* ClassificationDef store in v1 format.
*/
public class AtlasClassificationDefStoreV1 implements AtlasClassificationDefStore {
public class AtlasClassificationDefStoreV1 extends AtlasAbstractDefStoreV1 implements AtlasClassificationDefStore {
private static final Logger LOG = LoggerFactory.getLogger(AtlasClassificationDefStoreV1.class);
private final AtlasTypeDefGraphStoreV1 typeDefStore;
public AtlasClassificationDefStoreV1(AtlasTypeDefGraphStoreV1 typeDefStore) {
super();
this.typeDefStore = typeDefStore;
public AtlasClassificationDefStoreV1(AtlasTypeDefGraphStoreV1 typeDefStore, AtlasTypeRegistry typeRegistry) {
super(typeDefStore, typeRegistry);
}
@Override
......@@ -80,20 +78,17 @@ public class AtlasClassificationDefStoreV1 implements AtlasClassificationDefStor
if (LOG.isDebugEnabled()) {
LOG.debug("==> AtlasClassificationDefStoreV1.create({})", classificationDefs);
}
List<AtlasClassificationDef> classificationDefList = new LinkedList<>();
for (AtlasClassificationDef structDef : classificationDefs) {
try {
AtlasClassificationDef atlasClassificationDef = create(structDef);
classificationDefList.add(atlasClassificationDef);
} catch (AtlasBaseException baseException) {
LOG.error("Failed to create {}", structDef);
LOG.error("Exception: {}", baseException);
}
List<AtlasClassificationDef> ret = new ArrayList<>();
for (AtlasClassificationDef classificationDef : classificationDefs) {
ret.add(create(classificationDef));
}
if (LOG.isDebugEnabled()) {
LOG.debug("<== AtlasClassificationDefStoreV1.create({}, {})", classificationDefs, classificationDefList);
LOG.debug("<== AtlasClassificationDefStoreV1.create({}, {})", classificationDefs, ret);
}
return classificationDefList;
return ret;
}
@Override
......@@ -102,17 +97,17 @@ public class AtlasClassificationDefStoreV1 implements AtlasClassificationDefStor
LOG.debug("==> AtlasClassificationDefStoreV1.getAll()");
}
List<AtlasClassificationDef> classificationDefs = new LinkedList<>();
Iterator<AtlasVertex> verticesByCategory = typeDefStore.findTypeVerticesByCategory(TypeCategory.TRAIT);
while (verticesByCategory.hasNext()) {
AtlasClassificationDef classificationDef = toClassificationDef(verticesByCategory.next());
classificationDefs.add(classificationDef);
List<AtlasClassificationDef> ret = new ArrayList<>();
Iterator<AtlasVertex> vertices = typeDefStore.findTypeVerticesByCategory(TypeCategory.TRAIT);
while (vertices.hasNext()) {
ret.add(toClassificationDef(vertices.next()));
}
if (LOG.isDebugEnabled()) {
LOG.debug("<== AtlasClassificationDefStoreV1.getAll()");
LOG.debug("<== AtlasClassificationDefStoreV1.getAll(): count={}", ret.size());
}
return classificationDefs;
return ret;
}
@Override
......@@ -211,22 +206,17 @@ public class AtlasClassificationDefStoreV1 implements AtlasClassificationDefStor
LOG.debug("==> AtlasClassificationDefStoreV1.update({})", classificationDefs);
}
List<AtlasClassificationDef> updatedClassificationDefs = new ArrayList<>();
List<AtlasClassificationDef> ret = new ArrayList<>();
for (AtlasClassificationDef classificationDef : classificationDefs) {
try {
AtlasClassificationDef updatedDef = updateByName(classificationDef.getName(), classificationDef);
updatedClassificationDefs.add(updatedDef);
} catch (AtlasBaseException ex) {
LOG.error("Failed to update {}", classificationDef);
}
ret.add(updateByName(classificationDef.getName(), classificationDef));
}
if (LOG.isDebugEnabled()) {
LOG.debug("<== AtlasClassificationDefStoreV1.update({}): {}", classificationDefs, updatedClassificationDefs);
LOG.debug("<== AtlasClassificationDefStoreV1.update({}): {}", classificationDefs, ret);
}
return updatedClassificationDefs;
return ret;
}
@Override
......@@ -255,11 +245,7 @@ public class AtlasClassificationDefStoreV1 implements AtlasClassificationDefStor
}
for (String name : names) {
try {
deleteByName(name);
} catch (AtlasBaseException ex) {
LOG.error("Failed to delete {}", name);
}
}
if (LOG.isDebugEnabled()) {
......@@ -293,11 +279,7 @@ public class AtlasClassificationDefStoreV1 implements AtlasClassificationDefStor
}
for (String guid : guids) {
try {
deleteByGuid(guid);
} catch (AtlasBaseException ex) {
LOG.error("Failed to delete {}", guid);
}
}
if (LOG.isDebugEnabled()) {
......@@ -313,7 +295,6 @@ public class AtlasClassificationDefStoreV1 implements AtlasClassificationDefStor
}
List<AtlasClassificationDef> classificationDefs = new ArrayList<AtlasClassificationDef>();
Iterator<AtlasVertex> vertices = typeDefStore.findTypeVerticesByCategory(TypeCategory.TRAIT);
while(vertices.hasNext()) {
......@@ -321,14 +302,11 @@ public class AtlasClassificationDefStoreV1 implements AtlasClassificationDefStor
AtlasClassificationDef classificationDef = toClassificationDef(vertex);
if (classificationDef != null) {
classificationDefs.add(classificationDef); // TODO: add only if this passes filter
classificationDefs.add(classificationDef);
}
}
if (CollectionUtils.isNotEmpty(classificationDefs)) {
CollectionUtils.filter(classificationDefs, FilterUtil.getPredicateFromSearchFilter(filter));
}
AtlasClassificationDefs ret = new AtlasClassificationDefs(classificationDefs);
......@@ -339,8 +317,15 @@ public class AtlasClassificationDefStoreV1 implements AtlasClassificationDefStor
return ret;
}
private void toVertex(AtlasClassificationDef classificationDef, AtlasVertex vertex) {
AtlasStructDefStoreV1.toVertex(classificationDef, vertex, typeDefStore);
private void toVertex(AtlasClassificationDef classificationDef, AtlasVertex vertex) throws AtlasBaseException {
AtlasType type = typeRegistry.getType(classificationDef.getName());
if (type.getTypeCategory() != AtlasType.TypeCategory.CLASSIFICATION) {
throw new AtlasBaseException(classificationDef.getName() + ": not a classification type");
}
AtlasStructDefStoreV1.toVertex(classificationDef, (AtlasClassificationType)type,
vertex, typeDefStore, typeRegistry);
typeDefStore.createSuperTypeEdges(vertex, classificationDef.getSuperTypes());
}
......
repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityDefStoreV1.java
......@@ -21,11 +21,13 @@ import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.SearchFilter;
import org.apache.atlas.model.typedef.AtlasEntityDef;
import org.apache.atlas.model.typedef.AtlasEntityDef.AtlasEntityDefs;
import org.apache.atlas.model.typedef.AtlasStructDef;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.repository.store.graph.AtlasEntityDefStore;
import org.apache.atlas.repository.util.FilterUtil;
import org.apache.atlas.type.AtlasEntityType;
import org.apache.atlas.type.AtlasType;
import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.atlas.typesystem.types.DataTypes.TypeCategory;
import org.apache.commons.collections.CollectionUtils;
import org.slf4j.Logger;
......@@ -33,21 +35,16 @@ import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
/**
* EntityDef store in v1 format.
*/
public class AtlasEntityDefStoreV1 implements AtlasEntityDefStore {
public class AtlasEntityDefStoreV1 extends AtlasAbstractDefStoreV1 implements AtlasEntityDefStore {
private static final Logger LOG = LoggerFactory.getLogger(AtlasEntityDefStoreV1.class);
private final AtlasTypeDefGraphStoreV1 typeDefStore;
public AtlasEntityDefStoreV1(AtlasTypeDefGraphStoreV1 typeDefStore) {
super();
this.typeDefStore = typeDefStore;
public AtlasEntityDefStoreV1(AtlasTypeDefGraphStoreV1 typeDefStore, AtlasTypeRegistry typeRegistry) {
super(typeDefStore, typeRegistry);
}
@Override
......@@ -80,20 +77,15 @@ public class AtlasEntityDefStoreV1 implements AtlasEntityDefStore {
if (LOG.isDebugEnabled()) {
LOG.debug("==> AtlasEntityDefStoreV1.create({})", entityDefs);
}
List<AtlasEntityDef> entityDefList = new LinkedList<>();
for (AtlasEntityDef structDef : entityDefs) {
try {
AtlasEntityDef atlasEntityDef = create(structDef);
entityDefList.add(atlasEntityDef);
} catch (AtlasBaseException baseException) {
LOG.error("Failed to create {}", structDef);
LOG.error("Exception: {}", baseException);
}
List<AtlasEntityDef> ret = new ArrayList<>();
for (AtlasEntityDef entityDef : entityDefs) {
ret.add(create(entityDef));
}
if (LOG.isDebugEnabled()) {
LOG.debug("<== AtlasEntityDefStoreV1.create({}, {})", entityDefs, entityDefList);
LOG.debug("<== AtlasEntityDefStoreV1.create({}, {})", entityDefs, ret);
}
return entityDefList;
return ret;
}
@Override
......@@ -102,17 +94,18 @@ public class AtlasEntityDefStoreV1 implements AtlasEntityDefStore {
LOG.debug("==> AtlasEntityDefStoreV1.getAll()");
}
List<AtlasEntityDef> entityDefs = new LinkedList<>();
Iterator<AtlasVertex> verticesByCategory = typeDefStore.findTypeVerticesByCategory(TypeCategory.CLASS);
while (verticesByCategory.hasNext()) {
AtlasEntityDef atlasEntityDef = toEntityDef(verticesByCategory.next());
entityDefs.add(atlasEntityDef);
List<AtlasEntityDef> ret = new ArrayList<>();
Iterator<AtlasVertex> vertices = typeDefStore.findTypeVerticesByCategory(TypeCategory.CLASS);
while (vertices.hasNext()) {
ret.add(toEntityDef(vertices.next()));
}
if (LOG.isDebugEnabled()) {
LOG.debug("<== AtlasEntityDefStoreV1.getAll()");
LOG.debug("<== AtlasEntityDefStoreV1.getAll(): count={}", ret.size());
}
return entityDefs;
return ret;
}
@Override
......@@ -211,22 +204,17 @@ public class AtlasEntityDefStoreV1 implements AtlasEntityDefStore {
LOG.debug("==> AtlasEntityDefStoreV1.update({})", entityDefs);
}
List<AtlasEntityDef> updatedEntityDefs = new ArrayList<>();
List<AtlasEntityDef> ret = new ArrayList<>();
for (AtlasEntityDef entityDef : entityDefs) {
try {
AtlasEntityDef atlasEntityDef = updateByName(entityDef.getName(), entityDef);
updatedEntityDefs.add(atlasEntityDef);
} catch (AtlasBaseException ex) {
LOG.error("Failed to update {}", entityDef);
}
ret.add(updateByName(entityDef.getName(), entityDef));
}
if (LOG.isDebugEnabled()) {
LOG.debug("<== AtlasEntityDefStoreV1.update({}): {}", entityDefs, updatedEntityDefs);
LOG.debug("<== AtlasEntityDefStoreV1.update({}): {}", entityDefs, ret);
}
return updatedEntityDefs;
return ret;
}
@Override
......@@ -254,12 +242,8 @@ public class AtlasEntityDefStoreV1 implements AtlasEntityDefStore {
LOG.debug("==> AtlasEntityDefStoreV1.deleteByNames({})", names);
}
List<AtlasStructDef> updatedDefs = new ArrayList<>();
for (String name : names) {
try {
deleteByName(name);
} catch (AtlasBaseException ex) {}
}
if (LOG.isDebugEnabled()) {
......@@ -292,12 +276,8 @@ public class AtlasEntityDefStoreV1 implements AtlasEntityDefStore {
LOG.debug("==> AtlasEntityDefStoreV1.deleteByGuids({})", guids);
}
List<AtlasStructDef> updatedDefs = new ArrayList<>();
for (String guid : guids) {
try {
deleteByGuid(guid);
} catch (AtlasBaseException ex) {}
}
if (LOG.isDebugEnabled()) {
......@@ -312,7 +292,6 @@ public class AtlasEntityDefStoreV1 implements AtlasEntityDefStore {
}
List<AtlasEntityDef> entityDefs = new ArrayList<AtlasEntityDef>();
Iterator<AtlasVertex> vertices = typeDefStore.findTypeVerticesByCategory(TypeCategory.CLASS);
while(vertices.hasNext()) {
......@@ -324,10 +303,7 @@ public class AtlasEntityDefStoreV1 implements AtlasEntityDefStore {
}
}
if (CollectionUtils.isNotEmpty(entityDefs)) {
CollectionUtils.filter(entityDefs, FilterUtil.getPredicateFromSearchFilter(filter));
}
AtlasEntityDefs ret = new AtlasEntityDefs(entityDefs);
......@@ -338,8 +314,14 @@ public class AtlasEntityDefStoreV1 implements AtlasEntityDefStore {
return ret;
}
private void toVertex(AtlasEntityDef entityDef, AtlasVertex vertex) {
AtlasStructDefStoreV1.toVertex(entityDef, vertex, typeDefStore);
private void toVertex(AtlasEntityDef entityDef, AtlasVertex vertex) throws AtlasBaseException {
AtlasType type = typeRegistry.getType(entityDef.getName());
if (type.getTypeCategory() != AtlasType.TypeCategory.ENTITY) {
throw new AtlasBaseException(entityDef.getName() + ": not a entity type");
}
AtlasStructDefStoreV1.toVertex(entityDef, (AtlasEntityType)type, vertex, typeDefStore, typeRegistry);
typeDefStore.createSuperTypeEdges(vertex, entityDef.getSuperTypes());
}
......
repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasEnumDefStoreV1.java
......@@ -26,6 +26,7 @@ import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.repository.store.graph.AtlasEnumDefStore;
import org.apache.atlas.repository.util.FilterUtil;
import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.atlas.typesystem.types.DataTypes.TypeCategory;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
......@@ -34,21 +35,16 @@ import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
/**
* EnumDef store in v1 format.
*/
public class AtlasEnumDefStoreV1 implements AtlasEnumDefStore {
public class AtlasEnumDefStoreV1 extends AtlasAbstractDefStoreV1 implements AtlasEnumDefStore {
private static final Logger LOG = LoggerFactory.getLogger(AtlasEnumDefStoreV1.class);
private final AtlasTypeDefGraphStoreV1 typeDefStore;
public AtlasEnumDefStoreV1(AtlasTypeDefGraphStoreV1 typeDefStore) {
super();
this.typeDefStore = typeDefStore;
public AtlasEnumDefStoreV1(AtlasTypeDefGraphStoreV1 typeDefStore, AtlasTypeRegistry typeRegistry) {
super(typeDefStore, typeRegistry);
}
@Override
......@@ -77,24 +73,22 @@ public class AtlasEnumDefStoreV1 implements AtlasEnumDefStore {
}
@Override
public List<AtlasEnumDef> create(List<AtlasEnumDef> atlasEnumDefs) throws AtlasBaseException {
public List<AtlasEnumDef> create(List<AtlasEnumDef> enumDefs) throws AtlasBaseException {
if (LOG.isDebugEnabled()) {
LOG.debug("==> AtlasEnumDefStoreV1.create({})", atlasEnumDefs);
}
List<AtlasEnumDef> enumDefList = new LinkedList<>();
for (AtlasEnumDef enumDef : atlasEnumDefs) {
try {
AtlasEnumDef atlasEnumDef = create(enumDef);
enumDefList.add(atlasEnumDef);
} catch (AtlasBaseException baseException) {
LOG.error("Failed to create {}", enumDef);
LOG.error("Exception: {}", baseException);
LOG.debug("==> AtlasEnumDefStoreV1.create({})", enumDefs);
}
List<AtlasEnumDef> ret = new ArrayList<>();
for (AtlasEnumDef enumDef : enumDefs) {
ret.add(create(enumDef));
}
if (LOG.isDebugEnabled()) {
LOG.debug("<== AtlasEnumDefStoreV1.create({}, {})", atlasEnumDefs, enumDefList);
LOG.debug("<== AtlasEnumDefStoreV1.create({}): {}", enumDefs, ret);
}
return enumDefList;
return ret;
}
@Override
......@@ -103,17 +97,18 @@ public class AtlasEnumDefStoreV1 implements AtlasEnumDefStore {
LOG.debug("==> AtlasEnumDefStoreV1.getAll()");
}
List<AtlasEnumDef> enumDefs = new LinkedList<>();
Iterator<AtlasVertex> verticesByCategory = typeDefStore.findTypeVerticesByCategory(TypeCategory.ENUM);
while (verticesByCategory.hasNext()) {
AtlasEnumDef enumDef = toEnumDef(verticesByCategory.next());
enumDefs.add(enumDef);
List<AtlasEnumDef> ret = new ArrayList<>();
Iterator<AtlasVertex> vertices = typeDefStore.findTypeVerticesByCategory(TypeCategory.ENUM);
while (vertices.hasNext()) {
ret.add(toEnumDef(vertices.next()));
}
if (LOG.isDebugEnabled()) {
LOG.debug("<== AtlasEnumDefStoreV1.getAll()");
LOG.debug("<== AtlasEnumDefStoreV1.getAll(): count={}", ret.size());
}
return enumDefs;
return ret;
}
@Override
......@@ -212,22 +207,17 @@ public class AtlasEnumDefStoreV1 implements AtlasEnumDefStore {
LOG.debug("==> AtlasEnumDefStoreV1.update({})", enumDefs);
}
List<AtlasEnumDef> updatedEnumDefs = new ArrayList<>();
List<AtlasEnumDef> ret = new ArrayList<>();
for (AtlasEnumDef enumDef : enumDefs) {
try {
AtlasEnumDef updatedDef = updateByName(enumDef.getName(), enumDef);
updatedEnumDefs.add(updatedDef);
} catch (AtlasBaseException ex) {
LOG.error("Failed to update {}", enumDef);
}
ret.add(updateByName(enumDef.getName(), enumDef));
}
if (LOG.isDebugEnabled()) {
LOG.debug("<== AtlasEnumDefStoreV1.update({}): {}", enumDefs, updatedEnumDefs);
LOG.debug("<== AtlasEnumDefStoreV1.update({}): {}", enumDefs, ret);
}
return updatedEnumDefs;
return ret;
}
@Override
......@@ -256,11 +246,7 @@ public class AtlasEnumDefStoreV1 implements AtlasEnumDefStore {
}
for (String name : names) {
try {
deleteByName(name);
} catch (AtlasBaseException ex) {
LOG.error("Failed to delete {}", name);
}
}
if (LOG.isDebugEnabled()) {
......@@ -294,11 +280,7 @@ public class AtlasEnumDefStoreV1 implements AtlasEnumDefStore {
}
for (String guid : guids) {
try {
deleteByGuid(guid);
} catch (AtlasBaseException ex) {
LOG.error("Failed to delete {}", guid);
}
}
if (LOG.isDebugEnabled()) {
......@@ -313,7 +295,6 @@ public class AtlasEnumDefStoreV1 implements AtlasEnumDefStore {
}
List<AtlasEnumDef> enumDefs = new ArrayList<AtlasEnumDef>();
Iterator<AtlasVertex> vertices = typeDefStore.findTypeVerticesByCategory(TypeCategory.ENUM);
while(vertices.hasNext()) {
......@@ -325,9 +306,7 @@ public class AtlasEnumDefStoreV1 implements AtlasEnumDefStore {
}
}
if (CollectionUtils.isNotEmpty(enumDefs)) {
CollectionUtils.filter(enumDefs, FilterUtil.getPredicateFromSearchFilter(filter));
}
AtlasEnumDefs ret = new AtlasEnumDefs(enumDefs);
......
repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasStructDefStoreV1.java
......@@ -41,6 +41,7 @@ import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.atlas.type.AtlasTypeUtil;
import org.apache.atlas.typesystem.types.DataTypes.TypeCategory;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.compress.archivers.dump.DumpArchiveEntry;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
......@@ -48,7 +49,6 @@ import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;
......@@ -56,15 +56,11 @@ import java.util.Set;
/**
* StructDef store in v1 format.
*/
public class AtlasStructDefStoreV1 implements AtlasStructDefStore {
public class AtlasStructDefStoreV1 extends AtlasAbstractDefStoreV1 implements AtlasStructDefStore {
private static final Logger LOG = LoggerFactory.getLogger(AtlasStructDefStoreV1.class);
private final AtlasTypeDefGraphStoreV1 typeDefStore;
public AtlasStructDefStoreV1(AtlasTypeDefGraphStoreV1 typeDefStore) {
super();
this.typeDefStore = typeDefStore;
public AtlasStructDefStoreV1(AtlasTypeDefGraphStoreV1 typeDefStore, AtlasTypeRegistry typeRegistry) {
super(typeDefStore, typeRegistry);
}
@Override
......@@ -79,9 +75,15 @@ public class AtlasStructDefStoreV1 implements AtlasStructDefStore {
throw new AtlasBaseException(structDef.getName() + ": type already exists");
}
AtlasType type = typeRegistry.getType(structDef.getName());
if (type.getTypeCategory() != AtlasType.TypeCategory.STRUCT) {
throw new AtlasBaseException(structDef.getName() + ": not a struct type");
}
vertex = typeDefStore.createTypeVertex(structDef);
toVertex(structDef, vertex, typeDefStore);
toVertex(structDef, (AtlasStructType)type, vertex, typeDefStore, typeRegistry);
AtlasStructDef ret = toStructDef(vertex);
......@@ -97,20 +99,17 @@ public class AtlasStructDefStoreV1 implements AtlasStructDefStore {
if (LOG.isDebugEnabled()) {
LOG.debug("==> AtlasStructDefStoreV1.create({})", structDefs);
}
List<AtlasStructDef> structDefList = new LinkedList<>();
List<AtlasStructDef> ret = new ArrayList<>();
for (AtlasStructDef structDef : structDefs) {
try {
AtlasStructDef atlasStructDef = create(structDef);
structDefList.add(atlasStructDef);
} catch (AtlasBaseException baseException) {
LOG.error("Failed to create {}", structDef);
LOG.error("Exception: {}", baseException);
}
ret.add(create(structDef));
}
if (LOG.isDebugEnabled()) {
LOG.debug("<== AtlasStructDefStoreV1.create({}, {})", structDefs, structDefList);
LOG.debug("<== AtlasStructDefStoreV1.create({}, {})", structDefs, ret);
}
return structDefList;
return ret;
}
@Override
......@@ -119,17 +118,17 @@ public class AtlasStructDefStoreV1 implements AtlasStructDefStore {
LOG.debug("==> AtlasStructDefStoreV1.getAll()");
}
List<AtlasStructDef> structDefs = new LinkedList<>();
Iterator<AtlasVertex> verticesByCategory = typeDefStore.findTypeVerticesByCategory(TypeCategory.STRUCT);
while (verticesByCategory.hasNext()) {
AtlasStructDef atlasStructDef = toStructDef(verticesByCategory.next());
structDefs.add(atlasStructDef);
List<AtlasStructDef> ret = new ArrayList<>();
Iterator<AtlasVertex> vertices = typeDefStore.findTypeVerticesByCategory(TypeCategory.STRUCT);
while (vertices.hasNext()) {
ret.add(toStructDef(vertices.next()));
}
if (LOG.isDebugEnabled()) {
LOG.debug("<== AtlasStructDefStoreV1.getAll()");
LOG.debug("<== AtlasStructDefStoreV1.getAll(): count={}", ret.size());
}
return structDefs;
return ret;
}
@Override
......@@ -188,7 +187,13 @@ public class AtlasStructDefStoreV1 implements AtlasStructDefStore {
throw new AtlasBaseException("no structDef exists with name " + name);
}
toVertex(structDef, vertex);
AtlasType type = typeRegistry.getType(structDef.getName());
if (type.getTypeCategory() != AtlasType.TypeCategory.STRUCT) {
throw new AtlasBaseException(structDef.getName() + ": not a struct type");
}
toVertex(structDef, (AtlasStructType)type, vertex);
AtlasStructDef ret = toStructDef(vertex);
......@@ -211,7 +216,13 @@ public class AtlasStructDefStoreV1 implements AtlasStructDefStore {
throw new AtlasBaseException("no structDef exists with guid " + guid);
}
toVertex(structDef, vertex);
AtlasType type = typeRegistry.getType(structDef.getName());
if (type.getTypeCategory() != AtlasType.TypeCategory.STRUCT) {
throw new AtlasBaseException(structDef.getName() + ": not a struct type");
}
toVertex(structDef, (AtlasStructType)type, vertex);
AtlasStructDef ret = toStructDef(vertex);
......@@ -228,20 +239,17 @@ public class AtlasStructDefStoreV1 implements AtlasStructDefStore {
LOG.debug("==> AtlasStructDefStoreV1.update({})", structDefs);
}
List<AtlasStructDef> updatedDefs = new ArrayList<>();
List<AtlasStructDef> ret = new ArrayList<>();
for (AtlasStructDef structDef : structDefs) {
try {
AtlasStructDef updatedDef = updateByName(structDef.getName(), structDef);
updatedDefs.add(updatedDef);
} catch (AtlasBaseException ex) {}
ret.add(updateByName(structDef.getName(), structDef));
}
if (LOG.isDebugEnabled()) {
LOG.debug("<== AtlasStructDefStoreV1.update({}): {}", structDefs, updatedDefs);
LOG.debug("<== AtlasStructDefStoreV1.update({}): {}", structDefs, ret);
}
return updatedDefs;
return ret;
}
@Override
......@@ -269,12 +277,8 @@ public class AtlasStructDefStoreV1 implements AtlasStructDefStore {
LOG.debug("==> AtlasStructDefStoreV1.deleteByNames({})", names);
}
List<AtlasStructDef> updatedDefs = new ArrayList<>();
for (String name : names) {
try {
deleteByName(name);
} catch (AtlasBaseException ex) {}
}
if (LOG.isDebugEnabled()) {
......@@ -307,12 +311,8 @@ public class AtlasStructDefStoreV1 implements AtlasStructDefStore {
LOG.debug("==> AtlasStructDefStoreV1.deleteByGuids({})", guids);
}
List<AtlasStructDef> updatedDefs = new ArrayList<>();
for (String guid : guids) {
try {
deleteByGuid(guid);
} catch (AtlasBaseException ex) {}
}
if (LOG.isDebugEnabled()) {
......@@ -327,21 +327,18 @@ public class AtlasStructDefStoreV1 implements AtlasStructDefStore {
}
List<AtlasStructDef> structDefs = new ArrayList<AtlasStructDef>();
Iterator<AtlasVertex> vertices = typeDefStore.findTypeVerticesByCategory(TypeCategory.STRUCT);
while(vertices.hasNext()) {
AtlasVertex AtlasVertex = vertices.next();
AtlasStructDef structDef = toStructDef(AtlasVertex);
while (vertices.hasNext()) {
AtlasVertex vertex = vertices.next();
AtlasStructDef structDef = toStructDef(vertex);
if (structDef != null) {
structDefs.add(structDef);
}
}
if (CollectionUtils.isNotEmpty(structDefs)) {
CollectionUtils.filter(structDefs, FilterUtil.getPredicateFromSearchFilter(filter));
}
AtlasStructDefs ret = new AtlasStructDefs(structDefs);
......@@ -352,8 +349,8 @@ public class AtlasStructDefStoreV1 implements AtlasStructDefStore {
return ret;
}
private void toVertex(AtlasStructDef structDef, AtlasVertex vertex) {
toVertex(structDef, vertex, typeDefStore);
private void toVertex(AtlasStructDef structDef, AtlasStructType structType, AtlasVertex vertex) {
toVertex(structDef, structType, vertex, typeDefStore, typeRegistry);
}
private AtlasStructDef toStructDef(AtlasVertex vertex) throws AtlasBaseException {
......@@ -366,15 +363,14 @@ public class AtlasStructDefStoreV1 implements AtlasStructDefStore {
return ret;
}
public static void toVertex(AtlasStructDef structDef, AtlasVertex vertex, AtlasTypeDefGraphStoreV1 typeDefStore) {
AtlasTypeRegistry typeRegistry = typeDefStore.getTypeRegistry();
AtlasStructType structType = (AtlasStructType)typeRegistry.getType(structDef.getName());
public static void toVertex(AtlasStructDef structDef, AtlasStructType structType, AtlasVertex vertex,
AtlasTypeDefGraphStoreV1 typeDefStore, AtlasTypeRegistry typeRegistry) {
List<String> attrNames = new ArrayList<>(structDef.getAttributeDefs().size());
for (AtlasAttributeDef attributeDef : structDef.getAttributeDefs()) {
String propertyKey = AtlasGraphUtilsV1.getPropertyKey(structDef, attributeDef.getName());
AtlasGraphUtilsV1.setProperty(vertex, propertyKey, toJsonFromAttributeDef(attributeDef, structType, typeRegistry));
AtlasGraphUtilsV1.setProperty(vertex, propertyKey, toJsonFromAttributeDef(attributeDef, structType));
attrNames.add(attributeDef.getName());
addReferencesForAttribute(vertex, attributeDef, typeDefStore);
......@@ -382,8 +378,7 @@ public class AtlasStructDefStoreV1 implements AtlasStructDefStore {
AtlasGraphUtilsV1.setProperty(vertex, AtlasGraphUtilsV1.getPropertyKey(structDef), attrNames);
}
public static AtlasStructDef toStructDef(AtlasVertex vertex,
AtlasStructDef structDef,
public static AtlasStructDef toStructDef(AtlasVertex vertex, AtlasStructDef structDef,
AtlasTypeDefGraphStoreV1 typeDefStore) throws AtlasBaseException {
AtlasStructDef ret = (structDef != null) ? structDef :new AtlasStructDef();
......@@ -406,8 +401,7 @@ public class AtlasStructDefStoreV1 implements AtlasStructDefStore {
return ret;
}
private static void addReferencesForAttribute(AtlasVertex vertex,
AtlasAttributeDef attributeDef,
private static void addReferencesForAttribute(AtlasVertex vertex, AtlasAttributeDef attributeDef,
AtlasTypeDefGraphStoreV1 typeDefStore) {
Set<String> referencedTypeNames = AtlasTypeUtil.getReferencedTypeNames(attributeDef.getTypeName());
......@@ -415,36 +409,34 @@ public class AtlasStructDefStoreV1 implements AtlasStructDefStore {
for (String referencedTypeName : referencedTypeNames) {
if (!AtlasTypeUtil.isBuiltInType(referencedTypeName)) {
AtlasVertex referencedTypeAtlasVertex = typeDefStore.findTypeVertexByName(referencedTypeName);
AtlasVertex referencedTypeVertex = typeDefStore.findTypeVertexByName(referencedTypeName);
if (referencedTypeAtlasVertex == null) {
// create atlasVertex?
if (referencedTypeVertex == null) {
// create vertex?
}
if (referencedTypeAtlasVertex != null) {
if (referencedTypeVertex != null) {
String label = AtlasGraphUtilsV1.getEdgeLabel(typeName, attributeDef.getName());
typeDefStore.getOrCreateEdge(vertex, referencedTypeAtlasVertex, label);
typeDefStore.getOrCreateEdge(vertex, referencedTypeVertex, label);
}
}
}
}
private static String toJsonFromAttributeDef(AtlasAttributeDef attributeDef,
AtlasStructType structType,
AtlasTypeRegistry typeRegistry) {
private static String toJsonFromAttributeDef(AtlasAttributeDef attributeDef, AtlasStructType structType) {
boolean isForeignKey = structType.isForeignKeyAttribute(attributeDef.getName());
boolean isMappedFromRef = structType.isMappedFromRefAttribute(attributeDef.getName());
String reverseAttribName = null;
if (isForeignKey) { // check if the referenced entity has foreignKeyRef to this attribute
AtlasType attribType = typeRegistry.getType(attributeDef.getTypeName());
AtlasType attribType = structType.getAttributeType(attributeDef.getTypeName());
if (attribType instanceof AtlasArrayType) {
if (attribType.getTypeCategory() == AtlasType.TypeCategory.ARRAY) {
attribType = ((AtlasArrayType)attribType).getElementType();
}
if (attribType instanceof AtlasEntityType) {
if (attribType.getTypeCategory() == AtlasType.TypeCategory.ENTITY) {
reverseAttribName = ((AtlasStructType)attribType).getMappedFromRefAttribute(structType.getTypeName(),
attributeDef.getName());
}
......@@ -491,7 +483,7 @@ public class AtlasStructDefStoreV1 implements AtlasStructDefStore {
}
}
if (! AtlasTypeUtil.isBuiltInType(attrTypeName)) {
if (!AtlasTypeUtil.isBuiltInType(attrTypeName)) {
AtlasVertex attributeType = typeDefStore.findTypeVertexByName(attrTypeName);
// check for isComposite/reverseAttributeName for entity types
......@@ -499,16 +491,13 @@ public class AtlasStructDefStoreV1 implements AtlasStructDefStore {
String reverseAttribName = (String) attribInfo.get("reverseAttributeName");
Boolean isComposite = (Boolean) attribInfo.get("isComposite");
// find the attribute that refers to this attribute
if (StringUtils.isNotBlank(reverseAttribName) || isComposite) {
if (AtlasTypeUtil.isMapType(attrTypeName)) {
throw new AtlasBaseException(structDef.getName() + "." + ret.getName()
+ ": constraints not supported on map type " + attrTypeName);
}
if (attributeType != null) {
String refAttributeName = null;
List<String> attrNames = attributeType.getProperty(
AtlasGraphUtilsV1.getPropertyKey(attrTypeName), List.class);
......@@ -550,15 +539,12 @@ public class AtlasStructDefStoreV1 implements AtlasStructDefStore {
}
}
}
}
Map multiplicity = AtlasType.fromJson((String) attribInfo.get("multiplicity"), Map.class);
Number minCount = (Number) multiplicity.get("lower");
Number maxCount = (Number) multiplicity.get("upper");
Boolean isUnique = (Boolean) multiplicity.get("isUnique");
if (minCount == null || minCount.intValue() == 0) {
ret.setOptional(true);
ret.setValuesMinCount(0);
......
repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasTypeDefGraphStoreV1.java
......@@ -19,6 +19,7 @@ package org.apache.atlas.repository.store.graph.v1;
import com.google.common.base.Preconditions;
import com.google.inject.Inject;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.typedef.AtlasBaseTypeDef;
import org.apache.atlas.model.typedef.AtlasClassificationDef;
......@@ -31,7 +32,12 @@ import org.apache.atlas.repository.graphdb.AtlasEdge;
import org.apache.atlas.repository.graphdb.AtlasEdgeDirection;
import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.repository.store.graph.AtlasClassificationDefStore;
import org.apache.atlas.repository.store.graph.AtlasEntityDefStore;
import org.apache.atlas.repository.store.graph.AtlasEnumDefStore;
import org.apache.atlas.repository.store.graph.AtlasStructDefStore;
import org.apache.atlas.repository.store.graph.AtlasTypeDefGraphStore;
import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.atlas.typesystem.types.DataTypes.TypeCategory;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
......@@ -57,8 +63,9 @@ public class AtlasTypeDefGraphStoreV1 extends AtlasTypeDefGraphStore {
protected final AtlasGraph atlasGraph = AtlasGraphProvider.getGraphInstance();
public AtlasTypeDefGraphStoreV1() {
super();
@Inject
public AtlasTypeDefGraphStoreV1(AtlasTypeRegistry typeRegistry) {
super(typeRegistry);
LOG.info("==> AtlasTypeDefGraphStoreV1()");
......@@ -72,16 +79,31 @@ public class AtlasTypeDefGraphStoreV1 extends AtlasTypeDefGraphStore {
}
@Override
protected AtlasEnumDefStore getEnumDefStore(AtlasTypeRegistry typeRegistry) {
return new AtlasEnumDefStoreV1(this, typeRegistry);
}
@Override
protected AtlasStructDefStore getStructDefStore(AtlasTypeRegistry typeRegistry) {
return new AtlasStructDefStoreV1(this, typeRegistry);
}
@Override
protected AtlasClassificationDefStore getClassificationDefStore(AtlasTypeRegistry typeRegistry) {
return new AtlasClassificationDefStoreV1(this, typeRegistry);
}
@Override
protected AtlasEntityDefStore getEntityDefStore(AtlasTypeRegistry typeRegistry) {
return new AtlasEntityDefStoreV1(this, typeRegistry);
}
@Override
public void init() throws AtlasBaseException {
LOG.info("==> AtlasTypeDefGraphStoreV1.init()");
super.init();
super.init(new AtlasEnumDefStoreV1(this),
new AtlasStructDefStoreV1(this),
new AtlasClassificationDefStoreV1(this),
new AtlasEntityDefStoreV1(this));
LOG.info("<== AtlasTypeDefGraphStoreV1.init()");
}
......
webapp/src/main/java/org/apache/atlas/web/rest/TypesREST.java
......@@ -64,7 +64,7 @@ public class TypesREST {
private HttpServletRequest httpServletRequest;
@Inject
public TypesREST(AtlasTypeDefStore typeDefStore, AtlasTypeRegistry atlasTypeRegistry) {
public TypesREST(AtlasTypeDefStore typeDefStore) {
LOG.info("new TypesREST");
this.typeDefStore = typeDefStore;
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment