Skip to content

A
atlas
Commit 6e7aa6ed by Sarath Subramanian
Options

ATLAS-2662: Remove complex array and map attribute's edge information from entity vertex

parent a8fab3e7
Showing with 1283 additions and 720 deletions
common/src/main/java/org/apache/atlas/repository/Constants.java
...@@ -126,6 +126,8 @@ public final class Constants { ...@@ -126,6 +126,8 @@ public final class Constants {
public static final String CLASSIFICATION_EDGE_STATE_PROPERTY_KEY = STATE_PROPERTY_KEY; public static final String CLASSIFICATION_EDGE_STATE_PROPERTY_KEY = STATE_PROPERTY_KEY;
public static final String CLASSIFICATION_LABEL = "classifiedAs"; public static final String CLASSIFICATION_LABEL = "classifiedAs";
public static final String TERM_ASSIGNMENT_LABEL = "r:AtlasGlossarySemanticAssignment"; public static final String TERM_ASSIGNMENT_LABEL = "r:AtlasGlossarySemanticAssignment";
public static final String ATTRIBUTE_INDEX_PROPERTY_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "index";
public static final String ATTRIBUTE_KEY_PROPERTY_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "key";
public static final String VERTEX_ID_IN_IMPORT_KEY = "__vIdInImport"; public static final String VERTEX_ID_IN_IMPORT_KEY = "__vIdInImport";
public static final String EDGE_ID_IN_IMPORT_KEY = "__eIdInImport"; public static final String EDGE_ID_IN_IMPORT_KEY = "__eIdInImport";
......
intg/src/main/java/org/apache/atlas/AtlasErrorCode.java
...@@ -142,6 +142,7 @@ public enum AtlasErrorCode { ...@@ -142,6 +142,7 @@ public enum AtlasErrorCode {
RELATIONSHIP_END_IS_NULL(400, "ATLAS-400-00-07D", "Relationship end is invalid. Expected {0} but is NULL"), RELATIONSHIP_END_IS_NULL(400, "ATLAS-400-00-07D", "Relationship end is invalid. Expected {0} but is NULL"),
INVALID_TERM_RELATION_TO_SELF(400, "ATLAS-400-00-07E", "Invalid Term relationship: Term can't have a relationship with self"), INVALID_TERM_RELATION_TO_SELF(400, "ATLAS-400-00-07E", "Invalid Term relationship: Term can't have a relationship with self"),
INVALID_CHILD_CATEGORY_DIFFERENT_GLOSSARY(400, "ATLAS-400-00-07F", "Invalid child category relationship: Child category (guid = {0}) belongs to different glossary"), INVALID_CHILD_CATEGORY_DIFFERENT_GLOSSARY(400, "ATLAS-400-00-07F", "Invalid child category relationship: Child category (guid = {0}) belongs to different glossary"),
ATTRIBUTE_TYPE_INVALID(400, "ATLAS-400-00-080", "{0}.{1}: invalid attribute type. Attribute cannot be of type classification"),
UNAUTHORIZED_ACCESS(403, "ATLAS-403-00-001", "{0} is not authorized to perform {1}"), UNAUTHORIZED_ACCESS(403, "ATLAS-403-00-001", "{0} is not authorized to perform {1}"),
......
intg/src/main/java/org/apache/atlas/model/instance/AtlasObjectId.java
...@@ -34,6 +34,8 @@ import javax.xml.bind.annotation.XmlSeeAlso; ...@@ -34,6 +34,8 @@ import javax.xml.bind.annotation.XmlSeeAlso;
import org.apache.atlas.model.PList; import org.apache.atlas.model.PList;
import org.apache.atlas.model.SearchFilter.SortType; import org.apache.atlas.model.SearchFilter.SortType;
import org.apache.atlas.model.typedef.AtlasBaseTypeDef; import org.apache.atlas.model.typedef.AtlasBaseTypeDef;
import org.apache.commons.lang.StringUtils;
import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.NONE; import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.NONE;
import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.PUBLIC_ONLY; import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.PUBLIC_ONLY;
...@@ -163,13 +165,12 @@ public class AtlasObjectId implements Serializable { ...@@ -163,13 +165,12 @@ public class AtlasObjectId implements Serializable {
AtlasObjectId that = (AtlasObjectId) o; AtlasObjectId that = (AtlasObjectId) o;
// if guid is null, equality should be based on typeName/uniqueAttributes // if guid is empty/null, equality should be based on typeName/uniqueAttributes
if (guid != null && Objects.equals(guid, that.guid)) { if (StringUtils.isEmpty(guid) && StringUtils.isEmpty(that.guid)) {
return true; return Objects.equals(typeName, that.typeName) && Objects.equals(uniqueAttributes, that.uniqueAttributes);
} else {
return Objects.equals(guid, that.guid);
} }
return Objects.equals(typeName, that.typeName) &&
Objects.equals(uniqueAttributes, that.uniqueAttributes);
} }
@Override @Override
......
intg/src/main/java/org/apache/atlas/type/AtlasStructType.java
...@@ -98,6 +98,17 @@ public class AtlasStructType extends AtlasType { ...@@ -98,6 +98,17 @@ public class AtlasStructType extends AtlasType {
arrayType.setMaxCount(attributeDef.getValuesMaxCount()); arrayType.setMaxCount(attributeDef.getValuesMaxCount());
} }
//check if attribute type is not classification
if (attrType instanceof AtlasArrayType) {
attrType = ((AtlasArrayType) attrType).getElementType();
} else if (attrType instanceof AtlasMapType) {
attrType = ((AtlasMapType) attrType).getValueType();
}
if (attrType instanceof AtlasClassificationType) {
throw new AtlasBaseException(AtlasErrorCode.ATTRIBUTE_TYPE_INVALID, getTypeName(), attributeDef.getName());
}
a.put(attributeDef.getName(), attribute); a.put(attributeDef.getName(), attribute);
} }
......
intg/src/test/java/org/apache/atlas/TestUtilsV2.java
...@@ -30,7 +30,6 @@ import org.apache.atlas.model.typedef.AtlasEnumDef; ...@@ -30,7 +30,6 @@ import org.apache.atlas.model.typedef.AtlasEnumDef;
import org.apache.atlas.model.typedef.AtlasEnumDef.AtlasEnumElementDef; import org.apache.atlas.model.typedef.AtlasEnumDef.AtlasEnumElementDef;
import org.apache.atlas.model.typedef.AtlasStructDef; import org.apache.atlas.model.typedef.AtlasStructDef;
import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef; import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef;
import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef.Cardinality;
import org.apache.atlas.model.typedef.AtlasStructDef.AtlasConstraintDef; import org.apache.atlas.model.typedef.AtlasStructDef.AtlasConstraintDef;
import org.apache.atlas.model.typedef.AtlasTypesDef; import org.apache.atlas.model.typedef.AtlasTypesDef;
import org.apache.atlas.type.AtlasTypeUtil; import org.apache.atlas.type.AtlasTypeUtil;
...@@ -48,7 +47,17 @@ import java.util.List; ...@@ -48,7 +47,17 @@ import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicInteger;
import static org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef.Cardinality.LIST;
import static org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef.Cardinality.SINGLE;
import static org.apache.atlas.model.typedef.AtlasStructDef.AtlasConstraintDef.CONSTRAINT_PARAM_ATTRIBUTE;
import static org.apache.atlas.model.typedef.AtlasStructDef.AtlasConstraintDef.CONSTRAINT_TYPE_INVERSE_REF;
import static org.apache.atlas.model.typedef.AtlasStructDef.AtlasConstraintDef.CONSTRAINT_TYPE_OWNED_REF;
import static org.apache.atlas.type.AtlasTypeUtil.createClassTypeDef;
import static org.apache.atlas.type.AtlasTypeUtil.createOptionalAttrDef;
import static org.apache.atlas.type.AtlasTypeUtil.createRequiredAttrDef;
import static org.apache.atlas.type.AtlasTypeUtil.createStructTypeDef; import static org.apache.atlas.type.AtlasTypeUtil.createStructTypeDef;
import static org.apache.atlas.type.AtlasTypeUtil.createUniqueRequiredAttrDef;
import static org.apache.atlas.type.AtlasTypeUtil.getAtlasObjectId;
/** /**
...@@ -59,6 +68,8 @@ public final class TestUtilsV2 { ...@@ -59,6 +68,8 @@ public final class TestUtilsV2 {
public static final long TEST_DATE_IN_LONG = 1418265358440L; public static final long TEST_DATE_IN_LONG = 1418265358440L;
public static final String TEST_USER = "testUser"; public static final String TEST_USER = "testUser";
public static final String STRUCT_TYPE = "struct_type";
public static final String ENTITY_TYPE = "entity_type";
private static AtomicInteger seq = new AtomicInteger(); private static AtomicInteger seq = new AtomicInteger();
...@@ -85,68 +96,68 @@ public final class TestUtilsV2 { ...@@ -85,68 +96,68 @@ public final class TestUtilsV2 {
AtlasStructDef addressDetails = AtlasStructDef addressDetails =
createStructTypeDef("Address", "Address"+_description, createStructTypeDef("Address", "Address"+_description,
AtlasTypeUtil.createRequiredAttrDef("street", "string"), createRequiredAttrDef("street", "string"),
AtlasTypeUtil.createRequiredAttrDef("city", "string")); createRequiredAttrDef("city", "string"));
AtlasEntityDef deptTypeDef = AtlasEntityDef deptTypeDef =
AtlasTypeUtil.createClassTypeDef(DEPARTMENT_TYPE, "Department"+_description, Collections.<String>emptySet(), createClassTypeDef(DEPARTMENT_TYPE, "Department"+_description, Collections.<String>emptySet(),
AtlasTypeUtil.createUniqueRequiredAttrDef("name", "string"), createUniqueRequiredAttrDef("name", "string"),
new AtlasAttributeDef("employees", String.format("array<%s>", "Employee"), true, new AtlasAttributeDef("employees", String.format("array<%s>", "Employee"), true,
AtlasAttributeDef.Cardinality.SINGLE, 0, 1, false, false, false, SINGLE, 0, 1, false, false, false,
new ArrayList<AtlasStructDef.AtlasConstraintDef>() {{ new ArrayList<AtlasStructDef.AtlasConstraintDef>() {{
add(new AtlasStructDef.AtlasConstraintDef(AtlasConstraintDef.CONSTRAINT_TYPE_OWNED_REF)); add(new AtlasStructDef.AtlasConstraintDef(CONSTRAINT_TYPE_OWNED_REF));
}})); }}));
AtlasEntityDef personTypeDef = AtlasTypeUtil.createClassTypeDef("Person", "Person"+_description, Collections.<String>emptySet(), AtlasEntityDef personTypeDef = createClassTypeDef("Person", "Person"+_description, Collections.<String>emptySet(),
AtlasTypeUtil.createUniqueRequiredAttrDef("name", "string"), createUniqueRequiredAttrDef("name", "string"),
AtlasTypeUtil.createOptionalAttrDef("address", "Address"), createOptionalAttrDef("address", "Address"),
AtlasTypeUtil.createOptionalAttrDef("birthday", "date"), createOptionalAttrDef("birthday", "date"),
AtlasTypeUtil.createOptionalAttrDef("hasPets", "boolean"), createOptionalAttrDef("hasPets", "boolean"),
AtlasTypeUtil.createOptionalAttrDef("numberOfCars", "byte"), createOptionalAttrDef("numberOfCars", "byte"),
AtlasTypeUtil.createOptionalAttrDef("houseNumber", "short"), createOptionalAttrDef("houseNumber", "short"),
AtlasTypeUtil.createOptionalAttrDef("carMileage", "int"), createOptionalAttrDef("carMileage", "int"),
AtlasTypeUtil.createOptionalAttrDef("age", "float"), createOptionalAttrDef("age", "float"),
AtlasTypeUtil.createOptionalAttrDef("numberOfStarsEstimate", "biginteger"), createOptionalAttrDef("numberOfStarsEstimate", "biginteger"),
AtlasTypeUtil.createOptionalAttrDef("approximationOfPi", "bigdecimal") createOptionalAttrDef("approximationOfPi", "bigdecimal")
); );
AtlasEntityDef employeeTypeDef = AtlasTypeUtil.createClassTypeDef("Employee", "Employee"+_description, Collections.singleton("Person"), AtlasEntityDef employeeTypeDef = createClassTypeDef("Employee", "Employee"+_description, Collections.singleton("Person"),
AtlasTypeUtil.createOptionalAttrDef("orgLevel", "OrgLevel"), createOptionalAttrDef("orgLevel", "OrgLevel"),
new AtlasAttributeDef("department", "Department", false, new AtlasAttributeDef("department", "Department", false,
AtlasAttributeDef.Cardinality.SINGLE, 1, 1, SINGLE, 1, 1,
false, false, false, false, false, false,
new ArrayList<AtlasConstraintDef>()), new ArrayList<>()),
new AtlasAttributeDef("manager", "Manager", true, new AtlasAttributeDef("manager", "Manager", true,
AtlasAttributeDef.Cardinality.SINGLE, 0, 1, SINGLE, 0, 1,
false, false, false, false, false, false,
new ArrayList<AtlasConstraintDef>() {{ new ArrayList<AtlasConstraintDef>() {{
add(new AtlasConstraintDef( add(new AtlasConstraintDef(
AtlasConstraintDef.CONSTRAINT_TYPE_INVERSE_REF, new HashMap<String, Object>() {{ CONSTRAINT_TYPE_INVERSE_REF, new HashMap<String, Object>() {{
put(AtlasConstraintDef.CONSTRAINT_PARAM_ATTRIBUTE, "subordinates"); put(CONSTRAINT_PARAM_ATTRIBUTE, "subordinates");
}})); }}));
}}), }}),
new AtlasAttributeDef("mentor", EMPLOYEE_TYPE, true, new AtlasAttributeDef("mentor", EMPLOYEE_TYPE, true,
AtlasAttributeDef.Cardinality.SINGLE, 0, 1, SINGLE, 0, 1,
false, false, false, false, false, false,
Collections.<AtlasConstraintDef>emptyList()), Collections.<AtlasConstraintDef>emptyList()),
AtlasTypeUtil.createOptionalAttrDef("shares", "long"), createOptionalAttrDef("shares", "long"),
AtlasTypeUtil.createOptionalAttrDef("salary", "double") createOptionalAttrDef("salary", "double")
); );
employeeTypeDef.getAttribute("department").addConstraint( employeeTypeDef.getAttribute("department").addConstraint(
new AtlasConstraintDef( new AtlasConstraintDef(
AtlasConstraintDef.CONSTRAINT_TYPE_INVERSE_REF, new HashMap<String, Object>() {{ CONSTRAINT_TYPE_INVERSE_REF, new HashMap<String, Object>() {{
put(AtlasConstraintDef.CONSTRAINT_PARAM_ATTRIBUTE, "employees"); put(CONSTRAINT_PARAM_ATTRIBUTE, "employees");
}})); }}));
AtlasEntityDef managerTypeDef = AtlasTypeUtil.createClassTypeDef("Manager", "Manager"+_description, Collections.singleton("Employee"), AtlasEntityDef managerTypeDef = createClassTypeDef("Manager", "Manager"+_description, Collections.singleton("Employee"),
new AtlasAttributeDef("subordinates", String.format("array<%s>", "Employee"), false, AtlasAttributeDef.Cardinality.SET, new AtlasAttributeDef("subordinates", String.format("array<%s>", "Employee"), false, AtlasAttributeDef.Cardinality.SET,
1, 10, false, false, false, 1, 10, false, false, false,
Collections.<AtlasConstraintDef>emptyList())); Collections.<AtlasConstraintDef>emptyList()));
AtlasClassificationDef securityClearanceTypeDef = AtlasClassificationDef securityClearanceTypeDef =
AtlasTypeUtil.createTraitTypeDef("SecurityClearance", "SecurityClearance"+_description, Collections.<String>emptySet(), AtlasTypeUtil.createTraitTypeDef("SecurityClearance", "SecurityClearance"+_description, Collections.<String>emptySet(),
AtlasTypeUtil.createRequiredAttrDef("level", "int")); createRequiredAttrDef("level", "int"));
AtlasTypesDef ret = new AtlasTypesDef(Collections.singletonList(orgLevelEnum), Collections.singletonList(addressDetails), AtlasTypesDef ret = new AtlasTypesDef(Collections.singletonList(orgLevelEnum), Collections.singletonList(addressDetails),
Collections.singletonList(securityClearanceTypeDef), Collections.singletonList(securityClearanceTypeDef),
...@@ -158,27 +169,27 @@ public final class TestUtilsV2 { ...@@ -158,27 +169,27 @@ public final class TestUtilsV2 {
} }
public static AtlasTypesDef defineInverseReferenceTestTypes() { public static AtlasTypesDef defineInverseReferenceTestTypes() {
AtlasEntityDef aDef = AtlasTypeUtil.createClassTypeDef("A", Collections.<String>emptySet(), AtlasEntityDef aDef = createClassTypeDef("A", Collections.<String>emptySet(),
AtlasTypeUtil.createUniqueRequiredAttrDef("name", "string"), createUniqueRequiredAttrDef("name", "string"),
new AtlasAttributeDef("b", "B", true, Cardinality.SINGLE, 0, 1, false, false, false, Collections.<AtlasConstraintDef>emptyList()), // 1-1 new AtlasAttributeDef("b", "B", true, SINGLE, 0, 1, false, false, false, Collections.<AtlasConstraintDef>emptyList()), // 1-1
new AtlasAttributeDef("oneB", "B", true, Cardinality.SINGLE, 0, 1, false, false, false, Collections.<AtlasConstraintDef>emptyList()), // 1-* new AtlasAttributeDef("oneB", "B", true, SINGLE, 0, 1, false, false, false, Collections.<AtlasConstraintDef>emptyList()), // 1-*
new AtlasAttributeDef("manyB", AtlasBaseTypeDef.getArrayTypeName("B"), true, Cardinality.SINGLE, 0, 1, false, false, false, Collections.<AtlasConstraintDef>emptyList()), new AtlasAttributeDef("manyB", AtlasBaseTypeDef.getArrayTypeName("B"), true, SINGLE, 0, 1, false, false, false, Collections.<AtlasConstraintDef>emptyList()),
new AtlasAttributeDef("mapToB", AtlasBaseTypeDef.getMapTypeName("string", "B"), true, Cardinality.SINGLE, 0, 1, false, false, false, new AtlasAttributeDef("mapToB", AtlasBaseTypeDef.getMapTypeName("string", "B"), true, SINGLE, 0, 1, false, false, false,
Collections.<AtlasConstraintDef>singletonList(new AtlasConstraintDef( Collections.<AtlasConstraintDef>singletonList(new AtlasConstraintDef(
AtlasConstraintDef.CONSTRAINT_TYPE_INVERSE_REF, Collections.<String, Object>singletonMap(AtlasConstraintDef.CONSTRAINT_PARAM_ATTRIBUTE, "mappedFromA"))))); // *-* CONSTRAINT_TYPE_INVERSE_REF, Collections.<String, Object>singletonMap(CONSTRAINT_PARAM_ATTRIBUTE, "mappedFromA"))))); // *-*
AtlasEntityDef bDef = AtlasTypeUtil.createClassTypeDef("B", Collections.<String>emptySet(), AtlasEntityDef bDef = createClassTypeDef("B", Collections.<String>emptySet(),
AtlasTypeUtil.createUniqueRequiredAttrDef("name", "string"), createUniqueRequiredAttrDef("name", "string"),
new AtlasAttributeDef("a", "A", true, Cardinality.SINGLE, 0, 1, false, false, false, new AtlasAttributeDef("a", "A", true, SINGLE, 0, 1, false, false, false,
Collections.<AtlasConstraintDef>singletonList(new AtlasConstraintDef( Collections.<AtlasConstraintDef>singletonList(new AtlasConstraintDef(
AtlasConstraintDef.CONSTRAINT_TYPE_INVERSE_REF, Collections.<String, Object>singletonMap(AtlasConstraintDef.CONSTRAINT_PARAM_ATTRIBUTE, "b")))), CONSTRAINT_TYPE_INVERSE_REF, Collections.<String, Object>singletonMap(CONSTRAINT_PARAM_ATTRIBUTE, "b")))),
new AtlasAttributeDef("manyA", AtlasBaseTypeDef.getArrayTypeName("A"), true, Cardinality.SINGLE, 0, 1, false, false, false, new AtlasAttributeDef("manyA", AtlasBaseTypeDef.getArrayTypeName("A"), true, SINGLE, 0, 1, false, false, false,
Collections.<AtlasConstraintDef>singletonList(new AtlasConstraintDef( Collections.<AtlasConstraintDef>singletonList(new AtlasConstraintDef(
AtlasConstraintDef.CONSTRAINT_TYPE_INVERSE_REF, Collections.<String, Object>singletonMap(AtlasConstraintDef.CONSTRAINT_PARAM_ATTRIBUTE, "oneB")))), CONSTRAINT_TYPE_INVERSE_REF, Collections.<String, Object>singletonMap(CONSTRAINT_PARAM_ATTRIBUTE, "oneB")))),
new AtlasAttributeDef("manyToManyA", AtlasBaseTypeDef.getArrayTypeName("A"), true, Cardinality.SINGLE, 0, 1, false, false, false, new AtlasAttributeDef("manyToManyA", AtlasBaseTypeDef.getArrayTypeName("A"), true, SINGLE, 0, 1, false, false, false,
Collections.<AtlasConstraintDef>singletonList(new AtlasConstraintDef( Collections.<AtlasConstraintDef>singletonList(new AtlasConstraintDef(
AtlasConstraintDef.CONSTRAINT_TYPE_INVERSE_REF, Collections.<String, Object>singletonMap(AtlasConstraintDef.CONSTRAINT_PARAM_ATTRIBUTE, "manyB")))), CONSTRAINT_TYPE_INVERSE_REF, Collections.<String, Object>singletonMap(CONSTRAINT_PARAM_ATTRIBUTE, "manyB")))),
new AtlasAttributeDef("mappedFromA", "A", true, Cardinality.SINGLE, 0, 1, false, false, false, Collections.<AtlasConstraintDef>emptyList())); new AtlasAttributeDef("mappedFromA", "A", true, SINGLE, 0, 1, false, false, false, Collections.<AtlasConstraintDef>emptyList()));
AtlasTypesDef ret = new AtlasTypesDef(Collections.<AtlasEnumDef>emptyList(), Collections.<AtlasStructDef>emptyList(), Collections.<AtlasClassificationDef>emptyList(), Arrays.asList(aDef, bDef)); AtlasTypesDef ret = new AtlasTypesDef(Collections.<AtlasEnumDef>emptyList(), Collections.<AtlasStructDef>emptyList(), Collections.<AtlasClassificationDef>emptyList(), Arrays.asList(aDef, bDef));
...@@ -198,63 +209,63 @@ public final class TestUtilsV2 { ...@@ -198,63 +209,63 @@ public final class TestUtilsV2 {
AtlasStructDef addressDetails = AtlasStructDef addressDetails =
createStructTypeDef("Address", "Address"+_description, createStructTypeDef("Address", "Address"+_description,
AtlasTypeUtil.createRequiredAttrDef("street", "string"), createRequiredAttrDef("street", "string"),
AtlasTypeUtil.createRequiredAttrDef("city", "string"), createRequiredAttrDef("city", "string"),
AtlasTypeUtil.createOptionalAttrDef("zip", "int")); createOptionalAttrDef("zip", "int"));
AtlasEntityDef deptTypeDef = AtlasEntityDef deptTypeDef =
AtlasTypeUtil.createClassTypeDef(DEPARTMENT_TYPE, "Department"+_description, createClassTypeDef(DEPARTMENT_TYPE, "Department"+_description,
Collections.<String>emptySet(), Collections.<String>emptySet(),
AtlasTypeUtil.createUniqueRequiredAttrDef("name", "string"), createUniqueRequiredAttrDef("name", "string"),
AtlasTypeUtil.createOptionalAttrDef("dep-code", "string"), createOptionalAttrDef("dep-code", "string"),
new AtlasAttributeDef("employees", String.format("array<%s>", "Employee"), true, new AtlasAttributeDef("employees", String.format("array<%s>", "Employee"), true,
AtlasAttributeDef.Cardinality.SINGLE, 0, 1, false, false, false, SINGLE, 0, 1, false, false, false,
new ArrayList<AtlasStructDef.AtlasConstraintDef>() {{ new ArrayList<AtlasStructDef.AtlasConstraintDef>() {{
add(new AtlasStructDef.AtlasConstraintDef(AtlasConstraintDef.CONSTRAINT_TYPE_OWNED_REF)); add(new AtlasStructDef.AtlasConstraintDef(CONSTRAINT_TYPE_OWNED_REF));
}})); }}));
AtlasEntityDef personTypeDef = AtlasTypeUtil.createClassTypeDef("Person", "Person"+_description, AtlasEntityDef personTypeDef = createClassTypeDef("Person", "Person"+_description,
Collections.<String>emptySet(), Collections.<String>emptySet(),
AtlasTypeUtil.createUniqueRequiredAttrDef("name", "string"), createUniqueRequiredAttrDef("name", "string"),
AtlasTypeUtil.createOptionalAttrDef("email", "string"), createOptionalAttrDef("email", "string"),
AtlasTypeUtil.createOptionalAttrDef("address", "Address"), createOptionalAttrDef("address", "Address"),
AtlasTypeUtil.createOptionalAttrDef("birthday", "date"), createOptionalAttrDef("birthday", "date"),
AtlasTypeUtil.createOptionalAttrDef("hasPets", "boolean"), createOptionalAttrDef("hasPets", "boolean"),
AtlasTypeUtil.createOptionalAttrDef("numberOfCars", "byte"), createOptionalAttrDef("numberOfCars", "byte"),
AtlasTypeUtil.createOptionalAttrDef("houseNumber", "short"), createOptionalAttrDef("houseNumber", "short"),
AtlasTypeUtil.createOptionalAttrDef("carMileage", "int"), createOptionalAttrDef("carMileage", "int"),
AtlasTypeUtil.createOptionalAttrDef("age", "float"), createOptionalAttrDef("age", "float"),
AtlasTypeUtil.createOptionalAttrDef("numberOfStarsEstimate", "biginteger"), createOptionalAttrDef("numberOfStarsEstimate", "biginteger"),
AtlasTypeUtil.createOptionalAttrDef("approximationOfPi", "bigdecimal") createOptionalAttrDef("approximationOfPi", "bigdecimal")
); );
AtlasEntityDef employeeTypeDef = AtlasTypeUtil.createClassTypeDef("Employee", "Employee"+_description, AtlasEntityDef employeeTypeDef = createClassTypeDef("Employee", "Employee"+_description,
Collections.singleton("Person"), Collections.singleton("Person"),
AtlasTypeUtil.createOptionalAttrDef("orgLevel", "OrgLevel"), createOptionalAttrDef("orgLevel", "OrgLevel"),
AtlasTypeUtil.createOptionalAttrDef("empCode", "string"), createOptionalAttrDef("empCode", "string"),
new AtlasAttributeDef("department", "Department", false, new AtlasAttributeDef("department", "Department", false,
AtlasAttributeDef.Cardinality.SINGLE, 1, 1, SINGLE, 1, 1,
false, false, false, false, false, false,
Collections.<AtlasConstraintDef>emptyList()), Collections.<AtlasConstraintDef>emptyList()),
new AtlasAttributeDef("manager", "Manager", true, new AtlasAttributeDef("manager", "Manager", true,
AtlasAttributeDef.Cardinality.SINGLE, 0, 1, SINGLE, 0, 1,
false, false, false, false, false, false,
new ArrayList<AtlasConstraintDef>() {{ new ArrayList<AtlasConstraintDef>() {{
add(new AtlasConstraintDef( add(new AtlasConstraintDef(
AtlasConstraintDef.CONSTRAINT_TYPE_INVERSE_REF, new HashMap<String, Object>() {{ CONSTRAINT_TYPE_INVERSE_REF, new HashMap<String, Object>() {{
put(AtlasConstraintDef.CONSTRAINT_PARAM_ATTRIBUTE, "subordinates"); put(CONSTRAINT_PARAM_ATTRIBUTE, "subordinates");
}})); }}));
}}), }}),
new AtlasAttributeDef("mentor", EMPLOYEE_TYPE, true, new AtlasAttributeDef("mentor", EMPLOYEE_TYPE, true,
AtlasAttributeDef.Cardinality.SINGLE, 0, 1, SINGLE, 0, 1,
false, false, false, false, false, false,
Collections.<AtlasConstraintDef>emptyList()), Collections.<AtlasConstraintDef>emptyList()),
AtlasTypeUtil.createOptionalAttrDef("shares", "long"), createOptionalAttrDef("shares", "long"),
AtlasTypeUtil.createOptionalAttrDef("salary", "double") createOptionalAttrDef("salary", "double")
); );
AtlasEntityDef managerTypeDef = AtlasTypeUtil.createClassTypeDef("Manager", "Manager"+_description, AtlasEntityDef managerTypeDef = createClassTypeDef("Manager", "Manager"+_description,
Collections.singleton("Employee"), Collections.singleton("Employee"),
new AtlasAttributeDef("subordinates", String.format("array<%s>", "Employee"), false, AtlasAttributeDef.Cardinality.SET, new AtlasAttributeDef("subordinates", String.format("array<%s>", "Employee"), false, AtlasAttributeDef.Cardinality.SET,
1, 10, false, false, false, 1, 10, false, false, false,
...@@ -262,7 +273,7 @@ public final class TestUtilsV2 { ...@@ -262,7 +273,7 @@ public final class TestUtilsV2 {
AtlasClassificationDef securityClearanceTypeDef = AtlasClassificationDef securityClearanceTypeDef =
AtlasTypeUtil.createTraitTypeDef("SecurityClearance", "SecurityClearance"+_description, Collections.<String>emptySet(), AtlasTypeUtil.createTraitTypeDef("SecurityClearance", "SecurityClearance"+_description, Collections.<String>emptySet(),
AtlasTypeUtil.createRequiredAttrDef("level", "int")); createRequiredAttrDef("level", "int"));
AtlasTypesDef ret = new AtlasTypesDef(Collections.singletonList(orgLevelEnum), AtlasTypesDef ret = new AtlasTypesDef(Collections.singletonList(orgLevelEnum),
Collections.singletonList(addressDetails), Collections.singletonList(addressDetails),
...@@ -287,52 +298,52 @@ public final class TestUtilsV2 { ...@@ -287,52 +298,52 @@ public final class TestUtilsV2 {
AtlasStructDef addressDetails = AtlasStructDef addressDetails =
createStructTypeDef("Address", "Address"+_description, createStructTypeDef("Address", "Address"+_description,
AtlasTypeUtil.createRequiredAttrDef("street", "string"), createRequiredAttrDef("street", "string"),
AtlasTypeUtil.createRequiredAttrDef("city", "string"), createRequiredAttrDef("city", "string"),
AtlasTypeUtil.createRequiredAttrDef("zip", "int")); createRequiredAttrDef("zip", "int"));
AtlasEntityDef deptTypeDef = AtlasEntityDef deptTypeDef =
AtlasTypeUtil.createClassTypeDef(DEPARTMENT_TYPE, "Department"+_description, Collections.<String>emptySet(), createClassTypeDef(DEPARTMENT_TYPE, "Department"+_description, Collections.<String>emptySet(),
AtlasTypeUtil.createRequiredAttrDef("name", "string"), createRequiredAttrDef("name", "string"),
AtlasTypeUtil.createRequiredAttrDef("dep-code", "string"), createRequiredAttrDef("dep-code", "string"),
new AtlasAttributeDef("employees", String.format("array<%s>", "Person"), true, new AtlasAttributeDef("employees", String.format("array<%s>", "Person"), true,
AtlasAttributeDef.Cardinality.SINGLE, 0, 1, false, false, false, SINGLE, 0, 1, false, false, false,
new ArrayList<AtlasStructDef.AtlasConstraintDef>() {{ new ArrayList<AtlasStructDef.AtlasConstraintDef>() {{
add(new AtlasStructDef.AtlasConstraintDef(AtlasConstraintDef.CONSTRAINT_TYPE_OWNED_REF)); add(new AtlasStructDef.AtlasConstraintDef(CONSTRAINT_TYPE_OWNED_REF));
}})); }}));
AtlasEntityDef personTypeDef = AtlasTypeUtil.createClassTypeDef("Person", "Person"+_description, Collections.<String>emptySet(), AtlasEntityDef personTypeDef = createClassTypeDef("Person", "Person"+_description, Collections.<String>emptySet(),
AtlasTypeUtil.createRequiredAttrDef("name", "string"), createRequiredAttrDef("name", "string"),
AtlasTypeUtil.createRequiredAttrDef("emp-code", "string"), createRequiredAttrDef("emp-code", "string"),
AtlasTypeUtil.createOptionalAttrDef("orgLevel", "OrgLevel"), createOptionalAttrDef("orgLevel", "OrgLevel"),
AtlasTypeUtil.createOptionalAttrDef("address", "Address"), createOptionalAttrDef("address", "Address"),
new AtlasAttributeDef("department", "Department", false, new AtlasAttributeDef("department", "Department", false,
AtlasAttributeDef.Cardinality.SINGLE, 1, 1, SINGLE, 1, 1,
false, false, false, false, false, false,
Collections.<AtlasConstraintDef>emptyList()), Collections.<AtlasConstraintDef>emptyList()),
new AtlasAttributeDef("manager", "Manager", true, new AtlasAttributeDef("manager", "Manager", true,
AtlasAttributeDef.Cardinality.SINGLE, 0, 1, SINGLE, 0, 1,
false, false, false, false, false, false,
new ArrayList<AtlasConstraintDef>() {{ new ArrayList<AtlasConstraintDef>() {{
add(new AtlasConstraintDef( add(new AtlasConstraintDef(
AtlasConstraintDef.CONSTRAINT_TYPE_INVERSE_REF, new HashMap<String, Object>() {{ CONSTRAINT_TYPE_INVERSE_REF, new HashMap<String, Object>() {{
put(AtlasConstraintDef.CONSTRAINT_PARAM_ATTRIBUTE, "subordinates"); put(CONSTRAINT_PARAM_ATTRIBUTE, "subordinates");
}})); }}));
}}), }}),
new AtlasAttributeDef("mentor", "Person", true, new AtlasAttributeDef("mentor", "Person", true,
AtlasAttributeDef.Cardinality.SINGLE, 0, 1, SINGLE, 0, 1,
false, false, false, false, false, false,
Collections.<AtlasConstraintDef>emptyList()), Collections.<AtlasConstraintDef>emptyList()),
AtlasTypeUtil.createOptionalAttrDef("birthday", "date"), createOptionalAttrDef("birthday", "date"),
AtlasTypeUtil.createOptionalAttrDef("hasPets", "boolean"), createOptionalAttrDef("hasPets", "boolean"),
AtlasTypeUtil.createOptionalAttrDef("numberOfCars", "byte"), createOptionalAttrDef("numberOfCars", "byte"),
AtlasTypeUtil.createOptionalAttrDef("houseNumber", "short"), createOptionalAttrDef("houseNumber", "short"),
AtlasTypeUtil.createOptionalAttrDef("carMileage", "int"), createOptionalAttrDef("carMileage", "int"),
AtlasTypeUtil.createOptionalAttrDef("shares", "long"), createOptionalAttrDef("shares", "long"),
AtlasTypeUtil.createOptionalAttrDef("salary", "double"), createOptionalAttrDef("salary", "double"),
AtlasTypeUtil.createRequiredAttrDef("age", "float"), createRequiredAttrDef("age", "float"),
AtlasTypeUtil.createOptionalAttrDef("numberOfStarsEstimate", "biginteger"), createOptionalAttrDef("numberOfStarsEstimate", "biginteger"),
AtlasTypeUtil.createOptionalAttrDef("approximationOfPi", "bigdecimal") createOptionalAttrDef("approximationOfPi", "bigdecimal")
); );
AtlasTypesDef ret = new AtlasTypesDef(Collections.singletonList(orgLevelEnum), AtlasTypesDef ret = new AtlasTypesDef(Collections.singletonList(orgLevelEnum),
...@@ -355,7 +366,7 @@ public final class TestUtilsV2 { ...@@ -355,7 +366,7 @@ public final class TestUtilsV2 {
/******* Department - HR *******/ /******* Department - HR *******/
AtlasEntity hrDept = new AtlasEntity(DEPARTMENT_TYPE, "name", "hr"); AtlasEntity hrDept = new AtlasEntity(DEPARTMENT_TYPE, "name", "hr");
AtlasObjectId hrDeptId = AtlasTypeUtil.getAtlasObjectId(hrDept); AtlasObjectId hrDeptId = getAtlasObjectId(hrDept);
/******* Address Entities *******/ /******* Address Entities *******/
AtlasStruct janeAddr = new AtlasStruct(ADDRESS_TYPE); AtlasStruct janeAddr = new AtlasStruct(ADDRESS_TYPE);
...@@ -376,14 +387,14 @@ public final class TestUtilsV2 { ...@@ -376,14 +387,14 @@ public final class TestUtilsV2 {
/******* Manager - Jane (John and Max subordinates) *******/ /******* Manager - Jane (John and Max subordinates) *******/
AtlasEntity jane = new AtlasEntity(MANAGER_TYPE); AtlasEntity jane = new AtlasEntity(MANAGER_TYPE);
AtlasObjectId janeId = AtlasTypeUtil.getAtlasObjectId(jane); AtlasObjectId janeId = getAtlasObjectId(jane);
jane.setAttribute("name", "Jane"); jane.setAttribute("name", "Jane");
jane.setAttribute("department", hrDeptId); jane.setAttribute("department", hrDeptId);
jane.setAttribute("address", janeAddr); jane.setAttribute("address", janeAddr);
/******* Manager - Julius (no subordinates) *******/ /******* Manager - Julius (no subordinates) *******/
AtlasEntity julius = new AtlasEntity(MANAGER_TYPE); AtlasEntity julius = new AtlasEntity(MANAGER_TYPE);
AtlasObjectId juliusId = AtlasTypeUtil.getAtlasObjectId(julius); AtlasObjectId juliusId = getAtlasObjectId(julius);
julius.setAttribute("name", "Julius"); julius.setAttribute("name", "Julius");
julius.setAttribute("department", hrDeptId); julius.setAttribute("department", hrDeptId);
julius.setAttribute("address", juliusAddr); julius.setAttribute("address", juliusAddr);
...@@ -391,7 +402,7 @@ public final class TestUtilsV2 { ...@@ -391,7 +402,7 @@ public final class TestUtilsV2 {
/******* Employee - Max (Manager: Jane, Mentor: Julius) *******/ /******* Employee - Max (Manager: Jane, Mentor: Julius) *******/
AtlasEntity max = new AtlasEntity(EMPLOYEE_TYPE); AtlasEntity max = new AtlasEntity(EMPLOYEE_TYPE);
AtlasObjectId maxId = AtlasTypeUtil.getAtlasObjectId(max); AtlasObjectId maxId = getAtlasObjectId(max);
max.setAttribute("name", "Max"); max.setAttribute("name", "Max");
max.setAttribute("department", hrDeptId); max.setAttribute("department", hrDeptId);
max.setAttribute("address", maxAddr); max.setAttribute("address", maxAddr);
...@@ -410,7 +421,7 @@ public final class TestUtilsV2 { ...@@ -410,7 +421,7 @@ public final class TestUtilsV2 {
/******* Employee - John (Manager: Jane, Mentor: Max) *******/ /******* Employee - John (Manager: Jane, Mentor: Max) *******/
AtlasEntity john = new AtlasEntity(EMPLOYEE_TYPE); AtlasEntity john = new AtlasEntity(EMPLOYEE_TYPE);
AtlasObjectId johnId = AtlasTypeUtil.getAtlasObjectId(john); AtlasObjectId johnId = getAtlasObjectId(john);
john.setAttribute("name", "John"); john.setAttribute("name", "John");
john.setAttribute("department", hrDeptId); john.setAttribute("department", hrDeptId);
john.setAttribute("address", johnAddr); john.setAttribute("address", johnAddr);
...@@ -486,8 +497,8 @@ public final class TestUtilsV2 { ...@@ -486,8 +497,8 @@ public final class TestUtilsV2 {
julius.setAttribute("address", juliusAddr); julius.setAttribute("address", juliusAddr);
julius.setAttribute("subordinates", Collections.emptyList()); julius.setAttribute("subordinates", Collections.emptyList());
AtlasObjectId janeId = AtlasTypeUtil.getAtlasObjectId(jane); AtlasObjectId janeId = getAtlasObjectId(jane);
AtlasObjectId johnId = AtlasTypeUtil.getAtlasObjectId(john); AtlasObjectId johnId = getAtlasObjectId(john);
//TODO - Change to MANAGER_TYPE for JULIUS //TODO - Change to MANAGER_TYPE for JULIUS
AtlasObjectId maxId = new AtlasObjectId(max.getGuid(), EMPLOYEE_TYPE); AtlasObjectId maxId = new AtlasObjectId(max.getGuid(), EMPLOYEE_TYPE);
...@@ -542,16 +553,18 @@ public final class TestUtilsV2 { ...@@ -542,16 +553,18 @@ public final class TestUtilsV2 {
public static final String COLUMNS_MAP = "columnsMap"; public static final String COLUMNS_MAP = "columnsMap";
public static final String COLUMNS_ATTR_NAME = "columns"; public static final String COLUMNS_ATTR_NAME = "columns";
public static final String ENTITY_TYPE_WITH_NESTED_COLLECTION_ATTR = "entity_with_nested_collection_attr"; public static final String ENTITY_TYPE_WITH_NESTED_COLLECTION_ATTR = "entity_with_nested_collection_attr";
public static final String ENTITY_TYPE_WITH_COMPLEX_COLLECTION_ATTR = "entity_with_complex_collection_attr";
public static final String ENTITY_TYPE_WITH_COMPLEX_COLLECTION_ATTR_DELETE = "entity_with_complex_collection_attr_delete";
public static final String NAME = "name"; public static final String NAME = "name";
public static AtlasTypesDef simpleType(){ public static AtlasTypesDef simpleType(){
AtlasEntityDef superTypeDefinition = AtlasEntityDef superTypeDefinition =
AtlasTypeUtil.createClassTypeDef("h_type", Collections.<String>emptySet(), createClassTypeDef("h_type", Collections.<String>emptySet(),
AtlasTypeUtil.createOptionalAttrDef("attr", "string")); createOptionalAttrDef("attr", "string"));
AtlasStructDef structTypeDefinition = new AtlasStructDef("s_type", "structType", "1.0", AtlasStructDef structTypeDefinition = new AtlasStructDef("s_type", "structType", "1.0",
Arrays.asList(AtlasTypeUtil.createRequiredAttrDef("name", "string"))); Arrays.asList(createRequiredAttrDef("name", "string")));
AtlasClassificationDef traitTypeDefinition = AtlasClassificationDef traitTypeDefinition =
AtlasTypeUtil.createTraitTypeDef("t_type", "traitType", Collections.<String>emptySet()); AtlasTypeUtil.createTraitTypeDef("t_type", "traitType", Collections.<String>emptySet());
...@@ -569,15 +582,15 @@ public final class TestUtilsV2 { ...@@ -569,15 +582,15 @@ public final class TestUtilsV2 {
public static AtlasTypesDef simpleTypeUpdated(){ public static AtlasTypesDef simpleTypeUpdated(){
AtlasEntityDef superTypeDefinition = AtlasEntityDef superTypeDefinition =
AtlasTypeUtil.createClassTypeDef("h_type", Collections.<String>emptySet(), createClassTypeDef("h_type", Collections.<String>emptySet(),
AtlasTypeUtil.createOptionalAttrDef("attr", "string")); createOptionalAttrDef("attr", "string"));
AtlasEntityDef newSuperTypeDefinition = AtlasEntityDef newSuperTypeDefinition =
AtlasTypeUtil.createClassTypeDef("new_h_type", Collections.<String>emptySet(), createClassTypeDef("new_h_type", Collections.<String>emptySet(),
AtlasTypeUtil.createOptionalAttrDef("attr", "string")); createOptionalAttrDef("attr", "string"));
AtlasStructDef structTypeDefinition = new AtlasStructDef("s_type", "structType", "1.0", AtlasStructDef structTypeDefinition = new AtlasStructDef("s_type", "structType", "1.0",
Arrays.asList(AtlasTypeUtil.createRequiredAttrDef("name", "string"))); Arrays.asList(createRequiredAttrDef("name", "string")));
AtlasClassificationDef traitTypeDefinition = AtlasClassificationDef traitTypeDefinition =
AtlasTypeUtil.createTraitTypeDef("t_type", "traitType", Collections.<String>emptySet()); AtlasTypeUtil.createTraitTypeDef("t_type", "traitType", Collections.<String>emptySet());
...@@ -594,8 +607,8 @@ public final class TestUtilsV2 { ...@@ -594,8 +607,8 @@ public final class TestUtilsV2 {
public static AtlasTypesDef simpleTypeUpdatedDiff() { public static AtlasTypesDef simpleTypeUpdatedDiff() {
AtlasEntityDef newSuperTypeDefinition = AtlasEntityDef newSuperTypeDefinition =
AtlasTypeUtil.createClassTypeDef("new_h_type", Collections.<String>emptySet(), createClassTypeDef("new_h_type", Collections.<String>emptySet(),
AtlasTypeUtil.createOptionalAttrDef("attr", "string")); createOptionalAttrDef("attr", "string"));
AtlasTypesDef ret = AtlasTypeUtil.getTypesDef(Collections.<AtlasEnumDef>emptyList(), AtlasTypesDef ret = AtlasTypeUtil.getTypesDef(Collections.<AtlasEnumDef>emptyList(),
Collections.<AtlasStructDef>emptyList(), Collections.<AtlasStructDef>emptyList(),
...@@ -611,24 +624,24 @@ public final class TestUtilsV2 { ...@@ -611,24 +624,24 @@ public final class TestUtilsV2 {
public static AtlasTypesDef defineHiveTypes() { public static AtlasTypesDef defineHiveTypes() {
String _description = "_description"; String _description = "_description";
AtlasEntityDef superTypeDefinition = AtlasEntityDef superTypeDefinition =
AtlasTypeUtil.createClassTypeDef(SUPER_TYPE_NAME, "SuperType_description", Collections.<String>emptySet(), createClassTypeDef(SUPER_TYPE_NAME, "SuperType_description", Collections.<String>emptySet(),
AtlasTypeUtil.createOptionalAttrDef("namespace", "string"), createOptionalAttrDef("namespace", "string"),
AtlasTypeUtil.createOptionalAttrDef("cluster", "string"), createOptionalAttrDef("cluster", "string"),
AtlasTypeUtil.createOptionalAttrDef("colo", "string")); createOptionalAttrDef("colo", "string"));
AtlasEntityDef databaseTypeDefinition = AtlasEntityDef databaseTypeDefinition =
AtlasTypeUtil.createClassTypeDef(DATABASE_TYPE, DATABASE_TYPE + _description,Collections.singleton(SUPER_TYPE_NAME), createClassTypeDef(DATABASE_TYPE, DATABASE_TYPE + _description,Collections.singleton(SUPER_TYPE_NAME),
AtlasTypeUtil.createUniqueRequiredAttrDef(NAME, "string"), createUniqueRequiredAttrDef(NAME, "string"),
AtlasTypeUtil.createOptionalAttrDef("isReplicated", "boolean"), createOptionalAttrDef("isReplicated", "boolean"),
AtlasTypeUtil.createOptionalAttrDef("created", "string"), createOptionalAttrDef("created", "string"),
AtlasTypeUtil.createOptionalAttrDef("parameters", "map<string,string>"), createOptionalAttrDef("parameters", "map<string,string>"),
AtlasTypeUtil.createRequiredAttrDef("description", "string")); createRequiredAttrDef("description", "string"));
AtlasStructDef structTypeDefinition = new AtlasStructDef("serdeType", "serdeType" + _description, "1.0", AtlasStructDef structTypeDefinition = new AtlasStructDef("serdeType", "serdeType" + _description, "1.0",
Arrays.asList( Arrays.asList(
AtlasTypeUtil.createRequiredAttrDef("name", "string"), createRequiredAttrDef("name", "string"),
AtlasTypeUtil.createRequiredAttrDef("serde", "string"), createRequiredAttrDef("serde", "string"),
AtlasTypeUtil.createOptionalAttrDef("description", "string"))); createOptionalAttrDef("description", "string")));
AtlasEnumElementDef values[] = { AtlasEnumElementDef values[] = {
new AtlasEnumElementDef("MANAGED", "Element Description", 1), new AtlasEnumElementDef("MANAGED", "Element Description", 1),
...@@ -637,45 +650,45 @@ public final class TestUtilsV2 { ...@@ -637,45 +650,45 @@ public final class TestUtilsV2 {
AtlasEnumDef enumTypeDefinition = new AtlasEnumDef("tableType", "tableType" + _description, "1.0", Arrays.asList(values)); AtlasEnumDef enumTypeDefinition = new AtlasEnumDef("tableType", "tableType" + _description, "1.0", Arrays.asList(values));
AtlasEntityDef columnsDefinition = AtlasEntityDef columnsDefinition =
AtlasTypeUtil.createClassTypeDef(COLUMN_TYPE, COLUMN_TYPE + "_description", createClassTypeDef(COLUMN_TYPE, COLUMN_TYPE + "_description",
Collections.<String>emptySet(), Collections.<String>emptySet(),
AtlasTypeUtil.createUniqueRequiredAttrDef("name", "string"), createUniqueRequiredAttrDef("name", "string"),
AtlasTypeUtil.createRequiredAttrDef("type", "string"), createRequiredAttrDef("type", "string"),
AtlasTypeUtil.createOptionalAttrDef("description", "string"), createOptionalAttrDef("description", "string"),
new AtlasAttributeDef("table", TABLE_TYPE, new AtlasAttributeDef("table", TABLE_TYPE,
true, true,
AtlasAttributeDef.Cardinality.SINGLE, 0, 1, SINGLE, 0, 1,
false, false, false, false, false, false,
new ArrayList<AtlasStructDef.AtlasConstraintDef>() {{ new ArrayList<AtlasStructDef.AtlasConstraintDef>() {{
add(new AtlasStructDef.AtlasConstraintDef( add(new AtlasStructDef.AtlasConstraintDef(
AtlasConstraintDef.CONSTRAINT_TYPE_INVERSE_REF, new HashMap<String, Object>() {{ CONSTRAINT_TYPE_INVERSE_REF, new HashMap<String, Object>() {{
put(AtlasConstraintDef.CONSTRAINT_PARAM_ATTRIBUTE, "columns"); put(CONSTRAINT_PARAM_ATTRIBUTE, "columns");
}})); }}));
}}) }})
); );
AtlasStructDef partitionDefinition = new AtlasStructDef("partition_struct_type", "partition_struct_type" + _description, "1.0", AtlasStructDef partitionDefinition = new AtlasStructDef("partition_struct_type", "partition_struct_type" + _description, "1.0",
Arrays.asList(AtlasTypeUtil.createRequiredAttrDef("name", "string"))); Arrays.asList(createRequiredAttrDef("name", "string")));
AtlasAttributeDef[] attributeDefinitions = new AtlasAttributeDef[]{ AtlasAttributeDef[] attributeDefinitions = new AtlasAttributeDef[]{
new AtlasAttributeDef("location", "string", true, new AtlasAttributeDef("location", "string", true,
AtlasAttributeDef.Cardinality.SINGLE, 0, 1, SINGLE, 0, 1,
false, false, false, false, false, false,
Collections.<AtlasConstraintDef>emptyList()), Collections.<AtlasConstraintDef>emptyList()),
new AtlasAttributeDef("inputFormat", "string", true, new AtlasAttributeDef("inputFormat", "string", true,
AtlasAttributeDef.Cardinality.SINGLE, 0, 1, SINGLE, 0, 1,
false, false, false, false, false, false,
Collections.<AtlasConstraintDef>emptyList()), Collections.<AtlasConstraintDef>emptyList()),
new AtlasAttributeDef("outputFormat", "string", true, new AtlasAttributeDef("outputFormat", "string", true,
AtlasAttributeDef.Cardinality.SINGLE, 0, 1, SINGLE, 0, 1,
false, false, false, false, false, false,
Collections.<AtlasConstraintDef>emptyList()), Collections.<AtlasConstraintDef>emptyList()),
new AtlasAttributeDef("compressed", "boolean", false, new AtlasAttributeDef("compressed", "boolean", false,
AtlasAttributeDef.Cardinality.SINGLE, 1, 1, SINGLE, 1, 1,
false, false, false, false, false, false,
Collections.<AtlasConstraintDef>emptyList()), Collections.<AtlasConstraintDef>emptyList()),
new AtlasAttributeDef("numBuckets", "int", true, new AtlasAttributeDef("numBuckets", "int", true,
AtlasAttributeDef.Cardinality.SINGLE, 0, 1, SINGLE, 0, 1,
false, false, false, false, false, false,
Collections.<AtlasConstraintDef>emptyList()), Collections.<AtlasConstraintDef>emptyList()),
}; };
...@@ -687,32 +700,32 @@ public final class TestUtilsV2 { ...@@ -687,32 +700,32 @@ public final class TestUtilsV2 {
AtlasAttributeDef[] partClsAttributes = new AtlasAttributeDef[]{ AtlasAttributeDef[] partClsAttributes = new AtlasAttributeDef[]{
new AtlasAttributeDef("values", "array<string>", new AtlasAttributeDef("values", "array<string>",
true, true,
AtlasAttributeDef.Cardinality.SINGLE, 0, 1, SINGLE, 0, 1,
false, false, false, false, false, false,
Collections.<AtlasConstraintDef>emptyList()), Collections.<AtlasConstraintDef>emptyList()),
new AtlasAttributeDef("table", TABLE_TYPE, true, new AtlasAttributeDef("table", TABLE_TYPE, true,
AtlasAttributeDef.Cardinality.SINGLE, 0, 1, SINGLE, 0, 1,
false, false, false, false, false, false,
Collections.<AtlasConstraintDef>emptyList()), Collections.<AtlasConstraintDef>emptyList()),
new AtlasAttributeDef("createTime", "long", true, new AtlasAttributeDef("createTime", "long", true,
AtlasAttributeDef.Cardinality.SINGLE, 0, 1, SINGLE, 0, 1,
false, false, false, false, false, false,
Collections.<AtlasConstraintDef>emptyList()), Collections.<AtlasConstraintDef>emptyList()),
new AtlasAttributeDef("lastAccessTime", "long", true, new AtlasAttributeDef("lastAccessTime", "long", true,
AtlasAttributeDef.Cardinality.SINGLE, 0, 1, SINGLE, 0, 1,
false, false, false, false, false, false,
Collections.<AtlasConstraintDef>emptyList()), Collections.<AtlasConstraintDef>emptyList()),
new AtlasAttributeDef("sd", STORAGE_DESC_TYPE, false, new AtlasAttributeDef("sd", STORAGE_DESC_TYPE, false,
AtlasAttributeDef.Cardinality.SINGLE, 1, 1, SINGLE, 1, 1,
false, false, false, false, false, false,
Collections.<AtlasConstraintDef>emptyList()), Collections.<AtlasConstraintDef>emptyList()),
new AtlasAttributeDef("columns", String.format("array<%s>", COLUMN_TYPE), new AtlasAttributeDef("columns", String.format("array<%s>", COLUMN_TYPE),
true, true,
AtlasAttributeDef.Cardinality.SINGLE, 0, 1, SINGLE, 0, 1,
false, false, false, false, false, false,
Collections.<AtlasConstraintDef>emptyList()), Collections.<AtlasConstraintDef>emptyList()),
new AtlasAttributeDef("parameters", String.format("map<%s,%s>", "string", "string"), true, new AtlasAttributeDef("parameters", String.format("map<%s,%s>", "string", "string"), true,
AtlasAttributeDef.Cardinality.SINGLE, 0, 1, SINGLE, 0, 1,
false, false, false, false, false, false,
Collections.<AtlasConstraintDef>emptyList())}; Collections.<AtlasConstraintDef>emptyList())};
...@@ -723,87 +736,93 @@ public final class TestUtilsV2 { ...@@ -723,87 +736,93 @@ public final class TestUtilsV2 {
AtlasEntityDef processClsType = AtlasEntityDef processClsType =
new AtlasEntityDef(PROCESS_TYPE, PROCESS_TYPE + _description, "1.0", new AtlasEntityDef(PROCESS_TYPE, PROCESS_TYPE + _description, "1.0",
Arrays.asList(new AtlasAttributeDef("outputs", "array<" + TABLE_TYPE + ">", true, Arrays.asList(new AtlasAttributeDef("outputs", "array<" + TABLE_TYPE + ">", true,
AtlasAttributeDef.Cardinality.SINGLE, 0, 1, SINGLE, 0, 1,
false, false, false, false, false, false,
Collections.<AtlasConstraintDef>emptyList())), Collections.<AtlasConstraintDef>emptyList())),
Collections.<String>emptySet()); Collections.<String>emptySet());
AtlasEntityDef tableTypeDefinition = AtlasEntityDef tableTypeDefinition =
AtlasTypeUtil.createClassTypeDef(TABLE_TYPE, TABLE_TYPE + _description, Collections.singleton(SUPER_TYPE_NAME), createClassTypeDef(TABLE_TYPE, TABLE_TYPE + _description, Collections.singleton(SUPER_TYPE_NAME),
AtlasTypeUtil.createUniqueRequiredAttrDef("name", "string"), createUniqueRequiredAttrDef("name", "string"),
AtlasTypeUtil.createOptionalAttrDef("description", "string"), createOptionalAttrDef("description", "string"),
AtlasTypeUtil.createRequiredAttrDef("type", "string"), createRequiredAttrDef("type", "string"),
AtlasTypeUtil.createOptionalAttrDef("created", "date"), createOptionalAttrDef("created", "date"),
// enum // enum
new AtlasAttributeDef("tableType", "tableType", false, new AtlasAttributeDef("tableType", "tableType", false,
AtlasAttributeDef.Cardinality.SINGLE, 1, 1, SINGLE, 1, 1,
false, false, false, false, false, false,
Collections.<AtlasConstraintDef>emptyList()), Collections.<AtlasConstraintDef>emptyList()),
// array of strings // array of strings
new AtlasAttributeDef("columnNames", new AtlasAttributeDef("columnNames",
String.format("array<%s>", "string"), true, String.format("array<%s>", "string"), true,
AtlasAttributeDef.Cardinality.SINGLE, 0, 1, SINGLE, 0, 1,
false, false, false, false, false, false,
Collections.<AtlasConstraintDef>emptyList()), Collections.<AtlasConstraintDef>emptyList()),
// array of classes // array of classes
new AtlasAttributeDef("columns", String.format("array<%s>", COLUMN_TYPE), new AtlasAttributeDef("columns", String.format("array<%s>", COLUMN_TYPE),
true, true,
AtlasAttributeDef.Cardinality.SINGLE, 0, 1, SINGLE, 0, 1,
false, false, false, false, false, false,
new ArrayList<AtlasStructDef.AtlasConstraintDef>() {{ new ArrayList<AtlasStructDef.AtlasConstraintDef>() {{
add(new AtlasStructDef.AtlasConstraintDef(AtlasConstraintDef.CONSTRAINT_TYPE_OWNED_REF)); add(new AtlasStructDef.AtlasConstraintDef(CONSTRAINT_TYPE_OWNED_REF));
}}), }}),
// array of structs // array of structs
new AtlasAttributeDef("partitions", String.format("array<%s>", "partition_struct_type"), new AtlasAttributeDef("partitions", String.format("array<%s>", "partition_struct_type"),
true, true,
AtlasAttributeDef.Cardinality.SINGLE, 0, 1, SINGLE, 0, 1,
false, false, false,
Collections.<AtlasConstraintDef>emptyList()),
// map of structs
new AtlasAttributeDef("partitionsMap", String.format("map<%s,%s>", "string", "partition_struct_type"),
true,
SINGLE, 0, 1,
false, false, false, false, false, false,
Collections.<AtlasConstraintDef>emptyList()), Collections.<AtlasConstraintDef>emptyList()),
// map of primitives // map of primitives
new AtlasAttributeDef("parametersMap", String.format("map<%s,%s>", "string", "string"), new AtlasAttributeDef("parametersMap", String.format("map<%s,%s>", "string", "string"),
true, true,
AtlasAttributeDef.Cardinality.SINGLE, 0, 1, SINGLE, 0, 1,
false, false, false, false, false, false,
Collections.<AtlasConstraintDef>emptyList()), Collections.<AtlasConstraintDef>emptyList()),
//map of classes - //map of classes -
new AtlasAttributeDef(COLUMNS_MAP, new AtlasAttributeDef(COLUMNS_MAP,
String.format("map<%s,%s>", "string", COLUMN_TYPE), String.format("map<%s,%s>", "string", COLUMN_TYPE),
true, true,
AtlasAttributeDef.Cardinality.SINGLE, 0, 1, SINGLE, 0, 1,
false, false, false, false, false, false,
new ArrayList<AtlasStructDef.AtlasConstraintDef>() {{ new ArrayList<AtlasStructDef.AtlasConstraintDef>() {{
add(new AtlasStructDef.AtlasConstraintDef( add(new AtlasStructDef.AtlasConstraintDef(
AtlasConstraintDef.CONSTRAINT_TYPE_OWNED_REF)); CONSTRAINT_TYPE_OWNED_REF));
}} }}
), ),
//map of structs //map of structs
new AtlasAttributeDef("partitionsMap", new AtlasAttributeDef("partitionsMap",
String.format("map<%s,%s>", "string", "partition_struct_type"), String.format("map<%s,%s>", "string", "partition_struct_type"),
true, true,
AtlasAttributeDef.Cardinality.SINGLE, 0, 1, SINGLE, 0, 1,
false, false, false, false, false, false,
Collections.<AtlasConstraintDef>emptyList()), Collections.<AtlasConstraintDef>emptyList()),
// struct reference // struct reference
new AtlasAttributeDef("serde1", "serdeType", true, new AtlasAttributeDef("serde1", "serdeType", true,
AtlasAttributeDef.Cardinality.SINGLE, 0, 1, SINGLE, 0, 1,
false, false, false, false, false, false,
Collections.<AtlasConstraintDef>emptyList()), Collections.<AtlasConstraintDef>emptyList()),
new AtlasAttributeDef("serde2", "serdeType", true, new AtlasAttributeDef("serde2", "serdeType", true,
AtlasAttributeDef.Cardinality.SINGLE, 0, 1, SINGLE, 0, 1,
false, false, false, false, false, false,
Collections.<AtlasConstraintDef>emptyList()), Collections.<AtlasConstraintDef>emptyList()),
// class reference // class reference
new AtlasAttributeDef("database", DATABASE_TYPE, false, new AtlasAttributeDef("database", DATABASE_TYPE, false,
AtlasAttributeDef.Cardinality.SINGLE, 1, 1, SINGLE, 1, 1,
false, false, false, false, false, false,
Collections.<AtlasConstraintDef>emptyList()), Collections.<AtlasConstraintDef>emptyList()),
//class reference as composite //class reference as composite
new AtlasAttributeDef("databaseComposite", DATABASE_TYPE, true, new AtlasAttributeDef("databaseComposite", DATABASE_TYPE, true,
AtlasAttributeDef.Cardinality.SINGLE, 0, 1, SINGLE, 0, 1,
false, false, false, false, false, false,
new ArrayList<AtlasStructDef.AtlasConstraintDef>() {{ new ArrayList<AtlasStructDef.AtlasConstraintDef>() {{
add(new AtlasStructDef.AtlasConstraintDef( add(new AtlasStructDef.AtlasConstraintDef(
AtlasConstraintDef.CONSTRAINT_TYPE_OWNED_REF)); CONSTRAINT_TYPE_OWNED_REF));
}} }}
)); ));
...@@ -812,16 +831,16 @@ public final class TestUtilsV2 { ...@@ -812,16 +831,16 @@ public final class TestUtilsV2 {
AtlasClassificationDef classificationTypeDefinition = AtlasClassificationDef classificationTypeDefinition =
AtlasTypeUtil.createTraitTypeDef(CLASSIFICATION, CLASSIFICATION + _description, Collections.<String>emptySet(), AtlasTypeUtil.createTraitTypeDef(CLASSIFICATION, CLASSIFICATION + _description, Collections.<String>emptySet(),
AtlasTypeUtil.createRequiredAttrDef("tag", "string")); createRequiredAttrDef("tag", "string"));
AtlasClassificationDef fetlClassificationTypeDefinition = AtlasClassificationDef fetlClassificationTypeDefinition =
AtlasTypeUtil.createTraitTypeDef("fetl" + CLASSIFICATION, "fetl" + CLASSIFICATION + _description, Collections.singleton(CLASSIFICATION), AtlasTypeUtil.createTraitTypeDef("fetl" + CLASSIFICATION, "fetl" + CLASSIFICATION + _description, Collections.singleton(CLASSIFICATION),
AtlasTypeUtil.createRequiredAttrDef("tag", "string")); createRequiredAttrDef("tag", "string"));
AtlasClassificationDef phiTypeDefinition = AtlasTypeUtil.createTraitTypeDef(PHI, PHI + _description, Collections.<String>emptySet(), AtlasClassificationDef phiTypeDefinition = AtlasTypeUtil.createTraitTypeDef(PHI, PHI + _description, Collections.<String>emptySet(),
AtlasTypeUtil.createRequiredAttrDef("stringAttr", "string"), createRequiredAttrDef("stringAttr", "string"),
AtlasTypeUtil.createRequiredAttrDef("booleanAttr", "boolean"), createRequiredAttrDef("booleanAttr", "boolean"),
AtlasTypeUtil.createRequiredAttrDef("integerAttr", "int")); createRequiredAttrDef("integerAttr", "int"));
AtlasTypesDef ret = AtlasTypeUtil.getTypesDef(Collections.singletonList(enumTypeDefinition), AtlasTypesDef ret = AtlasTypeUtil.getTypesDef(Collections.singletonList(enumTypeDefinition),
Arrays.asList(structTypeDefinition, partitionDefinition), Arrays.asList(structTypeDefinition, partitionDefinition),
...@@ -835,90 +854,90 @@ public final class TestUtilsV2 { ...@@ -835,90 +854,90 @@ public final class TestUtilsV2 {
public static AtlasTypesDef defineTypeWithNestedCollectionAttributes() { public static AtlasTypesDef defineTypeWithNestedCollectionAttributes() {
AtlasEntityDef nestedCollectionAttributesEntityType = AtlasEntityDef nestedCollectionAttributesEntityType =
AtlasTypeUtil.createClassTypeDef(ENTITY_TYPE_WITH_NESTED_COLLECTION_ATTR, ENTITY_TYPE_WITH_NESTED_COLLECTION_ATTR + "_description", null, createClassTypeDef(ENTITY_TYPE_WITH_NESTED_COLLECTION_ATTR, ENTITY_TYPE_WITH_NESTED_COLLECTION_ATTR + "_description", null,
AtlasTypeUtil.createUniqueRequiredAttrDef("name", "string"), createUniqueRequiredAttrDef("name", "string"),
new AtlasAttributeDef("mapOfArrayOfStrings", "map<string,array<string>>", false, new AtlasAttributeDef("mapOfArrayOfStrings", "map<string,array<string>>", false,
AtlasAttributeDef.Cardinality.SINGLE, 1, 1, SINGLE, 1, 1,
false, false, false, false, false, false,
Collections.<AtlasConstraintDef>emptyList()), Collections.<AtlasConstraintDef>emptyList()),
new AtlasAttributeDef("mapOfArrayOfBooleans", "map<string,array<boolean>>", false, new AtlasAttributeDef("mapOfArrayOfBooleans", "map<string,array<boolean>>", false,
AtlasAttributeDef.Cardinality.SINGLE, 1, 1, SINGLE, 1, 1,
false, false, false, false, false, false,
Collections.<AtlasConstraintDef>emptyList()), Collections.<AtlasConstraintDef>emptyList()),
new AtlasAttributeDef("mapOfArrayOfInts", "map<string,array<int>>", false, new AtlasAttributeDef("mapOfArrayOfInts", "map<string,array<int>>", false,
AtlasAttributeDef.Cardinality.SINGLE, 1, 1, SINGLE, 1, 1,
false, false, false, false, false, false,
Collections.<AtlasConstraintDef>emptyList()), Collections.<AtlasConstraintDef>emptyList()),
new AtlasAttributeDef("mapOfArrayOfFloats", "map<string,array<float>>", false, new AtlasAttributeDef("mapOfArrayOfFloats", "map<string,array<float>>", false,
AtlasAttributeDef.Cardinality.SINGLE, 1, 1, SINGLE, 1, 1,
false, false, false, false, false, false,
Collections.<AtlasConstraintDef>emptyList()), Collections.<AtlasConstraintDef>emptyList()),
new AtlasAttributeDef("mapOfArrayOfDates", "map<string,array<date>>", false, new AtlasAttributeDef("mapOfArrayOfDates", "map<string,array<date>>", false,
AtlasAttributeDef.Cardinality.SINGLE, 1, 1, SINGLE, 1, 1,
false, false, false, false, false, false,
Collections.<AtlasConstraintDef>emptyList()), Collections.<AtlasConstraintDef>emptyList()),
new AtlasAttributeDef("mapOfMapOfStrings", "map<string,map<string,string>>", false, new AtlasAttributeDef("mapOfMapOfStrings", "map<string,map<string,string>>", false,
AtlasAttributeDef.Cardinality.SINGLE, 1, 1, SINGLE, 1, 1,
false, false, false, false, false, false,
Collections.<AtlasConstraintDef>emptyList()), Collections.<AtlasConstraintDef>emptyList()),
new AtlasAttributeDef("mapOfMapOfBooleans", "map<string,map<string,boolean>>", false, new AtlasAttributeDef("mapOfMapOfBooleans", "map<string,map<string,boolean>>", false,
AtlasAttributeDef.Cardinality.SINGLE, 1, 1, SINGLE, 1, 1,
false, false, false, false, false, false,
Collections.<AtlasConstraintDef>emptyList()), Collections.<AtlasConstraintDef>emptyList()),
new AtlasAttributeDef("mapOfMapOfInts", "map<string,map<string,int>>", false, new AtlasAttributeDef("mapOfMapOfInts", "map<string,map<string,int>>", false,
AtlasAttributeDef.Cardinality.SINGLE, 1, 1, SINGLE, 1, 1,
false, false, false, false, false, false,
Collections.<AtlasConstraintDef>emptyList()), Collections.<AtlasConstraintDef>emptyList()),
new AtlasAttributeDef("mapOfMapOfFloats", "map<string,map<string,float>>", false, new AtlasAttributeDef("mapOfMapOfFloats", "map<string,map<string,float>>", false,
AtlasAttributeDef.Cardinality.SINGLE, 1, 1, SINGLE, 1, 1,
false, false, false, false, false, false,
Collections.<AtlasConstraintDef>emptyList()), Collections.<AtlasConstraintDef>emptyList()),
new AtlasAttributeDef("mapOfMapOfDates", "map<string,map<string,date>>", false, new AtlasAttributeDef("mapOfMapOfDates", "map<string,map<string,date>>", false,
AtlasAttributeDef.Cardinality.SINGLE, 1, 1, SINGLE, 1, 1,
false, false, false, false, false, false,
Collections.<AtlasConstraintDef>emptyList()), Collections.<AtlasConstraintDef>emptyList()),
new AtlasAttributeDef("arrayOfArrayOfStrings", "array<array<string>>", false, new AtlasAttributeDef("arrayOfArrayOfStrings", "array<array<string>>", false,
AtlasAttributeDef.Cardinality.SINGLE, 1, 1, SINGLE, 1, 1,
false, false, false, false, false, false,
Collections.<AtlasConstraintDef>emptyList()), Collections.<AtlasConstraintDef>emptyList()),
new AtlasAttributeDef("arrayOfArrayOfBooleans", "array<array<boolean>>", false, new AtlasAttributeDef("arrayOfArrayOfBooleans", "array<array<boolean>>", false,
AtlasAttributeDef.Cardinality.SINGLE, 1, 1, SINGLE, 1, 1,
false, false, false, false, false, false,
Collections.<AtlasConstraintDef>emptyList()), Collections.<AtlasConstraintDef>emptyList()),
new AtlasAttributeDef("arrayOfArrayOfInts", "array<array<int>>", false, new AtlasAttributeDef("arrayOfArrayOfInts", "array<array<int>>", false,
AtlasAttributeDef.Cardinality.SINGLE, 1, 1, SINGLE, 1, 1,
false, false, false, false, false, false,
Collections.<AtlasConstraintDef>emptyList()), Collections.<AtlasConstraintDef>emptyList()),
new AtlasAttributeDef("arrayOfArrayOfFloats", "array<array<float>>", false, new AtlasAttributeDef("arrayOfArrayOfFloats", "array<array<float>>", false,
AtlasAttributeDef.Cardinality.SINGLE, 1, 1, SINGLE, 1, 1,
false, false, false, false, false, false,
Collections.<AtlasConstraintDef>emptyList()), Collections.<AtlasConstraintDef>emptyList()),
new AtlasAttributeDef("arrayOfArrayOfDates", "array<array<date>>", false, new AtlasAttributeDef("arrayOfArrayOfDates", "array<array<date>>", false,
AtlasAttributeDef.Cardinality.SINGLE, 1, 1, SINGLE, 1, 1,
false, false, false, false, false, false,
Collections.<AtlasConstraintDef>emptyList()), Collections.<AtlasConstraintDef>emptyList()),
new AtlasAttributeDef("arrayOfMapOfStrings", "array<map<string,string>>", false, new AtlasAttributeDef("arrayOfMapOfStrings", "array<map<string,string>>", false,
AtlasAttributeDef.Cardinality.SINGLE, 1, 1, SINGLE, 1, 1,
false, false, false, false, false, false,
Collections.<AtlasConstraintDef>emptyList()), Collections.<AtlasConstraintDef>emptyList()),
new AtlasAttributeDef("arrayOfMapOfBooleans", "array<map<string,boolean>>", false, new AtlasAttributeDef("arrayOfMapOfBooleans", "array<map<string,boolean>>", false,
AtlasAttributeDef.Cardinality.SINGLE, 1, 1, SINGLE, 1, 1,
false, false, false, false, false, false,
Collections.<AtlasConstraintDef>emptyList()), Collections.<AtlasConstraintDef>emptyList()),
new AtlasAttributeDef("arrayOfMapOfInts", "array<map<string,int>>", false, new AtlasAttributeDef("arrayOfMapOfInts", "array<map<string,int>>", false,
AtlasAttributeDef.Cardinality.SINGLE, 1, 1, SINGLE, 1, 1,
false, false, false, false, false, false,
Collections.<AtlasConstraintDef>emptyList()), Collections.<AtlasConstraintDef>emptyList()),
new AtlasAttributeDef("arrayOfMapOfFloats", "array<map<string,float>>", false, new AtlasAttributeDef("arrayOfMapOfFloats", "array<map<string,float>>", false,
AtlasAttributeDef.Cardinality.SINGLE, 1, 1, SINGLE, 1, 1,
false, false, false, false, false, false,
Collections.<AtlasConstraintDef>emptyList()), Collections.<AtlasConstraintDef>emptyList()),
new AtlasAttributeDef("arrayOfMapOfDates", "array<map<string,date>>", false, new AtlasAttributeDef("arrayOfMapOfDates", "array<map<string,date>>", false,
AtlasAttributeDef.Cardinality.SINGLE, 1, 1, SINGLE, 1, 1,
false, false, false, false, false, false,
Collections.<AtlasConstraintDef>emptyList()) Collections.<AtlasConstraintDef>emptyList())
); );
...@@ -933,6 +952,31 @@ public final class TestUtilsV2 { ...@@ -933,6 +952,31 @@ public final class TestUtilsV2 {
return ret; return ret;
} }
public static AtlasTypesDef defineTypeWithComplexCollectionAttributes() {
AtlasStructDef structType =
new AtlasStructDef(STRUCT_TYPE, "struct_type_description", "1.0", Arrays.asList(createRequiredAttrDef(NAME, "string")));
AtlasEntityDef entityType =
createClassTypeDef(ENTITY_TYPE, "entity_type_description", Collections.emptySet(), createUniqueRequiredAttrDef(NAME, "string"), createOptionalAttrDef("isReplicated", "boolean"));
AtlasEntityDef complexCollectionEntityType =
createClassTypeDef(ENTITY_TYPE_WITH_COMPLEX_COLLECTION_ATTR, ENTITY_TYPE_WITH_COMPLEX_COLLECTION_ATTR + "_description", null,
createUniqueRequiredAttrDef("name", "string"),
new AtlasAttributeDef("listOfStructs", String.format("array<%s>", STRUCT_TYPE), true, LIST, 1, 1, false, false, false, Collections.emptyList()),
new AtlasAttributeDef("listOfEntities", String.format("array<%s>", ENTITY_TYPE), true, LIST, 1, 1, false, false, false, new ArrayList<AtlasConstraintDef>() {{ add(new AtlasConstraintDef(CONSTRAINT_TYPE_OWNED_REF)); }}),
new AtlasAttributeDef("mapOfStructs", String.format("map<%s,%s>", "string", STRUCT_TYPE), true, SINGLE, 1, 1, false, false, false, Collections.emptyList()),
new AtlasAttributeDef("mapOfEntities", String.format("map<%s,%s>", "string", ENTITY_TYPE), true, SINGLE, 1, 1, false, false, false, new ArrayList<AtlasConstraintDef>() {{ add(new AtlasConstraintDef(CONSTRAINT_TYPE_OWNED_REF)); }}) );
AtlasTypesDef ret = AtlasTypeUtil.getTypesDef(Collections.emptyList(),
Arrays.asList(structType),
Collections.emptyList(),
Arrays.asList(entityType, complexCollectionEntityType));
populateSystemAttributes(ret);
return ret;
}
public static AtlasEntityWithExtInfo createNestedCollectionAttrEntity() { public static AtlasEntityWithExtInfo createNestedCollectionAttrEntity() {
AtlasEntity entity = new AtlasEntity(ENTITY_TYPE_WITH_NESTED_COLLECTION_ATTR); AtlasEntity entity = new AtlasEntity(ENTITY_TYPE_WITH_NESTED_COLLECTION_ATTR);
...@@ -977,6 +1021,44 @@ public final class TestUtilsV2 { ...@@ -977,6 +1021,44 @@ public final class TestUtilsV2 {
return new AtlasEntityWithExtInfo(entity); return new AtlasEntityWithExtInfo(entity);
} }
public static AtlasEntityWithExtInfo createComplexCollectionAttrEntity() {
AtlasEntity entity = new AtlasEntity(ENTITY_TYPE_WITH_COMPLEX_COLLECTION_ATTR);
entity.setAttribute(NAME, ENTITY_TYPE_WITH_COMPLEX_COLLECTION_ATTR);
entity.setAttribute("listOfStructs", Arrays.asList(new AtlasStruct("struct_type", "name", "structArray0"),
new AtlasStruct("struct_type", "name", "structArray1"),
new AtlasStruct("struct_type", "name", "structArray2")));
entity.setAttribute("mapOfStructs", new HashMap<String, AtlasStruct>() {{ put("key0", new AtlasStruct("struct_type", "name", "structMap0"));
put("key1", new AtlasStruct("struct_type", "name", "structMap1"));
put("key2", new AtlasStruct("struct_type", "name", "structMap2")); }});
AtlasEntity e1Array = new AtlasEntity(ENTITY_TYPE, new HashMap<String, Object>() {{ put(NAME, "entityArray0"); put("isReplicated", true); }});
AtlasEntity e2Array = new AtlasEntity(ENTITY_TYPE, new HashMap<String, Object>() {{ put(NAME, "entityArray1"); put("isReplicated", false); }});
AtlasEntity e3Array = new AtlasEntity(ENTITY_TYPE, new HashMap<String, Object>() {{ put(NAME, "entityArray2"); put("isReplicated", true); }});
entity.setAttribute("listOfEntities", Arrays.asList(getAtlasObjectId(e1Array), getAtlasObjectId(e2Array), getAtlasObjectId(e3Array)));
AtlasEntity e1MapValue = new AtlasEntity(ENTITY_TYPE, new HashMap<String, Object>() {{ put(NAME, "entityMapValue0"); put("isReplicated", false); }});
AtlasEntity e2MapValue = new AtlasEntity(ENTITY_TYPE, new HashMap<String, Object>() {{ put(NAME, "entityMapValue1"); put("isReplicated", true); }});
AtlasEntity e3MapValue = new AtlasEntity(ENTITY_TYPE, new HashMap<String, Object>() {{ put(NAME, "entityMapValue2"); put("isReplicated", false); }});
entity.setAttribute("mapOfEntities", new HashMap<String, Object>() {{ put("key0", getAtlasObjectId(e1MapValue));
put("key1", getAtlasObjectId(e2MapValue));
put("key2", getAtlasObjectId(e3MapValue)); }});
AtlasEntityWithExtInfo ret = new AtlasEntityWithExtInfo(entity);
ret.addReferredEntity(e1Array);
ret.addReferredEntity(e2Array);
ret.addReferredEntity(e3Array);
ret.addReferredEntity(e1MapValue);
ret.addReferredEntity(e2MapValue);
ret.addReferredEntity(e3MapValue);
return ret;
}
public static final String randomString() { public static final String randomString() {
return RandomStringUtils.randomAlphanumeric(10); return RandomStringUtils.randomAlphanumeric(10);
} }
...@@ -1018,7 +1100,7 @@ public final class TestUtilsV2 { ...@@ -1018,7 +1100,7 @@ public final class TestUtilsV2 {
entity.setAttribute("description", "random table"); entity.setAttribute("description", "random table");
entity.setAttribute("type", "type"); entity.setAttribute("type", "type");
entity.setAttribute("tableType", "MANAGED"); entity.setAttribute("tableType", "MANAGED");
entity.setAttribute("database", AtlasTypeUtil.getAtlasObjectId(dbEntity)); entity.setAttribute("database", getAtlasObjectId(dbEntity));
entity.setAttribute("created", new Date()); entity.setAttribute("created", new Date());
Map<String, Object> partAttributes = new HashMap<String, Object>() {{ Map<String, Object> partAttributes = new HashMap<String, Object>() {{
...@@ -1041,12 +1123,19 @@ public final class TestUtilsV2 { ...@@ -1041,12 +1123,19 @@ public final class TestUtilsV2 {
tblEntity.setAttribute("description", "random table"); tblEntity.setAttribute("description", "random table");
tblEntity.setAttribute("type", "type"); tblEntity.setAttribute("type", "type");
tblEntity.setAttribute("tableType", "MANAGED"); tblEntity.setAttribute("tableType", "MANAGED");
tblEntity.setAttribute("database", AtlasTypeUtil.getAtlasObjectId(dbEntity)); tblEntity.setAttribute("database", getAtlasObjectId(dbEntity));
tblEntity.setAttribute("created", new Date()); tblEntity.setAttribute("created", new Date());
final AtlasStruct partitionStruct = new AtlasStruct("partition_struct_type", "name", "part0"); final AtlasStruct partitionStruct = new AtlasStruct("partition_struct_type", "name", "part0");
tblEntity.setAttribute("partitions", new ArrayList<AtlasStruct>() {{ add(partitionStruct); }}); tblEntity.setAttribute("partitions", new ArrayList<AtlasStruct>() {{ add(partitionStruct); }});
tblEntity.setAttribute("partitionsMap", new HashMap<String, AtlasStruct>() {{
put("part0", new AtlasStruct("partition_struct_type", "name", "part0"));
put("part1", new AtlasStruct("partition_struct_type", "name", "part1"));
put("part2", new AtlasStruct("partition_struct_type", "name", "part2"));
}});
tblEntity.setAttribute("parametersMap", tblEntity.setAttribute("parametersMap",
new java.util.HashMap<String, String>() {{ put("key1", "value1"); }}); new java.util.HashMap<String, String>() {{ put("key1", "value1"); }});
...@@ -1120,7 +1209,7 @@ public final class TestUtilsV2 { ...@@ -1120,7 +1209,7 @@ public final class TestUtilsV2 {
AtlasEntity entity = new AtlasEntity(COLUMN_TYPE); AtlasEntity entity = new AtlasEntity(COLUMN_TYPE);
entity.setAttribute(NAME, colName); entity.setAttribute(NAME, colName);
entity.setAttribute("type", "VARCHAR(32)"); entity.setAttribute("type", "VARCHAR(32)");
entity.setAttribute("table", AtlasTypeUtil.getAtlasObjectId(tableEntity)); entity.setAttribute("table", getAtlasObjectId(tableEntity));
return entity; return entity;
} }
...@@ -1136,11 +1225,11 @@ public final class TestUtilsV2 { ...@@ -1136,11 +1225,11 @@ public final class TestUtilsV2 {
public static List<AtlasClassificationDef> getClassificationWithValidSuperType() { public static List<AtlasClassificationDef> getClassificationWithValidSuperType() {
AtlasClassificationDef securityClearanceTypeDef = AtlasClassificationDef securityClearanceTypeDef =
AtlasTypeUtil.createTraitTypeDef("SecurityClearance1", "SecurityClearance_description", Collections.<String>emptySet(), AtlasTypeUtil.createTraitTypeDef("SecurityClearance1", "SecurityClearance_description", Collections.<String>emptySet(),
AtlasTypeUtil.createRequiredAttrDef("level", "int")); createRequiredAttrDef("level", "int"));
AtlasClassificationDef janitorSecurityClearanceTypeDef = AtlasClassificationDef janitorSecurityClearanceTypeDef =
AtlasTypeUtil.createTraitTypeDef("JanitorClearance", "JanitorClearance_description", Collections.singleton("SecurityClearance1"), AtlasTypeUtil.createTraitTypeDef("JanitorClearance", "JanitorClearance_description", Collections.singleton("SecurityClearance1"),
AtlasTypeUtil.createRequiredAttrDef("level", "int")); createRequiredAttrDef("level", "int"));
List<AtlasClassificationDef> ret = Arrays.asList(securityClearanceTypeDef, janitorSecurityClearanceTypeDef); List<AtlasClassificationDef> ret = Arrays.asList(securityClearanceTypeDef, janitorSecurityClearanceTypeDef);
...@@ -1152,7 +1241,7 @@ public final class TestUtilsV2 { ...@@ -1152,7 +1241,7 @@ public final class TestUtilsV2 {
public static List<AtlasClassificationDef> getClassificationWithName(String name) { public static List<AtlasClassificationDef> getClassificationWithName(String name) {
AtlasClassificationDef classificationTypeDef = AtlasClassificationDef classificationTypeDef =
AtlasTypeUtil.createTraitTypeDef(name, "s_description", Collections.<String>emptySet(), AtlasTypeUtil.createTraitTypeDef(name, "s_description", Collections.<String>emptySet(),
AtlasTypeUtil.createRequiredAttrDef("level", "int")); createRequiredAttrDef("level", "int"));
List<AtlasClassificationDef> ret = Arrays.asList(classificationTypeDef); List<AtlasClassificationDef> ret = Arrays.asList(classificationTypeDef);
...@@ -1165,7 +1254,7 @@ public final class TestUtilsV2 { ...@@ -1165,7 +1254,7 @@ public final class TestUtilsV2 {
public static AtlasClassificationDef getSingleClassificationWithName(String name) { public static AtlasClassificationDef getSingleClassificationWithName(String name) {
AtlasClassificationDef classificaitonTypeDef = AtlasClassificationDef classificaitonTypeDef =
AtlasTypeUtil.createTraitTypeDef(name, "s_description", Collections.<String>emptySet(), AtlasTypeUtil.createTraitTypeDef(name, "s_description", Collections.<String>emptySet(),
AtlasTypeUtil.createRequiredAttrDef("level", "int")); createRequiredAttrDef("level", "int"));
populateSystemAttributes(classificaitonTypeDef); populateSystemAttributes(classificaitonTypeDef);
...@@ -1177,7 +1266,7 @@ public final class TestUtilsV2 { ...@@ -1177,7 +1266,7 @@ public final class TestUtilsV2 {
} }
public static List<AtlasEntityDef> getEntityWithValidSuperType() { public static List<AtlasEntityDef> getEntityWithValidSuperType() {
AtlasEntityDef developerTypeDef = AtlasTypeUtil.createClassTypeDef("Developer", "Developer_description", Collections.singleton("Employee"), AtlasEntityDef developerTypeDef = createClassTypeDef("Developer", "Developer_description", Collections.singleton("Employee"),
new AtlasAttributeDef("language", String.format("array<%s>", "string"), false, AtlasAttributeDef.Cardinality.SET, new AtlasAttributeDef("language", String.format("array<%s>", "string"), false, AtlasAttributeDef.Cardinality.SET,
1, 10, false, false, false, 1, 10, false, false, false,
Collections.<AtlasConstraintDef>emptyList())); Collections.<AtlasConstraintDef>emptyList()));
...@@ -1190,7 +1279,7 @@ public final class TestUtilsV2 { ...@@ -1190,7 +1279,7 @@ public final class TestUtilsV2 {
} }
public static List<AtlasEntityDef> getEntityWithName(String name) { public static List<AtlasEntityDef> getEntityWithName(String name) {
AtlasEntityDef developerTypeDef = AtlasTypeUtil.createClassTypeDef(name, "Developer_description", Collections.<String>emptySet(), AtlasEntityDef developerTypeDef = createClassTypeDef(name, "Developer_description", Collections.<String>emptySet(),
new AtlasAttributeDef("language", String.format("array<%s>", "string"), false, AtlasAttributeDef.Cardinality.SET, new AtlasAttributeDef("language", String.format("array<%s>", "string"), false, AtlasAttributeDef.Cardinality.SET,
1, 10, false, false, false, 1, 10, false, false, false,
Collections.<AtlasConstraintDef>emptyList())); Collections.<AtlasConstraintDef>emptyList()));
...@@ -1203,7 +1292,7 @@ public final class TestUtilsV2 { ...@@ -1203,7 +1292,7 @@ public final class TestUtilsV2 {
} }
public static AtlasEntityDef getSingleEntityWithName(String name) { public static AtlasEntityDef getSingleEntityWithName(String name) {
AtlasEntityDef developerTypeDef = AtlasTypeUtil.createClassTypeDef(name, "Developer_description", Collections.<String>emptySet(), AtlasEntityDef developerTypeDef = createClassTypeDef(name, "Developer_description", Collections.<String>emptySet(),
new AtlasAttributeDef("language", String.format("array<%s>", "string"), false, AtlasAttributeDef.Cardinality.SET, new AtlasAttributeDef("language", String.format("array<%s>", "string"), false, AtlasAttributeDef.Cardinality.SET,
1, 10, false, false, false, 1, 10, false, false, false,
Collections.<AtlasConstraintDef>emptyList())); Collections.<AtlasConstraintDef>emptyList()));
......
repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java
...@@ -26,12 +26,13 @@ import org.apache.atlas.AtlasErrorCode; ...@@ -26,12 +26,13 @@ import org.apache.atlas.AtlasErrorCode;
import org.apache.atlas.AtlasException; import org.apache.atlas.AtlasException;
import org.apache.atlas.RequestContextV1; import org.apache.atlas.RequestContextV1;
import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.instance.AtlasClassification;
import org.apache.atlas.model.instance.AtlasEntity.Status; import org.apache.atlas.model.instance.AtlasEntity.Status;
import org.apache.atlas.model.instance.AtlasObjectId; import org.apache.atlas.model.instance.AtlasObjectId;
import org.apache.atlas.model.instance.AtlasRelationship; import org.apache.atlas.model.instance.AtlasRelationship;
import org.apache.atlas.model.typedef.AtlasRelationshipDef; import org.apache.atlas.model.typedef.AtlasRelationshipDef;
import org.apache.atlas.repository.graphdb.AtlasVertexQuery; import org.apache.atlas.repository.graphdb.AtlasVertexQuery;
import org.apache.atlas.type.AtlasArrayType;
import org.apache.atlas.type.AtlasMapType;
import org.apache.atlas.util.AtlasGremlinQueryProvider; import org.apache.atlas.util.AtlasGremlinQueryProvider;
import org.apache.atlas.v1.model.instance.Id; import org.apache.atlas.v1.model.instance.Id;
import org.apache.atlas.v1.model.instance.Referenceable; import org.apache.atlas.v1.model.instance.Referenceable;
...@@ -54,6 +55,7 @@ import org.apache.atlas.exception.EntityNotFoundException; ...@@ -54,6 +55,7 @@ import org.apache.atlas.exception.EntityNotFoundException;
import org.apache.atlas.util.AttributeValueMap; import org.apache.atlas.util.AttributeValueMap;
import org.apache.atlas.util.IndexedInstance; import org.apache.atlas.util.IndexedInstance;
import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.IteratorUtils;
import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
...@@ -75,6 +77,8 @@ import java.util.Set; ...@@ -75,6 +77,8 @@ import java.util.Set;
import java.util.UUID; import java.util.UUID;
import static org.apache.atlas.model.instance.AtlasEntity.Status.DELETED; import static org.apache.atlas.model.instance.AtlasEntity.Status.DELETED;
import static org.apache.atlas.repository.Constants.ATTRIBUTE_INDEX_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.ATTRIBUTE_KEY_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.CLASSIFICATION_EDGE_IS_PROPAGATED_PROPERTY_KEY; import static org.apache.atlas.repository.Constants.CLASSIFICATION_EDGE_IS_PROPAGATED_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.CLASSIFICATION_ENTITY_GUID; import static org.apache.atlas.repository.Constants.CLASSIFICATION_ENTITY_GUID;
import static org.apache.atlas.repository.Constants.CLASSIFICATION_LABEL; import static org.apache.atlas.repository.Constants.CLASSIFICATION_LABEL;
...@@ -82,6 +86,7 @@ import static org.apache.atlas.repository.Constants.CLASSIFICATION_EDGE_NAME_PRO ...@@ -82,6 +86,7 @@ import static org.apache.atlas.repository.Constants.CLASSIFICATION_EDGE_NAME_PRO
import static org.apache.atlas.repository.Constants.CLASSIFICATION_VERTEX_NAME_KEY; import static org.apache.atlas.repository.Constants.CLASSIFICATION_VERTEX_NAME_KEY;
import static org.apache.atlas.repository.Constants.CLASSIFICATION_VERTEX_PROPAGATE_KEY; import static org.apache.atlas.repository.Constants.CLASSIFICATION_VERTEX_PROPAGATE_KEY;
import static org.apache.atlas.repository.Constants.PROPAGATED_TRAIT_NAMES_PROPERTY_KEY; import static org.apache.atlas.repository.Constants.PROPAGATED_TRAIT_NAMES_PROPERTY_KEY;
import static org.apache.atlas.repository.store.graph.v1.AtlasGraphUtilsV1.isReference;
import static org.apache.atlas.type.AtlasStructType.AtlasAttribute.AtlasRelationshipEdgeDirection.BOTH; import static org.apache.atlas.type.AtlasStructType.AtlasAttribute.AtlasRelationshipEdgeDirection.BOTH;
import static org.apache.atlas.type.AtlasStructType.AtlasAttribute.AtlasRelationshipEdgeDirection.IN; import static org.apache.atlas.type.AtlasStructType.AtlasAttribute.AtlasRelationshipEdgeDirection.IN;
import static org.apache.atlas.type.AtlasStructType.AtlasAttribute.AtlasRelationshipEdgeDirection.OUT; import static org.apache.atlas.type.AtlasStructType.AtlasAttribute.AtlasRelationshipEdgeDirection.OUT;
...@@ -1170,6 +1175,11 @@ public final class GraphHelper { ...@@ -1170,6 +1175,11 @@ public final class GraphHelper {
return AtlasGraphUtilsV1.getProperty(classificationVertex, CLASSIFICATION_ENTITY_GUID, String.class); return AtlasGraphUtilsV1.getProperty(classificationVertex, CLASSIFICATION_ENTITY_GUID, String.class);
} }
public static Integer getIndexValue(AtlasEdge edge) {
Integer index = edge.getProperty(ATTRIBUTE_INDEX_PROPERTY_KEY, Integer.class);
return (index == null) ? 0: index;
}
public static boolean isPropagatedClassificationEdge(AtlasEdge edge) { public static boolean isPropagatedClassificationEdge(AtlasEdge edge) {
boolean ret = false; boolean ret = false;
...@@ -1557,44 +1567,100 @@ public final class GraphHelper { ...@@ -1557,44 +1567,100 @@ public final class GraphHelper {
return typeName != null && typeName.startsWith(Constants.INTERNAL_PROPERTY_KEY_PREFIX); return typeName != null && typeName.startsWith(Constants.INTERNAL_PROPERTY_KEY_PREFIX);
} }
public static void setArrayElementsProperty(AtlasType elementType, AtlasVertex instanceVertex, String propertyName, List<Object> values) { public static Object getMapValueProperty(AtlasType elementType, AtlasVertex instanceVertex, String propertyName) {
String actualPropertyName = GraphHelper.encodePropertyKey(propertyName); String vertexPropertyName = GraphHelper.encodePropertyKey(propertyName);
if(AtlasGraphUtilsV1.isReference(elementType)) {
setListPropertyFromElementIds(instanceVertex, actualPropertyName, (List)values); if (!AtlasGraphUtilsV1.isReference(elementType)) {
} return instanceVertex.getProperty(vertexPropertyName, Object.class);
else {
setProperty(instanceVertex, actualPropertyName, values);
} }
return null;
} }
public static void setMapValueProperty(AtlasType elementType, AtlasVertex instanceVertex, String propertyName, Object value) { public static List<Object> getArrayElementsProperty(AtlasType elementType, AtlasVertex instanceVertex, String propertyName, AtlasAttribute attribute) {
String actualPropertyName = GraphHelper.encodePropertyKey(propertyName); String encodedPropertyName = GraphHelper.encodePropertyKey(propertyName);
if(AtlasGraphUtilsV1.isReference(elementType)) {
instanceVertex.setPropertyFromElementId(actualPropertyName, (AtlasEdge)value); if (isReference(elementType)) {
} return (List) getCollectionElementsUsingRelationship(instanceVertex, attribute);
else { } else {
instanceVertex.setProperty(actualPropertyName, value); return (List) instanceVertex.getListProperty(encodedPropertyName);
} }
} }
public static Object getMapValueProperty(AtlasType elementType, AtlasVertex instanceVertex, String propertyName) { public static Map<String, Object> getMapElementsProperty(AtlasMapType mapType, AtlasVertex instanceVertex, String propertyName, AtlasAttribute attribute) {
String vertexPropertyName = GraphHelper.encodePropertyKey(propertyName); AtlasType mapValueType = mapType.getValueType();
if (AtlasGraphUtilsV1.isReference(elementType)) { if (isReference(mapValueType)) {
return instanceVertex.getProperty(vertexPropertyName, AtlasEdge.class); return getReferenceMap(instanceVertex, attribute);
} else { } else {
return instanceVertex.getProperty(vertexPropertyName, Object.class); return getPrimitiveMap(instanceVertex, propertyName, mapValueType);
} }
} }
// newly added // map elements for reference types - AtlasObjectId, AtlasStruct
public static List<Object> getArrayElementsProperty(AtlasType elementType, AtlasVertex instanceVertex, String propertyName) { public static Map<String, Object> getReferenceMap(AtlasVertex instanceVertex, AtlasAttribute attribute) {
String encodedPropertyName = GraphHelper.encodePropertyKey(propertyName); Map<String, Object> ret = new HashMap<>();
if(AtlasGraphUtilsV1.isReference(elementType)) { List<AtlasEdge> referenceEdges = getCollectionElementsUsingRelationship(instanceVertex, attribute);
return (List)instanceVertex.getListProperty(encodedPropertyName, AtlasEdge.class);
for (AtlasEdge edge : referenceEdges) {
String key = edge.getProperty(ATTRIBUTE_KEY_PROPERTY_KEY, String.class);
if (StringUtils.isNotEmpty(key)) {
ret.put(key, edge);
}
} }
else {
return (List)instanceVertex.getListProperty(encodedPropertyName); return ret;
}
public static List<AtlasEdge> getMapValuesUsingRelationship(AtlasVertex vertex, AtlasAttribute attribute) {
String edgeLabel = attribute.getRelationshipEdgeLabel();
AtlasRelationshipEdgeDirection edgeDirection = attribute.getRelationshipEdgeDirection();
Iterator<AtlasEdge> edgesForLabel = getEdgesForLabel(vertex, edgeLabel, edgeDirection);
return (List<AtlasEdge>) IteratorUtils.toList(edgesForLabel);
}
// map elements for primitive types
public static Map<String,Object> getPrimitiveMap(AtlasVertex instanceVertex, String propertyName, AtlasType mapValueType) {
String encodedPropertyName = encodePropertyKey(propertyName);
List<String> currentKeys = getListProperty(instanceVertex, encodedPropertyName);
Map<String, Object> ret = new HashMap<>();
if (CollectionUtils.isNotEmpty(currentKeys)) {
for (String key : currentKeys) {
String propertyNameForKey = getQualifiedNameForMapKey(encodedPropertyName, encodePropertyKey(key));
Object propertyValueForKey = getMapValueProperty(mapValueType, instanceVertex, propertyNameForKey);
ret.put(key, propertyValueForKey);
}
}
return ret;
}
public static List<AtlasEdge> getCollectionElementsUsingRelationship(AtlasVertex vertex, AtlasAttribute attribute) {
List<AtlasEdge> ret;
String edgeLabel = attribute.getRelationshipEdgeLabel();
AtlasRelationshipEdgeDirection edgeDirection = attribute.getRelationshipEdgeDirection();
Iterator<AtlasEdge> edgesForLabel = getEdgesForLabel(vertex, edgeLabel, edgeDirection);
ret = IteratorUtils.toList(edgesForLabel);
sortCollectionElements(attribute, ret);
return ret;
}
private static void sortCollectionElements(AtlasAttribute attribute, List<AtlasEdge> edges) {
// sort array elements based on edge index
if (attribute.getAttributeType() instanceof AtlasArrayType && CollectionUtils.isNotEmpty(edges)) {
Collections.sort(edges, (e1, e2) -> {
Integer e1Index = getIndexValue(e1);
Integer e2Index = getIndexValue(e2);
return e1Index.compareTo(e2Index);
});
} }
} }
...@@ -1730,7 +1796,7 @@ public final class GraphHelper { ...@@ -1730,7 +1796,7 @@ public final class GraphHelper {
} }
public static void setListProperty(AtlasVertex instanceVertex, String propertyName, ArrayList<String> value) throws AtlasException { public static void setListProperty(AtlasVertex instanceVertex, String propertyName, ArrayList<String> value) {
String actualPropertyName = GraphHelper.encodePropertyKey(propertyName); String actualPropertyName = GraphHelper.encodePropertyKey(propertyName);
instanceVertex.setListProperty(actualPropertyName, value); instanceVertex.setListProperty(actualPropertyName, value);
} }
......
repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasGraphUtilsV1.java
...@@ -146,9 +146,8 @@ public class AtlasGraphUtilsV1 { ...@@ -146,9 +146,8 @@ public class AtlasGraphUtilsV1 {
public static boolean isReference(TypeCategory typeCategory) { public static boolean isReference(TypeCategory typeCategory) {
return typeCategory == TypeCategory.STRUCT || return typeCategory == TypeCategory.STRUCT ||
typeCategory == TypeCategory.ENTITY || typeCategory == TypeCategory.ENTITY ||
typeCategory == TypeCategory.CLASSIFICATION || typeCategory == TypeCategory.OBJECT_ID_TYPE;
typeCategory == TypeCategory.OBJECT_ID_TYPE;
} }
public static String encodePropertyKey(String key) { public static String encodePropertyKey(String key) {
......
repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1.java
...@@ -51,6 +51,11 @@ import org.slf4j.LoggerFactory; ...@@ -51,6 +51,11 @@ import org.slf4j.LoggerFactory;
import java.util.*; import java.util.*;
import static org.apache.atlas.model.TypeCategory.ARRAY;
import static org.apache.atlas.model.TypeCategory.CLASSIFICATION;
import static org.apache.atlas.model.TypeCategory.MAP;
import static org.apache.atlas.model.TypeCategory.OBJECT_ID_TYPE;
import static org.apache.atlas.model.TypeCategory.STRUCT;
import static org.apache.atlas.model.instance.AtlasEntity.Status.DELETED; import static org.apache.atlas.model.instance.AtlasEntity.Status.DELETED;
import static org.apache.atlas.model.typedef.AtlasRelationshipDef.PropagateTags.ONE_TO_TWO; import static org.apache.atlas.model.typedef.AtlasRelationshipDef.PropagateTags.ONE_TO_TWO;
import static org.apache.atlas.repository.Constants.CLASSIFICATION_EDGE_NAME_PROPERTY_KEY; import static org.apache.atlas.repository.Constants.CLASSIFICATION_EDGE_NAME_PROPERTY_KEY;
...@@ -58,7 +63,6 @@ import static org.apache.atlas.repository.Constants.CLASSIFICATION_LABEL; ...@@ -58,7 +63,6 @@ import static org.apache.atlas.repository.Constants.CLASSIFICATION_LABEL;
import static org.apache.atlas.repository.Constants.PROPAGATED_TRAIT_NAMES_PROPERTY_KEY; import static org.apache.atlas.repository.Constants.PROPAGATED_TRAIT_NAMES_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.RELATIONSHIP_GUID_PROPERTY_KEY; import static org.apache.atlas.repository.Constants.RELATIONSHIP_GUID_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.TRAIT_NAMES_PROPERTY_KEY; import static org.apache.atlas.repository.Constants.TRAIT_NAMES_PROPERTY_KEY;
import static org.apache.atlas.repository.graph.GraphHelper.EDGE_LABEL_PREFIX;
import static org.apache.atlas.repository.graph.GraphHelper.addToPropagatedTraitNames; import static org.apache.atlas.repository.graph.GraphHelper.addToPropagatedTraitNames;
import static org.apache.atlas.repository.graph.GraphHelper.getAllClassificationEdges; import static org.apache.atlas.repository.graph.GraphHelper.getAllClassificationEdges;
import static org.apache.atlas.repository.graph.GraphHelper.getAssociatedEntityVertex; import static org.apache.atlas.repository.graph.GraphHelper.getAssociatedEntityVertex;
...@@ -66,7 +70,9 @@ import static org.apache.atlas.repository.graph.GraphHelper.getClassificationEdg ...@@ -66,7 +70,9 @@ import static org.apache.atlas.repository.graph.GraphHelper.getClassificationEdg
import static org.apache.atlas.repository.graph.GraphHelper.getClassificationEntityGuid; import static org.apache.atlas.repository.graph.GraphHelper.getClassificationEntityGuid;
import static org.apache.atlas.repository.graph.GraphHelper.getClassificationName; import static org.apache.atlas.repository.graph.GraphHelper.getClassificationName;
import static org.apache.atlas.repository.graph.GraphHelper.getClassificationVertices; import static org.apache.atlas.repository.graph.GraphHelper.getClassificationVertices;
import static org.apache.atlas.repository.graph.GraphHelper.getCollectionElementsUsingRelationship;
import static org.apache.atlas.repository.graph.GraphHelper.getGuid; import static org.apache.atlas.repository.graph.GraphHelper.getGuid;
import static org.apache.atlas.repository.graph.GraphHelper.getMapValuesUsingRelationship;
import static org.apache.atlas.repository.graph.GraphHelper.getPropagatedClassificationEdge; import static org.apache.atlas.repository.graph.GraphHelper.getPropagatedClassificationEdge;
import static org.apache.atlas.repository.graph.GraphHelper.getPropagatedEdges; import static org.apache.atlas.repository.graph.GraphHelper.getPropagatedEdges;
import static org.apache.atlas.repository.graph.GraphHelper.getPropagationEnabledClassificationVertices; import static org.apache.atlas.repository.graph.GraphHelper.getPropagationEnabledClassificationVertices;
...@@ -77,7 +83,9 @@ import static org.apache.atlas.repository.graph.GraphHelper.isPropagatedClassifi ...@@ -77,7 +83,9 @@ import static org.apache.atlas.repository.graph.GraphHelper.isPropagatedClassifi
import static org.apache.atlas.repository.graph.GraphHelper.string; import static org.apache.atlas.repository.graph.GraphHelper.string;
import static org.apache.atlas.repository.graph.GraphHelper.updateModificationMetadata; import static org.apache.atlas.repository.graph.GraphHelper.updateModificationMetadata;
import static org.apache.atlas.repository.store.graph.v1.AtlasGraphUtilsV1.getIdFromEdge; import static org.apache.atlas.repository.store.graph.v1.AtlasGraphUtilsV1.getIdFromEdge;
import static org.apache.atlas.repository.store.graph.v1.AtlasGraphUtilsV1.getQualifiedAttributePropertyKey;
import static org.apache.atlas.repository.store.graph.v1.AtlasGraphUtilsV1.getState; import static org.apache.atlas.repository.store.graph.v1.AtlasGraphUtilsV1.getState;
import static org.apache.atlas.repository.store.graph.v1.AtlasGraphUtilsV1.isReference;
public abstract class DeleteHandlerV1 { public abstract class DeleteHandlerV1 {
public static final Logger LOG = LoggerFactory.getLogger(DeleteHandlerV1.class); public static final Logger LOG = LoggerFactory.getLogger(DeleteHandlerV1.class);
...@@ -198,63 +206,43 @@ public abstract class DeleteHandlerV1 { ...@@ -198,63 +206,43 @@ public abstract class DeleteHandlerV1 {
continue; continue;
} }
String edgeLabel = AtlasGraphUtilsV1.getAttributeEdgeLabel(entityType, attributeInfo.getName()); String edgeLabel = attributeInfo.getRelationshipEdgeLabel();
AtlasType attrType = attributeInfo.getAttributeType(); AtlasType attrType = attributeInfo.getAttributeType();
TypeCategory typeCategory = attrType.getTypeCategory();
switch (attrType.getTypeCategory()) { if (typeCategory == OBJECT_ID_TYPE) {
case OBJECT_ID_TYPE: { AtlasEdge edge = graphHelper.getEdgeForLabel(vertex, edgeLabel);
AtlasEdge edge = graphHelper.getEdgeForLabel(vertex, edgeLabel);
if (edge != null && getState(edge) == AtlasEntity.Status.ACTIVE) { if (edge == null || getState(edge) == DELETED) {
vertices.push(edge.getInVertex()); continue;
}
} }
break;
case ARRAY: {
AtlasArrayType arrType = (AtlasArrayType) attrType;
if (arrType.getElementType().getTypeCategory() != TypeCategory.OBJECT_ID_TYPE) {
continue;
}
Iterator<AtlasEdge> edges = graphHelper.getOutGoingEdgesByLabel(vertex, edgeLabel); vertices.push(edge.getInVertex());
if (edges != null) { } else if (typeCategory == ARRAY || typeCategory == MAP) {
while (edges.hasNext()) { TypeCategory elementType = null;
AtlasEdge edge = edges.next();
if (edge != null && getState(edge) == AtlasEntity.Status.ACTIVE) { if (typeCategory == ARRAY) {
vertices.push(edge.getInVertex()); elementType = ((AtlasArrayType) attrType).getElementType().getTypeCategory();
} } else if (typeCategory == MAP) {
} elementType = ((AtlasMapType) attrType).getValueType().getTypeCategory();
}
} }
break;
case MAP: {
AtlasMapType mapType = (AtlasMapType) attrType;
TypeCategory valueTypeCategory = mapType.getValueType().getTypeCategory();
if (valueTypeCategory != TypeCategory.OBJECT_ID_TYPE) {
continue;
}
String propertyName = AtlasGraphUtilsV1.getQualifiedAttributePropertyKey(entityType, attributeInfo.getName()); if (elementType != OBJECT_ID_TYPE) {
List<String> keys = vertex.getProperty(propertyName, List.class); continue;
}
if (keys != null) { List<AtlasEdge> edges = getCollectionElementsUsingRelationship(vertex, attributeInfo);
for (String key : keys) {
String mapEdgeLabel = GraphHelper.getQualifiedNameForMapKey(edgeLabel, key);
AtlasEdge edge = graphHelper.getEdgeForLabel(vertex, mapEdgeLabel);
if (edge != null && getState(edge) == AtlasEntity.Status.ACTIVE) { if (CollectionUtils.isNotEmpty(edges)) {
vertices.push(edge.getInVertex()); for (AtlasEdge edge : edges) {
} if (edge == null || getState(edge) == DELETED) {
continue;
} }
vertices.push(edge.getInVertex());
} }
} }
break;
} }
} }
} }
...@@ -284,14 +272,14 @@ public abstract class DeleteHandlerV1 { ...@@ -284,14 +272,14 @@ public abstract class DeleteHandlerV1 {
} }
boolean isInternalType = isInternalType(entityVertex); boolean isInternalType = isInternalType(entityVertex);
boolean forceDelete = (typeCategory == TypeCategory.STRUCT || typeCategory == TypeCategory.CLASSIFICATION) boolean forceDelete = (typeCategory == STRUCT || typeCategory == CLASSIFICATION)
&& (forceDeleteStructTrait || isInternalType); && (forceDeleteStructTrait || isInternalType);
if (LOG.isDebugEnabled()) { if (LOG.isDebugEnabled()) {
LOG.debug("isInternal = {}, forceDelete = {}", isInternalType, forceDelete); LOG.debug("isInternal = {}, forceDelete = {}", isInternalType, forceDelete);
} }
if (typeCategory == TypeCategory.STRUCT || typeCategory == TypeCategory.CLASSIFICATION || (typeCategory == TypeCategory.OBJECT_ID_TYPE && isOwned)) { if (typeCategory == STRUCT || typeCategory == CLASSIFICATION || (typeCategory == OBJECT_ID_TYPE && isOwned)) {
if (LOG.isDebugEnabled()) { if (LOG.isDebugEnabled()) {
LOG.debug("Processing delete for typeCategory={}, isOwned={}", typeCategory, isOwned); LOG.debug("Processing delete for typeCategory={}, isOwned={}", typeCategory, isOwned);
} }
...@@ -681,7 +669,7 @@ public abstract class DeleteHandlerV1 { ...@@ -681,7 +669,7 @@ public abstract class DeleteHandlerV1 {
boolean isOwned = isEntityType && attributeInfo.isOwnedRef(); boolean isOwned = isEntityType && attributeInfo.isOwnedRef();
AtlasType attrType = attributeInfo.getAttributeType(); AtlasType attrType = attributeInfo.getAttributeType();
String edgeLabel = AtlasGraphUtilsV1.getAttributeEdgeLabel(structType, attributeInfo.getName()); String edgeLabel = attributeInfo.getRelationshipEdgeLabel();
switch (attrType.getTypeCategory()) { switch (attrType.getTypeCategory()) {
case OBJECT_ID_TYPE: case OBJECT_ID_TYPE:
...@@ -699,13 +687,11 @@ public abstract class DeleteHandlerV1 { ...@@ -699,13 +687,11 @@ public abstract class DeleteHandlerV1 {
AtlasArrayType arrType = (AtlasArrayType) attrType; AtlasArrayType arrType = (AtlasArrayType) attrType;
AtlasType elemType = arrType.getElementType(); AtlasType elemType = arrType.getElementType();
if (AtlasGraphUtilsV1.isReference(elemType.getTypeCategory())) { if (isReference(elemType.getTypeCategory())) {
Iterator<AtlasEdge> edges = graphHelper.getOutGoingEdgesByLabel(instanceVertex, edgeLabel); List<AtlasEdge> edges = getCollectionElementsUsingRelationship(instanceVertex, attributeInfo);
if (edges != null) {
while (edges.hasNext()) {
AtlasEdge edge = edges.next();
if (CollectionUtils.isNotEmpty(edges)) {
for (AtlasEdge edge : edges) {
deleteEdgeReference(edge, elemType.getTypeCategory(), isOwned, false, instanceVertex); deleteEdgeReference(edge, elemType.getTypeCategory(), isOwned, false, instanceVertex);
} }
} }
...@@ -715,19 +701,13 @@ public abstract class DeleteHandlerV1 { ...@@ -715,19 +701,13 @@ public abstract class DeleteHandlerV1 {
case MAP: case MAP:
//For map attribute, if the value type is struct/class, delete all the references //For map attribute, if the value type is struct/class, delete all the references
AtlasMapType mapType = (AtlasMapType) attrType; AtlasMapType mapType = (AtlasMapType) attrType;
AtlasType keyType = mapType.getKeyType();
TypeCategory valueTypeCategory = mapType.getValueType().getTypeCategory(); TypeCategory valueTypeCategory = mapType.getValueType().getTypeCategory();
String propertyName = AtlasGraphUtilsV1.getQualifiedAttributePropertyKey(structType, attributeInfo.getName());
if (AtlasGraphUtilsV1.isReference(valueTypeCategory)) {
List<Object> keys = EntityGraphMapper.getArrayElementsProperty(keyType, instanceVertex, propertyName);
if (keys != null) { if (isReference(valueTypeCategory)) {
for (Object key : keys) { List<AtlasEdge> edges = getMapValuesUsingRelationship(instanceVertex, attributeInfo);
String mapEdgeLabel = GraphHelper.getQualifiedNameForMapKey(edgeLabel, (String) key);
deleteEdgeReference(instanceVertex, mapEdgeLabel, valueTypeCategory, isOwned); for (AtlasEdge edge : edges) {
} deleteEdgeReference(edge, valueTypeCategory, isOwned, false, instanceVertex);
} }
} }
break; break;
...@@ -772,8 +752,8 @@ public abstract class DeleteHandlerV1 { ...@@ -772,8 +752,8 @@ public abstract class DeleteHandlerV1 {
} }
AtlasStructType parentType = (AtlasStructType) typeRegistry.getType(typeName); AtlasStructType parentType = (AtlasStructType) typeRegistry.getType(typeName);
String propertyName = AtlasGraphUtilsV1.getQualifiedAttributePropertyKey(parentType, attribute.getName()); String propertyName = getQualifiedAttributePropertyKey(parentType, attribute.getName());
String edgeLabel = EDGE_LABEL_PREFIX + propertyName; String edgeLabel = attribute.getRelationshipEdgeLabel();
AtlasEdge edge = null; AtlasEdge edge = null;
AtlasAttributeDef attrDef = attribute.getAttributeDef(); AtlasAttributeDef attrDef = attribute.getAttributeDef();
AtlasType attrType = attribute.getAttributeType(); AtlasType attrType = attribute.getAttributeType();
...@@ -796,14 +776,12 @@ public abstract class DeleteHandlerV1 { ...@@ -796,14 +776,12 @@ public abstract class DeleteHandlerV1 {
case ARRAY: { case ARRAY: {
//If its array attribute, find the right edge between the two vertices and update array property //If its array attribute, find the right edge between the two vertices and update array property
List<String> elements = GraphHelper.getListProperty(outVertex, propertyName); List<AtlasEdge> elementEdges = getCollectionElementsUsingRelationship(outVertex, attribute);
if (elements != null) {
elements = new ArrayList<>(elements);
for (String elementEdgeId : elements) { if (elementEdges != null) {
AtlasEdge elementEdge = graphHelper.getEdgeByEdgeId(outVertex, edgeLabel, elementEdgeId); elementEdges = new ArrayList<>(elementEdges);
for (AtlasEdge elementEdge : elementEdges) {
if (elementEdge == null) { if (elementEdge == null) {
continue; continue;
} }
...@@ -814,26 +792,10 @@ public abstract class DeleteHandlerV1 { ...@@ -814,26 +792,10 @@ public abstract class DeleteHandlerV1 {
edge = elementEdge; edge = elementEdge;
//TODO element.size includes deleted items as well. should exclude //TODO element.size includes deleted items as well. should exclude
if (!attrDef.getIsOptional() && elements.size() <= attrDef.getValuesMinCount()) { if (!attrDef.getIsOptional() && elementEdges.size() <= attrDef.getValuesMinCount()) {
// Deleting this edge would violate the attribute's lower bound. // Deleting this edge would violate the attribute's lower bound.
throw new AtlasBaseException("Cannot remove array element from required attribute " + propertyName + " on " + GraphHelper.getVertexDetails(outVertex) + " " + GraphHelper.getEdgeDetails(elementEdge)); throw new AtlasBaseException("Cannot remove array element from required attribute " + propertyName + " on " + GraphHelper.getVertexDetails(outVertex) + " " + GraphHelper.getEdgeDetails(elementEdge));
} }
if (shouldUpdateInverseReferences) {
//if composite attribute, remove the reference as well. else, just remove the edge
//for example, when table is deleted, process still references the table
//but when column is deleted, table will not reference the deleted column
if (LOG.isDebugEnabled()) {
LOG.debug("Removing edge {} from the array attribute {}", string(elementEdge), attribute.getName());
}
// Remove all occurrences of the edge ID from the list.
// This prevents dangling edge IDs (i.e. edge IDs for deleted edges)
// from the remaining in the list if there are duplicates.
elements.removeAll(Collections.singletonList(elementEdge.getId().toString()));
GraphHelper.setProperty(outVertex, propertyName, elements);
break;
}
} }
} }
} }
...@@ -842,37 +804,22 @@ public abstract class DeleteHandlerV1 { ...@@ -842,37 +804,22 @@ public abstract class DeleteHandlerV1 {
case MAP: { case MAP: {
//If its map attribute, find the right edge between two vertices and update map property //If its map attribute, find the right edge between two vertices and update map property
List<String> keys = GraphHelper.getListProperty(outVertex, propertyName); List<AtlasEdge> mapEdges = getMapValuesUsingRelationship(outVertex, attribute);
if (keys != null) { if (mapEdges != null) {
keys = new ArrayList<>(keys); mapEdges = new ArrayList<>(mapEdges);
for (String key : keys) {
String keyPropertyName = GraphHelper.getQualifiedNameForMapKey(propertyName, key);
String mapEdgeId = GraphHelper.getSingleValuedProperty(outVertex, keyPropertyName, String.class);
AtlasEdge mapEdge = graphHelper.getEdgeByEdgeId(outVertex, keyPropertyName, mapEdgeId);
for (AtlasEdge mapEdge : mapEdges) {
if (mapEdge != null) { if (mapEdge != null) {
AtlasVertex mapVertex = mapEdge.getInVertex(); AtlasVertex mapVertex = mapEdge.getInVertex();
if (mapVertex.getId().toString().equals(inVertex.getId().toString())) { if (mapVertex.getId().toString().equals(inVertex.getId().toString())) {
//TODO keys.size includes deleted items as well. should exclude //TODO keys.size includes deleted items as well. should exclude
if (attrDef.getIsOptional() || keys.size() > attrDef.getValuesMinCount()) { if (attrDef.getIsOptional() || mapEdges.size() > attrDef.getValuesMinCount()) {
edge = mapEdge; edge = mapEdge;
} else { } else {
// Deleting this entry would violate the attribute's lower bound. // Deleting this entry would violate the attribute's lower bound.
throw new AtlasBaseException("Cannot remove map entry " + keyPropertyName + " from required attribute " + propertyName + " on " + GraphHelper.getVertexDetails(outVertex) + " " + GraphHelper.getEdgeDetails(mapEdge)); throw new AtlasBaseException("Cannot remove map entry " + propertyName + " from required attribute " + propertyName + " on " + GraphHelper.getVertexDetails(outVertex) + " " + GraphHelper.getEdgeDetails(mapEdge));
}
if (shouldUpdateInverseReferences) {
//remove this key
if (LOG.isDebugEnabled()) {
LOG.debug("Removing edge {}, key {} from the map attribute {}", string(mapEdge), key, attribute.getName());
}
keys.remove(key);
GraphHelper.setProperty(outVertex, propertyName, keys);
GraphHelper.setProperty(outVertex, keyPropertyName, null);
} }
break; break;
} }
...@@ -968,7 +915,7 @@ public abstract class DeleteHandlerV1 { ...@@ -968,7 +915,7 @@ public abstract class DeleteHandlerV1 {
removeTagPropagation(classificationVertex); removeTagPropagation(classificationVertex);
deleteEdgeReference(edge, TypeCategory.CLASSIFICATION, false, false, instanceVertex); deleteEdgeReference(edge, CLASSIFICATION, false, false, instanceVertex);
} }
} }
......
repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphMapper.java
...@@ -56,7 +56,6 @@ import org.apache.atlas.type.AtlasTypeRegistry; ...@@ -56,7 +56,6 @@ import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.atlas.type.AtlasTypeUtil; import org.apache.atlas.type.AtlasTypeUtil;
import org.apache.atlas.utils.AtlasJson; import org.apache.atlas.utils.AtlasJson;
import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.IteratorUtils;
import org.apache.commons.collections.MapUtils; import org.apache.commons.collections.MapUtils;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger; import org.slf4j.Logger;
...@@ -68,26 +67,36 @@ import java.util.*; ...@@ -68,26 +67,36 @@ import java.util.*;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import static org.apache.atlas.model.TypeCategory.CLASSIFICATION; import static org.apache.atlas.model.TypeCategory.CLASSIFICATION;
import static org.apache.atlas.model.instance.AtlasEntity.Status.DELETED;
import static org.apache.atlas.model.instance.AtlasRelatedObjectId.KEY_RELATIONSHIP_ATTRIBUTES; import static org.apache.atlas.model.instance.AtlasRelatedObjectId.KEY_RELATIONSHIP_ATTRIBUTES;
import static org.apache.atlas.model.instance.EntityMutations.EntityOperation.CREATE; import static org.apache.atlas.model.instance.EntityMutations.EntityOperation.CREATE;
import static org.apache.atlas.model.instance.EntityMutations.EntityOperation.DELETE; import static org.apache.atlas.model.instance.EntityMutations.EntityOperation.DELETE;
import static org.apache.atlas.model.instance.EntityMutations.EntityOperation.PARTIAL_UPDATE; import static org.apache.atlas.model.instance.EntityMutations.EntityOperation.PARTIAL_UPDATE;
import static org.apache.atlas.model.instance.EntityMutations.EntityOperation.UPDATE; import static org.apache.atlas.model.instance.EntityMutations.EntityOperation.UPDATE;
import static org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef.Cardinality.SET; import static org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef.Cardinality.SET;
import static org.apache.atlas.repository.Constants.ATTRIBUTE_KEY_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.CLASSIFICATION_LABEL; import static org.apache.atlas.repository.Constants.CLASSIFICATION_LABEL;
import static org.apache.atlas.repository.Constants.STATE_PROPERTY_KEY; import static org.apache.atlas.repository.Constants.STATE_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.TRAIT_NAMES_PROPERTY_KEY; import static org.apache.atlas.repository.Constants.TRAIT_NAMES_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.ATTRIBUTE_INDEX_PROPERTY_KEY;
import static org.apache.atlas.repository.graph.GraphHelper.getCollectionElementsUsingRelationship;
import static org.apache.atlas.repository.graph.GraphHelper.getClassificationEdge; import static org.apache.atlas.repository.graph.GraphHelper.getClassificationEdge;
import static org.apache.atlas.repository.graph.GraphHelper.getClassificationVertex; import static org.apache.atlas.repository.graph.GraphHelper.getClassificationVertex;
import static org.apache.atlas.repository.graph.GraphHelper.getListProperty;
import static org.apache.atlas.repository.graph.GraphHelper.getMapElementsProperty;
import static org.apache.atlas.repository.graph.GraphHelper.getQualifiedNameForMapKey;
import static org.apache.atlas.repository.graph.GraphHelper.getStatus;
import static org.apache.atlas.repository.graph.GraphHelper.getTraitLabel; import static org.apache.atlas.repository.graph.GraphHelper.getTraitLabel;
import static org.apache.atlas.repository.graph.GraphHelper.getTraitNames; import static org.apache.atlas.repository.graph.GraphHelper.getTraitNames;
import static org.apache.atlas.repository.graph.GraphHelper.getTypeName; import static org.apache.atlas.repository.graph.GraphHelper.getTypeName;
import static org.apache.atlas.repository.graph.GraphHelper.getTypeNames; import static org.apache.atlas.repository.graph.GraphHelper.getTypeNames;
import static org.apache.atlas.repository.graph.GraphHelper.isPropagationEnabled; import static org.apache.atlas.repository.graph.GraphHelper.isPropagationEnabled;
import static org.apache.atlas.repository.graph.GraphHelper.isRelationshipEdge; import static org.apache.atlas.repository.graph.GraphHelper.isRelationshipEdge;
import static org.apache.atlas.repository.graph.GraphHelper.setListProperty;
import static org.apache.atlas.repository.graph.GraphHelper.string; import static org.apache.atlas.repository.graph.GraphHelper.string;
import static org.apache.atlas.repository.graph.GraphHelper.updateModificationMetadata; import static org.apache.atlas.repository.graph.GraphHelper.updateModificationMetadata;
import static org.apache.atlas.repository.store.graph.v1.AtlasGraphUtilsV1.getIdFromVertex; import static org.apache.atlas.repository.store.graph.v1.AtlasGraphUtilsV1.getIdFromVertex;
import static org.apache.atlas.repository.store.graph.v1.AtlasGraphUtilsV1.isReference;
import static org.apache.atlas.type.AtlasStructType.AtlasAttribute.AtlasRelationshipEdgeDirection.IN; import static org.apache.atlas.type.AtlasStructType.AtlasAttribute.AtlasRelationshipEdgeDirection.IN;
import static org.apache.atlas.type.AtlasStructType.AtlasAttribute.AtlasRelationshipEdgeDirection.OUT; import static org.apache.atlas.type.AtlasStructType.AtlasAttribute.AtlasRelationshipEdgeDirection.OUT;
...@@ -794,64 +803,59 @@ public class EntityGraphMapper { ...@@ -794,64 +803,59 @@ public class EntityGraphMapper {
LOG.debug("==> mapMapValue({})", ctx); LOG.debug("==> mapMapValue({})", ctx);
} }
@SuppressWarnings("unchecked") Map<Object, Object> newVal = (Map<Object, Object>) ctx.getValue();
Map<Object, Object> newVal = (Map<Object, Object>) ctx.getValue(); Map<String, Object> newMap = new HashMap<>();
Map<String, Object> newMap = new HashMap<>(); AtlasMapType mapType = (AtlasMapType) ctx.getAttrType();
AtlasMapType mapType = (AtlasMapType) ctx.getAttrType(); AtlasAttribute attribute = ctx.getAttribute();
Map<String, Object> currentMap = getMapElementsProperty(mapType, ctx.getReferringVertex(), ctx.getVertexProperty(), attribute);
boolean isReference = isReference(mapType.getValueType());
try { if (MapUtils.isNotEmpty(newVal)) {
AtlasAttribute attribute = ctx.getAttribute(); AtlasAttribute inverseRefAttribute = attribute.getInverseRefAttribute();
List<String> currentKeys = GraphHelper.getListProperty(ctx.getReferringVertex(), ctx.getVertexProperty());
Map<String, Object> currentMap = new HashMap<>();
if (CollectionUtils.isNotEmpty(currentKeys)) {
for (String key : currentKeys) {
String propertyNameForKey = GraphHelper.getQualifiedNameForMapKey(ctx.getVertexProperty(), GraphHelper.encodePropertyKey(key));
Object propertyValueForKey = getMapValueProperty(mapType.getValueType(), ctx.getReferringVertex(), propertyNameForKey);
currentMap.put(key, propertyValueForKey); for (Map.Entry<Object, Object> entry : newVal.entrySet()) {
} String key = entry.getKey().toString();
} String propertyName = (isReference) ? ctx.getVertexProperty() : getQualifiedNameForMapKey(ctx.getVertexProperty(), GraphHelper.encodePropertyKey(key));
AtlasEdge existingEdge = getEdgeIfExists(mapType, currentMap, key);
if (MapUtils.isNotEmpty(newVal)) { AttributeMutationContext mapCtx = new AttributeMutationContext(ctx.getOp(), ctx.getReferringVertex(), attribute, entry.getValue(),
boolean isReference = AtlasGraphUtilsV1.isReference(mapType.getValueType()); propertyName, mapType.getValueType(), existingEdge);
AtlasAttribute inverseRefAttribute = attribute.getInverseRefAttribute();
for (Map.Entry<Object, Object> entry : newVal.entrySet()) {
String key = entry.getKey().toString();
String propertyName = GraphHelper.getQualifiedNameForMapKey(ctx.getVertexProperty(), GraphHelper.encodePropertyKey(key));
AtlasEdge existingEdge = getEdgeIfExists(mapType, currentMap, key);
AttributeMutationContext mapCtx = new AttributeMutationContext(ctx.getOp(), ctx.getReferringVertex(), attribute, entry.getValue(), propertyName, mapType.getValueType(), existingEdge); //Add/Update/Remove property value
Object newEntry = mapCollectionElementsToVertex(mapCtx, context);
//Add/Update/Remove property value if (isReference) {
Object newEntry = mapCollectionElementsToVertex(mapCtx, context); AtlasEdge edge = (AtlasEdge) newEntry;
setMapValueProperty(mapType.getValueType(), ctx.getReferringVertex(), propertyName, newEntry); edge.setProperty(ATTRIBUTE_KEY_PROPERTY_KEY, key);
} else {
ctx.getReferringVertex().setProperty(propertyName, newEntry);
}
newMap.put(key, newEntry); newMap.put(key, newEntry);
// If value type indicates this attribute is a reference, and the attribute has an inverse reference attribute, // If value type indicates this attribute is a reference, and the attribute has an inverse reference attribute,
// update the inverse reference value. // update the inverse reference value.
if (isReference && newEntry instanceof AtlasEdge && inverseRefAttribute != null) { if (isReference && newEntry instanceof AtlasEdge && inverseRefAttribute != null) {
AtlasEdge newEdge = (AtlasEdge) newEntry; AtlasEdge newEdge = (AtlasEdge) newEntry;
addInverseReference(context, inverseRefAttribute, newEdge, getRelationshipAttributes(ctx.getValue())); addInverseReference(context, inverseRefAttribute, newEdge, getRelationshipAttributes(ctx.getValue()));
}
} }
} }
}
Map<String, Object> finalMap = removeUnusedMapEntries(attribute, ctx.getReferringVertex(), ctx.getVertexProperty(), currentMap, newMap); Map<String, Object> finalMap = removeUnusedMapEntries(attribute, ctx.getReferringVertex(), ctx.getVertexProperty(), currentMap, newMap);
for (Object newEntry : newMap.values()) { for (Object newEntry : newMap.values()) {
updateInConsistentOwnedMapVertices(ctx, mapType, newEntry); updateInConsistentOwnedMapVertices(ctx, mapType, newEntry);
} }
// for dereference on way out for primitive map types
if (!isReference) {
Set<String> newKeys = new LinkedHashSet<>(newMap.keySet()); Set<String> newKeys = new LinkedHashSet<>(newMap.keySet());
newKeys.addAll(finalMap.keySet()); newKeys.addAll(finalMap.keySet());
// for dereference on way out setListProperty(ctx.getReferringVertex(), ctx.getVertexProperty(), new ArrayList<>(newKeys));
GraphHelper.setListProperty(ctx.getReferringVertex(), ctx.getVertexProperty(), new ArrayList<>(newKeys));
} catch (AtlasException e) {
throw new AtlasBaseException(AtlasErrorCode.INTERNAL_ERROR, e);
} }
if (LOG.isDebugEnabled()) { if (LOG.isDebugEnabled()) {
...@@ -870,16 +874,16 @@ public class EntityGraphMapper { ...@@ -870,16 +874,16 @@ public class EntityGraphMapper {
List newElements = (List) ctx.getValue(); List newElements = (List) ctx.getValue();
AtlasArrayType arrType = (AtlasArrayType) attribute.getAttributeType(); AtlasArrayType arrType = (AtlasArrayType) attribute.getAttributeType();
AtlasType elementType = arrType.getElementType(); AtlasType elementType = arrType.getElementType();
boolean isReference = AtlasGraphUtilsV1.isReference(elementType); boolean isReference = isReference(elementType);
AtlasAttribute inverseRefAttribute = attribute.getInverseRefAttribute(); AtlasAttribute inverseRefAttribute = attribute.getInverseRefAttribute();
Cardinality cardinality = attribute.getAttributeDef().getCardinality(); Cardinality cardinality = attribute.getAttributeDef().getCardinality();
List<Object> newElementsCreated = new ArrayList<>(); List<Object> newElementsCreated = new ArrayList<>();
List<Object> currentElements; List<Object> currentElements;
if (isRelationshipAttribute(attribute)) { if (isReference) {
currentElements = getArrayElementsUsingRelationship(ctx.getReferringVertex(), attribute, elementType); currentElements = (List) getCollectionElementsUsingRelationship(ctx.getReferringVertex(), attribute);
} else { } else {
currentElements = getArrayElementsProperty(elementType, ctx.getReferringVertex(), ctx.getVertexProperty()); currentElements = (List) getArrayElementsProperty(elementType, ctx.getReferringVertex(), ctx.getVertexProperty());
} }
if (CollectionUtils.isNotEmpty(newElements)) { if (CollectionUtils.isNotEmpty(newElements)) {
...@@ -912,6 +916,15 @@ public class EntityGraphMapper { ...@@ -912,6 +916,15 @@ public class EntityGraphMapper {
newElementsCreated.addAll(additionalEdges); newElementsCreated.addAll(additionalEdges);
} }
// add index to attributes of array type
for (int index = 0; index < newElementsCreated.size(); index++) {
Object element = newElementsCreated.get(index);
if (element instanceof AtlasEdge) {
AtlasGraphUtilsV1.setProperty((AtlasEdge) element, ATTRIBUTE_INDEX_PROPERTY_KEY, index);
}
}
// for dereference on way out // for dereference on way out
setArrayElementsProperty(elementType, ctx.getReferringVertex(), ctx.getVertexProperty(), newElementsCreated); setArrayElementsProperty(elementType, ctx.getReferringVertex(), ctx.getVertexProperty(), newElementsCreated);
...@@ -922,22 +935,6 @@ public class EntityGraphMapper { ...@@ -922,22 +935,6 @@ public class EntityGraphMapper {
return newElementsCreated; return newElementsCreated;
} }
private boolean isRelationshipAttribute(AtlasAttribute attribute) {
boolean ret = false;
if (attribute != null) {
AtlasStructType structType = attribute.getDefinedInType();
String attributeName = attribute.getName();
if (structType instanceof AtlasEntityType) {
ret = ((AtlasEntityType) structType).hasRelationshipAttribute(attributeName);
}
}
return ret;
}
private AtlasEdge createVertex(AtlasStruct struct, AtlasVertex referringVertex, String edgeLabel, EntityMutationContext context) throws AtlasBaseException { private AtlasEdge createVertex(AtlasStruct struct, AtlasVertex referringVertex, String edgeLabel, EntityMutationContext context) throws AtlasBaseException {
AtlasVertex vertex = createStructVertex(struct); AtlasVertex vertex = createStructVertex(struct);
...@@ -1089,7 +1086,7 @@ public class EntityGraphMapper { ...@@ -1089,7 +1086,7 @@ public class EntityGraphMapper {
public static Object getMapValueProperty(AtlasType elementType, AtlasVertex vertex, String vertexPropertyName) { public static Object getMapValueProperty(AtlasType elementType, AtlasVertex vertex, String vertexPropertyName) {
if (AtlasGraphUtilsV1.isReference(elementType)) { if (isReference(elementType)) {
return vertex.getProperty(vertexPropertyName, AtlasEdge.class); return vertex.getProperty(vertexPropertyName, AtlasEdge.class);
} else if (elementType instanceof AtlasArrayType) { } else if (elementType instanceof AtlasArrayType) {
return vertex.getProperty(vertexPropertyName, List.class); return vertex.getProperty(vertexPropertyName, List.class);
...@@ -1102,7 +1099,7 @@ public class EntityGraphMapper { ...@@ -1102,7 +1099,7 @@ public class EntityGraphMapper {
} }
private static void setMapValueProperty(AtlasType elementType, AtlasVertex vertex, String vertexPropertyName, Object value) { private static void setMapValueProperty(AtlasType elementType, AtlasVertex vertex, String vertexPropertyName, Object value) {
if (AtlasGraphUtilsV1.isReference(elementType)) { if (isReference(elementType)) {
vertex.setPropertyFromElementId(vertexPropertyName, (AtlasEdge)value); vertex.setPropertyFromElementId(vertexPropertyName, (AtlasEdge)value);
} }
else { else {
...@@ -1113,16 +1110,18 @@ public class EntityGraphMapper { ...@@ -1113,16 +1110,18 @@ public class EntityGraphMapper {
//Remove unused entries from map //Remove unused entries from map
private Map<String, Object> removeUnusedMapEntries(AtlasAttribute attribute, AtlasVertex vertex, String propertyName, private Map<String, Object> removeUnusedMapEntries(AtlasAttribute attribute, AtlasVertex vertex, String propertyName,
Map<String, Object> currentMap, Map<String, Object> newMap) Map<String, Object> currentMap, Map<String, Object> newMap)
throws AtlasException, AtlasBaseException { throws AtlasBaseException {
AtlasMapType mapType = (AtlasMapType) attribute.getAttributeType(); AtlasMapType mapType = (AtlasMapType) attribute.getAttributeType();
boolean isReference = isReference(mapType.getValueType());
Map<String, Object> additionalMap = new HashMap<>(); Map<String, Object> additionalMap = new HashMap<>();
for (String currentKey : currentMap.keySet()) { for (String currentKey : currentMap.keySet()) {
boolean shouldDeleteKey = !newMap.containsKey(currentKey); boolean shouldDeleteKey = !newMap.containsKey(currentKey);
if (AtlasGraphUtilsV1.isReference(mapType.getValueType())) { if (isReference) {
//Delete the edge reference if its not part of new edges created/updated //Delete the edge reference if its not part of new edges created/updated
AtlasEdge currentEdge = (AtlasEdge)currentMap.get(currentKey); AtlasEdge currentEdge = (AtlasEdge) currentMap.get(currentKey);
if (!newMap.values().contains(currentEdge)) { if (!newMap.values().contains(currentEdge)) {
boolean deleted = deleteHandler.deleteEdgeReference(currentEdge, mapType.getValueType().getTypeCategory(), attribute.isOwnedRef(), true, vertex); boolean deleted = deleteHandler.deleteEdgeReference(currentEdge, mapType.getValueType().getTypeCategory(), attribute.isOwnedRef(), true, vertex);
...@@ -1134,8 +1133,9 @@ public class EntityGraphMapper { ...@@ -1134,8 +1133,9 @@ public class EntityGraphMapper {
} }
} }
if (shouldDeleteKey) { if (!isReference && shouldDeleteKey) {
String propertyNameForKey = GraphHelper.getQualifiedNameForMapKey(propertyName, GraphHelper.encodePropertyKey(currentKey)); String propertyNameForKey = getQualifiedNameForMapKey(propertyName, GraphHelper.encodePropertyKey(currentKey));
GraphHelper.setProperty(vertex, propertyNameForKey, null); GraphHelper.setProperty(vertex, propertyNameForKey, null);
} }
} }
...@@ -1146,7 +1146,7 @@ public class EntityGraphMapper { ...@@ -1146,7 +1146,7 @@ public class EntityGraphMapper {
private static AtlasEdge getEdgeIfExists(AtlasMapType mapType, Map<String, Object> currentMap, String keyStr) { private static AtlasEdge getEdgeIfExists(AtlasMapType mapType, Map<String, Object> currentMap, String keyStr) {
AtlasEdge ret = null; AtlasEdge ret = null;
if (AtlasGraphUtilsV1.isReference(mapType.getValueType())) { if (isReference(mapType.getValueType())) {
Object val = currentMap.get(keyStr); Object val = currentMap.get(keyStr);
if (val != null) { if (val != null) {
...@@ -1232,7 +1232,7 @@ public class EntityGraphMapper { ...@@ -1232,7 +1232,7 @@ public class EntityGraphMapper {
} }
public static List<Object> getArrayElementsProperty(AtlasType elementType, AtlasVertex vertex, String vertexPropertyName) { public static List<Object> getArrayElementsProperty(AtlasType elementType, AtlasVertex vertex, String vertexPropertyName) {
if (AtlasGraphUtilsV1.isReference(elementType)) { if (isReference(elementType)) {
return (List)vertex.getListProperty(vertexPropertyName, AtlasEdge.class); return (List)vertex.getListProperty(vertexPropertyName, AtlasEdge.class);
} }
else { else {
...@@ -1240,25 +1240,10 @@ public class EntityGraphMapper { ...@@ -1240,25 +1240,10 @@ public class EntityGraphMapper {
} }
} }
public static List<Object> getArrayElementsUsingRelationship(AtlasVertex vertex, AtlasAttribute attribute, AtlasType elementType) {
List<Object> ret = null;
if (AtlasGraphUtilsV1.isReference(elementType)) {
AtlasRelationshipEdgeDirection edgeDirection = attribute.getRelationshipEdgeDirection();
String edgeLabel = attribute.getRelationshipEdgeLabel();
Iterator<AtlasEdge> edgesForLabel = GraphHelper.getEdgesForLabel(vertex, edgeLabel, edgeDirection);
ret = IteratorUtils.toList(edgesForLabel);
}
return ret;
}
private AtlasEdge getEdgeAt(List<Object> currentElements, int index, AtlasType elemType) { private AtlasEdge getEdgeAt(List<Object> currentElements, int index, AtlasType elemType) {
AtlasEdge ret = null; AtlasEdge ret = null;
if (AtlasGraphUtilsV1.isReference(elemType)) { if (isReference(elemType)) {
if (currentElements != null && index < currentElements.size()) { if (currentElements != null && index < currentElements.size()) {
ret = (AtlasEdge) currentElements.get(index); ret = (AtlasEdge) currentElements.get(index);
} }
...@@ -1273,7 +1258,7 @@ public class EntityGraphMapper { ...@@ -1273,7 +1258,7 @@ public class EntityGraphMapper {
if (CollectionUtils.isNotEmpty(currentEntries)) { if (CollectionUtils.isNotEmpty(currentEntries)) {
AtlasType entryType = ((AtlasArrayType) attribute.getAttributeType()).getElementType(); AtlasType entryType = ((AtlasArrayType) attribute.getAttributeType()).getElementType();
if (AtlasGraphUtilsV1.isReference(entryType)) { if (isReference(entryType)) {
Collection<AtlasEdge> edgesToRemove = CollectionUtils.subtract(currentEntries, newEntries); Collection<AtlasEdge> edgesToRemove = CollectionUtils.subtract(currentEntries, newEntries);
if (CollectionUtils.isNotEmpty(edgesToRemove)) { if (CollectionUtils.isNotEmpty(edgesToRemove)) {
...@@ -1296,15 +1281,11 @@ public class EntityGraphMapper { ...@@ -1296,15 +1281,11 @@ public class EntityGraphMapper {
return Collections.emptyList(); return Collections.emptyList();
} }
private void setArrayElementsProperty(AtlasType elementType, AtlasVertex vertex, String vertexPropertyName, List<Object> values) { private void setArrayElementsProperty(AtlasType elementType, AtlasVertex vertex, String vertexPropertyName, List<Object> values) {
if (AtlasGraphUtilsV1.isReference(elementType)) { if (!isReference(elementType)) {
GraphHelper.setListPropertyFromElementIds(vertex, vertexPropertyName, (List) values);
}
else {
GraphHelper.setProperty(vertex, vertexPropertyName, values); GraphHelper.setProperty(vertex, vertexPropertyName, values);
} }
} }
private AtlasEntityHeader constructHeader(AtlasEntity entity, final AtlasEntityType type, AtlasVertex vertex) { private AtlasEntityHeader constructHeader(AtlasEntity entity, final AtlasEntityType type, AtlasVertex vertex) {
AtlasEntityHeader header = new AtlasEntityHeader(entity.getTypeName()); AtlasEntityHeader header = new AtlasEntityHeader(entity.getTypeName());
...@@ -1324,9 +1305,9 @@ public class EntityGraphMapper { ...@@ -1324,9 +1305,9 @@ public class EntityGraphMapper {
private void updateInConsistentOwnedMapVertices(AttributeMutationContext ctx, AtlasMapType mapType, Object val) { private void updateInConsistentOwnedMapVertices(AttributeMutationContext ctx, AtlasMapType mapType, Object val) {
if (mapType.getValueType().getTypeCategory() == TypeCategory.OBJECT_ID_TYPE) { if (mapType.getValueType().getTypeCategory() == TypeCategory.OBJECT_ID_TYPE) {
AtlasEdge edge = (AtlasEdge) val; AtlasEdge edge = (AtlasEdge) val;
if (ctx.getAttribute().isOwnedRef() &&
GraphHelper.getStatus(edge) == AtlasEntity.Status.DELETED && if (ctx.getAttribute().isOwnedRef() && getStatus(edge) == DELETED && getStatus(edge.getInVertex()) == DELETED) {
GraphHelper.getStatus(edge.getInVertex()) == AtlasEntity.Status.DELETED) {
//Resurrect the vertex and edge to ACTIVE state //Resurrect the vertex and edge to ACTIVE state
GraphHelper.setProperty(edge, STATE_PROPERTY_KEY, AtlasEntity.Status.ACTIVE.name()); GraphHelper.setProperty(edge, STATE_PROPERTY_KEY, AtlasEntity.Status.ACTIVE.name());
GraphHelper.setProperty(edge.getInVertex(), STATE_PROPERTY_KEY, AtlasEntity.Status.ACTIVE.name()); GraphHelper.setProperty(edge.getInVertex(), STATE_PROPERTY_KEY, AtlasEntity.Status.ACTIVE.name());
...@@ -1813,6 +1794,7 @@ public class EntityGraphMapper { ...@@ -1813,6 +1794,7 @@ public class EntityGraphMapper {
currentEntityId = getIdFromOutVertex(currentEdge); currentEntityId = getIdFromOutVertex(currentEdge);
} }
return currentEntityId; return currentEntityId;
} }
......
repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphRetriever.java
...@@ -54,6 +54,7 @@ import org.apache.atlas.type.AtlasTypeRegistry; ...@@ -54,6 +54,7 @@ import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.atlas.type.AtlasTypeUtil; import org.apache.atlas.type.AtlasTypeUtil;
import org.apache.atlas.utils.AtlasJson; import org.apache.atlas.utils.AtlasJson;
import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.MapUtils;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
...@@ -97,15 +98,19 @@ import static org.apache.atlas.repository.graph.GraphHelper.getAdjacentEdgesByLa ...@@ -97,15 +98,19 @@ import static org.apache.atlas.repository.graph.GraphHelper.getAdjacentEdgesByLa
import static org.apache.atlas.repository.graph.GraphHelper.getAllClassificationEdges; import static org.apache.atlas.repository.graph.GraphHelper.getAllClassificationEdges;
import static org.apache.atlas.repository.graph.GraphHelper.getAllTraitNames; import static org.apache.atlas.repository.graph.GraphHelper.getAllTraitNames;
import static org.apache.atlas.repository.graph.GraphHelper.getBlockedClassificationIds; import static org.apache.atlas.repository.graph.GraphHelper.getBlockedClassificationIds;
import static org.apache.atlas.repository.graph.GraphHelper.getArrayElementsProperty;
import static org.apache.atlas.repository.graph.GraphHelper.getClassificationVertices; import static org.apache.atlas.repository.graph.GraphHelper.getClassificationVertices;
import static org.apache.atlas.repository.graph.GraphHelper.getGuid; import static org.apache.atlas.repository.graph.GraphHelper.getGuid;
import static org.apache.atlas.repository.graph.GraphHelper.getIncomingEdgesByLabel; import static org.apache.atlas.repository.graph.GraphHelper.getIncomingEdgesByLabel;
import static org.apache.atlas.repository.graph.GraphHelper.getPrimitiveMap;
import static org.apache.atlas.repository.graph.GraphHelper.getReferenceMap;
import static org.apache.atlas.repository.graph.GraphHelper.getOutGoingEdgesByLabel; import static org.apache.atlas.repository.graph.GraphHelper.getOutGoingEdgesByLabel;
import static org.apache.atlas.repository.graph.GraphHelper.getPropagateTags; import static org.apache.atlas.repository.graph.GraphHelper.getPropagateTags;
import static org.apache.atlas.repository.graph.GraphHelper.getRelationshipGuid; import static org.apache.atlas.repository.graph.GraphHelper.getRelationshipGuid;
import static org.apache.atlas.repository.graph.GraphHelper.getTypeName; import static org.apache.atlas.repository.graph.GraphHelper.getTypeName;
import static org.apache.atlas.repository.graph.GraphHelper.isPropagationEnabled; import static org.apache.atlas.repository.graph.GraphHelper.isPropagationEnabled;
import static org.apache.atlas.repository.store.graph.v1.AtlasGraphUtilsV1.getIdFromVertex; import static org.apache.atlas.repository.store.graph.v1.AtlasGraphUtilsV1.getIdFromVertex;
import static org.apache.atlas.repository.store.graph.v1.AtlasGraphUtilsV1.isReference;
import static org.apache.atlas.type.AtlasStructType.AtlasAttribute.AtlasRelationshipEdgeDirection; import static org.apache.atlas.type.AtlasStructType.AtlasAttribute.AtlasRelationshipEdgeDirection;
import static org.apache.atlas.type.AtlasStructType.AtlasAttribute.AtlasRelationshipEdgeDirection.BOTH; import static org.apache.atlas.type.AtlasStructType.AtlasAttribute.AtlasRelationshipEdgeDirection.BOTH;
import static org.apache.atlas.type.AtlasStructType.AtlasAttribute.AtlasRelationshipEdgeDirection.IN; import static org.apache.atlas.type.AtlasStructType.AtlasAttribute.AtlasRelationshipEdgeDirection.IN;
...@@ -614,10 +619,10 @@ public final class EntityGraphRetriever { ...@@ -614,10 +619,10 @@ public final class EntityGraphRetriever {
ret = mapVertexToObjectId(entityVertex, edgeLabel, null, entityExtInfo, isOwnedAttribute, edgeDirection); ret = mapVertexToObjectId(entityVertex, edgeLabel, null, entityExtInfo, isOwnedAttribute, edgeDirection);
break; break;
case ARRAY: case ARRAY:
ret = mapVertexToArray(entityVertex, (AtlasArrayType) attrType, vertexPropertyName, entityExtInfo, isOwnedAttribute, edgeDirection); ret = mapVertexToArray(entityVertex, vertexPropertyName, entityExtInfo, isOwnedAttribute, attribute);
break; break;
case MAP: case MAP:
ret = mapVertexToMap(entityVertex, (AtlasMapType) attrType, vertexPropertyName, entityExtInfo, isOwnedAttribute, edgeDirection); ret = mapVertexToMap(entityVertex, vertexPropertyName, entityExtInfo, isOwnedAttribute, attribute);
break; break;
case CLASSIFICATION: case CLASSIFICATION:
// do nothing // do nothing
...@@ -627,45 +632,50 @@ public final class EntityGraphRetriever { ...@@ -627,45 +632,50 @@ public final class EntityGraphRetriever {
return ret; return ret;
} }
private Map<String, Object> mapVertexToMap(AtlasVertex entityVertex, AtlasMapType atlasMapType, final String propertyName, private Map<String, Object> mapVertexToMap(AtlasVertex entityVertex, final String propertyName, AtlasEntityExtInfo entityExtInfo,
AtlasEntityExtInfo entityExtInfo, boolean isOwnedAttribute, boolean isOwnedAttribute, AtlasAttribute attribute) throws AtlasBaseException {
AtlasRelationshipEdgeDirection edgeDirection) throws AtlasBaseException {
List<String> mapKeys = GraphHelper.getListProperty(entityVertex, propertyName); Map<String, Object> ret = null;
AtlasMapType mapType = (AtlasMapType) attribute.getAttributeType();
if (CollectionUtils.isEmpty(mapKeys)) { AtlasType mapValueType = mapType.getValueType();
return null;
}
if (LOG.isDebugEnabled()) { if (LOG.isDebugEnabled()) {
LOG.debug("Mapping map attribute {} for vertex {}", atlasMapType.getTypeName(), entityVertex); LOG.debug("Mapping map attribute {} for vertex {}", mapType.getTypeName(), entityVertex);
} }
Map<String, Object> ret = new HashMap<>(mapKeys.size()); if (isReference(mapValueType)) {
AtlasType mapValueType = atlasMapType.getValueType(); Map<String, Object> currentMap = getReferenceMap(entityVertex, attribute);
for (String mapKey : mapKeys) { if (MapUtils.isNotEmpty(currentMap)) {
final String keyPropertyName = propertyName + "." + mapKey; ret = new HashMap<>();
final String edgeLabel = EDGE_LABEL_PREFIX + keyPropertyName;
final Object keyValue = GraphHelper.getMapValueProperty(mapValueType, entityVertex, keyPropertyName);
Object mapValue = mapVertexToCollectionEntry(entityVertex, mapValueType, keyValue, edgeLabel, for (Map.Entry<String, Object> entry : currentMap.entrySet()) {
entityExtInfo, isOwnedAttribute, edgeDirection); String mapKey = entry.getKey();
Object keyValue = entry.getValue();
if (mapValue != null) { Object mapValue = mapVertexToCollectionEntry(entityVertex, mapValueType, keyValue, attribute.getRelationshipEdgeLabel(),
ret.put(mapKey, mapValue); entityExtInfo, isOwnedAttribute, attribute.getRelationshipEdgeDirection());
if (mapValue != null) {
ret.put(mapKey, mapValue);
}
}
} }
} else {
ret = getPrimitiveMap(entityVertex, propertyName, mapValueType);
}
if (MapUtils.isEmpty(ret)) {
ret = null;
} }
return ret; return ret;
} }
private List<Object> mapVertexToArray(AtlasVertex entityVertex, AtlasArrayType arrayType, String propertyName, private List<Object> mapVertexToArray(AtlasVertex entityVertex, String propertyName, AtlasEntityExtInfo entityExtInfo,
AtlasEntityExtInfo entityExtInfo, boolean isOwnedAttribute, boolean isOwnedAttribute, AtlasAttribute attribute) throws AtlasBaseException {
AtlasRelationshipEdgeDirection edgeDirection) throws AtlasBaseException {
AtlasType arrayElementType = arrayType.getElementType(); AtlasArrayType arrayType = (AtlasArrayType) attribute.getAttributeType();
List<Object> arrayElements = GraphHelper.getArrayElementsProperty(arrayElementType, entityVertex, propertyName); AtlasType arrayElementType = arrayType.getElementType();
List<Object> arrayElements = getArrayElementsProperty(arrayElementType, entityVertex, propertyName, attribute);
if (CollectionUtils.isEmpty(arrayElements)) { if (CollectionUtils.isEmpty(arrayElements)) {
return null; return null;
...@@ -675,8 +685,9 @@ public final class EntityGraphRetriever { ...@@ -675,8 +685,9 @@ public final class EntityGraphRetriever {
LOG.debug("Mapping array attribute {} for vertex {}", arrayElementType.getTypeName(), entityVertex); LOG.debug("Mapping array attribute {} for vertex {}", arrayElementType.getTypeName(), entityVertex);
} }
List arrValues = new ArrayList(arrayElements.size()); List arrValues = new ArrayList(arrayElements.size());
String edgeLabel = EDGE_LABEL_PREFIX + propertyName; String edgeLabel = attribute.getRelationshipEdgeLabel();
AtlasRelationshipEdgeDirection edgeDirection = attribute.getRelationshipEdgeDirection();
for (Object element : arrayElements) { for (Object element : arrayElements) {
// When internal types are deleted, sometimes the collection type attribute will contain a null value // When internal types are deleted, sometimes the collection type attribute will contain a null value
......
repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasComplexAttributesTest.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.repository.store.graph.v1;
import org.apache.atlas.TestModules;
import org.apache.atlas.TestUtilsV2;
import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.model.instance.AtlasEntity.AtlasEntitiesWithExtInfo;
import org.apache.atlas.model.instance.AtlasEntity.AtlasEntityWithExtInfo;
import org.apache.atlas.model.instance.AtlasEntityHeader;
import org.apache.atlas.model.instance.AtlasObjectId;
import org.apache.atlas.model.instance.AtlasStruct;
import org.apache.atlas.model.instance.EntityMutationResponse;
import org.apache.atlas.model.typedef.AtlasTypesDef;
import org.apache.atlas.repository.graphdb.AtlasEdge;
import org.apache.atlas.repository.graphdb.AtlasEdgeDirection;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Guice;
import org.testng.annotations.Test;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import static org.apache.atlas.TestUtilsV2.ENTITY_TYPE;
import static org.apache.atlas.TestUtilsV2.ENTITY_TYPE_WITH_COMPLEX_COLLECTION_ATTR;
import static org.apache.atlas.TestUtilsV2.ENTITY_TYPE_WITH_COMPLEX_COLLECTION_ATTR_DELETE;
import static org.apache.atlas.TestUtilsV2.NAME;
import static org.apache.atlas.repository.graph.GraphHelper.getStatus;
import static org.apache.atlas.type.AtlasTypeUtil.getAtlasObjectId;
import static org.testng.Assert.assertNull;
import static org.testng.AssertJUnit.assertEquals;
@Guice(modules = TestModules.TestOnlyModule.class)
public class AtlasComplexAttributesTest extends AtlasEntityTestBase {
private AtlasEntityWithExtInfo complexCollectionAttrEntity;
private AtlasEntityWithExtInfo complexCollectionAttrEntityForDelete;
@BeforeClass
public void setUp() throws Exception {
super.setUp();
// create typeDefs
AtlasTypesDef[] testTypesDefs = new AtlasTypesDef[] { TestUtilsV2.defineTypeWithComplexCollectionAttributes() };
createTypesDef(testTypesDefs);
// create entity
complexCollectionAttrEntity = TestUtilsV2.createComplexCollectionAttrEntity();
complexCollectionAttrEntityForDelete = TestUtilsV2.createComplexCollectionAttrEntity();
}
@Test
public void testCreateComplexAttributeEntity() throws Exception {
init();
EntityMutationResponse response = entityStore.createOrUpdate(new AtlasEntityStream(complexCollectionAttrEntity), false);
AtlasEntityHeader entityCreated = response.getFirstCreatedEntityByTypeName(ENTITY_TYPE_WITH_COMPLEX_COLLECTION_ATTR);
validateEntity(complexCollectionAttrEntity, getEntityFromStore(entityCreated));
}
@Test(dependsOnMethods = "testCreateComplexAttributeEntity")
public void testStructArray() throws Exception {
init();
AtlasEntity complexEntity = getEntityFromStore(complexCollectionAttrEntity.getEntity().getGuid());
AtlasEntitiesWithExtInfo complexEntitiesInfo = new AtlasEntitiesWithExtInfo(complexEntity);
// Modify array of structs
List<AtlasStruct> structList = new ArrayList<>(Arrays.asList(new AtlasStruct("struct_type", "name", "structArray00"),
new AtlasStruct("struct_type", "name", "structArray11"),
new AtlasStruct("struct_type", "name", "structArray22")));
complexEntity.setAttribute("listOfStructs", structList);
EntityMutationResponse response = entityStore.createOrUpdate(new AtlasEntityStream(complexEntitiesInfo), false);
AtlasEntityHeader updatedComplexEntity = response.getFirstUpdatedEntityByTypeName(ENTITY_TYPE_WITH_COMPLEX_COLLECTION_ATTR);
validateEntity(complexEntitiesInfo, getEntityFromStore(updatedComplexEntity));
// add a new element to array of struct
init();
structList.add(new AtlasStruct("struct_type", "name", "structArray33"));
complexEntity.setAttribute("listOfStructs", structList);
response = entityStore.createOrUpdate(new AtlasEntityStream(complexEntitiesInfo), false);
updatedComplexEntity = response.getFirstUpdatedEntityByTypeName(ENTITY_TYPE_WITH_COMPLEX_COLLECTION_ATTR);
validateEntity(complexEntitiesInfo, getEntityFromStore(updatedComplexEntity));
// remove one of the struct values - structArray00
init();
structList.remove(0);
complexEntity.setAttribute("listOfStructs", structList);
response = entityStore.createOrUpdate(new AtlasEntityStream(complexEntitiesInfo), false);
updatedComplexEntity = response.getFirstUpdatedEntityByTypeName(ENTITY_TYPE_WITH_COMPLEX_COLLECTION_ATTR);
validateEntity(complexEntitiesInfo, getEntityFromStore(updatedComplexEntity));
// Update struct value within array of struct
init();
structList.get(0).setAttribute(NAME, "structArray11-edit");
complexEntity.setAttribute("listOfStructs", structList);
response = entityStore.createOrUpdate(new AtlasEntityStream(complexEntitiesInfo), false);
updatedComplexEntity = response.getFirstUpdatedEntityByTypeName(ENTITY_TYPE_WITH_COMPLEX_COLLECTION_ATTR);
validateEntity(complexEntitiesInfo, getEntityFromStore(updatedComplexEntity));
// add a repeated element to array of struct
init();
structList.add(new AtlasStruct("struct_type", "name", "structArray33"));
complexEntity.setAttribute("listOfStructs", structList);
response = entityStore.createOrUpdate(new AtlasEntityStream(complexEntitiesInfo), false);
updatedComplexEntity = response.getFirstUpdatedEntityByTypeName(ENTITY_TYPE_WITH_COMPLEX_COLLECTION_ATTR);
validateEntity(complexEntitiesInfo, getEntityFromStore(updatedComplexEntity));
// Remove all elements. Should set array attribute to null
init();
structList.clear();
complexEntity.setAttribute("listOfStructs", structList);
response = entityStore.createOrUpdate(new AtlasEntityStream(complexEntitiesInfo), false);
updatedComplexEntity = response.getFirstUpdatedEntityByTypeName(ENTITY_TYPE_WITH_COMPLEX_COLLECTION_ATTR);
validateEntity(complexEntitiesInfo, getEntityFromStore(updatedComplexEntity));
}
@Test(dependsOnMethods = "testStructArray")
public void testEntityArray() throws Exception {
init();
AtlasEntity complexEntity = getEntityFromStore(complexCollectionAttrEntity.getEntity().getGuid());
AtlasEntitiesWithExtInfo complexEntitiesInfo = new AtlasEntitiesWithExtInfo(complexEntity);
// Modify array of entities
AtlasEntity e0Array = new AtlasEntity(ENTITY_TYPE, new HashMap<String, Object>() {{ put(NAME, "entityArray00"); put("isReplicated", true); }});
AtlasEntity e1Array = new AtlasEntity(ENTITY_TYPE, new HashMap<String, Object>() {{ put(NAME, "entityArray11"); put("isReplicated", false); }});
AtlasEntity e2Array = new AtlasEntity(ENTITY_TYPE, new HashMap<String, Object>() {{ put(NAME, "entityArray22"); put("isReplicated", true); }});
List<AtlasObjectId> entityList = new ArrayList<>(Arrays.asList(getAtlasObjectId(e0Array), getAtlasObjectId(e1Array), getAtlasObjectId(e2Array)));
complexEntity.setAttribute("listOfEntities", entityList);
complexEntitiesInfo.addReferredEntity(e0Array);
complexEntitiesInfo.addReferredEntity(e1Array);
complexEntitiesInfo.addReferredEntity(e2Array);
init();
EntityMutationResponse response = entityStore.createOrUpdate(new AtlasEntityStream(complexEntitiesInfo), false);
AtlasEntityHeader updatedComplexEntity = response.getFirstUpdatedEntityByTypeName(ENTITY_TYPE_WITH_COMPLEX_COLLECTION_ATTR);
validateEntity(complexEntitiesInfo, getEntityFromStore(updatedComplexEntity));
// add a new element to array of entities
init();
AtlasEntity e3Array = new AtlasEntity(ENTITY_TYPE, new HashMap<String, Object>() {{ put(NAME, "entityArray33"); put("isReplicated", true); }});
entityList.add(getAtlasObjectId(e3Array));
complexEntity.setAttribute("listOfEntities", entityList);
complexEntitiesInfo.addReferredEntity(e3Array);
response = entityStore.createOrUpdate(new AtlasEntityStream(complexEntitiesInfo), false);
updatedComplexEntity = response.getFirstUpdatedEntityByTypeName(ENTITY_TYPE_WITH_COMPLEX_COLLECTION_ATTR);
validateEntity(complexEntitiesInfo, getEntityFromStore(updatedComplexEntity));
// remove one of the entity values - entityArray00
init();
entityList.remove(0);
complexEntity.setAttribute("listOfEntities", entityList);
response = entityStore.createOrUpdate(new AtlasEntityStream(complexEntitiesInfo), false);
updatedComplexEntity = response.getFirstUpdatedEntityByTypeName(ENTITY_TYPE_WITH_COMPLEX_COLLECTION_ATTR);
validateEntity(complexEntitiesInfo, getEntityFromStore(updatedComplexEntity));
// Update entity value within array of struct
init();
e1Array.setAttribute(NAME, "entityArray11-edit");
complexEntity.setAttribute("listOfEntities", entityList);
response = entityStore.createOrUpdate(new AtlasEntityStream(complexEntitiesInfo), false);
updatedComplexEntity = response.getFirstUpdatedEntityByTypeName(ENTITY_TYPE_WITH_COMPLEX_COLLECTION_ATTR);
validateEntity(complexEntitiesInfo, getEntityFromStore(updatedComplexEntity));
// add a repeated element to array of struct
init();
AtlasEntity e3Array_duplicate = new AtlasEntity(ENTITY_TYPE, new HashMap<String, Object>() {{ put(NAME, "entityArray33"); put("isReplicated", true); }});
entityList.add(getAtlasObjectId(e3Array_duplicate));
complexEntity.setAttribute("listOfEntities", entityList);
complexEntitiesInfo.addReferredEntity(e3Array_duplicate);
response = entityStore.createOrUpdate(new AtlasEntityStream(complexEntitiesInfo), false);
updatedComplexEntity = response.getFirstUpdatedEntityByTypeName(ENTITY_TYPE_WITH_COMPLEX_COLLECTION_ATTR);
validateEntity(complexEntitiesInfo, getEntityFromStore(updatedComplexEntity));
// Remove all elements. Should set array attribute to null
init();
entityList.clear();
complexEntity.setAttribute("listOfEntities", entityList);
response = entityStore.createOrUpdate(new AtlasEntityStream(complexEntitiesInfo), false);
updatedComplexEntity = response.getFirstUpdatedEntityByTypeName(ENTITY_TYPE_WITH_COMPLEX_COLLECTION_ATTR);
validateEntity(complexEntitiesInfo, getEntityFromStore(updatedComplexEntity));
}
@Test(dependsOnMethods = "testEntityArray")
public void testStructMap() throws Exception {
init();
AtlasEntity complexEntity = getEntityFromStore(complexCollectionAttrEntity.getEntity().getGuid());
AtlasEntitiesWithExtInfo complexEntitiesInfo = new AtlasEntitiesWithExtInfo(complexEntity);
// Modify map of structs
HashMap<String, AtlasStruct> structMap = new HashMap<String, AtlasStruct>() {{
put("key00", new AtlasStruct("struct_type", "name", "structMap00"));
put("key11", new AtlasStruct("struct_type", "name", "structMap11"));
put("key22", new AtlasStruct("struct_type", "name", "structMap22")); }};
complexEntity.setAttribute("mapOfStructs", structMap);
EntityMutationResponse response = entityStore.createOrUpdate(new AtlasEntityStream(complexEntitiesInfo), false);
AtlasEntityHeader updatedComplexEntity = response.getFirstUpdatedEntityByTypeName(ENTITY_TYPE_WITH_COMPLEX_COLLECTION_ATTR);
validateEntity(complexEntitiesInfo, getEntityFromStore(updatedComplexEntity));
// add a new element to map of struct - structMap6
init();
structMap.put("key33", new AtlasStruct("struct_type", "name", "structMap33"));
complexEntity.setAttribute("mapOfStructs", structMap);
response = entityStore.createOrUpdate(new AtlasEntityStream(complexEntitiesInfo), false);
updatedComplexEntity = response.getFirstUpdatedEntityByTypeName(ENTITY_TYPE_WITH_COMPLEX_COLLECTION_ATTR);
validateEntity(complexEntitiesInfo, getEntityFromStore(updatedComplexEntity));
// remove one of the entity values - structMap
init();
structMap.remove("key00");
complexEntity.setAttribute("mapOfStructs", structMap);
response = entityStore.createOrUpdate(new AtlasEntityStream(complexEntitiesInfo), false);
updatedComplexEntity = response.getFirstUpdatedEntityByTypeName(ENTITY_TYPE_WITH_COMPLEX_COLLECTION_ATTR);
validateEntity(complexEntitiesInfo, getEntityFromStore(updatedComplexEntity));
// Update struct value within map of struct
init();
structMap.get("key11").setAttribute("name", "structMap11-edit");
complexEntity.setAttribute("mapOfStructs", structMap);
response = entityStore.createOrUpdate(new AtlasEntityStream(complexEntitiesInfo), false);
updatedComplexEntity = response.getFirstUpdatedEntityByTypeName(ENTITY_TYPE_WITH_COMPLEX_COLLECTION_ATTR);
validateEntity(complexEntitiesInfo, getEntityFromStore(updatedComplexEntity));
// add a repeated element to array of struct
init();
structMap.put("key33", new AtlasStruct("struct_type", "name", "structMap33"));
complexEntity.setAttribute("mapOfStructs", structMap);
response = entityStore.createOrUpdate(new AtlasEntityStream(complexEntitiesInfo), false);
updatedComplexEntity = response.getFirstUpdatedEntityByTypeName(ENTITY_TYPE_WITH_COMPLEX_COLLECTION_ATTR);
// no update since duplicate entry
assertNull(updatedComplexEntity);
// Remove all elements. Should set array attribute to null
init();
structMap.clear();
complexEntity.setAttribute("mapOfStructs", structMap);
response = entityStore.createOrUpdate(new AtlasEntityStream(complexEntitiesInfo), false);
updatedComplexEntity = response.getFirstUpdatedEntityByTypeName(ENTITY_TYPE_WITH_COMPLEX_COLLECTION_ATTR);
validateEntity(complexEntitiesInfo, getEntityFromStore(updatedComplexEntity));
}
@Test(dependsOnMethods = "testStructMap")
public void testEntityMap() throws Exception {
init();
AtlasEntity complexEntity = getEntityFromStore(complexCollectionAttrEntity.getEntity().getGuid());
AtlasEntitiesWithExtInfo complexEntitiesInfo = new AtlasEntitiesWithExtInfo(complexEntity);
// Modify map of entities
AtlasEntity e0MapValue = new AtlasEntity(ENTITY_TYPE, new HashMap<String, Object>() {{ put(NAME, "entityMapValue00"); put("isReplicated", false); }});
AtlasEntity e1MapValue = new AtlasEntity(ENTITY_TYPE, new HashMap<String, Object>() {{ put(NAME, "entityMapValue11"); put("isReplicated", true); }});
AtlasEntity e2MapValue = new AtlasEntity(ENTITY_TYPE, new HashMap<String, Object>() {{ put(NAME, "entityMapValue22"); put("isReplicated", false); }});
HashMap<String, Object> entityMap = new HashMap<String, Object>() {{ put("key00", getAtlasObjectId(e0MapValue));
put("key11", getAtlasObjectId(e1MapValue));
put("key22", getAtlasObjectId(e2MapValue)); }};
complexEntity.setAttribute("mapOfEntities", entityMap);
complexEntitiesInfo.addReferredEntity(e0MapValue);
complexEntitiesInfo.addReferredEntity(e1MapValue);
complexEntitiesInfo.addReferredEntity(e2MapValue);
init();
EntityMutationResponse response = entityStore.createOrUpdate(new AtlasEntityStream(complexEntitiesInfo), false);
AtlasEntityHeader updatedComplexEntity = response.getFirstUpdatedEntityByTypeName(ENTITY_TYPE_WITH_COMPLEX_COLLECTION_ATTR);
validateEntity(complexEntitiesInfo, getEntityFromStore(updatedComplexEntity));
// add a new element to map of entities
init();
AtlasEntity e3MapValue = new AtlasEntity(ENTITY_TYPE, new HashMap<String, Object>() {{ put(NAME, "entityMapValue33"); put("isReplicated", false); }});
entityMap.put("key33", getAtlasObjectId(e3MapValue));
complexEntity.setAttribute("mapOfEntities", entityMap);
complexEntitiesInfo.addReferredEntity(e3MapValue);
response = entityStore.createOrUpdate(new AtlasEntityStream(complexEntitiesInfo), false);
updatedComplexEntity = response.getFirstUpdatedEntityByTypeName(ENTITY_TYPE_WITH_COMPLEX_COLLECTION_ATTR);
validateEntity(complexEntitiesInfo, getEntityFromStore(updatedComplexEntity));
// remove one of the entity values - [key00 : entityMapValue00]
init();
entityMap.remove("key00");
complexEntity.setAttribute("mapOfEntities", entityMap);
complexEntitiesInfo.addReferredEntity(e3MapValue);
response = entityStore.createOrUpdate(new AtlasEntityStream(complexEntitiesInfo), false);
updatedComplexEntity = response.getFirstUpdatedEntityByTypeName(ENTITY_TYPE_WITH_COMPLEX_COLLECTION_ATTR);
validateEntity(complexEntitiesInfo, getEntityFromStore(updatedComplexEntity));
// Update entity value within map of entities
init();
AtlasEntity e1MapValueEdit = new AtlasEntity(ENTITY_TYPE, new HashMap<String, Object>() {{ put(NAME, "entityMapValue11-edit"); put("isReplicated", false); }});
entityMap.clear();
entityMap.put("key11", getAtlasObjectId(e1MapValueEdit));
entityMap.put("key22", getAtlasObjectId(e2MapValue));
entityMap.put("key33", getAtlasObjectId(e3MapValue));
complexEntity.setAttribute("mapOfEntities", entityMap);
complexEntitiesInfo.addReferredEntity(e1MapValueEdit);
response = entityStore.createOrUpdate(new AtlasEntityStream(complexEntitiesInfo), false);
updatedComplexEntity = response.getFirstUpdatedEntityByTypeName(ENTITY_TYPE_WITH_COMPLEX_COLLECTION_ATTR);
validateEntity(complexEntitiesInfo, getEntityFromStore(updatedComplexEntity));
// add a repeated element to map of entities
init();
e3MapValue = new AtlasEntity(ENTITY_TYPE, new HashMap<String, Object>() {{ put(NAME, "entityMapValue33"); put("isReplicated", false); }});
entityMap.put("key33", getAtlasObjectId(e3MapValue));
complexEntity.setAttribute("mapOfEntities", entityMap);
complexEntitiesInfo.addReferredEntity(e3MapValue);
response = entityStore.createOrUpdate(new AtlasEntityStream(complexEntitiesInfo), false);
updatedComplexEntity = response.getFirstUpdatedEntityByTypeName(ENTITY_TYPE_WITH_COMPLEX_COLLECTION_ATTR);
validateEntity(complexEntitiesInfo, getEntityFromStore(updatedComplexEntity));
// Remove all elements. Should set map attribute to null
init();
entityMap.clear();
complexEntity.setAttribute("mapOfEntities", entityMap);
complexEntitiesInfo.addReferredEntity(e3MapValue);
response = entityStore.createOrUpdate(new AtlasEntityStream(complexEntitiesInfo), false);
updatedComplexEntity = response.getFirstUpdatedEntityByTypeName(ENTITY_TYPE_WITH_COMPLEX_COLLECTION_ATTR);
validateEntity(complexEntitiesInfo, getEntityFromStore(updatedComplexEntity));
}
@Test(dependsOnMethods = "testEntityMap")
public void testDeleteEntityRemoveReferences() throws Exception {
init();
complexCollectionAttrEntityForDelete.getEntity().setAttribute(NAME, ENTITY_TYPE_WITH_COMPLEX_COLLECTION_ATTR_DELETE);
EntityMutationResponse response = entityStore.createOrUpdate(new AtlasEntityStream(complexCollectionAttrEntityForDelete), false);
AtlasEntityHeader entityCreated = response.getFirstCreatedEntityByTypeName(ENTITY_TYPE_WITH_COMPLEX_COLLECTION_ATTR);
validateEntity(complexCollectionAttrEntityForDelete, getEntityFromStore(entityCreated));
// delete entity and check if referenced complex attribute edges are deleted
response = entityStore.deleteById(entityCreated.getGuid());
AtlasEntityHeader entityDeleted = response.getFirstDeletedEntityByTypeName(ENTITY_TYPE_WITH_COMPLEX_COLLECTION_ATTR);
AtlasEntityWithExtInfo deletedEntityWithExtInfo = entityStore.getById(entityDeleted.getGuid());
AtlasVertex deletedEntityVertex = AtlasGraphUtilsV1.findByGuid(entityDeleted.getGuid());
Iterator<AtlasEdge> edges = deletedEntityVertex.getEdges(AtlasEdgeDirection.OUT).iterator();
// validate all attribute edges are deleted
while (edges != null && edges.hasNext()) {
assertEquals(getStatus(edges.next()), AtlasEntity.Status.DELETED);
}
AtlasEntity deletedEntity = deletedEntityWithExtInfo.getEntity();
List<AtlasObjectId> listOfEntities = (List<AtlasObjectId>) deletedEntity.getAttribute("listOfEntities");
Map<String, AtlasObjectId> mapOfEntities = (Map<String, AtlasObjectId>) deletedEntity.getAttribute("mapOfEntities");
// validate entity attributes are deleted
for (AtlasObjectId o : listOfEntities) {
AtlasEntity entity = deletedEntityWithExtInfo.getEntity(o.getGuid());
assertEquals(entity.getStatus(), AtlasEntity.Status.DELETED);
}
for (AtlasObjectId o : mapOfEntities.values()) {
AtlasEntity entity = deletedEntityWithExtInfo.getEntity(o.getGuid());
assertEquals(entity.getStatus(), AtlasEntity.Status.DELETED);
}
}
}
\ No newline at end of file
repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityStoreV1Test.java
...@@ -19,15 +19,12 @@ package org.apache.atlas.repository.store.graph.v1; ...@@ -19,15 +19,12 @@ package org.apache.atlas.repository.store.graph.v1;
import com.google.common.collect.ImmutableSet; import com.google.common.collect.ImmutableSet;
import org.apache.atlas.AtlasErrorCode; import org.apache.atlas.AtlasErrorCode;
import org.apache.atlas.AtlasException;
import org.apache.atlas.TestModules; import org.apache.atlas.TestModules;
import org.apache.atlas.RequestContextV1;
import org.apache.atlas.TestUtilsV2; import org.apache.atlas.TestUtilsV2;
import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.instance.AtlasClassification; import org.apache.atlas.model.instance.AtlasClassification;
import org.apache.atlas.model.instance.AtlasEntity; import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.model.instance.AtlasEntity.AtlasEntitiesWithExtInfo; import org.apache.atlas.model.instance.AtlasEntity.AtlasEntitiesWithExtInfo;
import org.apache.atlas.model.instance.AtlasEntity.AtlasEntityExtInfo;
import org.apache.atlas.model.instance.AtlasEntity.AtlasEntityWithExtInfo; import org.apache.atlas.model.instance.AtlasEntity.AtlasEntityWithExtInfo;
import org.apache.atlas.model.instance.AtlasEntityHeader; import org.apache.atlas.model.instance.AtlasEntityHeader;
import org.apache.atlas.model.instance.AtlasObjectId; import org.apache.atlas.model.instance.AtlasObjectId;
...@@ -37,63 +34,35 @@ import org.apache.atlas.model.instance.EntityMutations; ...@@ -37,63 +34,35 @@ import org.apache.atlas.model.instance.EntityMutations;
import org.apache.atlas.model.instance.EntityMutations.EntityOperation; import org.apache.atlas.model.instance.EntityMutations.EntityOperation;
import org.apache.atlas.model.typedef.AtlasClassificationDef; import org.apache.atlas.model.typedef.AtlasClassificationDef;
import org.apache.atlas.model.typedef.AtlasEntityDef; import org.apache.atlas.model.typedef.AtlasEntityDef;
import org.apache.atlas.model.typedef.AtlasStructDef;
import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef;
import org.apache.atlas.model.typedef.AtlasTypesDef; import org.apache.atlas.model.typedef.AtlasTypesDef;
import org.apache.atlas.repository.graph.AtlasGraphProvider;
import org.apache.atlas.repository.graph.GraphBackedSearchIndexer;
import org.apache.atlas.repository.store.bootstrap.AtlasTypeDefStoreInitializer;
import org.apache.atlas.repository.store.graph.AtlasEntityStore;
import org.apache.atlas.runner.LocalSolrRunner;
import org.apache.atlas.store.AtlasTypeDefStore;
import org.apache.atlas.type.AtlasArrayType;
import org.apache.atlas.type.AtlasMapType;
import org.apache.atlas.type.AtlasStructType;
import org.apache.atlas.type.AtlasType;
import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.atlas.type.AtlasTypeUtil; import org.apache.atlas.type.AtlasTypeUtil;
import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.MapUtils;
import org.apache.commons.collections.Transformer;
import org.apache.commons.configuration.Configuration;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.testng.Assert; import org.testng.Assert;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass; import org.testng.annotations.BeforeClass;
import org.testng.annotations.BeforeTest;
import org.testng.annotations.Guice; import org.testng.annotations.Guice;
import org.testng.annotations.Test; import org.testng.annotations.Test;
import javax.inject.Inject; import javax.inject.Inject;
import java.util.*; import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import static org.apache.atlas.TestUtilsV2.COLUMNS_ATTR_NAME; import static org.apache.atlas.TestUtilsV2.COLUMNS_ATTR_NAME;
import static org.apache.atlas.TestUtilsV2.COLUMN_TYPE; import static org.apache.atlas.TestUtilsV2.COLUMN_TYPE;
import static org.apache.atlas.TestUtilsV2.NAME; import static org.apache.atlas.TestUtilsV2.NAME;
import static org.apache.atlas.TestUtilsV2.randomString;
import static org.apache.atlas.TestUtilsV2.TABLE_TYPE; import static org.apache.atlas.TestUtilsV2.TABLE_TYPE;
import static org.apache.atlas.graph.GraphSandboxUtil.useLocalSolr;
import static org.mockito.Mockito.mock; import static org.mockito.Mockito.mock;
import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertTrue; import static org.testng.Assert.assertTrue;
import static org.testng.Assert.fail; import static org.testng.Assert.fail;
@Guice(modules = TestModules.TestOnlyModule.class) @Guice(modules = TestModules.TestOnlyModule.class)
public class AtlasEntityStoreV1Test { public class AtlasEntityStoreV1Test extends AtlasEntityTestBase {
private static final Logger LOG = LoggerFactory.getLogger(AtlasEntityStoreV1Test.class);
@Inject
AtlasTypeRegistry typeRegistry;
@Inject
AtlasTypeDefStore typeDefStore;
AtlasEntityStore entityStore;
@Inject
DeleteHandlerV1 deleteHandler;
private AtlasEntitiesWithExtInfo deptEntity; private AtlasEntitiesWithExtInfo deptEntity;
private AtlasEntityWithExtInfo dbEntity; private AtlasEntityWithExtInfo dbEntity;
private AtlasEntityWithExtInfo tblEntity; private AtlasEntityWithExtInfo tblEntity;
...@@ -101,86 +70,55 @@ public class AtlasEntityStoreV1Test { ...@@ -101,86 +70,55 @@ public class AtlasEntityStoreV1Test {
private AtlasEntityWithExtInfo primitiveEntity; private AtlasEntityWithExtInfo primitiveEntity;
AtlasEntityChangeNotifier mockChangeNotifier = mock(AtlasEntityChangeNotifier.class); AtlasEntityChangeNotifier mockChangeNotifier = mock(AtlasEntityChangeNotifier.class);
@Inject @Inject
private EntityGraphMapper graphMapper; private EntityGraphMapper graphMapper;
@Inject @Inject
private Configuration configuration;
private String dbEntityGuid; private String dbEntityGuid;
private String tblEntityGuid; private String tblEntityGuid;
@BeforeClass @BeforeClass
public void setUp() throws Exception { public void setUp() throws Exception {
RequestContextV1.clear(); super.setUp();
RequestContextV1.get().setUser(TestUtilsV2.TEST_USER, null);
new GraphBackedSearchIndexer(typeRegistry);
AtlasTypesDef[] testTypesDefs = new AtlasTypesDef[] { TestUtilsV2.defineDeptEmployeeTypes(), AtlasTypesDef[] testTypesDefs = new AtlasTypesDef[] { TestUtilsV2.defineDeptEmployeeTypes(),
TestUtilsV2.defineHiveTypes(), TestUtilsV2.defineHiveTypes(),
TestUtilsV2.defineTypeWithNestedCollectionAttributes(), TestUtilsV2.defineTypeWithNestedCollectionAttributes(),
}; };
createTypesDef(testTypesDefs);
for (AtlasTypesDef typesDef : testTypesDefs) { deptEntity = TestUtilsV2.createDeptEg2();
AtlasTypesDef typesToCreate = AtlasTypeDefStoreInitializer.getTypesToCreate(typesDef, typeRegistry); dbEntity = TestUtilsV2.createDBEntityV2();
tblEntity = TestUtilsV2.createTableEntityV2(dbEntity.getEntity());
if (!typesToCreate.isEmpty()) {
typeDefStore.createTypesDef(typesToCreate);
}
}
deptEntity = TestUtilsV2.createDeptEg2();
dbEntity = TestUtilsV2.createDBEntityV2();
tblEntity = TestUtilsV2.createTableEntityV2(dbEntity.getEntity());
nestedCollectionAttrEntity = TestUtilsV2.createNestedCollectionAttrEntity(); nestedCollectionAttrEntity = TestUtilsV2.createNestedCollectionAttrEntity();
primitiveEntity = TestUtilsV2.createprimitiveEntityV2();
AtlasTypesDef typesDef11 = new AtlasTypesDef();
List primitiveEntityDef = new ArrayList<AtlasEntityDef>();
AtlasTypesDef typesDef11 = new AtlasTypesDef();
List primitiveEntityDef = new ArrayList<AtlasEntityDef>();
primitiveEntityDef.add(TestUtilsV2.createPrimitiveEntityDef()); primitiveEntityDef.add(TestUtilsV2.createPrimitiveEntityDef());
typesDef11.setEntityDefs(primitiveEntityDef); typesDef11.setEntityDefs(primitiveEntityDef);
typeDefStore.createTypesDef( typesDef11 );
primitiveEntity = TestUtilsV2.createprimitiveEntityV2(); typeDefStore.createTypesDef(typesDef11);
}
@AfterClass
public void clear() throws Exception {
AtlasGraphProvider.cleanup();
if (useLocalSolr()) {
LocalSolrRunner.stop();
}
}
@BeforeTest
public void init() throws Exception {
entityStore = new AtlasEntityStoreV1(deleteHandler, typeRegistry, mockChangeNotifier, graphMapper);
RequestContextV1.clear();
RequestContextV1.get().setUser(TestUtilsV2.TEST_USER, null);
} }
@Test @Test
public void testDefaultValueForPrimitiveTypes() throws Exception { public void testDefaultValueForPrimitiveTypes() throws Exception {
init(); init();
EntityMutationResponse response = entityStore.createOrUpdate(new AtlasEntityStream(primitiveEntity), false); EntityMutationResponse response = entityStore.createOrUpdate(new AtlasEntityStream(primitiveEntity), false);
List<AtlasEntityHeader> entitiesCreatedResponse = response.getEntitiesByOperation(EntityOperation.CREATE); List<AtlasEntityHeader> entitiesCreatedResponse = response.getEntitiesByOperation(EntityOperation.CREATE);
final Map<EntityOperation, List<AtlasEntityHeader>> entitiesMutated = response.getMutatedEntities(); List<AtlasEntityHeader> entitiesCreatedwithdefault = response.getMutatedEntities().get(EntityOperation.CREATE);
List<AtlasEntityHeader> entitiesCreatedwithdefault = entitiesMutated.get(EntityOperation.CREATE); AtlasEntity entityCreated = getEntityFromStore(entitiesCreatedResponse.get(0));
AtlasEntity entityCreated = getEntityFromStore(entitiesCreatedResponse.get(0));
Map attributesMap = entityCreated.getAttributes();
String description = (String) attributesMap.get("description");
String check = (String) attributesMap.get("check");
String sourceCode = (String) attributesMap.get("sourcecode");
float diskUsage = (float) attributesMap.get("diskUsage");
boolean isstoreUse = (boolean) attributesMap.get("isstoreUse");
int cost = (int)attributesMap.get("Cost");
Map attributesMap = entityCreated.getAttributes();
String description = (String) attributesMap.get("description");
String check = (String) attributesMap.get("check");
String sourceCode = (String) attributesMap.get("sourcecode");
float diskUsage = (float) attributesMap.get("diskUsage");
boolean isstoreUse = (boolean) attributesMap.get("isstoreUse");
int cost = (int) attributesMap.get("Cost");
assertEquals(description,"test"); assertEquals(description,"test");
assertEquals(check,"check"); assertEquals(check,"check");
...@@ -192,20 +130,20 @@ public class AtlasEntityStoreV1Test { ...@@ -192,20 +130,20 @@ public class AtlasEntityStoreV1Test {
assertEquals(cost,30); assertEquals(cost,30);
} }
@Test @Test
public void testCreate() throws Exception { public void testCreate() throws Exception {
init(); init();
EntityMutationResponse response = entityStore.createOrUpdate(new AtlasEntityStream(deptEntity), false);
EntityMutationResponse response = entityStore.createOrUpdate(new AtlasEntityStream(deptEntity), false);
validateMutationResponse(response, EntityOperation.CREATE, 5); validateMutationResponse(response, EntityOperation.CREATE, 5);
AtlasEntityHeader dept1 = response.getFirstCreatedEntityByTypeName(TestUtilsV2.DEPARTMENT_TYPE); AtlasEntityHeader dept1 = response.getFirstCreatedEntityByTypeName(TestUtilsV2.DEPARTMENT_TYPE);
validateEntity(deptEntity, getEntityFromStore(dept1), deptEntity.getEntities().get(0)); validateEntity(deptEntity, getEntityFromStore(dept1), deptEntity.getEntities().get(0));
final Map<EntityOperation, List<AtlasEntityHeader>> entitiesMutated = response.getMutatedEntities(); final Map<EntityOperation, List<AtlasEntityHeader>> entitiesMutated = response.getMutatedEntities();
List<AtlasEntityHeader> entitiesCreated = entitiesMutated.get(EntityOperation.CREATE); List<AtlasEntityHeader> entitiesCreated = entitiesMutated.get(EntityOperation.CREATE);
Assert.assertTrue(entitiesCreated.size() >= deptEntity.getEntities().size()); assertTrue(entitiesCreated.size() >= deptEntity.getEntities().size());
for (int i = 0; i < deptEntity.getEntities().size(); i++) { for (int i = 0; i < deptEntity.getEntities().size(); i++) {
AtlasEntity expected = deptEntity.getEntities().get(i); AtlasEntity expected = deptEntity.getEntities().get(i);
...@@ -252,7 +190,6 @@ public class AtlasEntityStoreV1Test { ...@@ -252,7 +190,6 @@ public class AtlasEntityStoreV1Test {
List<AtlasObjectId> columns = new ArrayList<>(); List<AtlasObjectId> columns = new ArrayList<>();
AtlasEntitiesWithExtInfo entitiesInfo = new AtlasEntitiesWithExtInfo(tableEntity); AtlasEntitiesWithExtInfo entitiesInfo = new AtlasEntitiesWithExtInfo(tableEntity);
AtlasEntity col1 = TestUtilsV2.createColumnEntity(tableEntity); AtlasEntity col1 = TestUtilsV2.createColumnEntity(tableEntity);
col1.setAttribute(TestUtilsV2.NAME, "col1"); col1.setAttribute(TestUtilsV2.NAME, "col1");
...@@ -947,10 +884,10 @@ public class AtlasEntityStoreV1Test { ...@@ -947,10 +884,10 @@ public class AtlasEntityStoreV1Test {
@Test(dependsOnMethods = "testCreate") @Test(dependsOnMethods = "testCreate")
public void associateMultipleTagsToOneEntity() throws AtlasBaseException { public void associateMultipleTagsToOneEntity() throws AtlasBaseException {
final String TAG_NAME = "tag_xy"; final String TAG_NAME = "tag_xy";
final String TAG_NAME_2 = TAG_NAME + "_2"; final String TAG_NAME_2 = TAG_NAME + "_2";
final String TAG_ATTRIBUTE_NAME = "testAttribute"; final String TAG_ATTRIBUTE_NAME = "testAttribute";
final String TAG_ATTRIBUTE_VALUE = "test-string"; final String TAG_ATTRIBUTE_VALUE = "test-string";
final String TAG_ATTRIBUTE_VALUE_2 = TAG_ATTRIBUTE_VALUE + "-2"; final String TAG_ATTRIBUTE_VALUE_2 = TAG_ATTRIBUTE_VALUE + "-2";
createTag(TAG_NAME, "string"); createTag(TAG_NAME, "string");
...@@ -973,8 +910,8 @@ public class AtlasEntityStoreV1Test { ...@@ -973,8 +910,8 @@ public class AtlasEntityStoreV1Test {
@Test(dependsOnMethods = "testCreate") @Test(dependsOnMethods = "testCreate")
public void associateSameTagToMultipleEntities() throws AtlasBaseException { public void associateSameTagToMultipleEntities() throws AtlasBaseException {
final String TAG_NAME = "tagx"; final String TAG_NAME = "tagx";
final String TAG_ATTRIBUTE_NAME = "testAttribute"; final String TAG_ATTRIBUTE_NAME = "testAttribute";
final String TAG_ATTRIBUTE_VALUE = "test-string"; final String TAG_ATTRIBUTE_VALUE = "test-string";
createTag(TAG_NAME, "string"); createTag(TAG_NAME, "string");
...@@ -984,16 +921,16 @@ public class AtlasEntityStoreV1Test { ...@@ -984,16 +921,16 @@ public class AtlasEntityStoreV1Test {
entityStore.addClassifications(dbEntityGuid, addedClassifications); entityStore.addClassifications(dbEntityGuid, addedClassifications);
entityStore.addClassifications(tblEntityGuid, addedClassifications); entityStore.addClassifications(tblEntityGuid, addedClassifications);
AtlasEntity dbEntityFromDb = getEntityFromStore(dbEntityGuid); AtlasEntity dbEntityFromDb = getEntityFromStore(dbEntityGuid);
AtlasEntity tblEntityFromDb = getEntityFromStore(tblEntityGuid); AtlasEntity tblEntityFromDb = getEntityFromStore(tblEntityGuid);
Set<String> actualDBClassifications = new HashSet<>(CollectionUtils.collect(dbEntityFromDb.getClassifications(), o -> ((AtlasClassification) o).getTypeName())); Set<String> actualDBClassifications = new HashSet<>(CollectionUtils.collect(dbEntityFromDb.getClassifications(), o -> ((AtlasClassification) o).getTypeName()));
Set<String> actualTblClassifications = new HashSet<>(CollectionUtils.collect(tblEntityFromDb.getClassifications(), o -> ((AtlasClassification) o).getTypeName())); Set<String> actualTblClassifications = new HashSet<>(CollectionUtils.collect(tblEntityFromDb.getClassifications(), o -> ((AtlasClassification) o).getTypeName()));
assertTrue(actualDBClassifications.contains(TAG_NAME)); assertTrue(actualDBClassifications.contains(TAG_NAME));
assertTrue(actualTblClassifications.contains(TAG_NAME)); assertTrue(actualTblClassifications.contains(TAG_NAME));
Set<String> actualDBAssociatedEntityGuid = new HashSet<>(CollectionUtils.collect(dbEntityFromDb.getClassifications(), o -> ((AtlasClassification) o).getEntityGuid())); Set<String> actualDBAssociatedEntityGuid = new HashSet<>(CollectionUtils.collect(dbEntityFromDb.getClassifications(), o -> ((AtlasClassification) o).getEntityGuid()));
Set<String> actualTblAssociatedEntityGuid = new HashSet<>(CollectionUtils.collect(tblEntityFromDb.getClassifications(), o -> ((AtlasClassification) o).getEntityGuid())); Set<String> actualTblAssociatedEntityGuid = new HashSet<>(CollectionUtils.collect(tblEntityFromDb.getClassifications(), o -> ((AtlasClassification) o).getEntityGuid()));
assertTrue(actualDBAssociatedEntityGuid.contains(dbEntityGuid)); assertTrue(actualDBAssociatedEntityGuid.contains(dbEntityGuid));
...@@ -1002,128 +939,4 @@ public class AtlasEntityStoreV1Test { ...@@ -1002,128 +939,4 @@ public class AtlasEntityStoreV1Test {
entityStore.deleteClassifications(dbEntityGuid, Collections.singletonList(TAG_NAME)); entityStore.deleteClassifications(dbEntityGuid, Collections.singletonList(TAG_NAME));
entityStore.deleteClassifications(tblEntityGuid, Collections.singletonList(TAG_NAME)); entityStore.deleteClassifications(tblEntityGuid, Collections.singletonList(TAG_NAME));
} }
}
\ No newline at end of file
private AtlasClassificationDef getTagWithName(AtlasTypesDef typesDef, String tagName, String attributeType) {
AtlasClassificationDef aTag = new AtlasClassificationDef(tagName);
AtlasAttributeDef attributeDef = new AtlasAttributeDef("testAttribute", attributeType, true,
AtlasAttributeDef.Cardinality.SINGLE, 0, 1, false, true, false,
Collections.emptyList());
aTag.addAttribute(attributeDef);
typesDef.setClassificationDefs(Arrays.asList(aTag));
return aTag;
}
private void createTag(String tagName, String attributeType) {
try {
AtlasTypesDef typesDef = new AtlasTypesDef();
getTagWithName(typesDef, tagName, attributeType);
typeDefStore.createTypesDef(typesDef);
} catch (AtlasBaseException e) {
fail("Tag creation should've succeeded");
}
}
private String randomStrWithReservedChars() {
return randomString() + "\"${}%";
}
private void validateMutationResponse(EntityMutationResponse response, EntityMutations.EntityOperation op, int expectedNumCreated) {
List<AtlasEntityHeader> entitiesCreated = response.getEntitiesByOperation(op);
Assert.assertNotNull(entitiesCreated);
Assert.assertEquals(entitiesCreated.size(), expectedNumCreated);
}
private void validateEntity(AtlasEntityExtInfo entityExtInfo, AtlasEntity actual) throws AtlasBaseException, AtlasException {
validateEntity(entityExtInfo, actual, entityExtInfo.getEntity(actual.getGuid()));
}
private void validateEntity(AtlasEntityExtInfo entityExtInfo, AtlasStruct actual, AtlasStruct expected) throws AtlasBaseException, AtlasException {
if (expected == null) {
Assert.assertNull(actual, "expected null instance. Found " + actual);
return;
}
Assert.assertNotNull(actual, "found null instance");
AtlasStructType entityType = (AtlasStructType) typeRegistry.getType(actual.getTypeName());
for (String attrName : expected.getAttributes().keySet()) {
Object expectedVal = expected.getAttribute(attrName);
Object actualVal = actual.getAttribute(attrName);
AtlasType attrType = entityType.getAttributeType(attrName);
validateAttribute(entityExtInfo, actualVal, expectedVal, attrType, attrName);
}
}
private void validateAttribute(AtlasEntityExtInfo entityExtInfo, Object actual, Object expected, AtlasType attributeType, String attrName) throws AtlasBaseException, AtlasException {
switch(attributeType.getTypeCategory()) {
case OBJECT_ID_TYPE:
Assert.assertTrue(actual instanceof AtlasObjectId);
String guid = ((AtlasObjectId) actual).getGuid();
Assert.assertTrue(AtlasTypeUtil.isAssignedGuid(guid), "expected assigned guid. found " + guid);
break;
case PRIMITIVE:
case ENUM:
Assert.assertEquals(actual, expected);
break;
case MAP:
AtlasMapType mapType = (AtlasMapType) attributeType;
AtlasType valueType = mapType.getValueType();
Map actualMap = (Map) actual;
Map expectedMap = (Map) expected;
if (MapUtils.isNotEmpty(expectedMap)) {
Assert.assertTrue(MapUtils.isNotEmpty(actualMap));
// deleted entries are included in the attribute; hence use >=
Assert.assertTrue(actualMap.size() >= expectedMap.size());
for (Object key : expectedMap.keySet()) {
validateAttribute(entityExtInfo, actualMap.get(key), expectedMap.get(key), valueType, attrName);
}
}
break;
case ARRAY:
AtlasArrayType arrType = (AtlasArrayType) attributeType;
AtlasType elemType = arrType.getElementType();
List actualList = (List) actual;
List expectedList = (List) expected;
if (CollectionUtils.isNotEmpty(expectedList)) {
Assert.assertTrue(CollectionUtils.isNotEmpty(actualList));
//actual list could have deleted entities. Hence size may not match.
Assert.assertTrue(actualList.size() >= expectedList.size());
for (int i = 0; i < expectedList.size(); i++) {
validateAttribute(entityExtInfo, actualList.get(i), expectedList.get(i), elemType, attrName);
}
}
break;
case STRUCT:
AtlasStruct expectedStruct = (AtlasStruct) expected;
AtlasStruct actualStruct = (AtlasStruct) actual;
validateEntity(entityExtInfo, actualStruct, expectedStruct);
break;
default:
Assert.fail("Unknown type category");
}
}
private AtlasEntity getEntityFromStore(AtlasEntityHeader header) throws AtlasBaseException {
return header != null ? getEntityFromStore(header.getGuid()) : null;
}
private AtlasEntity getEntityFromStore(String guid) throws AtlasBaseException {
AtlasEntityWithExtInfo entity = guid != null ? entityStore.getById(guid) : null;
return entity != null ? entity.getEntity() : null;
}
}
repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityTestBase.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.repository.store.graph.v1;
import org.apache.atlas.AtlasException;
import org.apache.atlas.RequestContextV1;
import org.apache.atlas.TestModules;
import org.apache.atlas.TestUtilsV2;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.model.instance.AtlasEntity.AtlasEntityExtInfo;
import org.apache.atlas.model.instance.AtlasEntity.AtlasEntityWithExtInfo;
import org.apache.atlas.model.instance.AtlasEntityHeader;
import org.apache.atlas.model.instance.AtlasObjectId;
import org.apache.atlas.model.instance.AtlasStruct;
import org.apache.atlas.model.instance.EntityMutationResponse;
import org.apache.atlas.model.instance.EntityMutations.EntityOperation;
import org.apache.atlas.model.typedef.AtlasClassificationDef;
import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef;
import org.apache.atlas.model.typedef.AtlasTypesDef;
import org.apache.atlas.repository.graph.AtlasGraphProvider;
import org.apache.atlas.repository.graph.GraphBackedSearchIndexer;
import org.apache.atlas.repository.store.bootstrap.AtlasTypeDefStoreInitializer;
import org.apache.atlas.repository.store.graph.AtlasEntityStore;
import org.apache.atlas.runner.LocalSolrRunner;
import org.apache.atlas.store.AtlasTypeDefStore;
import org.apache.atlas.type.AtlasArrayType;
import org.apache.atlas.type.AtlasMapType;
import org.apache.atlas.type.AtlasStructType;
import org.apache.atlas.type.AtlasType;
import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.atlas.type.AtlasTypeUtil;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.MapUtils;
import org.testng.Assert;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.BeforeTest;
import org.testng.annotations.Guice;
import javax.inject.Inject;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import static org.apache.atlas.TestUtilsV2.randomString;
import static org.apache.atlas.graph.GraphSandboxUtil.useLocalSolr;
import static org.mockito.Mockito.mock;
import static org.testng.Assert.fail;
@Guice(modules = TestModules.TestOnlyModule.class)
public class AtlasEntityTestBase {
@Inject
AtlasTypeRegistry typeRegistry;
@Inject
AtlasTypeDefStore typeDefStore;
@Inject
AtlasEntityStore entityStore;
@Inject
DeleteHandlerV1 deleteHandler;
@Inject
private EntityGraphMapper graphMapper;
AtlasEntityChangeNotifier mockChangeNotifier = mock(AtlasEntityChangeNotifier.class);
@BeforeClass
public void setUp() throws Exception {
RequestContextV1.clear();
RequestContextV1.get().setUser(TestUtilsV2.TEST_USER, null);
new GraphBackedSearchIndexer(typeRegistry);
}
@AfterClass
public void clear() throws Exception {
AtlasGraphProvider.cleanup();
if (useLocalSolr()) {
LocalSolrRunner.stop();
}
}
@BeforeTest
public void init() throws Exception {
entityStore = new AtlasEntityStoreV1(deleteHandler, typeRegistry, mockChangeNotifier, graphMapper);
RequestContextV1.clear();
RequestContextV1.get().setUser(TestUtilsV2.TEST_USER, null);
}
protected AtlasClassificationDef getTagWithName(AtlasTypesDef typesDef, String tagName, String attributeType) {
AtlasClassificationDef aTag = new AtlasClassificationDef(tagName);
AtlasAttributeDef attributeDef = new AtlasAttributeDef("testAttribute", attributeType, true,
AtlasAttributeDef.Cardinality.SINGLE, 0, 1, false, true, false,
Collections.emptyList());
aTag.addAttribute(attributeDef);
typesDef.setClassificationDefs(Arrays.asList(aTag));
return aTag;
}
protected void createTag(String tagName, String attributeType) {
try {
AtlasTypesDef typesDef = new AtlasTypesDef();
getTagWithName(typesDef, tagName, attributeType);
typeDefStore.createTypesDef(typesDef);
} catch (AtlasBaseException e) {
fail("Tag creation should've succeeded");
}
}
protected String randomStrWithReservedChars() {
return randomString() + "\"${}%";
}
protected void validateMutationResponse(EntityMutationResponse response, EntityOperation op, int expectedNumCreated) {
List<AtlasEntityHeader> entitiesCreated = response.getEntitiesByOperation(op);
Assert.assertNotNull(entitiesCreated);
Assert.assertEquals(entitiesCreated.size(), expectedNumCreated);
}
protected void validateEntity(AtlasEntityExtInfo entityExtInfo, AtlasEntity actual) throws AtlasBaseException, AtlasException {
validateEntity(entityExtInfo, actual, entityExtInfo.getEntity(actual.getGuid()));
}
protected void validateEntity(AtlasEntityExtInfo entityExtInfo, AtlasStruct actual, AtlasStruct expected) throws AtlasBaseException, AtlasException {
if (expected == null) {
Assert.assertNull(actual, "expected null instance. Found " + actual);
return;
}
Assert.assertNotNull(actual, "found null instance");
AtlasStructType entityType = (AtlasStructType) typeRegistry.getType(actual.getTypeName());
for (String attrName : expected.getAttributes().keySet()) {
Object expectedVal = expected.getAttribute(attrName);
Object actualVal = actual.getAttribute(attrName);
AtlasType attrType = entityType.getAttributeType(attrName);
validateAttribute(entityExtInfo, actualVal, expectedVal, attrType, attrName);
}
}
protected void validateAttribute(AtlasEntityExtInfo entityExtInfo, Object actual, Object expected, AtlasType attributeType, String attrName) throws AtlasBaseException, AtlasException {
switch(attributeType.getTypeCategory()) {
case OBJECT_ID_TYPE:
Assert.assertTrue(actual instanceof AtlasObjectId);
String guid = ((AtlasObjectId) actual).getGuid();
Assert.assertTrue(AtlasTypeUtil.isAssignedGuid(guid), "expected assigned guid. found " + guid);
break;
case PRIMITIVE:
case ENUM:
Assert.assertEquals(actual, expected);
break;
case MAP:
AtlasMapType mapType = (AtlasMapType) attributeType;
AtlasType valueType = mapType.getValueType();
Map actualMap = (Map) actual;
Map expectedMap = (Map) expected;
if (MapUtils.isNotEmpty(expectedMap)) {
Assert.assertTrue(MapUtils.isNotEmpty(actualMap));
// deleted entries are included in the attribute; hence use >=
Assert.assertTrue(actualMap.size() >= expectedMap.size());
for (Object key : expectedMap.keySet()) {
validateAttribute(entityExtInfo, actualMap.get(key), expectedMap.get(key), valueType, attrName);
}
}
break;
case ARRAY:
AtlasArrayType arrType = (AtlasArrayType) attributeType;
AtlasType elemType = arrType.getElementType();
List actualList = (List) actual;
List expectedList = (List) expected;
if (CollectionUtils.isNotEmpty(expectedList)) {
Assert.assertTrue(CollectionUtils.isNotEmpty(actualList));
//actual list could have deleted entities. Hence size may not match.
Assert.assertTrue(actualList.size() >= expectedList.size());
for (int i = 0; i < expectedList.size(); i++) {
validateAttribute(entityExtInfo, actualList.get(i), expectedList.get(i), elemType, attrName);
}
}
break;
case STRUCT:
AtlasStruct expectedStruct = (AtlasStruct) expected;
AtlasStruct actualStruct = (AtlasStruct) actual;
validateEntity(entityExtInfo, actualStruct, expectedStruct);
break;
default:
Assert.fail("Unknown type category");
}
}
protected AtlasEntity getEntityFromStore(AtlasEntityHeader header) throws AtlasBaseException {
return header != null ? getEntityFromStore(header.getGuid()) : null;
}
protected AtlasEntity getEntityFromStore(String guid) throws AtlasBaseException {
AtlasEntityWithExtInfo entity = guid != null ? entityStore.getById(guid) : null;
return entity != null ? entity.getEntity() : null;
}
protected void createTypesDef(AtlasTypesDef[] testTypesDefs) throws AtlasBaseException {
for (AtlasTypesDef typesDef : testTypesDefs) {
AtlasTypesDef typesToCreate = AtlasTypeDefStoreInitializer.getTypesToCreate(typesDef, typeRegistry);
if (!typesToCreate.isEmpty()) {
typeDefStore.createTypesDef(typesToCreate);
}
}
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment