Commit bebe746b by Sarath Subramanian

ATLAS-3431: Ability to create and store user defined key-value pairs in Atlas entity instances

parent 74bf4cda
...@@ -91,6 +91,7 @@ public final class Constants { ...@@ -91,6 +91,7 @@ public final class Constants {
public static final String CLASSIFICATION_TEXT_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "classificationsText"); public static final String CLASSIFICATION_TEXT_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "classificationsText");
public static final String CLASSIFICATION_NAMES_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "classificationNames"); public static final String CLASSIFICATION_NAMES_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "classificationNames");
public static final String PROPAGATED_CLASSIFICATION_NAMES_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "propagatedClassificationNames"); public static final String PROPAGATED_CLASSIFICATION_NAMES_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "propagatedClassificationNames");
public static final String CUSTOM_ATTRIBUTES_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "customAttributes");
public static final String MODIFIED_BY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "modifiedBy"); public static final String MODIFIED_BY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "modifiedBy");
......
...@@ -59,6 +59,8 @@ public enum AtlasConfiguration { ...@@ -59,6 +59,8 @@ public enum AtlasConfiguration {
SEARCH_MAX_LIMIT("atlas.search.maxlimit", 10000), SEARCH_MAX_LIMIT("atlas.search.maxlimit", 10000),
SEARCH_DEFAULT_LIMIT("atlas.search.defaultlimit", 100), SEARCH_DEFAULT_LIMIT("atlas.search.defaultlimit", 100),
CUSTOM_ATTRIBUTE_KEY_MAX_LENGTH("atlas.custom.attribute.key.max.length", 50),
CUSTOM_ATTRIBUTE_VALUE_MAX_LENGTH("atlas.custom.attribute.value.max.length", 500),
IMPORT_TEMP_DIRECTORY("atlas.import.temp.directory", ""); IMPORT_TEMP_DIRECTORY("atlas.import.temp.directory", "");
private static final Configuration APPLICATION_PROPERTIES; private static final Configuration APPLICATION_PROPERTIES;
......
...@@ -154,7 +154,10 @@ public enum AtlasErrorCode { ...@@ -154,7 +154,10 @@ public enum AtlasErrorCode {
INVALID_TIMEBOUNDRY_DATERANGE(400, "ATLAS-400-00-87D", "Invalid dateRange: startTime {0} must be before endTime {1}"), INVALID_TIMEBOUNDRY_DATERANGE(400, "ATLAS-400-00-87D", "Invalid dateRange: startTime {0} must be before endTime {1}"),
PROPAGATED_CLASSIFICATION_REMOVAL_NOT_SUPPORTED(400, "ATLAS-400-00-87E", "Removal of classification {0}, which is propagated from entity {1}, is not supported"), PROPAGATED_CLASSIFICATION_REMOVAL_NOT_SUPPORTED(400, "ATLAS-400-00-87E", "Removal of classification {0}, which is propagated from entity {1}, is not supported"),
IMPORT_ATTEMPTING_EMPTY_ZIP(400, "ATLAS-400-00-87F", "Attempting to import empty ZIP file."), IMPORT_ATTEMPTING_EMPTY_ZIP(400, "ATLAS-400-00-87F", "Attempting to import empty ZIP file."),
PATCH_MISSING_RELATIONSHIP_LABEL(400, "ATLAS-400-00-880", "{0} - must include relationship label for type {1}"), PATCH_MISSING_RELATIONSHIP_LABEL(400, "ATLAS-400-00-88", "{0} - must include relationship label for type {1}"),
INVALID_CUSTOM_ATTRIBUTE_KEY_LENGTH(400, "ATLAS-400-00-89", "Invalid key: {0} in custom attribute, key size should not be greater than 50"),
INVALID_CUSTOM_ATTRIBUTE_KEY_CHARACTERS(400, "ATLAS-400-00-90", "Invalid key: {0} in custom attribute, key should only contain alphanumeric characters, '_' or '-'"),
INVALID_CUSTOM_ATTRIBUTE_VALUE(400, "ATLAS-400-00-9A", "Invalid value: {0} in custom attribute, value length is greater than {1}"),
UNAUTHORIZED_ACCESS(403, "ATLAS-403-00-001", "{0} is not authorized to perform {1}"), UNAUTHORIZED_ACCESS(403, "ATLAS-403-00-001", "{0} is not authorized to perform {1}"),
......
...@@ -91,6 +91,7 @@ public class AtlasEntity extends AtlasStruct implements Serializable { ...@@ -91,6 +91,7 @@ public class AtlasEntity extends AtlasStruct implements Serializable {
private Map<String, Object> relationshipAttributes; private Map<String, Object> relationshipAttributes;
private List<AtlasClassification> classifications; private List<AtlasClassification> classifications;
private List<AtlasTermAssignmentHeader> meanings; private List<AtlasTermAssignmentHeader> meanings;
private Map<String, String> customAttributes;
@JsonIgnore @JsonIgnore
private static AtomicLong s_nextId = new AtomicLong(System.nanoTime()); private static AtomicLong s_nextId = new AtomicLong(System.nanoTime());
...@@ -213,6 +214,7 @@ public class AtlasEntity extends AtlasStruct implements Serializable { ...@@ -213,6 +214,7 @@ public class AtlasEntity extends AtlasStruct implements Serializable {
setClassifications(other.getClassifications()); setClassifications(other.getClassifications());
setRelationshipAttributes(other.getRelationshipAttributes()); setRelationshipAttributes(other.getRelationshipAttributes());
setMeanings(other.getMeanings()); setMeanings(other.getMeanings());
setCustomAttributes(other.getCustomAttributes());
} }
} }
...@@ -335,6 +337,14 @@ public class AtlasEntity extends AtlasStruct implements Serializable { ...@@ -335,6 +337,14 @@ public class AtlasEntity extends AtlasStruct implements Serializable {
return r != null ? r.containsKey(name) : false; return r != null ? r.containsKey(name) : false;
} }
public Map<String, String> getCustomAttributes() {
return customAttributes;
}
public void setCustomAttributes(Map<String, String> customAttributes) {
this.customAttributes = customAttributes;
}
public List<AtlasClassification> getClassifications() { return classifications; } public List<AtlasClassification> getClassifications() { return classifications; }
public void setClassifications(List<AtlasClassification> classifications) { this.classifications = classifications; } public void setClassifications(List<AtlasClassification> classifications) { this.classifications = classifications; }
...@@ -382,6 +392,7 @@ public class AtlasEntity extends AtlasStruct implements Serializable { ...@@ -382,6 +392,7 @@ public class AtlasEntity extends AtlasStruct implements Serializable {
setUpdateTime(null); setUpdateTime(null);
setClassifications(null); setClassifications(null);
setMeanings(null); setMeanings(null);
setCustomAttributes(null);
} }
private static String nextInternalId() { private static String nextInternalId() {
...@@ -416,6 +427,9 @@ public class AtlasEntity extends AtlasStruct implements Serializable { ...@@ -416,6 +427,9 @@ public class AtlasEntity extends AtlasStruct implements Serializable {
sb.append(", meanings=["); sb.append(", meanings=[");
AtlasBaseTypeDef.dumpObjects(meanings, sb); AtlasBaseTypeDef.dumpObjects(meanings, sb);
sb.append(']'); sb.append(']');
sb.append(", customAttributes=[");
dumpObjects(customAttributes, sb);
sb.append("]");
sb.append('}'); sb.append('}');
return sb; return sb;
...@@ -440,13 +454,14 @@ public class AtlasEntity extends AtlasStruct implements Serializable { ...@@ -440,13 +454,14 @@ public class AtlasEntity extends AtlasStruct implements Serializable {
Objects.equals(updateTime, that.updateTime) && Objects.equals(updateTime, that.updateTime) &&
Objects.equals(version, that.version) && Objects.equals(version, that.version) &&
Objects.equals(relationshipAttributes, that.relationshipAttributes) && Objects.equals(relationshipAttributes, that.relationshipAttributes) &&
Objects.equals(customAttributes, that.customAttributes) &&
Objects.equals(classifications, that.classifications); Objects.equals(classifications, that.classifications);
} }
@Override @Override
public int hashCode() { public int hashCode() {
return Objects.hash(super.hashCode(), guid, homeId, isProxy, isIncomplete, provenanceType, status, return Objects.hash(super.hashCode(), guid, homeId, isProxy, isIncomplete, provenanceType, status,
createdBy, updatedBy, createTime, updateTime, version, relationshipAttributes, classifications); createdBy, updatedBy, createTime, updateTime, version, relationshipAttributes, classifications, customAttributes);
} }
@Override @Override
......
...@@ -324,6 +324,7 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang ...@@ -324,6 +324,7 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
createCommonVertexIndex(management, TRAIT_NAMES_PROPERTY_KEY, UniqueKind.NONE, String.class, SET, true, true); createCommonVertexIndex(management, TRAIT_NAMES_PROPERTY_KEY, UniqueKind.NONE, String.class, SET, true, true);
createCommonVertexIndex(management, PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, UniqueKind.NONE, String.class, LIST, true, true); createCommonVertexIndex(management, PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, UniqueKind.NONE, String.class, LIST, true, true);
createCommonVertexIndex(management, IS_INCOMPLETE_PROPERTY_KEY, UniqueKind.NONE, Integer.class, SINGLE, true, true); createCommonVertexIndex(management, IS_INCOMPLETE_PROPERTY_KEY, UniqueKind.NONE, Integer.class, SINGLE, true, true);
createCommonVertexIndex(management, CUSTOM_ATTRIBUTES_PROPERTY_KEY, UniqueKind.NONE, String.class, SINGLE, true, false);
createCommonVertexIndex(management, PATCH_ID_PROPERTY_KEY, UniqueKind.GLOBAL_UNIQUE, String.class, SINGLE, true, false); createCommonVertexIndex(management, PATCH_ID_PROPERTY_KEY, UniqueKind.GLOBAL_UNIQUE, String.class, SINGLE, true, false);
createCommonVertexIndex(management, PATCH_DESCRIPTION_PROPERTY_KEY, UniqueKind.NONE, String.class, SINGLE, true, false); createCommonVertexIndex(management, PATCH_DESCRIPTION_PROPERTY_KEY, UniqueKind.NONE, String.class, SINGLE, true, false);
......
...@@ -1065,6 +1065,17 @@ public final class GraphHelper { ...@@ -1065,6 +1065,17 @@ public final class GraphHelper {
return ret; return ret;
} }
public static Map getCustomAttributes(AtlasElement element) {
Map ret = null;
String customAttrsString = element.getProperty(CUSTOM_ATTRIBUTES_PROPERTY_KEY, String.class);
if (customAttrsString != null) {
ret = AtlasType.fromJson(customAttrsString, Map.class);
}
return ret;
}
public static Integer getProvenanceType(AtlasElement element) { public static Integer getProvenanceType(AtlasElement element) {
return element.getProperty(Constants.PROVENANCE_TYPE_KEY, Integer.class); return element.getProperty(Constants.PROVENANCE_TYPE_KEY, Integer.class);
} }
......
...@@ -50,6 +50,7 @@ import java.util.List; ...@@ -50,6 +50,7 @@ import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Set; import java.util.Set;
import static org.apache.atlas.repository.store.graph.v2.EntityGraphMapper.validateCustomAttributes;
public class AtlasEntityGraphDiscoveryV2 implements EntityGraphDiscovery { public class AtlasEntityGraphDiscoveryV2 implements EntityGraphDiscovery {
private static final Logger LOG = LoggerFactory.getLogger(AtlasEntityGraphDiscoveryV2.class); private static final Logger LOG = LoggerFactory.getLogger(AtlasEntityGraphDiscoveryV2.class);
...@@ -84,7 +85,7 @@ public class AtlasEntityGraphDiscoveryV2 implements EntityGraphDiscovery { ...@@ -84,7 +85,7 @@ public class AtlasEntityGraphDiscoveryV2 implements EntityGraphDiscovery {
public void validateAndNormalize(AtlasEntity entity) throws AtlasBaseException { public void validateAndNormalize(AtlasEntity entity) throws AtlasBaseException {
List<String> messages = new ArrayList<>(); List<String> messages = new ArrayList<>();
if (! AtlasTypeUtil.isValidGuid(entity.getGuid())) { if (!AtlasTypeUtil.isValidGuid(entity.getGuid())) {
throw new AtlasBaseException(AtlasErrorCode.INVALID_OBJECT_ID, "invalid guid " + entity.getGuid()); throw new AtlasBaseException(AtlasErrorCode.INVALID_OBJECT_ID, "invalid guid " + entity.getGuid());
} }
...@@ -94,6 +95,8 @@ public class AtlasEntityGraphDiscoveryV2 implements EntityGraphDiscovery { ...@@ -94,6 +95,8 @@ public class AtlasEntityGraphDiscoveryV2 implements EntityGraphDiscovery {
throw new AtlasBaseException(AtlasErrorCode.TYPE_NAME_INVALID, TypeCategory.ENTITY.name(), entity.getTypeName()); throw new AtlasBaseException(AtlasErrorCode.TYPE_NAME_INVALID, TypeCategory.ENTITY.name(), entity.getTypeName());
} }
validateCustomAttributes(entity);
type.validateValue(entity, entity.getTypeName(), messages); type.validateValue(entity, entity.getTypeName(), messages);
if (!messages.isEmpty()) { if (!messages.isEmpty()) {
...@@ -107,7 +110,7 @@ public class AtlasEntityGraphDiscoveryV2 implements EntityGraphDiscovery { ...@@ -107,7 +110,7 @@ public class AtlasEntityGraphDiscoveryV2 implements EntityGraphDiscovery {
public void validateAndNormalizeForUpdate(AtlasEntity entity) throws AtlasBaseException { public void validateAndNormalizeForUpdate(AtlasEntity entity) throws AtlasBaseException {
List<String> messages = new ArrayList<>(); List<String> messages = new ArrayList<>();
if (! AtlasTypeUtil.isValidGuid(entity.getGuid())) { if (!AtlasTypeUtil.isValidGuid(entity.getGuid())) {
throw new AtlasBaseException(AtlasErrorCode.INVALID_OBJECT_ID, "invalid guid " + entity.getGuid()); throw new AtlasBaseException(AtlasErrorCode.INVALID_OBJECT_ID, "invalid guid " + entity.getGuid());
} }
...@@ -117,6 +120,8 @@ public class AtlasEntityGraphDiscoveryV2 implements EntityGraphDiscovery { ...@@ -117,6 +120,8 @@ public class AtlasEntityGraphDiscoveryV2 implements EntityGraphDiscovery {
throw new AtlasBaseException(AtlasErrorCode.TYPE_NAME_INVALID, TypeCategory.ENTITY.name(), entity.getTypeName()); throw new AtlasBaseException(AtlasErrorCode.TYPE_NAME_INVALID, TypeCategory.ENTITY.name(), entity.getTypeName());
} }
validateCustomAttributes(entity);
type.validateValueForUpdate(entity, entity.getTypeName(), messages); type.validateValueForUpdate(entity, entity.getTypeName(), messages);
if (!messages.isEmpty()) { if (!messages.isEmpty()) {
......
...@@ -59,6 +59,7 @@ import static java.lang.Boolean.FALSE; ...@@ -59,6 +59,7 @@ import static java.lang.Boolean.FALSE;
import static org.apache.atlas.model.instance.EntityMutations.EntityOperation.DELETE; import static org.apache.atlas.model.instance.EntityMutations.EntityOperation.DELETE;
import static org.apache.atlas.model.instance.EntityMutations.EntityOperation.UPDATE; import static org.apache.atlas.model.instance.EntityMutations.EntityOperation.UPDATE;
import static org.apache.atlas.repository.Constants.IS_INCOMPLETE_PROPERTY_KEY; import static org.apache.atlas.repository.Constants.IS_INCOMPLETE_PROPERTY_KEY;
import static org.apache.atlas.repository.graph.GraphHelper.getCustomAttributes;
import static org.apache.atlas.repository.graph.GraphHelper.isEntityIncomplete; import static org.apache.atlas.repository.graph.GraphHelper.isEntityIncomplete;
...@@ -814,6 +815,15 @@ public class AtlasEntityStoreV2 implements AtlasEntityStore { ...@@ -814,6 +815,15 @@ public class AtlasEntityStoreV2 implements AtlasEntityStore {
} }
} }
if (!hasUpdates && entity.getCustomAttributes() != null) {
Map<String, String> currCustomAttributes = getCustomAttributes(vertex);
Map<String, String> newCustomAttributes = entity.getCustomAttributes();
if (!Objects.equals(currCustomAttributes, newCustomAttributes)) {
hasUpdates = true;
}
}
// if classifications are to be replaced, then skip updates only when no change in classifications // if classifications are to be replaced, then skip updates only when no change in classifications
if (!hasUpdates && replaceClassifications) { if (!hasUpdates && replaceClassifications) {
List<AtlasClassification> newVal = entity.getClassifications(); List<AtlasClassification> newVal = entity.getClassifications();
...@@ -921,6 +931,8 @@ public class AtlasEntityStoreV2 implements AtlasEntityStore { ...@@ -921,6 +931,8 @@ public class AtlasEntityStoreV2 implements AtlasEntityStore {
requestContext.recordEntityGuidUpdate(entity, guid); requestContext.recordEntityGuidUpdate(entity, guid);
} }
entityGraphMapper.setCustomAttributes(vertex, entity);
context.addUpdated(guid, entity, entityType, vertex); context.addUpdated(guid, entity, entityType, vertex);
} else { } else {
graphDiscoverer.validateAndNormalize(entity); graphDiscoverer.validateAndNormalize(entity);
......
...@@ -72,6 +72,8 @@ import org.springframework.stereotype.Component; ...@@ -72,6 +72,8 @@ import org.springframework.stereotype.Component;
import javax.inject.Inject; import javax.inject.Inject;
import java.util.*; import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import static org.apache.atlas.model.TypeCategory.CLASSIFICATION; import static org.apache.atlas.model.TypeCategory.CLASSIFICATION;
...@@ -109,10 +111,13 @@ import static org.apache.atlas.type.AtlasStructType.AtlasAttribute.AtlasRelation ...@@ -109,10 +111,13 @@ import static org.apache.atlas.type.AtlasStructType.AtlasAttribute.AtlasRelation
public class EntityGraphMapper { public class EntityGraphMapper {
private static final Logger LOG = LoggerFactory.getLogger(EntityGraphMapper.class); private static final Logger LOG = LoggerFactory.getLogger(EntityGraphMapper.class);
private static final String SOFT_REF_FORMAT = "%s:%s"; private static final String SOFT_REF_FORMAT = "%s:%s";
private static final int INDEXED_STR_SAFE_LEN = AtlasConfiguration.GRAPHSTORE_INDEXED_STRING_SAFE_LENGTH.getInt(); private static final int INDEXED_STR_SAFE_LEN = AtlasConfiguration.GRAPHSTORE_INDEXED_STRING_SAFE_LENGTH.getInt();
private static final boolean WARN_ON_NO_RELATIONSHIP = AtlasConfiguration.RELATIONSHIP_WARN_NO_RELATIONSHIPS.getBoolean(); private static final boolean WARN_ON_NO_RELATIONSHIP = AtlasConfiguration.RELATIONSHIP_WARN_NO_RELATIONSHIPS.getBoolean();
private static final String CLASSIFICATION_NAME_DELIMITER = "|"; private static final String CLASSIFICATION_NAME_DELIMITER = "|";
private static final Pattern CUSTOM_ATTRIBUTE_KEY_REGEX = Pattern.compile("^[a-zA-Z0-9_-]*$");
private static final int CUSTOM_ATTRIBUTE_KEY_MAX_LENGTH = AtlasConfiguration.CUSTOM_ATTRIBUTE_KEY_MAX_LENGTH.getInt();
private static final int CUSTOM_ATTRIBUTE_VALUE_MAX_LENGTH = AtlasConfiguration.CUSTOM_ATTRIBUTE_VALUE_MAX_LENGTH.getInt();
private final GraphHelper graphHelper = GraphHelper.getInstance(); private final GraphHelper graphHelper = GraphHelper.getInstance();
private final AtlasGraph graph; private final AtlasGraph graph;
...@@ -138,7 +143,7 @@ public class EntityGraphMapper { ...@@ -138,7 +143,7 @@ public class EntityGraphMapper {
this.fullTextMapperV2 = fullTextMapperV2; this.fullTextMapperV2 = fullTextMapperV2;
} }
public AtlasVertex createVertex(AtlasEntity entity) { public AtlasVertex createVertex(AtlasEntity entity) throws AtlasBaseException {
final String guid = UUID.randomUUID().toString(); final String guid = UUID.randomUUID().toString();
return createVertexWithGuid(entity, guid); return createVertexWithGuid(entity, guid);
} }
...@@ -179,7 +184,7 @@ public class EntityGraphMapper { ...@@ -179,7 +184,7 @@ public class EntityGraphMapper {
return ret; return ret;
} }
public AtlasVertex createVertexWithGuid(AtlasEntity entity, String guid) { public AtlasVertex createVertexWithGuid(AtlasEntity entity, String guid) throws AtlasBaseException {
if (LOG.isDebugEnabled()) { if (LOG.isDebugEnabled()) {
LOG.debug("==> createVertexWithGuid({})", entity.getTypeName()); LOG.debug("==> createVertexWithGuid({})", entity.getTypeName());
} }
...@@ -194,12 +199,14 @@ public class EntityGraphMapper { ...@@ -194,12 +199,14 @@ public class EntityGraphMapper {
AtlasGraphUtilsV2.setEncodedProperty(ret, GUID_PROPERTY_KEY, guid); AtlasGraphUtilsV2.setEncodedProperty(ret, GUID_PROPERTY_KEY, guid);
AtlasGraphUtilsV2.setEncodedProperty(ret, VERSION_PROPERTY_KEY, getEntityVersion(entity)); AtlasGraphUtilsV2.setEncodedProperty(ret, VERSION_PROPERTY_KEY, getEntityVersion(entity));
setCustomAttributes(ret, entity);
GraphTransactionInterceptor.addToVertexCache(guid, ret); GraphTransactionInterceptor.addToVertexCache(guid, ret);
return ret; return ret;
} }
public void updateSystemAttributes(AtlasVertex vertex, AtlasEntity entity) { public void updateSystemAttributes(AtlasVertex vertex, AtlasEntity entity) throws AtlasBaseException {
if (entity.getVersion() != null) { if (entity.getVersion() != null) {
AtlasGraphUtilsV2.setEncodedProperty(vertex, VERSION_PROPERTY_KEY, entity.getVersion()); AtlasGraphUtilsV2.setEncodedProperty(vertex, VERSION_PROPERTY_KEY, entity.getVersion());
} }
...@@ -231,6 +238,10 @@ public class EntityGraphMapper { ...@@ -231,6 +238,10 @@ public class EntityGraphMapper {
if (entity.getProvenanceType() != null) { if (entity.getProvenanceType() != null) {
AtlasGraphUtilsV2.setEncodedProperty(vertex, PROVENANCE_TYPE_KEY, entity.getProvenanceType()); AtlasGraphUtilsV2.setEncodedProperty(vertex, PROVENANCE_TYPE_KEY, entity.getProvenanceType());
} }
if (entity.getCustomAttributes() != null) {
setCustomAttributes(vertex, entity);
}
} }
public EntityMutationResponse mapAttributesAndClassifications(EntityMutationContext context, final boolean isPartialUpdate, final boolean replaceClassifications) throws AtlasBaseException { public EntityMutationResponse mapAttributesAndClassifications(EntityMutationContext context, final boolean isPartialUpdate, final boolean replaceClassifications) throws AtlasBaseException {
...@@ -308,6 +319,14 @@ public class EntityGraphMapper { ...@@ -308,6 +319,14 @@ public class EntityGraphMapper {
return resp; return resp;
} }
public void setCustomAttributes(AtlasVertex vertex, AtlasEntity entity) throws AtlasBaseException {
String customAttributesString = getCustomAttributesString(entity);
if (customAttributesString != null) {
AtlasGraphUtilsV2.setEncodedProperty(vertex, CUSTOM_ATTRIBUTES_PROPERTY_KEY, customAttributesString);
}
}
private AtlasVertex createStructVertex(AtlasStruct struct) { private AtlasVertex createStructVertex(AtlasStruct struct) {
return createStructVertex(struct.getTypeName()); return createStructVertex(struct.getTypeName());
} }
...@@ -1150,6 +1169,17 @@ public class EntityGraphMapper { ...@@ -1150,6 +1169,17 @@ public class EntityGraphMapper {
return (ret != null) ? ret : 0; return (ret != null) ? ret : 0;
} }
private String getCustomAttributesString(AtlasEntity entity) {
String ret = null;
Map<String, String> customAttributes = entity.getCustomAttributes();
if (customAttributes != null) {
ret = AtlasType.toJson(customAttributes);
}
return ret;
}
private AtlasStructType getStructType(String typeName) throws AtlasBaseException { private AtlasStructType getStructType(String typeName) throws AtlasBaseException {
AtlasType objType = typeRegistry.getType(typeName); AtlasType objType = typeRegistry.getType(typeName);
...@@ -2151,4 +2181,29 @@ public class EntityGraphMapper { ...@@ -2151,4 +2181,29 @@ public class EntityGraphMapper {
} }
return relGuidsSet; return relGuidsSet;
} }
public static void validateCustomAttributes(AtlasEntity entity) throws AtlasBaseException {
Map<String, String> customAttributes = entity.getCustomAttributes();
if (MapUtils.isNotEmpty(customAttributes)) {
for (Map.Entry<String, String> entry : customAttributes.entrySet()) {
String key = entry.getKey();
String value = entry.getValue();
if (key.length() > CUSTOM_ATTRIBUTE_KEY_MAX_LENGTH) {
throw new AtlasBaseException(AtlasErrorCode.INVALID_CUSTOM_ATTRIBUTE_KEY_LENGTH, key);
}
Matcher matcher = CUSTOM_ATTRIBUTE_KEY_REGEX.matcher(key);
if (!matcher.matches()) {
throw new AtlasBaseException(AtlasErrorCode.INVALID_CUSTOM_ATTRIBUTE_KEY_CHARACTERS, key);
}
if (value.length() > CUSTOM_ATTRIBUTE_VALUE_MAX_LENGTH) {
throw new AtlasBaseException(AtlasErrorCode.INVALID_CUSTOM_ATTRIBUTE_VALUE, value, String.valueOf(CUSTOM_ATTRIBUTE_VALUE_MAX_LENGTH));
}
}
}
}
} }
...@@ -582,6 +582,7 @@ public class EntityGraphRetriever { ...@@ -582,6 +582,7 @@ public class EntityGraphRetriever {
entity.setIsIncomplete(isEntityIncomplete(entityVertex)); entity.setIsIncomplete(isEntityIncomplete(entityVertex));
entity.setProvenanceType(GraphHelper.getProvenanceType(entityVertex)); entity.setProvenanceType(GraphHelper.getProvenanceType(entityVertex));
entity.setCustomAttributes(getCustomAttributes(entityVertex));
return entity; return entity;
} }
......
...@@ -55,10 +55,12 @@ import java.util.List; ...@@ -55,10 +55,12 @@ import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Set; import java.util.Set;
import static org.apache.atlas.AtlasErrorCode.*;
import static org.apache.atlas.TestUtilsV2.COLUMNS_ATTR_NAME; import static org.apache.atlas.TestUtilsV2.COLUMNS_ATTR_NAME;
import static org.apache.atlas.TestUtilsV2.COLUMN_TYPE; import static org.apache.atlas.TestUtilsV2.COLUMN_TYPE;
import static org.apache.atlas.TestUtilsV2.NAME; import static org.apache.atlas.TestUtilsV2.NAME;
import static org.apache.atlas.TestUtilsV2.TABLE_TYPE; import static org.apache.atlas.TestUtilsV2.TABLE_TYPE;
import static org.apache.commons.lang.RandomStringUtils.randomAlphanumeric;
import static org.mockito.Mockito.mock; import static org.mockito.Mockito.mock;
import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertTrue; import static org.testng.Assert.assertTrue;
...@@ -982,4 +984,128 @@ public class AtlasEntityStoreV2Test extends AtlasEntityTestBase { ...@@ -982,4 +984,128 @@ public class AtlasEntityStoreV2Test extends AtlasEntityTestBase {
entityStore.deleteClassification(dbEntityGuid, TAG_NAME); entityStore.deleteClassification(dbEntityGuid, TAG_NAME);
entityStore.deleteClassification(tblEntityGuid, TAG_NAME); entityStore.deleteClassification(tblEntityGuid, TAG_NAME);
} }
@Test (dependsOnMethods = "testCreate")
public void addCustomAttributesToEntity() throws AtlasBaseException {
AtlasEntity tblEntity = getEntityFromStore(tblEntityGuid);
Map<String, String> customAttributes = new HashMap<>();
customAttributes.put("key1", "val1");
customAttributes.put("key2", "val2");
customAttributes.put("key3", "val3");
customAttributes.put("key4", "val4");
customAttributes.put("key5", "val5");
tblEntity.setCustomAttributes(customAttributes);
entityStore.createOrUpdate(new AtlasEntityStream(tblEntity), false);
tblEntity = getEntityFromStore(tblEntityGuid);
assertEquals(customAttributes, tblEntity.getCustomAttributes());
}
@Test (dependsOnMethods = "addCustomAttributesToEntity")
public void updateCustomAttributesToEntity() throws AtlasBaseException {
AtlasEntity tblEntity = getEntityFromStore(tblEntityGuid);
// update custom attributes, remove key3, key4 and key5
Map<String, String> customAttributes = new HashMap<>();
customAttributes.put("key1", "val1");
customAttributes.put("key2", "val2");
tblEntity.setCustomAttributes(customAttributes);
entityStore.createOrUpdate(new AtlasEntityStream(tblEntity), false);
tblEntity = getEntityFromStore(tblEntityGuid);
assertEquals(customAttributes, tblEntity.getCustomAttributes());
}
@Test (dependsOnMethods = "updateCustomAttributesToEntity")
public void deleteCustomAttributesToEntity() throws AtlasBaseException {
AtlasEntity tblEntity = getEntityFromStore(tblEntityGuid);
Map<String, String> emptyCustomAttributes = new HashMap<>();
// remove all custom attributes
tblEntity.setCustomAttributes(emptyCustomAttributes);
entityStore.createOrUpdate(new AtlasEntityStream(tblEntity), false);
tblEntity = getEntityFromStore(tblEntityGuid);
assertEquals(emptyCustomAttributes, tblEntity.getCustomAttributes());
}
@Test (dependsOnMethods = "deleteCustomAttributesToEntity")
public void nullCustomAttributesToEntity() throws AtlasBaseException {
AtlasEntity tblEntity = getEntityFromStore(tblEntityGuid);
Map<String, String> customAttributes = new HashMap<>();
customAttributes.put("key1", "val1");
customAttributes.put("key2", "val2");
tblEntity.setCustomAttributes(customAttributes);
entityStore.createOrUpdate(new AtlasEntityStream(tblEntity), false);
// assign custom attributes to null
tblEntity.setCustomAttributes(null);
entityStore.createOrUpdate(new AtlasEntityStream(tblEntity), false);
tblEntity = getEntityFromStore(tblEntityGuid);
assertEquals(customAttributes, tblEntity.getCustomAttributes());
}
@Test (dependsOnMethods = "nullCustomAttributesToEntity")
public void addInvalidKeysToEntityCustomAttributes() throws AtlasBaseException {
AtlasEntity tblEntity = getEntityFromStore(tblEntityGuid);
// key should contain 1 to 50 alphanumeric characters, '_' or '-'
Map<String, String> invalidCustomAttributes = new HashMap<>();
invalidCustomAttributes.put("key0_65765-6565", "val0");
invalidCustomAttributes.put("key1-aaa_bbb-ccc", "val1");
invalidCustomAttributes.put("key2!@#$%&*()", "val2"); // invalid key characters
tblEntity.setCustomAttributes(invalidCustomAttributes);
try {
entityStore.createOrUpdate(new AtlasEntityStream(tblEntity), false);
} catch (AtlasBaseException ex) {
assertEquals(ex.getAtlasErrorCode(), INVALID_CUSTOM_ATTRIBUTE_KEY_CHARACTERS);
}
invalidCustomAttributes = new HashMap<>();
invalidCustomAttributes.put("bigValue_lengthEquals_50", randomAlphanumeric(50));
invalidCustomAttributes.put("bigValue_lengthEquals_51", randomAlphanumeric(51));
tblEntity.setCustomAttributes(invalidCustomAttributes);
try {
entityStore.createOrUpdate(new AtlasEntityStream(tblEntity), false);
} catch (AtlasBaseException ex) {
assertEquals(ex.getAtlasErrorCode(), INVALID_CUSTOM_ATTRIBUTE_KEY_LENGTH);
}
}
@Test (dependsOnMethods = "addInvalidKeysToEntityCustomAttributes")
public void addInvalidValuesToEntityCustomAttributes() throws AtlasBaseException {
AtlasEntity tblEntity = getEntityFromStore(tblEntityGuid);
// value length is greater than 500
Map<String, String> invalidCustomAttributes = new HashMap<>();
invalidCustomAttributes.put("key1", randomAlphanumeric(500));
invalidCustomAttributes.put("key2", randomAlphanumeric(501));
tblEntity.setCustomAttributes(invalidCustomAttributes);
try {
entityStore.createOrUpdate(new AtlasEntityStream(tblEntity), false);
} catch (AtlasBaseException ex) {
assertEquals(ex.getAtlasErrorCode(), INVALID_CUSTOM_ATTRIBUTE_VALUE);
}
}
} }
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment