Commit bebe746b by Sarath Subramanian

ATLAS-3431: Ability to create and store user defined key-value pairs in Atlas entity instances

parent 74bf4cda
......@@ -91,6 +91,7 @@ public final class Constants {
public static final String CLASSIFICATION_TEXT_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "classificationsText");
public static final String CLASSIFICATION_NAMES_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "classificationNames");
public static final String PROPAGATED_CLASSIFICATION_NAMES_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "propagatedClassificationNames");
public static final String CUSTOM_ATTRIBUTES_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "customAttributes");
public static final String MODIFIED_BY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "modifiedBy");
......
......@@ -59,6 +59,8 @@ public enum AtlasConfiguration {
SEARCH_MAX_LIMIT("atlas.search.maxlimit", 10000),
SEARCH_DEFAULT_LIMIT("atlas.search.defaultlimit", 100),
CUSTOM_ATTRIBUTE_KEY_MAX_LENGTH("atlas.custom.attribute.key.max.length", 50),
CUSTOM_ATTRIBUTE_VALUE_MAX_LENGTH("atlas.custom.attribute.value.max.length", 500),
IMPORT_TEMP_DIRECTORY("atlas.import.temp.directory", "");
private static final Configuration APPLICATION_PROPERTIES;
......
......@@ -154,7 +154,10 @@ public enum AtlasErrorCode {
INVALID_TIMEBOUNDRY_DATERANGE(400, "ATLAS-400-00-87D", "Invalid dateRange: startTime {0} must be before endTime {1}"),
PROPAGATED_CLASSIFICATION_REMOVAL_NOT_SUPPORTED(400, "ATLAS-400-00-87E", "Removal of classification {0}, which is propagated from entity {1}, is not supported"),
IMPORT_ATTEMPTING_EMPTY_ZIP(400, "ATLAS-400-00-87F", "Attempting to import empty ZIP file."),
PATCH_MISSING_RELATIONSHIP_LABEL(400, "ATLAS-400-00-880", "{0} - must include relationship label for type {1}"),
PATCH_MISSING_RELATIONSHIP_LABEL(400, "ATLAS-400-00-88", "{0} - must include relationship label for type {1}"),
INVALID_CUSTOM_ATTRIBUTE_KEY_LENGTH(400, "ATLAS-400-00-89", "Invalid key: {0} in custom attribute, key size should not be greater than 50"),
INVALID_CUSTOM_ATTRIBUTE_KEY_CHARACTERS(400, "ATLAS-400-00-90", "Invalid key: {0} in custom attribute, key should only contain alphanumeric characters, '_' or '-'"),
INVALID_CUSTOM_ATTRIBUTE_VALUE(400, "ATLAS-400-00-9A", "Invalid value: {0} in custom attribute, value length is greater than {1}"),
UNAUTHORIZED_ACCESS(403, "ATLAS-403-00-001", "{0} is not authorized to perform {1}"),
......
......@@ -91,6 +91,7 @@ public class AtlasEntity extends AtlasStruct implements Serializable {
private Map<String, Object> relationshipAttributes;
private List<AtlasClassification> classifications;
private List<AtlasTermAssignmentHeader> meanings;
private Map<String, String> customAttributes;
@JsonIgnore
private static AtomicLong s_nextId = new AtomicLong(System.nanoTime());
......@@ -213,6 +214,7 @@ public class AtlasEntity extends AtlasStruct implements Serializable {
setClassifications(other.getClassifications());
setRelationshipAttributes(other.getRelationshipAttributes());
setMeanings(other.getMeanings());
setCustomAttributes(other.getCustomAttributes());
}
}
......@@ -335,6 +337,14 @@ public class AtlasEntity extends AtlasStruct implements Serializable {
return r != null ? r.containsKey(name) : false;
}
public Map<String, String> getCustomAttributes() {
return customAttributes;
}
public void setCustomAttributes(Map<String, String> customAttributes) {
this.customAttributes = customAttributes;
}
public List<AtlasClassification> getClassifications() { return classifications; }
public void setClassifications(List<AtlasClassification> classifications) { this.classifications = classifications; }
......@@ -382,6 +392,7 @@ public class AtlasEntity extends AtlasStruct implements Serializable {
setUpdateTime(null);
setClassifications(null);
setMeanings(null);
setCustomAttributes(null);
}
private static String nextInternalId() {
......@@ -416,6 +427,9 @@ public class AtlasEntity extends AtlasStruct implements Serializable {
sb.append(", meanings=[");
AtlasBaseTypeDef.dumpObjects(meanings, sb);
sb.append(']');
sb.append(", customAttributes=[");
dumpObjects(customAttributes, sb);
sb.append("]");
sb.append('}');
return sb;
......@@ -440,13 +454,14 @@ public class AtlasEntity extends AtlasStruct implements Serializable {
Objects.equals(updateTime, that.updateTime) &&
Objects.equals(version, that.version) &&
Objects.equals(relationshipAttributes, that.relationshipAttributes) &&
Objects.equals(customAttributes, that.customAttributes) &&
Objects.equals(classifications, that.classifications);
}
@Override
public int hashCode() {
return Objects.hash(super.hashCode(), guid, homeId, isProxy, isIncomplete, provenanceType, status,
createdBy, updatedBy, createTime, updateTime, version, relationshipAttributes, classifications);
createdBy, updatedBy, createTime, updateTime, version, relationshipAttributes, classifications, customAttributes);
}
@Override
......
......@@ -324,6 +324,7 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
createCommonVertexIndex(management, TRAIT_NAMES_PROPERTY_KEY, UniqueKind.NONE, String.class, SET, true, true);
createCommonVertexIndex(management, PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, UniqueKind.NONE, String.class, LIST, true, true);
createCommonVertexIndex(management, IS_INCOMPLETE_PROPERTY_KEY, UniqueKind.NONE, Integer.class, SINGLE, true, true);
createCommonVertexIndex(management, CUSTOM_ATTRIBUTES_PROPERTY_KEY, UniqueKind.NONE, String.class, SINGLE, true, false);
createCommonVertexIndex(management, PATCH_ID_PROPERTY_KEY, UniqueKind.GLOBAL_UNIQUE, String.class, SINGLE, true, false);
createCommonVertexIndex(management, PATCH_DESCRIPTION_PROPERTY_KEY, UniqueKind.NONE, String.class, SINGLE, true, false);
......
......@@ -1065,6 +1065,17 @@ public final class GraphHelper {
return ret;
}
public static Map getCustomAttributes(AtlasElement element) {
Map ret = null;
String customAttrsString = element.getProperty(CUSTOM_ATTRIBUTES_PROPERTY_KEY, String.class);
if (customAttrsString != null) {
ret = AtlasType.fromJson(customAttrsString, Map.class);
}
return ret;
}
public static Integer getProvenanceType(AtlasElement element) {
return element.getProperty(Constants.PROVENANCE_TYPE_KEY, Integer.class);
}
......
......@@ -50,6 +50,7 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import static org.apache.atlas.repository.store.graph.v2.EntityGraphMapper.validateCustomAttributes;
public class AtlasEntityGraphDiscoveryV2 implements EntityGraphDiscovery {
private static final Logger LOG = LoggerFactory.getLogger(AtlasEntityGraphDiscoveryV2.class);
......@@ -84,7 +85,7 @@ public class AtlasEntityGraphDiscoveryV2 implements EntityGraphDiscovery {
public void validateAndNormalize(AtlasEntity entity) throws AtlasBaseException {
List<String> messages = new ArrayList<>();
if (! AtlasTypeUtil.isValidGuid(entity.getGuid())) {
if (!AtlasTypeUtil.isValidGuid(entity.getGuid())) {
throw new AtlasBaseException(AtlasErrorCode.INVALID_OBJECT_ID, "invalid guid " + entity.getGuid());
}
......@@ -94,6 +95,8 @@ public class AtlasEntityGraphDiscoveryV2 implements EntityGraphDiscovery {
throw new AtlasBaseException(AtlasErrorCode.TYPE_NAME_INVALID, TypeCategory.ENTITY.name(), entity.getTypeName());
}
validateCustomAttributes(entity);
type.validateValue(entity, entity.getTypeName(), messages);
if (!messages.isEmpty()) {
......@@ -107,7 +110,7 @@ public class AtlasEntityGraphDiscoveryV2 implements EntityGraphDiscovery {
public void validateAndNormalizeForUpdate(AtlasEntity entity) throws AtlasBaseException {
List<String> messages = new ArrayList<>();
if (! AtlasTypeUtil.isValidGuid(entity.getGuid())) {
if (!AtlasTypeUtil.isValidGuid(entity.getGuid())) {
throw new AtlasBaseException(AtlasErrorCode.INVALID_OBJECT_ID, "invalid guid " + entity.getGuid());
}
......@@ -117,6 +120,8 @@ public class AtlasEntityGraphDiscoveryV2 implements EntityGraphDiscovery {
throw new AtlasBaseException(AtlasErrorCode.TYPE_NAME_INVALID, TypeCategory.ENTITY.name(), entity.getTypeName());
}
validateCustomAttributes(entity);
type.validateValueForUpdate(entity, entity.getTypeName(), messages);
if (!messages.isEmpty()) {
......
......@@ -59,6 +59,7 @@ import static java.lang.Boolean.FALSE;
import static org.apache.atlas.model.instance.EntityMutations.EntityOperation.DELETE;
import static org.apache.atlas.model.instance.EntityMutations.EntityOperation.UPDATE;
import static org.apache.atlas.repository.Constants.IS_INCOMPLETE_PROPERTY_KEY;
import static org.apache.atlas.repository.graph.GraphHelper.getCustomAttributes;
import static org.apache.atlas.repository.graph.GraphHelper.isEntityIncomplete;
......@@ -814,6 +815,15 @@ public class AtlasEntityStoreV2 implements AtlasEntityStore {
}
}
if (!hasUpdates && entity.getCustomAttributes() != null) {
Map<String, String> currCustomAttributes = getCustomAttributes(vertex);
Map<String, String> newCustomAttributes = entity.getCustomAttributes();
if (!Objects.equals(currCustomAttributes, newCustomAttributes)) {
hasUpdates = true;
}
}
// if classifications are to be replaced, then skip updates only when no change in classifications
if (!hasUpdates && replaceClassifications) {
List<AtlasClassification> newVal = entity.getClassifications();
......@@ -921,6 +931,8 @@ public class AtlasEntityStoreV2 implements AtlasEntityStore {
requestContext.recordEntityGuidUpdate(entity, guid);
}
entityGraphMapper.setCustomAttributes(vertex, entity);
context.addUpdated(guid, entity, entityType, vertex);
} else {
graphDiscoverer.validateAndNormalize(entity);
......
......@@ -72,6 +72,8 @@ import org.springframework.stereotype.Component;
import javax.inject.Inject;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import static org.apache.atlas.model.TypeCategory.CLASSIFICATION;
......@@ -113,6 +115,9 @@ public class EntityGraphMapper {
private static final int INDEXED_STR_SAFE_LEN = AtlasConfiguration.GRAPHSTORE_INDEXED_STRING_SAFE_LENGTH.getInt();
private static final boolean WARN_ON_NO_RELATIONSHIP = AtlasConfiguration.RELATIONSHIP_WARN_NO_RELATIONSHIPS.getBoolean();
private static final String CLASSIFICATION_NAME_DELIMITER = "|";
private static final Pattern CUSTOM_ATTRIBUTE_KEY_REGEX = Pattern.compile("^[a-zA-Z0-9_-]*$");
private static final int CUSTOM_ATTRIBUTE_KEY_MAX_LENGTH = AtlasConfiguration.CUSTOM_ATTRIBUTE_KEY_MAX_LENGTH.getInt();
private static final int CUSTOM_ATTRIBUTE_VALUE_MAX_LENGTH = AtlasConfiguration.CUSTOM_ATTRIBUTE_VALUE_MAX_LENGTH.getInt();
private final GraphHelper graphHelper = GraphHelper.getInstance();
private final AtlasGraph graph;
......@@ -138,7 +143,7 @@ public class EntityGraphMapper {
this.fullTextMapperV2 = fullTextMapperV2;
}
public AtlasVertex createVertex(AtlasEntity entity) {
public AtlasVertex createVertex(AtlasEntity entity) throws AtlasBaseException {
final String guid = UUID.randomUUID().toString();
return createVertexWithGuid(entity, guid);
}
......@@ -179,7 +184,7 @@ public class EntityGraphMapper {
return ret;
}
public AtlasVertex createVertexWithGuid(AtlasEntity entity, String guid) {
public AtlasVertex createVertexWithGuid(AtlasEntity entity, String guid) throws AtlasBaseException {
if (LOG.isDebugEnabled()) {
LOG.debug("==> createVertexWithGuid({})", entity.getTypeName());
}
......@@ -194,12 +199,14 @@ public class EntityGraphMapper {
AtlasGraphUtilsV2.setEncodedProperty(ret, GUID_PROPERTY_KEY, guid);
AtlasGraphUtilsV2.setEncodedProperty(ret, VERSION_PROPERTY_KEY, getEntityVersion(entity));
setCustomAttributes(ret, entity);
GraphTransactionInterceptor.addToVertexCache(guid, ret);
return ret;
}
public void updateSystemAttributes(AtlasVertex vertex, AtlasEntity entity) {
public void updateSystemAttributes(AtlasVertex vertex, AtlasEntity entity) throws AtlasBaseException {
if (entity.getVersion() != null) {
AtlasGraphUtilsV2.setEncodedProperty(vertex, VERSION_PROPERTY_KEY, entity.getVersion());
}
......@@ -231,6 +238,10 @@ public class EntityGraphMapper {
if (entity.getProvenanceType() != null) {
AtlasGraphUtilsV2.setEncodedProperty(vertex, PROVENANCE_TYPE_KEY, entity.getProvenanceType());
}
if (entity.getCustomAttributes() != null) {
setCustomAttributes(vertex, entity);
}
}
public EntityMutationResponse mapAttributesAndClassifications(EntityMutationContext context, final boolean isPartialUpdate, final boolean replaceClassifications) throws AtlasBaseException {
......@@ -308,6 +319,14 @@ public class EntityGraphMapper {
return resp;
}
public void setCustomAttributes(AtlasVertex vertex, AtlasEntity entity) throws AtlasBaseException {
String customAttributesString = getCustomAttributesString(entity);
if (customAttributesString != null) {
AtlasGraphUtilsV2.setEncodedProperty(vertex, CUSTOM_ATTRIBUTES_PROPERTY_KEY, customAttributesString);
}
}
private AtlasVertex createStructVertex(AtlasStruct struct) {
return createStructVertex(struct.getTypeName());
}
......@@ -1150,6 +1169,17 @@ public class EntityGraphMapper {
return (ret != null) ? ret : 0;
}
private String getCustomAttributesString(AtlasEntity entity) {
String ret = null;
Map<String, String> customAttributes = entity.getCustomAttributes();
if (customAttributes != null) {
ret = AtlasType.toJson(customAttributes);
}
return ret;
}
private AtlasStructType getStructType(String typeName) throws AtlasBaseException {
AtlasType objType = typeRegistry.getType(typeName);
......@@ -2151,4 +2181,29 @@ public class EntityGraphMapper {
}
return relGuidsSet;
}
public static void validateCustomAttributes(AtlasEntity entity) throws AtlasBaseException {
Map<String, String> customAttributes = entity.getCustomAttributes();
if (MapUtils.isNotEmpty(customAttributes)) {
for (Map.Entry<String, String> entry : customAttributes.entrySet()) {
String key = entry.getKey();
String value = entry.getValue();
if (key.length() > CUSTOM_ATTRIBUTE_KEY_MAX_LENGTH) {
throw new AtlasBaseException(AtlasErrorCode.INVALID_CUSTOM_ATTRIBUTE_KEY_LENGTH, key);
}
Matcher matcher = CUSTOM_ATTRIBUTE_KEY_REGEX.matcher(key);
if (!matcher.matches()) {
throw new AtlasBaseException(AtlasErrorCode.INVALID_CUSTOM_ATTRIBUTE_KEY_CHARACTERS, key);
}
if (value.length() > CUSTOM_ATTRIBUTE_VALUE_MAX_LENGTH) {
throw new AtlasBaseException(AtlasErrorCode.INVALID_CUSTOM_ATTRIBUTE_VALUE, value, String.valueOf(CUSTOM_ATTRIBUTE_VALUE_MAX_LENGTH));
}
}
}
}
}
......@@ -582,6 +582,7 @@ public class EntityGraphRetriever {
entity.setIsIncomplete(isEntityIncomplete(entityVertex));
entity.setProvenanceType(GraphHelper.getProvenanceType(entityVertex));
entity.setCustomAttributes(getCustomAttributes(entityVertex));
return entity;
}
......
......@@ -55,10 +55,12 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import static org.apache.atlas.AtlasErrorCode.*;
import static org.apache.atlas.TestUtilsV2.COLUMNS_ATTR_NAME;
import static org.apache.atlas.TestUtilsV2.COLUMN_TYPE;
import static org.apache.atlas.TestUtilsV2.NAME;
import static org.apache.atlas.TestUtilsV2.TABLE_TYPE;
import static org.apache.commons.lang.RandomStringUtils.randomAlphanumeric;
import static org.mockito.Mockito.mock;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertTrue;
......@@ -982,4 +984,128 @@ public class AtlasEntityStoreV2Test extends AtlasEntityTestBase {
entityStore.deleteClassification(dbEntityGuid, TAG_NAME);
entityStore.deleteClassification(tblEntityGuid, TAG_NAME);
}
@Test (dependsOnMethods = "testCreate")
public void addCustomAttributesToEntity() throws AtlasBaseException {
AtlasEntity tblEntity = getEntityFromStore(tblEntityGuid);
Map<String, String> customAttributes = new HashMap<>();
customAttributes.put("key1", "val1");
customAttributes.put("key2", "val2");
customAttributes.put("key3", "val3");
customAttributes.put("key4", "val4");
customAttributes.put("key5", "val5");
tblEntity.setCustomAttributes(customAttributes);
entityStore.createOrUpdate(new AtlasEntityStream(tblEntity), false);
tblEntity = getEntityFromStore(tblEntityGuid);
assertEquals(customAttributes, tblEntity.getCustomAttributes());
}
@Test (dependsOnMethods = "addCustomAttributesToEntity")
public void updateCustomAttributesToEntity() throws AtlasBaseException {
AtlasEntity tblEntity = getEntityFromStore(tblEntityGuid);
// update custom attributes, remove key3, key4 and key5
Map<String, String> customAttributes = new HashMap<>();
customAttributes.put("key1", "val1");
customAttributes.put("key2", "val2");
tblEntity.setCustomAttributes(customAttributes);
entityStore.createOrUpdate(new AtlasEntityStream(tblEntity), false);
tblEntity = getEntityFromStore(tblEntityGuid);
assertEquals(customAttributes, tblEntity.getCustomAttributes());
}
@Test (dependsOnMethods = "updateCustomAttributesToEntity")
public void deleteCustomAttributesToEntity() throws AtlasBaseException {
AtlasEntity tblEntity = getEntityFromStore(tblEntityGuid);
Map<String, String> emptyCustomAttributes = new HashMap<>();
// remove all custom attributes
tblEntity.setCustomAttributes(emptyCustomAttributes);
entityStore.createOrUpdate(new AtlasEntityStream(tblEntity), false);
tblEntity = getEntityFromStore(tblEntityGuid);
assertEquals(emptyCustomAttributes, tblEntity.getCustomAttributes());
}
@Test (dependsOnMethods = "deleteCustomAttributesToEntity")
public void nullCustomAttributesToEntity() throws AtlasBaseException {
AtlasEntity tblEntity = getEntityFromStore(tblEntityGuid);
Map<String, String> customAttributes = new HashMap<>();
customAttributes.put("key1", "val1");
customAttributes.put("key2", "val2");
tblEntity.setCustomAttributes(customAttributes);
entityStore.createOrUpdate(new AtlasEntityStream(tblEntity), false);
// assign custom attributes to null
tblEntity.setCustomAttributes(null);
entityStore.createOrUpdate(new AtlasEntityStream(tblEntity), false);
tblEntity = getEntityFromStore(tblEntityGuid);
assertEquals(customAttributes, tblEntity.getCustomAttributes());
}
@Test (dependsOnMethods = "nullCustomAttributesToEntity")
public void addInvalidKeysToEntityCustomAttributes() throws AtlasBaseException {
AtlasEntity tblEntity = getEntityFromStore(tblEntityGuid);
// key should contain 1 to 50 alphanumeric characters, '_' or '-'
Map<String, String> invalidCustomAttributes = new HashMap<>();
invalidCustomAttributes.put("key0_65765-6565", "val0");
invalidCustomAttributes.put("key1-aaa_bbb-ccc", "val1");
invalidCustomAttributes.put("key2!@#$%&*()", "val2"); // invalid key characters
tblEntity.setCustomAttributes(invalidCustomAttributes);
try {
entityStore.createOrUpdate(new AtlasEntityStream(tblEntity), false);
} catch (AtlasBaseException ex) {
assertEquals(ex.getAtlasErrorCode(), INVALID_CUSTOM_ATTRIBUTE_KEY_CHARACTERS);
}
invalidCustomAttributes = new HashMap<>();
invalidCustomAttributes.put("bigValue_lengthEquals_50", randomAlphanumeric(50));
invalidCustomAttributes.put("bigValue_lengthEquals_51", randomAlphanumeric(51));
tblEntity.setCustomAttributes(invalidCustomAttributes);
try {
entityStore.createOrUpdate(new AtlasEntityStream(tblEntity), false);
} catch (AtlasBaseException ex) {
assertEquals(ex.getAtlasErrorCode(), INVALID_CUSTOM_ATTRIBUTE_KEY_LENGTH);
}
}
@Test (dependsOnMethods = "addInvalidKeysToEntityCustomAttributes")
public void addInvalidValuesToEntityCustomAttributes() throws AtlasBaseException {
AtlasEntity tblEntity = getEntityFromStore(tblEntityGuid);
// value length is greater than 500
Map<String, String> invalidCustomAttributes = new HashMap<>();
invalidCustomAttributes.put("key1", randomAlphanumeric(500));
invalidCustomAttributes.put("key2", randomAlphanumeric(501));
tblEntity.setCustomAttributes(invalidCustomAttributes);
try {
entityStore.createOrUpdate(new AtlasEntityStream(tblEntity), false);
} catch (AtlasBaseException ex) {
assertEquals(ex.getAtlasErrorCode(), INVALID_CUSTOM_ATTRIBUTE_VALUE);
}
}
}
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment