Commit cacf361e by Sarath Subramanian

ATLAS-3443: Enhancements to support 'Labels' in Atlas

parent 678043f8
......@@ -92,6 +92,7 @@ public final class Constants {
public static final String CLASSIFICATION_NAMES_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "classificationNames");
public static final String PROPAGATED_CLASSIFICATION_NAMES_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "propagatedClassificationNames");
public static final String CUSTOM_ATTRIBUTES_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "customAttributes");
public static final String LABELS_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "labels");
public static final String MODIFIED_BY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "modifiedBy");
......@@ -184,6 +185,7 @@ public final class Constants {
public static final String CLASSIFICATION_EDGE_STATE_PROPERTY_KEY = STATE_PROPERTY_KEY;
public static final String CLASSIFICATION_LABEL = "classifiedAs";
public static final String CLASSIFICATION_NAME_DELIMITER = "|";
public static final String LABEL_NAME_DELIMITER = CLASSIFICATION_NAME_DELIMITER;
public static final String TERM_ASSIGNMENT_LABEL = "r:AtlasGlossarySemanticAssignment";
public static final String ATTRIBUTE_INDEX_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "index");
public static final String ATTRIBUTE_KEY_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "key");
......
......@@ -61,6 +61,7 @@ public enum AtlasConfiguration {
CUSTOM_ATTRIBUTE_KEY_MAX_LENGTH("atlas.custom.attribute.key.max.length", 50),
CUSTOM_ATTRIBUTE_VALUE_MAX_LENGTH("atlas.custom.attribute.value.max.length", 500),
LABEL_MAX_LENGTH("atlas.entity.label.max.length", 50),
IMPORT_TEMP_DIRECTORY("atlas.import.temp.directory", "");
private static final Configuration APPLICATION_PROPERTIES;
......
......@@ -158,6 +158,8 @@ public enum AtlasErrorCode {
INVALID_CUSTOM_ATTRIBUTE_KEY_LENGTH(400, "ATLAS-400-00-89", "Invalid key: {0} in custom attribute, key size should not be greater than 50"),
INVALID_CUSTOM_ATTRIBUTE_KEY_CHARACTERS(400, "ATLAS-400-00-90", "Invalid key: {0} in custom attribute, key should only contain alphanumeric characters, '_' or '-'"),
INVALID_CUSTOM_ATTRIBUTE_VALUE(400, "ATLAS-400-00-9A", "Invalid value: {0} in custom attribute, value length is greater than {1}"),
INVALID_LABEL_LENGTH(400, "ATLAS-400-00-9B", "Invalid label: {0}, label size should not be greater than {1}"),
INVALID_LABEL_CHARACTERS(400, "ATLAS-400-00-9C", "Invalid label: {0}, label should contain alphanumeric characters, '_' or '-'"),
UNAUTHORIZED_ACCESS(403, "ATLAS-403-00-001", "{0} is not authorized to perform {1}"),
......
......@@ -42,6 +42,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.atomic.AtomicLong;
import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.NONE;
......@@ -92,6 +93,7 @@ public class AtlasEntity extends AtlasStruct implements Serializable {
private List<AtlasClassification> classifications;
private List<AtlasTermAssignmentHeader> meanings;
private Map<String, String> customAttributes;
private Set<String> labels;
@JsonIgnore
private static AtomicLong s_nextId = new AtomicLong(System.nanoTime());
......@@ -215,6 +217,7 @@ public class AtlasEntity extends AtlasStruct implements Serializable {
setRelationshipAttributes(other.getRelationshipAttributes());
setMeanings(other.getMeanings());
setCustomAttributes(other.getCustomAttributes());
setLabels(other.getLabels());
}
}
......@@ -345,6 +348,14 @@ public class AtlasEntity extends AtlasStruct implements Serializable {
this.customAttributes = customAttributes;
}
public Set<String> getLabels() {
return labels;
}
public void setLabels(Set<String> labels) {
this.labels = labels;
}
public List<AtlasClassification> getClassifications() { return classifications; }
public void setClassifications(List<AtlasClassification> classifications) { this.classifications = classifications; }
......@@ -393,6 +404,7 @@ public class AtlasEntity extends AtlasStruct implements Serializable {
setClassifications(null);
setMeanings(null);
setCustomAttributes(null);
setLabels(null);
}
private static String nextInternalId() {
......@@ -430,6 +442,9 @@ public class AtlasEntity extends AtlasStruct implements Serializable {
sb.append(", customAttributes=[");
dumpObjects(customAttributes, sb);
sb.append("]");
sb.append(", labels=[");
dumpObjects(labels, sb);
sb.append("]");
sb.append('}');
return sb;
......@@ -455,13 +470,14 @@ public class AtlasEntity extends AtlasStruct implements Serializable {
Objects.equals(version, that.version) &&
Objects.equals(relationshipAttributes, that.relationshipAttributes) &&
Objects.equals(customAttributes, that.customAttributes) &&
Objects.equals(labels, that.labels) &&
Objects.equals(classifications, that.classifications);
}
@Override
public int hashCode() {
return Objects.hash(super.hashCode(), guid, homeId, isProxy, isIncomplete, provenanceType, status,
createdBy, updatedBy, createTime, updateTime, version, relationshipAttributes, classifications, customAttributes);
return Objects.hash(super.hashCode(), guid, homeId, isProxy, isIncomplete, provenanceType, status, createdBy,
updatedBy, createTime, updateTime, version, relationshipAttributes, classifications, customAttributes, labels);
}
@Override
......
......@@ -37,6 +37,7 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.NONE;
import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.PUBLIC_ONLY;
......@@ -61,6 +62,7 @@ public class AtlasEntityHeader extends AtlasStruct implements Serializable {
private List<String> meaningNames = null;
private List<AtlasTermAssignmentHeader> meanings = null;
private Boolean isIncomplete = Boolean.FALSE;
private Set<String> labels = null;
public AtlasEntityHeader() {
this(null, null);
......@@ -79,6 +81,7 @@ public class AtlasEntityHeader extends AtlasStruct implements Serializable {
setClassificationNames(null);
setClassifications(null);
setLabels(null);
}
......@@ -87,6 +90,7 @@ public class AtlasEntityHeader extends AtlasStruct implements Serializable {
setGuid(guid);
setClassificationNames(null);
setClassifications(null);
setLabels(null);
}
......@@ -100,6 +104,7 @@ public class AtlasEntityHeader extends AtlasStruct implements Serializable {
setClassificationNames(other.getClassificationNames());
setClassifications(other.getClassifications());
setIsIncomplete(other.getIsIncomplete());
setLabels(other.getLabels());
}
}
......@@ -117,6 +122,10 @@ public class AtlasEntityHeader extends AtlasStruct implements Serializable {
this.classificationNames.add(classification.getTypeName());
}
}
if (CollectionUtils.isNotEmpty(entity.getLabels())) {
setLabels(entity.getLabels());
}
}
public String getGuid() {
......@@ -159,6 +168,14 @@ public class AtlasEntityHeader extends AtlasStruct implements Serializable {
this.classifications = classifications;
}
public Set<String> getLabels() {
return labels;
}
public void setLabels(Set<String> labels) {
this.labels = labels;
}
public Boolean getIsIncomplete() {
return isIncomplete;
}
......@@ -183,6 +200,9 @@ public class AtlasEntityHeader extends AtlasStruct implements Serializable {
sb.append("classifications=[");
AtlasBaseTypeDef.dumpObjects(classifications, sb);
sb.append("], ");
sb.append("labels=[");
dumpObjects(labels, sb);
sb.append("], ");
sb.append("isIncomplete=").append(isIncomplete);
super.toString(sb);
sb.append('}');
......@@ -202,13 +222,14 @@ public class AtlasEntityHeader extends AtlasStruct implements Serializable {
Objects.equals(classificationNames, that.classificationNames) &&
Objects.equals(meaningNames, that.classificationNames) &&
Objects.equals(classifications, that.classifications) &&
Objects.equals(labels, that.labels) &&
Objects.equals(isIncomplete, that.isIncomplete) &&
Objects.equals(meanings, that.meanings);
}
@Override
public int hashCode() {
return Objects.hash(super.hashCode(), guid, status, displayText, classificationNames, classifications, meaningNames, meanings, isIncomplete);
return Objects.hash(super.hashCode(), guid, status, displayText, classificationNames, classifications, meaningNames, meanings, isIncomplete, labels);
}
@Override
......
......@@ -325,6 +325,7 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
createCommonVertexIndex(management, PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, UniqueKind.NONE, String.class, LIST, true, true);
createCommonVertexIndex(management, IS_INCOMPLETE_PROPERTY_KEY, UniqueKind.NONE, Integer.class, SINGLE, true, true);
createCommonVertexIndex(management, CUSTOM_ATTRIBUTES_PROPERTY_KEY, UniqueKind.NONE, String.class, SINGLE, true, false);
createCommonVertexIndex(management, LABELS_PROPERTY_KEY, UniqueKind.NONE, String.class, SINGLE, true, false);
createCommonVertexIndex(management, PATCH_ID_PROPERTY_KEY, UniqueKind.GLOBAL_UNIQUE, String.class, SINGLE, true, false);
createCommonVertexIndex(management, PATCH_DESCRIPTION_PROPERTY_KEY, UniqueKind.NONE, String.class, SINGLE, true, false);
......
......@@ -1076,6 +1076,10 @@ public final class GraphHelper {
return ret;
}
public static Set<String> getLabels(AtlasElement element) {
return parseLabelsString(element.getProperty(LABELS_PROPERTY_KEY, String.class));
}
public static Integer getProvenanceType(AtlasElement element) {
return element.getProperty(Constants.PROVENANCE_TYPE_KEY, Integer.class);
}
......@@ -1845,4 +1849,20 @@ public final class GraphHelper {
}
return ret;
}
private static Set<String> parseLabelsString(String labels) {
Set<String> ret = null;
if (StringUtils.isNotEmpty(labels)) {
ret = new HashSet<>();
for (String label : labels.split("\\" + LABEL_NAME_DELIMITER)) {
if (StringUtils.isNotEmpty(label)) {
ret.add(label);
}
}
}
return ret;
}
}
\ No newline at end of file
......@@ -32,6 +32,7 @@ import org.apache.atlas.type.AtlasEntityType;
import java.util.List;
import java.util.Map;
import java.util.Set;
/**
* Persistence/Retrieval API for AtlasEntity
......@@ -231,4 +232,9 @@ public interface AtlasEntityStore {
AtlasClassification getClassification(String guid, String classificationName) throws AtlasBaseException;
String setClassifications(AtlasEntityHeaders entityHeaders);
/**
* Set Labels
*/
void setLabels(String guid, Set<String> labels) throws AtlasBaseException;
}
......@@ -51,6 +51,7 @@ import java.util.Map;
import java.util.Set;
import static org.apache.atlas.repository.store.graph.v2.EntityGraphMapper.validateCustomAttributes;
import static org.apache.atlas.repository.store.graph.v2.EntityGraphMapper.validateLabels;
public class AtlasEntityGraphDiscoveryV2 implements EntityGraphDiscovery {
private static final Logger LOG = LoggerFactory.getLogger(AtlasEntityGraphDiscoveryV2.class);
......@@ -97,6 +98,8 @@ public class AtlasEntityGraphDiscoveryV2 implements EntityGraphDiscovery {
validateCustomAttributes(entity);
validateLabels(entity.getLabels());
type.validateValue(entity, entity.getTypeName(), messages);
if (!messages.isEmpty()) {
......@@ -122,6 +125,8 @@ public class AtlasEntityGraphDiscoveryV2 implements EntityGraphDiscovery {
validateCustomAttributes(entity);
validateLabels(entity.getLabels());
type.validateValueForUpdate(entity, entity.getTypeName(), messages);
if (!messages.isEmpty()) {
......
......@@ -22,15 +22,22 @@ import org.apache.atlas.AtlasErrorCode;
import org.apache.atlas.GraphTransactionInterceptor;
import org.apache.atlas.RequestContext;
import org.apache.atlas.annotation.GraphTransaction;
import org.apache.atlas.authorize.AtlasAuthorizationUtils;
import org.apache.atlas.authorize.AtlasEntityAccessRequest;
import org.apache.atlas.authorize.AtlasPrivilege;
import org.apache.atlas.authorize.AtlasAuthorizationUtils;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.TypeCategory;
import org.apache.atlas.model.instance.*;
import org.apache.atlas.model.instance.AtlasCheckStateRequest;
import org.apache.atlas.model.instance.AtlasCheckStateResult;
import org.apache.atlas.model.instance.AtlasClassification;
import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.model.instance.AtlasEntity.AtlasEntitiesWithExtInfo;
import org.apache.atlas.model.instance.AtlasEntity.AtlasEntityWithExtInfo;
import org.apache.atlas.model.instance.AtlasEntity.Status;
import org.apache.atlas.model.instance.AtlasEntityHeader;
import org.apache.atlas.model.instance.AtlasEntityHeaders;
import org.apache.atlas.model.instance.AtlasObjectId;
import org.apache.atlas.model.instance.EntityMutationResponse;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.repository.store.graph.AtlasEntityStore;
import org.apache.atlas.repository.store.graph.EntityGraphDiscovery;
......@@ -53,7 +60,13 @@ import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Component;
import javax.inject.Inject;
import java.util.*;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import static java.lang.Boolean.FALSE;
import static org.apache.atlas.model.instance.EntityMutations.EntityOperation.DELETE;
......@@ -61,6 +74,7 @@ import static org.apache.atlas.model.instance.EntityMutations.EntityOperation.UP
import static org.apache.atlas.repository.Constants.IS_INCOMPLETE_PROPERTY_KEY;
import static org.apache.atlas.repository.graph.GraphHelper.getCustomAttributes;
import static org.apache.atlas.repository.graph.GraphHelper.isEntityIncomplete;
import static org.apache.atlas.repository.store.graph.v2.EntityGraphMapper.validateLabels;
@Component
......@@ -727,6 +741,32 @@ public class AtlasEntityStoreV2 implements AtlasEntityStore {
return associator.setClassifications(entityHeaders.getGuidHeaderMap());
}
@Override
@GraphTransaction
public void setLabels(String guid, Set<String> labels) throws AtlasBaseException {
if (LOG.isDebugEnabled()) {
LOG.debug("==> setLabels()");
}
if (StringUtils.isEmpty(guid)) {
throw new AtlasBaseException(AtlasErrorCode.INVALID_PARAMETERS, "guid is null/empty");
}
AtlasVertex entityVertex = AtlasGraphUtilsV2.findByGuid(guid);
if (entityVertex == null) {
throw new AtlasBaseException(AtlasErrorCode.INSTANCE_GUID_NOT_FOUND, guid);
}
validateLabels(labels);
entityGraphMapper.setLabels(entityVertex, labels);
if (LOG.isDebugEnabled()) {
LOG.debug("<== setLabels()");
}
}
private EntityMutationResponse createOrUpdate(EntityStream entityStream, boolean isPartialUpdate, boolean replaceClassifications) throws AtlasBaseException {
if (LOG.isDebugEnabled()) {
LOG.debug("==> createOrUpdate()");
......
......@@ -76,6 +76,7 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import static org.apache.atlas.AtlasConfiguration.LABEL_MAX_LENGTH;
import static org.apache.atlas.model.TypeCategory.CLASSIFICATION;
import static org.apache.atlas.model.instance.AtlasEntity.Status.ACTIVE;
import static org.apache.atlas.model.instance.AtlasEntity.Status.DELETED;
......@@ -116,6 +117,7 @@ public class EntityGraphMapper {
private static final boolean WARN_ON_NO_RELATIONSHIP = AtlasConfiguration.RELATIONSHIP_WARN_NO_RELATIONSHIPS.getBoolean();
private static final String CLASSIFICATION_NAME_DELIMITER = "|";
private static final Pattern CUSTOM_ATTRIBUTE_KEY_REGEX = Pattern.compile("^[a-zA-Z0-9_-]*$");
private static final Pattern LABEL_REGEX = Pattern.compile("^[a-zA-Z0-9_-]*$");
private static final int CUSTOM_ATTRIBUTE_KEY_MAX_LENGTH = AtlasConfiguration.CUSTOM_ATTRIBUTE_KEY_MAX_LENGTH.getInt();
private static final int CUSTOM_ATTRIBUTE_VALUE_MAX_LENGTH = AtlasConfiguration.CUSTOM_ATTRIBUTE_VALUE_MAX_LENGTH.getInt();
......@@ -201,6 +203,8 @@ public class EntityGraphMapper {
setCustomAttributes(ret, entity);
setLabels(ret, entity.getLabels());
GraphTransactionInterceptor.addToVertexCache(guid, ret);
return ret;
......@@ -319,7 +323,7 @@ public class EntityGraphMapper {
return resp;
}
public void setCustomAttributes(AtlasVertex vertex, AtlasEntity entity) throws AtlasBaseException {
public void setCustomAttributes(AtlasVertex vertex, AtlasEntity entity) {
String customAttributesString = getCustomAttributesString(entity);
if (customAttributesString != null) {
......@@ -327,6 +331,24 @@ public class EntityGraphMapper {
}
}
public void setLabels(AtlasVertex vertex, Set<String> labels) {
if (CollectionUtils.isNotEmpty(labels)) {
AtlasGraphUtilsV2.setEncodedProperty(vertex, LABELS_PROPERTY_KEY, getLabelString(labels));
} else {
vertex.removeProperty(LABELS_PROPERTY_KEY);
}
}
private String getLabelString(Set<String> labels) {
String ret = null;
if (!labels.isEmpty()) {
ret = LABEL_NAME_DELIMITER + String.join(LABEL_NAME_DELIMITER, labels) + LABEL_NAME_DELIMITER;
}
return ret;
}
private AtlasVertex createStructVertex(AtlasStruct struct) {
return createStructVertex(struct.getTypeName());
}
......@@ -2206,4 +2228,20 @@ public class EntityGraphMapper {
}
}
}
public static void validateLabels(Set<String> labels) throws AtlasBaseException {
if (CollectionUtils.isNotEmpty(labels)) {
for (String label : labels) {
if (label.length() > LABEL_MAX_LENGTH.getInt()) {
throw new AtlasBaseException(AtlasErrorCode.INVALID_LABEL_LENGTH, label, String.valueOf(LABEL_MAX_LENGTH.getInt()));
}
Matcher matcher = LABEL_REGEX.matcher(label);
if (!matcher.matches()) {
throw new AtlasBaseException(AtlasErrorCode.INVALID_LABEL_CHARACTERS, label);
}
}
}
}
}
......@@ -583,6 +583,7 @@ public class EntityGraphRetriever {
entity.setProvenanceType(GraphHelper.getProvenanceType(entityVertex));
entity.setCustomAttributes(getCustomAttributes(entityVertex));
entity.setLabels(getLabels(entityVertex));
return entity;
}
......
......@@ -1108,4 +1108,81 @@ public class AtlasEntityStoreV2Test extends AtlasEntityTestBase {
assertEquals(ex.getAtlasErrorCode(), INVALID_CUSTOM_ATTRIBUTE_VALUE);
}
}
@Test(dependsOnMethods = "testCreate")
public void addLabelsToEntity() throws AtlasBaseException {
Set<String> labels = new HashSet<>();
labels.add("label_1");
labels.add("label_2");
labels.add("label_3");
labels.add("label_4");
labels.add("label_5");
entityStore.setLabels(tblEntityGuid, labels);
AtlasEntity tblEntity = getEntityFromStore(tblEntityGuid);
assertEquals(labels, tblEntity.getLabels());
}
@Test (dependsOnMethods = "addLabelsToEntity")
public void updateLabelsToEntity() throws AtlasBaseException {
Set<String> labels = new HashSet<>();
labels.add("label_1_update");
labels.add("label_2_update");
labels.add("label_3_update");
entityStore.setLabels(tblEntityGuid, labels);
AtlasEntity tblEntity = getEntityFromStore(tblEntityGuid);
assertEquals(labels, tblEntity.getLabels());
}
@Test (dependsOnMethods = "updateLabelsToEntity")
public void clearLabelsToEntity() throws AtlasBaseException {
HashSet<String> emptyLabels = new HashSet<>();
entityStore.setLabels(tblEntityGuid, emptyLabels);
AtlasEntity tblEntity = getEntityFromStore(tblEntityGuid);
Assert.assertNull(tblEntity.getLabels());
}
@Test (dependsOnMethods = "clearLabelsToEntity")
public void nullLabelsToEntity() throws AtlasBaseException {
entityStore.setLabels(tblEntityGuid, null);
AtlasEntity tblEntity = getEntityFromStore(tblEntityGuid);
Assert.assertNull(tblEntity.getLabels());
}
@Test (dependsOnMethods = "nullLabelsToEntity")
public void invalidLabelLengthToEntity() throws AtlasBaseException {
Set<String> labels = new HashSet<>();
labels.add(randomAlphanumeric(50));
labels.add(randomAlphanumeric(51));
try {
entityStore.setLabels(tblEntityGuid, labels);
} catch (AtlasBaseException ex) {
assertEquals(ex.getAtlasErrorCode(), INVALID_LABEL_LENGTH);
}
}
@Test (dependsOnMethods = "invalidLabelLengthToEntity")
public void invalidLabelCharactersToEntity() throws AtlasBaseException {
Set<String> labels = new HashSet<>();
labels.add("label-1_100_45");
labels.add("LABEL-1_200-55");
labels.add("LaBeL-1-)(*U&%^%#$@!~");
try {
entityStore.setLabels(tblEntityGuid, labels);
} catch (AtlasBaseException ex) {
assertEquals(ex.getAtlasErrorCode(), INVALID_LABEL_CHARACTERS);
}
}
}
\ No newline at end of file
......@@ -67,6 +67,7 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
/**
......@@ -813,6 +814,30 @@ public class EntityREST {
}
}
/**
* Set labels to a given entity
* @param guid - Unique entity identifier
* @param labels - set of labels to be set to the entity
* @throws AtlasBaseException
*/
@POST
@Path("/guid/{guid}/labels")
@Produces(Servlets.JSON_MEDIA_TYPE)
@Consumes(Servlets.JSON_MEDIA_TYPE)
public void setLabels(@PathParam("guid") final String guid, Set<String> labels) throws AtlasBaseException {
AtlasPerfTracer perf = null;
try {
if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) {
perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "EntityREST.setLabels()");
}
entitiesStore.setLabels(guid, labels);
} finally {
AtlasPerfTracer.log(perf);
}
}
private AtlasEntityType ensureEntityType(String typeName) throws AtlasBaseException {
AtlasEntityType ret = typeRegistry.getEntityTypeByName(typeName);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment