Skip to content

A
atlas
Commit 371f967f by Sidharth Committed by Sarath Subramanian
Options

ATLAS-3518: Added new Atlas Audit Framework with Purge Operation and Unit Test

Signed-off-by: 's avatarSarath Subramanian <sarath@apache.org>
parent cc601d73
Showing with 1268 additions and 146 deletions
addons/models/0000-Area0/0010-base_model.json
{
"enumDefs": [],
"enumDefs": [
{
"name": "atlas_operation",
"description": "Defines audit operations in Atlas",
"typeVersion": "1.0",
"serviceType": "atlas_core",
"elementDefs": [
{
"ordinal": 0,
"value": "OTHERS"
},
{
"ordinal": 1,
"value": "PURGE"
},
{
"ordinal": 2,
"value": "EXPORT"
},
{
"ordinal": 3,
"value": "IMPORT"
},
{
"ordinal": 4,
"value": "IMPORT_DELETE_REPL"
}
]
}
],
"structDefs": [],
"classificationDefs": [],
"entityDefs": [
......@@ -388,6 +417,72 @@
]
},
{
"name": "__AtlasAuditEntry",
"serviceType": "atlas_core",
"typeVersion": "1.0",
"superTypes": [
"__internal"
],
"attributeDefs": [
{
"name": "userName",
"typeName": "string",
"cardinality": "SINGLE",
"isIndexable": true,
"isOptional": false,
"isUnique": false
},
{
"name": "operation",
"typeName": "atlas_operation",
"cardinality": "SINGLE",
"isIndexable": true,
"isOptional": false,
"isUnique": false
},
{
"name": "startTime",
"typeName": "date",
"cardinality": "SINGLE",
"isIndexable": true,
"isOptional": false,
"isUnique": false
},
{
"name": "endTime",
"typeName": "date",
"cardinality": "SINGLE",
"isIndexable": true,
"isOptional": true,
"isUnique": false
},
{
"name": "clientId",
"typeName": "string",
"cardinality": "SINGLE",
"isIndexable": true,
"isOptional": true,
"isUnique": false
},
{
"name": "params",
"typeName": "string",
"cardinality": "SINGLE",
"isIndexable": true,
"isOptional": true,
"isUnique": false
},
{
"name": "result",
"typeName": "string",
"cardinality": "SINGLE",
"isIndexable": false,
"isOptional": true,
"isUnique": false
}
]
},
{
"name": "ProcessExecution",
"superTypes": [
"Asset"
......
client/client-v2/src/main/java/org/apache/atlas/AtlasClientV2.java
......@@ -17,19 +17,23 @@
*/
package org.apache.atlas;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.google.common.annotations.VisibleForTesting;
import com.sun.jersey.api.client.WebResource;
import com.sun.jersey.core.util.MultivaluedMapImpl;
import org.apache.atlas.model.SearchFilter;
import org.apache.atlas.model.audit.AtlasAuditEntry;
import org.apache.atlas.model.audit.AuditSearchParameters;
import org.apache.atlas.model.discovery.AtlasSearchResult;
import org.apache.atlas.model.discovery.SearchParameters;
import org.apache.atlas.model.instance.AtlasClassification;
import org.apache.atlas.model.instance.AtlasClassification.AtlasClassifications;
import org.apache.atlas.model.instance.AtlasEntity.AtlasEntitiesWithExtInfo;
import org.apache.atlas.model.instance.AtlasEntity.AtlasEntityWithExtInfo;
import org.apache.atlas.model.instance.AtlasEntityHeaders;
import org.apache.atlas.model.instance.AtlasRelationship;
import org.apache.atlas.model.instance.AtlasRelationship.AtlasRelationshipWithExtInfo;
import org.apache.atlas.model.instance.AtlasEntityHeaders;
import org.apache.atlas.model.instance.EntityMutationResponse;
import org.apache.atlas.model.lineage.AtlasLineageInfo;
import org.apache.atlas.model.lineage.AtlasLineageInfo.LineageDirection;
......@@ -47,6 +51,7 @@ import javax.ws.rs.HttpMethod;
import javax.ws.rs.core.Cookie;
import javax.ws.rs.core.MultivaluedMap;
import javax.ws.rs.core.Response;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Set;
......@@ -65,9 +70,10 @@ public class AtlasClientV2 extends AtlasBaseClient {
private static final String GET_BY_GUID_TEMPLATE = TYPES_API + "%s/guid/%s";
private static final String ENTITY_BULK_API = ENTITY_API + "bulk/";
//Admin Entity Purge
//Admin Entity
private static final String ADMIN_API = BASE_URI + "admin/";
private static final String ENTITY_PURGE_API = ADMIN_API + "purge/";
private static final String ATLAS_AUDIT_API = ADMIN_API + "audits/";
// Lineage APIs
private static final String LINEAGE_URI = BASE_URI + "v2/lineage/";
......@@ -479,11 +485,37 @@ public class AtlasClientV2 extends AtlasBaseClient {
return callAPI(API_V2.UPDATE_RELATIONSHIP, AtlasRelationship.class, relationship);
}
public List<AtlasAuditEntry> getAtlasAuditByOperation(final AuditSearchParameters auditSearchParameters) throws AtlasServiceException {
ArrayNode response = callAPI(API_V2.GET_ATLAS_AUDITS, ArrayNode.class, auditSearchParameters);
return extractResults(response, new ExtractOperation<AtlasAuditEntry, ObjectNode>() {
@Override
AtlasAuditEntry extractElement(ObjectNode element) {
return AtlasType.fromV1Json(element.toString(), AtlasAuditEntry.class);
}
});
}
@Override
protected API formatPathParameters(final API api, final String... params) {
return new API(String.format(api.getPath(), params), api.getMethod(), api.getExpectedStatus());
}
private class ExtractOperation<T, U> {
T extractElement(U element) {
return (T) element;
}
}
private <T, U> List<T> extractResults(ArrayNode jsonResponse, ExtractOperation<T, U> extractInterafce) {
ArrayList<T> resultsList = new ArrayList<>();
for (int index = 0; index < jsonResponse.size(); index++) {
Object element = jsonResponse.get(index);
resultsList.add(extractInterafce.extractElement((U) element));
}
return resultsList;
}
private MultivaluedMap<String, String> attributesToQueryParams(Map<String, String> attributes) {
return attributesToQueryParams(attributes, null);
}
......@@ -580,8 +612,9 @@ public class AtlasClientV2 extends AtlasBaseClient {
public static final API_V2 DELETE_RELATIONSHIP_BY_GUID = new API_V2(RELATIONSHIPS_URI + "guid/", HttpMethod.DELETE, Response.Status.NO_CONTENT);
public static final API_V2 CREATE_RELATIONSHIP = new API_V2(RELATIONSHIPS_URI , HttpMethod.POST, Response.Status.OK);
public static final API_V2 UPDATE_RELATIONSHIP = new API_V2(RELATIONSHIPS_URI , HttpMethod.PUT, Response.Status.OK);
public static final API_V2 GET_BULK_HEADERS = new API_V2(ENTITY_API + BULK_HEADERS, HttpMethod.GET, Response.Status.OK);
public static final API_V2 GET_BULK_HEADERS = new API_V2(ENTITY_API + BULK_HEADERS, HttpMethod.GET, Response.Status.OK);
public static final API_V2 UPDATE_BULK_SET_CLASSIFICATIONS = new API_V2(ENTITY_API + AtlasClientV2.BULK_SET_CLASSIFICATIONS, HttpMethod.POST, Response.Status.OK);
public static final API_V2 GET_ATLAS_AUDITS = new API_V2(ATLAS_AUDIT_API, HttpMethod.POST, Response.Status.OK);
private API_V2(String path, String method, Response.Status status) {
super(path, method, status);
......
dashboardv2/public/js/utils/Enums.js
......@@ -37,7 +37,8 @@ define(['require'], function(require) {
TERM_ADD: "Term Added",
TERM_DELETE: "Term Deleted",
LABEL_ADD: "Label(s) Added",
LABEL_DELETE: "Label(s) Deleted"
LABEL_DELETE: "Label(s) Deleted",
ENTITY_PURGE: "Entity Purged"
}
Enums.entityStateReadOnly = {
......@@ -211,4 +212,4 @@ define(['require'], function(require) {
1: "true"
};
return Enums;
});
\ No newline at end of file
});
dashboardv3/public/js/utils/Enums.js
......@@ -37,7 +37,8 @@ define(['require'], function(require) {
TERM_ADD: "Term Added",
TERM_DELETE: "Term Deleted",
LABEL_ADD: "Label(s) Added",
LABEL_DELETE: "Label(s) Deleted"
LABEL_DELETE: "Label(s) Deleted",
ENTITY_PURGE: "Entity Purged"
}
Enums.entityStateReadOnly = {
......@@ -211,4 +212,4 @@ define(['require'], function(require) {
1: "true"
};
return Enums;
});
\ No newline at end of file
});
intg/src/main/java/org/apache/atlas/model/audit/AtlasAuditEntry.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.model.audit;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
import org.apache.atlas.model.AtlasBaseModelObject;
import java.io.Serializable;
import java.util.Date;
import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.NONE;
import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.PUBLIC_ONLY;
@JsonAutoDetect(getterVisibility = PUBLIC_ONLY, setterVisibility = PUBLIC_ONLY, fieldVisibility = NONE)
@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
@JsonIgnoreProperties(ignoreUnknown = true)
public class AtlasAuditEntry extends AtlasBaseModelObject implements Serializable {
private static final long serialVersionUID = 1L;
public enum AuditOperation {
PURGE,
EXPORT,
IMPORT,
IMPORT_DELETE_REPL
}
private String userName;
private AuditOperation operation;
private String params;
private Date startTime;
private Date endTime;
private String clientId;
private String result;
public AtlasAuditEntry() {
}
public AtlasAuditEntry(AuditOperation operation, String userName, String clientId) {
this.operation = operation;
this.userName = userName;
this.clientId = clientId;
}
public AuditOperation getOperation() {
return operation;
}
public void setOperation(AuditOperation operation) {
this.operation = operation;
}
public void setUserName(String userName) {
this.userName = userName;
}
public String getUserName() {
return this.userName;
}
public void setParams(String params) {
this.params = params;
}
public String getParams() {
return this.params;
}
public void setStartTime(Date startTime) {
this.startTime = startTime;
}
public Date getStartTime() {
return startTime;
}
public void setEndTime(Date endTime) {
this.endTime = endTime;
}
public Date getEndTime() {
return this.endTime;
}
public String getClientId() {
return this.clientId;
}
public void setClientId(String clientId) {
this.clientId = clientId;
}
public String getResult() {
return result;
}
public void setResult(String result) {
this.result = result;
}
@Override
public StringBuilder toString(StringBuilder sb) {
sb.append(", userName: ").append(userName);
sb.append(", operation: ").append(operation);
sb.append(", params: ").append(params);
sb.append(", clientId: ").append(clientId);
sb.append(", startTime: ").append(startTime);
sb.append(", endTime: ").append(endTime);
sb.append(", result: ").append(result);
return sb;
}
}
intg/src/main/java/org/apache/atlas/model/audit/AuditSearchParameters.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.model.audit;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
import org.apache.atlas.SortOrder;
import org.apache.atlas.model.discovery.SearchParameters.FilterCriteria;
import java.io.Serializable;
import java.util.Objects;
import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.NONE;
import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.PUBLIC_ONLY;
@JsonAutoDetect(getterVisibility = PUBLIC_ONLY, setterVisibility = PUBLIC_ONLY, fieldVisibility = NONE)
@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
@JsonIgnoreProperties(ignoreUnknown = true)
public class AuditSearchParameters implements Serializable {
private static final long serialVersionUID = 1L;
public static long getSerialVersionUID() {
return serialVersionUID;
}
AuditSearchParameters() {}
private FilterCriteria auditFilters;
private int limit;
private int offset;
private String sortBy;
private SortOrder sortOrder;
/**
* Entity attribute filters for the type (if type name is specified)
* @return
*/
public FilterCriteria getAuditFilters() {
return auditFilters;
}
/**
* Filter the entities on this criteria
* @param auditFilters
*/
public void setAuditFilters(FilterCriteria auditFilters) {
this.auditFilters = auditFilters;
}
/**
* @return Max number of results to be returned
*/
public int getLimit() {
return limit;
}
/**
* Restrict the results to the specified limit
* @param limit max number of results
*/
public void setLimit(int limit) {
this.limit = limit;
}
/**
* @return Offset(pagination) of the results
*/
public int getOffset() {
return offset;
}
/**
* @param offset
*/
public void setOffset(int offset) {
this.offset = offset;
}
/**
* @return Attribute on which to sort the results
*/
public String getSortBy() { return sortBy; }
/**
* Sort the results based on sortBy attribute
* @param sortBy Attribute on which to sort the results
*/
public void setSortBy(String sortBy) { this.sortBy = sortBy; }
/**
* @return Sorting order of the results
*/
public SortOrder getSortOrder() {
return sortOrder;
}
/**
* Sorting order to sort the results
* @param sortOrder ASCENDING vs DESCENDING
*/
public void setSortOrder(SortOrder sortOrder) { this.sortOrder = sortOrder; }
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
AuditSearchParameters that = (AuditSearchParameters) o;
return Objects.equals(auditFilters, that.auditFilters) &&
limit == that.limit &&
offset == that.offset &&
Objects.equals(sortBy, that.sortBy) &&
Objects.equals(sortOrder, that.sortOrder);
}
@Override
public int hashCode() {
return Objects.hash(auditFilters, limit, offset, sortBy, sortOrder);
}
@Override
public String toString() {
final StringBuilder sb = new StringBuilder("AuditSearchParameters{");
sb.append("auditFilters=").append(auditFilters);
sb.append(", limit=").append(limit);
sb.append(", offset=").append(offset);
sb.append(", sortBy='").append(sortBy).append('\'');
sb.append(", sortOrder=").append(sortOrder);
sb.append('}');
return sb.toString();
}
}
intg/src/main/java/org/apache/atlas/model/instance/EntityMutationResponse.java
......@@ -22,21 +22,16 @@ import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import org.apache.atlas.model.instance.EntityMutations.EntityOperation;
import org.apache.atlas.model.typedef.AtlasBaseTypeDef;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang.StringUtils;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlRootElement;
import org.apache.atlas.model.typedef.AtlasBaseTypeDef;
import org.apache.atlas.model.instance.EntityMutations.EntityOperation;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang.StringUtils;
import java.util.*;
import java.util.stream.Collectors;
import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.NONE;
import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.PUBLIC_ONLY;
......@@ -123,6 +118,20 @@ public class EntityMutationResponse {
}
@JsonIgnore
public String getPurgedEntitiesIds() {
String ret = null;
List<AtlasEntityHeader> purgedEntities = getPurgedEntities();
if (CollectionUtils.isNotEmpty(purgedEntities)) {
List<String> entityIds = purgedEntities.stream().map(entity -> entity.getGuid()).collect(Collectors.toList());
ret = String.join(",", entityIds);
}
return ret;
}
@JsonIgnore
public AtlasEntityHeader getFirstEntityCreated() {
final List<AtlasEntityHeader> entitiesByOperation = getEntitiesByOperation(EntityOperation.CREATE);
if ( entitiesByOperation != null && entitiesByOperation.size() > 0) {
......
repository/src/main/java/org/apache/atlas/repository/audit/AtlasAuditService.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.repository.audit;
import org.apache.atlas.AtlasErrorCode;
import org.apache.atlas.annotation.AtlasService;
import org.apache.atlas.annotation.GraphTransaction;
import org.apache.atlas.discovery.AtlasDiscoveryService;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.audit.AtlasAuditEntry;
import org.apache.atlas.model.audit.AtlasAuditEntry.AuditOperation;
import org.apache.atlas.model.audit.AuditSearchParameters;
import org.apache.atlas.model.discovery.AtlasSearchResult;
import org.apache.atlas.model.discovery.SearchParameters;
import org.apache.atlas.model.instance.AtlasEntityHeader;
import org.apache.atlas.repository.ogm.AtlasAuditEntryDTO;
import org.apache.atlas.repository.ogm.DataAccess;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.inject.Inject;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.Set;
@AtlasService
public class AtlasAuditService {
private static final Logger LOG = LoggerFactory.getLogger(AtlasAuditService.class);
private static final String ENTITY_TYPE_AUDIT_ENTRY = "__AtlasAuditEntry";
private final DataAccess dataAccess;
private final AtlasDiscoveryService discoveryService;
@Inject
public AtlasAuditService(DataAccess dataAccess, AtlasDiscoveryService discoveryService) {
this.dataAccess = dataAccess;
this.discoveryService = discoveryService;
}
@GraphTransaction
public void save(AtlasAuditEntry entry) throws AtlasBaseException {
dataAccess.saveNoLoad(entry);
}
public void add(String userName, AuditOperation operation, String clientId, Date startTime,
Date endTime, String params, String result) throws AtlasBaseException {
if (LOG.isDebugEnabled()) {
LOG.debug("==> AtlasAuditService.add()");
}
AtlasAuditEntry entry = new AtlasAuditEntry();
entry.setUserName(userName);
entry.setOperation(operation);
entry.setClientId(clientId);
entry.setStartTime(startTime);
entry.setEndTime(endTime);
entry.setParams(params);
entry.setResult(result);
save(entry);
if (LOG.isDebugEnabled()) {
LOG.debug("addAuditEntry: user: {}, clientId: {}, operation: {} ", entry.getUserName(),
entry.getClientId(), entry.getOperation());
LOG.debug("<== AtlasAuditService.add({})");
}
}
public AtlasAuditEntry get(AtlasAuditEntry entry) throws AtlasBaseException {
if(entry.getGuid() == null) {
throw new AtlasBaseException("Entity does not have GUID set. load cannot proceed.");
}
return dataAccess.load(entry);
}
public List<AtlasAuditEntry> get(AuditSearchParameters auditSearchParameters) throws AtlasBaseException {
if (auditSearchParameters == null) {
throw new AtlasBaseException(AtlasErrorCode.INVALID_PARAMETERS, "Audit Search Parameters not specified");
}
SearchParameters searchParameters = getSearchParameters(auditSearchParameters);
searchParameters.setAttributes(getAuditEntityAttributes());
AtlasSearchResult result = discoveryService.searchWithParameters(searchParameters);
return toAtlasAuditEntry(result);
}
private Set<String> getAuditEntityAttributes() {
return AtlasAuditEntryDTO.getAttributes();
}
private List<AtlasAuditEntry> toAtlasAuditEntry(AtlasSearchResult result) {
List<AtlasAuditEntry> ret = new ArrayList<>();
if(CollectionUtils.isNotEmpty(result.getEntities())) {
for (AtlasEntityHeader entityHeader : result.getEntities()) {
AtlasAuditEntry entry = AtlasAuditEntryDTO.from(entityHeader.getGuid(),
entityHeader.getAttributes());
if (entry == null) {
continue;
}
ret.add(entry);
}
}
return ret;
}
private SearchParameters getSearchParameters(AuditSearchParameters auditSearchParameters) throws AtlasBaseException {
SearchParameters searchParameters = new SearchParameters();
searchParameters.setTypeName(ENTITY_TYPE_AUDIT_ENTRY);
SearchParameters.FilterCriteria validFilter = getNonEmptyFilter(auditSearchParameters.getAuditFilters());
searchParameters.setEntityFilters(validFilter);
searchParameters.setLimit(auditSearchParameters.getLimit());
searchParameters.setOffset(auditSearchParameters.getOffset());
String sortBy = auditSearchParameters.getSortBy();
validateSortByParameter(sortBy);
searchParameters.setSortBy(auditSearchParameters.getSortBy());
searchParameters.setSortOrder(auditSearchParameters.getSortOrder());
return searchParameters;
}
private void validateSortByParameter(String sortBy) throws AtlasBaseException{
if (StringUtils.isNotEmpty(sortBy) && !AtlasAuditEntryDTO.getAttributes().contains(sortBy)) {
throw new AtlasBaseException(AtlasErrorCode.UNKNOWN_ATTRIBUTE, sortBy, "Atlas Audit Entry");
}
}
private SearchParameters.FilterCriteria getNonEmptyFilter(SearchParameters.FilterCriteria auditFilter) throws AtlasBaseException {
SearchParameters.FilterCriteria outCriteria = new SearchParameters.FilterCriteria();
outCriteria.setCriterion(new ArrayList<>());
if(auditFilter != null) {
outCriteria.setCondition(auditFilter.getCondition());
List<SearchParameters.FilterCriteria> givenFilterCriterion = auditFilter.getCriterion();
for (SearchParameters.FilterCriteria each : givenFilterCriterion) {
if (StringUtils.isNotEmpty(each.getAttributeName()) && !AtlasAuditEntryDTO.getAttributes().contains(each.getAttributeName())) {
throw new AtlasBaseException(AtlasErrorCode.UNKNOWN_ATTRIBUTE, each.getAttributeName(), "Atlas Audit Entry");
}
addParameterIfValueNotEmpty(outCriteria, each.getAttributeName(), each.getOperator(), each.getAttributeValue());
}
}
return outCriteria;
}
private void addParameterIfValueNotEmpty(SearchParameters.FilterCriteria criteria, String attributeName,
SearchParameters.Operator operator, String value) {
if(StringUtils.isNotEmpty(value)) {
SearchParameters.FilterCriteria filterCriteria = new SearchParameters.FilterCriteria();
filterCriteria.setAttributeName(attributeName);
filterCriteria.setAttributeValue(value);
filterCriteria.setOperator(operator);
criteria.getCriterion().add(filterCriteria);
}
}
}
\ No newline at end of file
repository/src/main/java/org/apache/atlas/repository/ogm/AtlasAuditEntryDTO.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.repository.ogm;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.audit.AtlasAuditEntry;
import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.type.AtlasTypeRegistry;
import org.springframework.stereotype.Component;
import javax.inject.Inject;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import java.util.Date;
import java.util.Arrays;
@Component
public class AtlasAuditEntryDTO extends AbstractDataTransferObject<AtlasAuditEntry> {
public static final String ATTRIBUTE_USER_NAME = "userName";
public static final String ATTRIBUTE_OPERATION = "operation";
public static final String ATTRIBUTE_PARAMS = "params";
public static final String ATTRIBUTE_START_TIME = "startTime";
public static final String ATTRIBUTE_END_TIME = "endTime";
public static final String ATTRIBUTE_CLIENT_ID = "clientId";
public static final String ATTRIBUTE_RESULT = "result";
private static final Set<String> ATTRIBUTE_NAMES = new HashSet<>(Arrays.asList(ATTRIBUTE_USER_NAME,
ATTRIBUTE_OPERATION, ATTRIBUTE_PARAMS,
ATTRIBUTE_START_TIME, ATTRIBUTE_END_TIME,
ATTRIBUTE_CLIENT_ID, ATTRIBUTE_RESULT));
@Inject
public AtlasAuditEntryDTO(AtlasTypeRegistry typeRegistry) {
super(typeRegistry, AtlasAuditEntry.class,
Constants.INTERNAL_PROPERTY_KEY_PREFIX + AtlasAuditEntry.class.getSimpleName());
}
public static Set<String> getAttributes() {
return ATTRIBUTE_NAMES;
}
public static AtlasAuditEntry from(String guid, Map<String,Object> attributes) {
AtlasAuditEntry entry = new AtlasAuditEntry();
entry.setGuid(guid);
entry.setUserName((String) attributes.get(ATTRIBUTE_USER_NAME));
entry.setOperation(AtlasAuditEntry.AuditOperation.valueOf((String)attributes.get(ATTRIBUTE_OPERATION)));
entry.setParams((String) attributes.get(ATTRIBUTE_PARAMS));
entry.setStartTime((Date) attributes.get(ATTRIBUTE_START_TIME));
entry.setEndTime((Date) attributes.get(ATTRIBUTE_END_TIME));
entry.setClientId((String) attributes.get(ATTRIBUTE_CLIENT_ID));
entry.setResult((String) attributes.get(ATTRIBUTE_RESULT));
return entry;
}
@Override
public AtlasAuditEntry from(AtlasEntity entity) {
return from(entity.getGuid(), entity.getAttributes());
}
@Override
public AtlasAuditEntry from(AtlasEntity.AtlasEntityWithExtInfo entityWithExtInfo) {
return from(entityWithExtInfo.getEntity());
}
@Override
public AtlasEntity toEntity(AtlasAuditEntry obj) {
AtlasEntity entity = getDefaultAtlasEntity(obj);
entity.setAttribute(ATTRIBUTE_USER_NAME, obj.getUserName());
entity.setAttribute(ATTRIBUTE_OPERATION, obj.getOperation());
entity.setAttribute(ATTRIBUTE_PARAMS, obj.getParams());
entity.setAttribute(ATTRIBUTE_START_TIME, obj.getStartTime());
entity.setAttribute(ATTRIBUTE_END_TIME, obj.getEndTime());
entity.setAttribute(ATTRIBUTE_CLIENT_ID, obj.getClientId());
entity.setAttribute(ATTRIBUTE_RESULT, obj.getResult());
return entity;
}
@Override
public AtlasEntity.AtlasEntityWithExtInfo toEntityWithExtInfo(AtlasAuditEntry obj) throws AtlasBaseException {
return new AtlasEntity.AtlasEntityWithExtInfo(toEntity(obj));
}
@Override
public Map<String, Object> getUniqueAttributes(final AtlasAuditEntry obj) {
return null;
}
}
repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1.java
......@@ -17,7 +17,6 @@
*/
package org.apache.atlas.repository.store.graph.v1;
import org.apache.atlas.AtlasErrorCode;
import org.apache.atlas.AtlasException;
import org.apache.atlas.RequestContext;
......@@ -25,7 +24,6 @@ import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.TypeCategory;
import org.apache.atlas.model.instance.AtlasClassification;
import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.model.instance.AtlasEntity.Status;
import org.apache.atlas.model.instance.AtlasEntityHeader;
import org.apache.atlas.model.instance.AtlasObjectId;
import org.apache.atlas.model.typedef.AtlasRelationshipDef.PropagateTags;
......@@ -37,15 +35,9 @@ import org.apache.atlas.repository.graphdb.AtlasEdgeDirection;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2;
import org.apache.atlas.repository.store.graph.v2.EntityGraphRetriever;
import org.apache.atlas.type.AtlasArrayType;
import org.apache.atlas.type.AtlasClassificationType;
import org.apache.atlas.type.AtlasEntityType;
import org.apache.atlas.type.AtlasMapType;
import org.apache.atlas.type.AtlasStructType;
import org.apache.atlas.type.*;
import org.apache.atlas.type.AtlasStructType.AtlasAttribute;
import org.apache.atlas.type.AtlasStructType.AtlasAttribute.AtlasRelationshipEdgeDirection;
import org.apache.atlas.type.AtlasType;
import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.atlas.utils.AtlasEntityUtil;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.MapUtils;
......@@ -59,20 +51,11 @@ import static org.apache.atlas.model.TypeCategory.*;
import static org.apache.atlas.model.instance.AtlasEntity.Status.ACTIVE;
import static org.apache.atlas.model.instance.AtlasEntity.Status.DELETED;
import static org.apache.atlas.model.typedef.AtlasRelationshipDef.PropagateTags.ONE_TO_TWO;
import static org.apache.atlas.repository.Constants.CLASSIFICATION_EDGE_NAME_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.CLASSIFICATION_ENTITY_STATUS;
import static org.apache.atlas.repository.Constants.CLASSIFICATION_LABEL;
import static org.apache.atlas.repository.Constants.CLASSIFICATION_NAME_DELIMITER;
import static org.apache.atlas.repository.Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.MODIFIED_BY_KEY;
import static org.apache.atlas.repository.Constants.PROPAGATED_CLASSIFICATION_NAMES_KEY;
import static org.apache.atlas.repository.Constants.PROPAGATED_TRAIT_NAMES_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.RELATIONSHIP_GUID_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.*;
import static org.apache.atlas.repository.graph.GraphHelper.getTypeName;
import static org.apache.atlas.repository.graph.GraphHelper.*;
import static org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2.getIdFromEdge;
import static org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2.getQualifiedAttributePropertyKey;
import static org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2.getState;
import static org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2.isReference;
import static org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2.*;
import static org.apache.atlas.type.AtlasStructType.AtlasAttribute.AtlasRelationshipEdgeDirection.OUT;
public abstract class DeleteHandlerV1 {
......@@ -105,16 +88,8 @@ public abstract class DeleteHandlerV1 {
for (AtlasVertex instanceVertex : instanceVertices) {
final String guid = AtlasGraphUtilsV2.getIdFromVertex(instanceVertex);
final AtlasEntity.Status state = getState(instanceVertex);
final boolean needToSkip;
if (isPurgeRequested) {
needToSkip = state == ACTIVE || requestContext.isPurgedEntity(guid);
} else {
needToSkip = state == DELETED || requestContext.isDeletedEntity(guid);
}
if (needToSkip) {
if (skipVertexForDelete(instanceVertex)) {
if (LOG.isDebugEnabled()) {
if (isPurgeRequested) {
LOG.debug("Skipping purging of entity={} as it is active or already purged", guid);
......@@ -122,7 +97,6 @@ public abstract class DeleteHandlerV1 {
LOG.debug("Skipping deletion of entity={} as it is already deleted", guid);
}
}
continue;
}
......@@ -855,23 +829,11 @@ public abstract class DeleteHandlerV1 {
LOG.debug("Removing edge from {} to {} with attribute name {}", string(outVertex), string(inVertex), attribute.getName());
}
final RequestContext requestContext = RequestContext.get();
final String typeName = GraphHelper.getTypeName(outVertex);
final String outId = GraphHelper.getGuid(outVertex);
final Status state = getState(outVertex);
final boolean needToSkip;
if (requestContext.isPurgeRequested()) {
needToSkip = state == ACTIVE || (outId != null && requestContext.isPurgedEntity(outId));
} else {
needToSkip = state == DELETED || (outId != null && requestContext.isDeletedEntity(outId));
}
if (needToSkip) {
if (skipVertexForDelete(outVertex)) {
return;
}
AtlasStructType parentType = (AtlasStructType) typeRegistry.getType(typeName);
AtlasStructType parentType = (AtlasStructType) typeRegistry.getType(GraphHelper.getTypeName(outVertex));
String propertyName = getQualifiedAttributePropertyKey(parentType, attribute.getName());
String edgeLabel = attribute.getRelationshipEdgeLabel();
AtlasEdge edge = null;
......@@ -960,6 +922,9 @@ public abstract class DeleteHandlerV1 {
if (edge != null) {
deleteEdge(edge, isInternalType(inVertex) && isInternalType(outVertex));
final RequestContext requestContext = RequestContext.get();
final String outId = GraphHelper.getGuid(outVertex);
if (! requestContext.isUpdatedEntity(outId)) {
AtlasGraphUtilsV2.setEncodedProperty(outVertex, MODIFICATION_TIMESTAMP_PROPERTY_KEY, requestContext.getRequestTime());
AtlasGraphUtilsV2.setEncodedProperty(outVertex, MODIFIED_BY_KEY, requestContext.getUser());
......@@ -1080,4 +1045,28 @@ public abstract class DeleteHandlerV1 {
deleteEdgeReference(edge, CLASSIFICATION, false, false, instanceVertex);
}
}
private boolean skipVertexForDelete(AtlasVertex vertex) {
boolean ret = true;
if(vertex != null) {
try {
final RequestContext reqContext = RequestContext.get();
final String guid = AtlasGraphUtilsV2.getIdFromVertex(vertex);
if(guid != null && !reqContext.isDeletedEntity(guid)) {
final AtlasEntity.Status vertexState = getState(vertex);
if (reqContext.isPurgeRequested()) {
ret = vertexState == ACTIVE; // skip purging ACTIVE vertices
} else {
ret = vertexState == DELETED; // skip deleting DELETED vertices
}
}
} catch (IllegalStateException excp) {
LOG.warn("skipVertexForDelete(): failed guid/state for the vertex", excp);
}
}
return ret;
}
}
repository/src/test/java/org/apache/atlas/TestModules.java
......@@ -41,16 +41,17 @@ import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.repository.graphdb.GraphDBMigrator;
import org.apache.atlas.repository.graphdb.janus.migration.GraphDBGraphSONMigrator;
import org.apache.atlas.repository.impexp.ExportService;
import org.apache.atlas.repository.ogm.AtlasAuditEntryDTO;
import org.apache.atlas.repository.ogm.AtlasServerDTO;
import org.apache.atlas.repository.ogm.ExportImportAuditEntryDTO;
import org.apache.atlas.repository.ogm.profiles.AtlasSavedSearchDTO;
import org.apache.atlas.repository.ogm.profiles.AtlasUserProfileDTO;
import org.apache.atlas.repository.ogm.DTORegistry;
import org.apache.atlas.repository.ogm.DataAccess;
import org.apache.atlas.repository.ogm.DataTransferObject;
import org.apache.atlas.repository.ogm.ExportImportAuditEntryDTO;
import org.apache.atlas.repository.ogm.glossary.AtlasGlossaryCategoryDTO;
import org.apache.atlas.repository.ogm.glossary.AtlasGlossaryDTO;
import org.apache.atlas.repository.ogm.glossary.AtlasGlossaryTermDTO;
import org.apache.atlas.repository.ogm.profiles.AtlasSavedSearchDTO;
import org.apache.atlas.repository.ogm.profiles.AtlasUserProfileDTO;
import org.apache.atlas.repository.store.graph.AtlasEntityStore;
import org.apache.atlas.repository.store.graph.AtlasRelationshipStore;
import org.apache.atlas.repository.store.graph.BulkImporter;
......@@ -169,6 +170,7 @@ public class TestModules {
availableDTOs.addBinding().to(AtlasGlossaryCategoryDTO.class);
availableDTOs.addBinding().to(AtlasServerDTO.class);
availableDTOs.addBinding().to(ExportImportAuditEntryDTO.class);
availableDTOs.addBinding().to(AtlasAuditEntryDTO.class);
bind(DTORegistry.class).asEagerSingleton();
bind(DataAccess.class).asEagerSingleton();
......
repository/src/test/java/org/apache/atlas/glossary/GlossaryServiceTest.java
......@@ -36,7 +36,7 @@ import org.apache.atlas.model.instance.AtlasRelatedObjectId;
import org.apache.atlas.model.instance.EntityMutationResponse;
import org.apache.atlas.model.typedef.AtlasClassificationDef;
import org.apache.atlas.model.typedef.AtlasTypesDef;
import org.apache.atlas.repository.impexp.ZipFileResourceTestUtils;
import org.apache.atlas.utils.TestLoadModelUtils;
import org.apache.atlas.repository.store.graph.AtlasEntityStore;
import org.apache.atlas.repository.store.graph.v2.AtlasEntityStream;
import org.apache.atlas.store.AtlasTypeDefStore;
......@@ -93,7 +93,7 @@ public class GlossaryServiceTest {
@BeforeClass
public void setupSampleGlossary() {
try {
ZipFileResourceTestUtils.loadAllModels("0000-Area0", typeDefStore, typeRegistry);
TestLoadModelUtils.loadAllModels("0000-Area0", typeDefStore, typeRegistry);
} catch (AtlasBaseException | IOException e) {
throw new SkipException("SubjectArea model loading failed");
}
......
repository/src/test/java/org/apache/atlas/query/BasicTestSetup.java
......@@ -36,7 +36,7 @@ import java.util.*;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import static org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.loadModelFromJson;
import static org.apache.atlas.utils.TestLoadModelUtils.loadModelFromJson;
import static org.testng.Assert.fail;
public abstract class BasicTestSetup {
......
repository/src/test/java/org/apache/atlas/repository/impexp/ExportImportTestBase.java → repository/src/test/java/org/apache/atlas/repository/AtlasTestBase.java
......@@ -16,11 +16,12 @@
* limitations under the License.
*/
package org.apache.atlas.repository.impexp;
package org.apache.atlas.repository;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.impexp.ExportImportAuditEntry;
import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.repository.impexp.ExportImportAuditService;
import org.apache.atlas.repository.store.graph.v2.AtlasEntityStoreV2;
import org.apache.atlas.store.AtlasTypeDefStore;
import org.apache.atlas.type.AtlasTypeRegistry;
......@@ -30,15 +31,15 @@ import java.io.IOException;
import java.util.Arrays;
import java.util.List;
import static org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.createAtlasEntity;
import static org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.loadBaseModel;
import static org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.loadEntity;
import static org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.loadHiveModel;
import static org.apache.atlas.utils.TestLoadModelUtils.createAtlasEntity;
import static org.apache.atlas.utils.TestLoadModelUtils.loadBaseModel;
import static org.apache.atlas.utils.TestLoadModelUtils.loadEntity;
import static org.apache.atlas.utils.TestLoadModelUtils.loadHiveModel;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertNotNull;
import static org.testng.Assert.assertTrue;
public class ExportImportTestBase {
public class AtlasTestBase {
protected static final String ENTITIES_SUB_DIR = "stocksDB-Entities";
protected static final String DB_GUID = "1637a33e-6512-447b-ade7-249c8cb5344b";
protected static final String TABLE_GUID = "df122fc3-5555-40f8-a30f-3090b8a622f8";
......@@ -69,7 +70,7 @@ public class ExportImportTestBase {
}
}
protected void assertAuditEntry(ExportImportAuditService auditService) throws InterruptedException {
protected void assertExportImportAuditEntry(ExportImportAuditService auditService) throws InterruptedException {
pauseForIndexCreation();
List<ExportImportAuditEntry> result = null;
try {
......
repository/src/test/java/org/apache/atlas/repository/audit/AdminPurgeTest.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.repository.audit;
import org.apache.atlas.RequestContext;
import org.apache.atlas.TestModules;
import org.apache.atlas.TestUtilsV2;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.audit.AtlasAuditEntry;
import org.apache.atlas.model.audit.AuditSearchParameters;
import org.apache.atlas.model.instance.*;
import org.apache.atlas.model.typedef.AtlasTypesDef;
import org.apache.atlas.repository.AtlasTestBase;
import org.apache.atlas.repository.graph.AtlasGraphProvider;
import org.apache.atlas.repository.store.bootstrap.AtlasTypeDefStoreInitializer;
import org.apache.atlas.repository.store.graph.v2.AtlasEntityStoreV2;
import org.apache.atlas.repository.store.graph.v2.AtlasEntityStream;
import org.apache.atlas.runner.LocalSolrRunner;
import org.apache.atlas.store.AtlasTypeDefStore;
import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.atlas.utils.TestResourceFileUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.testng.Assert;
import org.testng.SkipException;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeTest;
import org.testng.annotations.Guice;
import org.testng.annotations.Test;
import javax.inject.Inject;
import java.io.IOException;
import java.util.*;
import java.util.stream.Collectors;
import static org.apache.atlas.graph.GraphSandboxUtil.useLocalSolr;
import static org.testng.Assert.assertNotNull;
import static org.testng.Assert.assertTrue;
import static org.testng.Assert.fail;
@Guice(modules = TestModules.TestOnlyModule.class)
public class AdminPurgeTest extends AtlasTestBase {
private static final Logger LOG = LoggerFactory.getLogger(AdminPurgeTest.class);
private static final String CLIENT_HOST = "127.0.0.0";
private static final String DEFAULT_USER = "Admin";
private static final String AUDIT_PARAMETER_RESOURCE_DIR = "auditSearchParameters";
@Inject
AtlasTypeRegistry typeRegistry;
@Inject
private AtlasTypeDefStore typeDefStore;
@Inject
private AtlasAuditService auditService;
@Inject
private AtlasEntityStoreV2 entityStore;
@BeforeTest
public void setupTest() throws IOException, AtlasBaseException {
RequestContext.clear();
RequestContext.get().setUser(TestUtilsV2.TEST_USER, null);
basicSetup(typeDefStore, typeRegistry);
}
@AfterClass
public void clear() throws Exception {
Thread.sleep(1000);
AtlasGraphProvider.cleanup();
if (useLocalSolr()) {
LocalSolrRunner.stop();
}
}
@Test
public void testDeleteEntitiesDoesNotLookupDeletedEntity() throws Exception {
AtlasTypesDef sampleTypes = TestUtilsV2.defineDeptEmployeeTypes();
AtlasTypesDef typesToCreate = AtlasTypeDefStoreInitializer.getTypesToCreate(sampleTypes, typeRegistry);
if (!typesToCreate.isEmpty()) {
typeDefStore.createTypesDef(typesToCreate);
}
AtlasEntity.AtlasEntitiesWithExtInfo deptEg2 = TestUtilsV2.createDeptEg2();
AtlasEntityStream entityStream = new AtlasEntityStream(deptEg2);
EntityMutationResponse emr = entityStore.createOrUpdate(entityStream, false);
pauseForIndexCreation();
assertNotNull(emr);
assertNotNull(emr.getCreatedEntities());
assertTrue(emr.getCreatedEntities().size() > 0);
List<String> guids = emr.getCreatedEntities().stream()
.map(p -> new String(p.getGuid())).collect(Collectors.toList());
EntityMutationResponse response = entityStore.deleteByIds(guids);
pauseForIndexCreation();
List<AtlasEntityHeader> responseDeletedEntities = response.getDeletedEntities();
assertNotNull(responseDeletedEntities);
responseDeletedEntities.sort((l,r) -> l.getGuid().compareTo(r.getGuid()));
List<AtlasEntityHeader> toBeDeletedEntities = emr.getCreatedEntities();
toBeDeletedEntities.sort((l,r) -> l.getGuid().compareTo(r.getGuid()));
Assert.assertEquals(responseDeletedEntities.size(), emr.getCreatedEntities().size());
for(int index = 0 ; index < responseDeletedEntities.size(); index++)
Assert.assertEquals(responseDeletedEntities.get(index).getGuid(), emr.getCreatedEntities().get(index).getGuid());
Date startTimestamp = new Date();
response = entityStore.purgeByIds(new HashSet<>(guids));
pauseForIndexCreation();
List<AtlasEntityHeader> responsePurgedEntities = response.getPurgedEntities();
responsePurgedEntities.sort((l,r) -> l.getGuid().compareTo(r.getGuid()));
Assert.assertEquals(responsePurgedEntities.size(), responseDeletedEntities.size());
for(int index = 0 ; index < responsePurgedEntities.size(); index++)
Assert.assertEquals(responsePurgedEntities.get(index).getGuid(), responseDeletedEntities.get(index).getGuid());
auditService.add(DEFAULT_USER, AtlasAuditEntry.AuditOperation.PURGE,
CLIENT_HOST, startTimestamp, new Date(), guids.toString(),
response.getPurgedEntitiesIds());
AuditSearchParameters auditParameterNull = createAuditParameter("audit-search-parameter-without-filter");
assertAuditEntry(auditService, auditParameterNull);
AuditSearchParameters auditSearchParameters = createAuditParameter("audit-search-parameter-purge");
assertAuditEntry(auditService, auditSearchParameters);
}
private AuditSearchParameters createAuditParameter(String fileName) {
try {
return TestResourceFileUtils.readObjectFromJson(AUDIT_PARAMETER_RESOURCE_DIR, fileName, AuditSearchParameters.class);
} catch (IOException e) {
fail(e.getMessage());
}
return null;
}
private void assertAuditEntry(AtlasAuditService auditService, AuditSearchParameters auditSearchParameters) throws InterruptedException {
pauseForIndexCreation();
List<AtlasAuditEntry> result = null;
try {
result = auditService.get(auditSearchParameters);
} catch (Exception e) {
throw new SkipException("audit entries not retrieved.");
}
assertNotNull(result);
assertTrue(result.size() > 0);
}
}
repository/src/test/java/org/apache/atlas/repository/audit/AtlasAuditServiceTest.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.repository.audit;
import org.apache.atlas.TestModules;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.audit.AtlasAuditEntry;
import org.apache.atlas.model.audit.AtlasAuditEntry.AuditOperation;
import org.apache.atlas.model.audit.AuditSearchParameters;
import org.apache.atlas.store.AtlasTypeDefStore;
import org.apache.atlas.type.AtlasType;
import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.atlas.utils.TestResourceFileUtils;
import org.testng.SkipException;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Guice;
import org.testng.annotations.Test;
import javax.inject.Inject;
import java.io.IOException;
import java.util.Date;
import java.util.List;
import static org.apache.atlas.utils.TestLoadModelUtils.loadBaseModel;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertNotEquals;
import static org.testng.Assert.assertNotNull;
import static org.testng.Assert.assertTrue;
import static org.testng.Assert.fail;
@Guice(modules = TestModules.TestOnlyModule.class)
public class AtlasAuditServiceTest {
private static final int WAIT_TIME_FOR_INDEX_CREATION_IN_MILLI = 5000;
private static final String AUDIT_PARAMETER_RESOURCE_DIR = "auditSearchParameters";
private static final String DEFAULT_USER = "admin";
@Inject
AtlasTypeRegistry typeRegistry;
@Inject
private AtlasTypeDefStore typeDefStore;
@Inject
AtlasAuditService auditService;
@BeforeClass
public void setup() throws IOException, AtlasBaseException {
loadBaseModel(typeDefStore, typeRegistry);
}
@Test
public void checkTypeRegistered() throws AtlasBaseException {
AtlasType auditEntryType = typeRegistry.getType("__" + AtlasAuditEntry.class.getSimpleName());
assertNotNull(auditEntryType);
}
@Test
public void checkStoringOfAuditEntry() throws AtlasBaseException {
final String clientId1 = "client1";
AtlasAuditEntry entryTobeStored1 = saveEntry(AuditOperation.PURGE, clientId1);
String clientId2 = "client2";
AtlasAuditEntry entryTobeStored2 = saveEntry(AuditOperation.PURGE, clientId2);
waitForIndexCreation();
AtlasAuditEntry storedEntry1 = retrieveEntry(entryTobeStored1);
AtlasAuditEntry storedEntry2 = retrieveEntry(entryTobeStored2);
assertNotEquals(storedEntry1.getGuid(), storedEntry2.getGuid());
assertNotNull(storedEntry1.getGuid());
assertNotNull(storedEntry2.getGuid());
assertEquals(storedEntry1.getUserName(), DEFAULT_USER);
assertEquals(storedEntry2.getUserName(), DEFAULT_USER);
assertEquals(storedEntry1.getClientId(), entryTobeStored1.getClientId());
assertEquals(storedEntry2.getClientId(), entryTobeStored2.getClientId());
assertEquals(storedEntry1.getOperation(), entryTobeStored1.getOperation());
assertEquals(storedEntry2.getOperation(), entryTobeStored2.getOperation());
}
@Test
public void checkStoringMultipleAuditEntries() throws AtlasBaseException, InterruptedException {
final String clientId = "client1";
final int MAX_ENTRIES = 5;
final int LIMIT_PARAM = 3;
for (int i = 0; i < MAX_ENTRIES; i++) {
saveEntry(AuditOperation.PURGE, clientId);
}
waitForIndexCreation();
AuditSearchParameters auditSearchParameters = createAuditParameter("audit-search-parameter-purge");
auditSearchParameters.setLimit(LIMIT_PARAM);
auditSearchParameters.setOffset(0);
List<AtlasAuditEntry> resultLimitedByParam = auditService.get(auditSearchParameters);
assertTrue(resultLimitedByParam.size() == LIMIT_PARAM);
auditSearchParameters.setLimit(MAX_ENTRIES);
auditSearchParameters.setOffset(LIMIT_PARAM);
List<AtlasAuditEntry> results = auditService.get(auditSearchParameters);
assertTrue(results.size() == (MAX_ENTRIES - LIMIT_PARAM));
}
private AuditSearchParameters createAuditParameter(String fileName) {
try {
return TestResourceFileUtils.readObjectFromJson(AUDIT_PARAMETER_RESOURCE_DIR, fileName, AuditSearchParameters.class);
} catch (IOException e) {
fail(e.getMessage());
}
return null;
}
private AtlasAuditEntry retrieveEntry(AtlasAuditEntry entry) throws AtlasBaseException {
AuditSearchParameters auditSearchParameters = createAuditParameter("audit-search-parameter-purge");
AtlasAuditEntry result = auditService.get(entry);
assertNotNull(result);
entry.setGuid(result.getGuid());
return auditService.get(entry);
}
private AtlasAuditEntry saveEntry(AuditOperation operation, String clientId) throws AtlasBaseException {
AtlasAuditEntry entry = new AtlasAuditEntry(operation, DEFAULT_USER, clientId);
entry.setStartTime(new Date());
entry.setEndTime(new Date());
auditService.save(entry);
return entry;
}
protected void waitForIndexCreation() {
try {
Thread.sleep(WAIT_TIME_FOR_INDEX_CREATION_IN_MILLI);
} catch (InterruptedException ex) {
throw new SkipException("Wait interrupted.");
}
}
}
repository/src/test/java/org/apache/atlas/repository/impexp/AtlasServerServiceTest.java
......@@ -35,7 +35,7 @@ import java.util.HashMap;
import java.util.Map;
import static org.apache.atlas.repository.Constants.ATTR_NAME_REFERENCEABLE;
import static org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.loadBaseModel;
import static org.apache.atlas.utils.TestLoadModelUtils.loadBaseModel;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertNotEquals;
import static org.testng.Assert.assertNotNull;
......
repository/src/test/java/org/apache/atlas/repository/impexp/ExportImportAuditServiceTest.java
......@@ -21,6 +21,7 @@ package org.apache.atlas.repository.impexp;
import org.apache.atlas.TestModules;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.impexp.ExportImportAuditEntry;
import org.apache.atlas.repository.AtlasTestBase;
import org.apache.atlas.store.AtlasTypeDefStore;
import org.apache.atlas.type.AtlasType;
import org.apache.atlas.type.AtlasTypeRegistry;
......@@ -32,14 +33,14 @@ import javax.inject.Inject;
import java.io.IOException;
import java.util.List;
import static org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.loadBaseModel;
import static org.apache.atlas.utils.TestLoadModelUtils.loadBaseModel;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertNotEquals;
import static org.testng.Assert.assertNotNull;
import static org.testng.Assert.assertTrue;
@Guice(modules = TestModules.TestOnlyModule.class)
public class ExportImportAuditServiceTest extends ExportImportTestBase {
public class ExportImportAuditServiceTest extends AtlasTestBase {
@Inject
AtlasTypeRegistry typeRegistry;
......
repository/src/test/java/org/apache/atlas/repository/impexp/ExportIncrementalTest.java
......@@ -27,6 +27,7 @@ import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.impexp.AtlasExportRequest;
import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.model.instance.AtlasObjectId;
import org.apache.atlas.repository.AtlasTestBase;
import org.apache.atlas.repository.store.graph.v2.AtlasEntityStoreV2;
import org.apache.atlas.repository.util.UniqueList;
import org.apache.atlas.store.AtlasTypeDefStore;
......@@ -53,7 +54,7 @@ import java.util.List;
import java.util.Map;
import static org.apache.atlas.model.impexp.AtlasExportRequest.FETCH_TYPE_INCREMENTAL_CHANGE_MARKER;
import static org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.createTypes;
import static org.apache.atlas.utils.TestLoadModelUtils.createTypes;
import static org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.getEntities;
import static org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.getZipSource;
import static org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.runExportWithParameters;
......@@ -63,7 +64,7 @@ import static org.testng.Assert.assertNotNull;
import static org.testng.Assert.assertTrue;
@Guice(modules = TestModules.TestOnlyModule.class)
public class ExportIncrementalTest extends ExportImportTestBase {
public class ExportIncrementalTest extends AtlasTestBase {
@Inject
AtlasTypeRegistry typeRegistry;
......
repository/src/test/java/org/apache/atlas/repository/impexp/ExportServiceTest.java
......@@ -28,6 +28,7 @@ import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.model.instance.AtlasObjectId;
import org.apache.atlas.model.instance.EntityMutationResponse;
import org.apache.atlas.model.typedef.AtlasTypesDef;
import org.apache.atlas.repository.AtlasTestBase;
import org.apache.atlas.repository.graph.AtlasGraphProvider;
import org.apache.atlas.repository.store.bootstrap.AtlasTypeDefStoreInitializer;
import org.apache.atlas.repository.store.graph.v2.AtlasEntityStoreV2;
......@@ -61,7 +62,7 @@ import static org.testng.Assert.assertNotNull;
import static org.testng.Assert.assertTrue;
@Guice(modules = TestModules.TestOnlyModule.class)
public class ExportServiceTest extends ExportImportTestBase {
public class ExportServiceTest extends AtlasTestBase {
private static final Logger LOG = LoggerFactory.getLogger(ExportServiceTest.class);
@Inject
......@@ -106,7 +107,7 @@ public class ExportServiceTest extends ExportImportTestBase {
@AfterClass
public void clear() throws Exception {
Thread.sleep(1000);
assertAuditEntry(auditService);
assertExportImportAuditEntry(auditService);
AtlasGraphProvider.cleanup();
if (useLocalSolr()) {
......
repository/src/test/java/org/apache/atlas/repository/impexp/ExportSkipLineageTest.java
......@@ -25,6 +25,7 @@ import org.apache.atlas.TestUtilsV2;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.impexp.AtlasExportRequest;
import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.repository.AtlasTestBase;
import org.apache.atlas.repository.store.graph.v1.DeleteHandlerDelegate;
import org.apache.atlas.repository.store.graph.v2.AtlasEntityChangeNotifier;
import org.apache.atlas.repository.store.graph.v2.AtlasEntityStoreV2;
......@@ -43,8 +44,8 @@ import java.io.IOException;
import java.io.InputStream;
import java.util.Map;
import static org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.loadBaseModel;
import static org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.loadHiveModel;
import static org.apache.atlas.utils.TestLoadModelUtils.loadBaseModel;
import static org.apache.atlas.utils.TestLoadModelUtils.loadHiveModel;
import static org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.runExportWithParameters;
import static org.mockito.Mockito.mock;
import static org.testng.Assert.assertEquals;
......@@ -52,7 +53,7 @@ import static org.testng.Assert.assertNotNull;
import static org.testng.AssertJUnit.fail;
@Guice(modules = TestModules.TestOnlyModule.class)
public class ExportSkipLineageTest extends ExportImportTestBase {
public class ExportSkipLineageTest extends AtlasTestBase {
@Inject
AtlasTypeRegistry typeRegistry;
......
repository/src/test/java/org/apache/atlas/repository/impexp/HdfsPathEntityCreatorTest.java
......@@ -20,6 +20,7 @@ package org.apache.atlas.repository.impexp;
import org.apache.atlas.TestModules;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.repository.AtlasTestBase;
import org.apache.atlas.store.AtlasTypeDefStore;
import org.apache.atlas.type.AtlasTypeRegistry;
import org.testng.annotations.BeforeClass;
......@@ -33,12 +34,12 @@ import static org.apache.atlas.repository.impexp.HdfsPathEntityCreator.HDFS_PATH
import static org.apache.atlas.repository.impexp.HdfsPathEntityCreator.HDFS_PATH_ATTRIBUTE_NAME_NAME;
import static org.apache.atlas.repository.impexp.HdfsPathEntityCreator.HDFS_PATH_ATTRIBUTE_QUALIFIED_NAME;
import static org.apache.atlas.repository.impexp.HdfsPathEntityCreator.getQualifiedName;
import static org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.loadFsModel;
import static org.apache.atlas.utils.TestLoadModelUtils.loadFsModel;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertNotNull;
@Guice(modules = TestModules.TestOnlyModule.class)
public class HdfsPathEntityCreatorTest extends ExportImportTestBase {
public class HdfsPathEntityCreatorTest extends AtlasTestBase {
@Inject
AtlasTypeRegistry typeRegistry;
......
repository/src/test/java/org/apache/atlas/repository/impexp/ImportReactivateTableTest.java
......@@ -27,6 +27,7 @@ import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.model.instance.AtlasRelatedObjectId;
import org.apache.atlas.model.instance.AtlasRelationship;
import org.apache.atlas.model.instance.EntityMutationResponse;
import org.apache.atlas.repository.AtlasTestBase;
import org.apache.atlas.repository.graph.AtlasGraphProvider;
import org.apache.atlas.repository.store.graph.AtlasEntityStore;
import org.apache.atlas.repository.store.graph.v2.AtlasEntityStream;
......@@ -48,15 +49,15 @@ import static org.apache.atlas.AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME;
import static org.apache.atlas.graph.GraphSandboxUtil.useLocalSolr;
import static org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.getInputStreamFrom;
import static org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.getDefaultImportRequest;
import static org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.loadFsModel;
import static org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.loadHiveModel;
import static org.apache.atlas.utils.TestLoadModelUtils.loadFsModel;
import static org.apache.atlas.utils.TestLoadModelUtils.loadHiveModel;
import static org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.runImportWithParameters;
import static org.apache.atlas.type.AtlasTypeUtil.toAtlasRelatedObjectId;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertNotNull;
@Guice(modules = TestModules.TestOnlyModule.class)
public class ImportReactivateTableTest extends ExportImportTestBase {
public class ImportReactivateTableTest extends AtlasTestBase {
private static final Logger LOG = LoggerFactory.getLogger(ImportReactivateTableTest.class);
private static final String ENTITY_TYPE_COL = "hive_column";
......
repository/src/test/java/org/apache/atlas/repository/impexp/ImportServiceTest.java
......@@ -33,6 +33,7 @@ import org.apache.atlas.model.instance.AtlasRelationship;
import org.apache.atlas.model.instance.AtlasRelatedObjectId;
import org.apache.atlas.model.instance.EntityMutationResponse;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.AtlasTestBase;
import org.apache.atlas.repository.graph.AtlasGraphProvider;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.repository.store.graph.AtlasEntityStore;
......@@ -63,8 +64,8 @@ import static org.apache.atlas.graph.GraphSandboxUtil.useLocalSolr;
import static org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.getDefaultImportRequest;
import static org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.getZipSource;
import static org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.getInputStreamFrom;
import static org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.loadModelFromJson;
import static org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.loadModelFromResourcesJson;
import static org.apache.atlas.utils.TestLoadModelUtils.loadModelFromJson;
import static org.apache.atlas.utils.TestLoadModelUtils.loadModelFromResourcesJson;
import static org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.runAndVerifyQuickStart_v1_Import;
import static org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.runImportWithNoParameters;
import static org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.runImportWithParameters;
......@@ -82,7 +83,7 @@ import static org.testng.Assert.assertNotNull;
import static org.testng.Assert.assertTrue;
@Guice(modules = TestModules.TestOnlyModule.class)
public class ImportServiceTest extends ExportImportTestBase {
public class ImportServiceTest extends AtlasTestBase {
private static final int DEFAULT_LIMIT = 25;
private final ImportService importService;
......@@ -123,7 +124,7 @@ public class ImportServiceTest extends ExportImportTestBase {
@AfterTest
public void postTest() throws InterruptedException {
assertAuditEntry(auditService);
assertExportImportAuditEntry(auditService);
}
@DataProvider(name = "sales")
......
repository/src/test/java/org/apache/atlas/repository/impexp/ImportTransformsShaperTest.java
......@@ -23,6 +23,7 @@ import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.impexp.AtlasImportRequest;
import org.apache.atlas.model.impexp.AtlasImportResult;
import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.repository.AtlasTestBase;
import org.apache.atlas.repository.store.graph.AtlasEntityStore;
import org.apache.atlas.store.AtlasTypeDefStore;
import org.apache.atlas.type.AtlasClassificationType;
......@@ -35,13 +36,13 @@ import javax.inject.Inject;
import java.io.IOException;
import java.util.List;
import static org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.loadFsModel;
import static org.apache.atlas.utils.TestLoadModelUtils.loadFsModel;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertTrue;
import static org.testng.AssertJUnit.assertNotNull;
@Guice(modules = TestModules.TestOnlyModule.class)
public class ImportTransformsShaperTest extends ExportImportTestBase {
public class ImportTransformsShaperTest extends AtlasTestBase {
@Inject
AtlasTypeRegistry typeRegistry;
......
repository/src/test/java/org/apache/atlas/repository/impexp/IncrementalExportEntityProviderTest.java
......@@ -22,6 +22,7 @@ package org.apache.atlas.repository.impexp;
import org.apache.atlas.RequestContext;
import org.apache.atlas.TestModules;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.repository.AtlasTestBase;
import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.repository.store.graph.v2.AtlasEntityStoreV2;
import org.apache.atlas.repository.util.UniqueList;
......@@ -40,7 +41,7 @@ import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertTrue;
@Guice(modules = TestModules.TestOnlyModule.class)
public class IncrementalExportEntityProviderTest extends ExportImportTestBase {
public class IncrementalExportEntityProviderTest extends AtlasTestBase {
@Inject
AtlasTypeRegistry typeRegistry;
......
repository/src/test/java/org/apache/atlas/repository/impexp/RelationshipAttributesExtractorTest.java
......@@ -25,6 +25,7 @@ import org.apache.atlas.model.impexp.AtlasExportRequest;
import org.apache.atlas.model.impexp.AtlasExportResult;
import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.model.instance.AtlasObjectId;
import org.apache.atlas.repository.AtlasTestBase;
import org.apache.atlas.repository.graph.AtlasGraphProvider;
import org.apache.atlas.repository.store.graph.v2.AtlasEntityStoreV2;
import org.apache.atlas.runner.LocalSolrRunner;
......@@ -50,7 +51,7 @@ import java.util.HashMap;
import static org.apache.atlas.graph.GraphSandboxUtil.useLocalSolr;
import static org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.getZipSource;
import static org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.loadModelFromJson;
import static org.apache.atlas.utils.TestLoadModelUtils.loadModelFromJson;
import static org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.runImportWithNoParameters;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertFalse;
......@@ -58,7 +59,7 @@ import static org.testng.Assert.assertNotNull;
import static org.testng.Assert.assertTrue;
@Guice(modules = TestModules.TestOnlyModule.class)
public class RelationshipAttributesExtractorTest extends ExportImportTestBase {
public class RelationshipAttributesExtractorTest extends AtlasTestBase {
private static final String EXPORT_FULL = "full";
private static final String EXPORT_CONNECTED = "connected";
......
repository/src/test/java/org/apache/atlas/repository/impexp/ReplicationEntityAttributeTest.java
......@@ -31,9 +31,8 @@ import org.apache.atlas.model.impexp.AtlasServer;
import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.model.typedef.AtlasEntityDef;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.store.graph.v2.AtlasEntityChangeNotifier;
import org.apache.atlas.repository.AtlasTestBase;
import org.apache.atlas.repository.store.graph.v2.AtlasEntityStoreV2;
import org.apache.atlas.repository.store.graph.v2.BulkImporterImpl;
import org.apache.atlas.repository.store.graph.v2.EntityGraphMapper;
import org.apache.atlas.store.AtlasTypeDefStore;
import org.apache.atlas.type.AtlasEntityType;
......@@ -63,7 +62,7 @@ import static org.testng.Assert.assertNotNull;
import static org.testng.Assert.assertTrue;
@Guice(modules = TestModules.TestOnlyModule.class)
public class ReplicationEntityAttributeTest extends ExportImportTestBase {
public class ReplicationEntityAttributeTest extends AtlasTestBase {
private final String ENTITIES_SUB_DIR = "stocksDB-Entities";
private final String EXPORT_REQUEST_FILE = "export-replicatedTo";
private final String IMPORT_REQUEST_FILE = "import-replicatedFrom";
......
repository/src/test/java/org/apache/atlas/repository/impexp/StartEntityFetchByExportRequestTest.java
......@@ -22,6 +22,7 @@ import org.apache.atlas.TestModules;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.impexp.AtlasExportRequest;
import org.apache.atlas.model.instance.AtlasObjectId;
import org.apache.atlas.repository.AtlasTestBase;
import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.store.AtlasTypeDefStore;
import org.apache.atlas.type.AtlasType;
......@@ -43,7 +44,7 @@ import static org.apache.atlas.repository.impexp.StartEntityFetchByExportRequest
import static org.testng.Assert.assertEquals;
@Guice(modules = TestModules.TestOnlyModule.class)
public class StartEntityFetchByExportRequestTest extends ExportImportTestBase {
public class StartEntityFetchByExportRequestTest extends AtlasTestBase {
@Inject
private AtlasGraph atlasGraph;
......
repository/src/test/java/org/apache/atlas/repository/impexp/TableReplicationRequestProcessorTest.java
......@@ -24,6 +24,7 @@ import org.apache.atlas.TestUtilsV2;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.impexp.AtlasImportRequest;
import org.apache.atlas.model.impexp.ExportImportAuditEntry;
import org.apache.atlas.repository.AtlasTestBase;
import org.apache.atlas.repository.graph.AtlasGraphProvider;
import org.apache.atlas.repository.store.graph.AtlasEntityStore;
import org.apache.atlas.runner.LocalSolrRunner;
......@@ -55,7 +56,7 @@ import static org.testng.Assert.assertNotNull;
@Guice(modules = TestModules.TestOnlyModule.class)
public class TableReplicationRequestProcessorTest extends ExportImportTestBase {
public class TableReplicationRequestProcessorTest extends AtlasTestBase {
private static final Logger LOG = LoggerFactory.getLogger(TableReplicationRequestProcessorTest.class);
private static final String ENTITY_GUID_REPLICATED = "718a6d12-35a8-4731-aff8-3a64637a43a3";
......
repository/src/test/java/org/apache/atlas/repository/migration/MigrationBaseAsserts.java
......@@ -37,7 +37,7 @@ import java.io.IOException;
import java.util.Iterator;
import static org.apache.atlas.graph.GraphSandboxUtil.useLocalSolr;
import static org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.loadModelFromJson;
import static org.apache.atlas.utils.TestLoadModelUtils.loadModelFromJson;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertNotNull;
import static org.testng.Assert.assertTrue;
......
repository/src/test/java/org/apache/atlas/repository/migration/RelationshipCacheGeneratorTest.java
......@@ -38,7 +38,7 @@ import java.util.Map;
import static org.apache.atlas.model.typedef.AtlasRelationshipDef.PropagateTags.ONE_TO_TWO;
import static org.apache.atlas.model.typedef.AtlasRelationshipDef.PropagateTags.TWO_TO_ONE;
import static org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.loadModelFromJson;
import static org.apache.atlas.utils.TestLoadModelUtils.loadModelFromJson;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertTrue;
......
repository/src/test/java/org/apache/atlas/repository/store/graph/v2/AtlasNamespaceDefStoreV2Test.java
......@@ -38,7 +38,10 @@ import java.io.IOException;
import java.util.*;
import static org.apache.atlas.model.typedef.AtlasNamespaceDef.ATTR_OPTION_APPLICABLE_ENTITY_TYPES;
import static org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.*;
import static org.apache.atlas.utils.TestLoadModelUtils.loadBaseModel;
import static org.apache.atlas.utils.TestLoadModelUtils.loadFsModel;
import static org.apache.atlas.utils.TestLoadModelUtils.loadHiveModel;
/* Please note that for these tests, since the typeRegistry can be injected only once,
* any new tests should make sure that they flush the type registry at the end of the test.
......
repository/src/test/java/org/apache/atlas/repository/tagpropagation/ClassificationPropagationTest.java
......@@ -66,7 +66,7 @@ import static org.apache.atlas.model.typedef.AtlasRelationshipDef.PropagateTags.
import static org.apache.atlas.model.typedef.AtlasRelationshipDef.PropagateTags.NONE;
import static org.apache.atlas.model.typedef.AtlasRelationshipDef.PropagateTags.ONE_TO_TWO;
import static org.apache.atlas.model.typedef.AtlasRelationshipDef.PropagateTags.TWO_TO_ONE;
import static org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.loadModelFromJson;
import static org.apache.atlas.utils.TestLoadModelUtils.loadModelFromJson;
import static org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.runImportWithNoParameters;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertNotNull;
......
repository/src/test/java/org/apache/atlas/repository/userprofile/UserProfileServiceTest.java
......@@ -44,7 +44,7 @@ import java.util.Optional;
import static org.apache.atlas.graph.GraphSandboxUtil.useLocalSolr;
import static org.apache.atlas.model.profile.AtlasUserSavedSearch.SavedSearchType.BASIC;
import static org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.loadModelFromJson;
import static org.apache.atlas.utils.TestLoadModelUtils.loadModelFromJson;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertFalse;
import static org.testng.Assert.assertNotNull;
......
repository/src/test/java/org/apache/atlas/services/MetricsServiceTest.java
......@@ -24,7 +24,6 @@ import org.apache.atlas.model.metrics.AtlasMetrics;
import org.apache.atlas.repository.graph.AtlasGraphProvider;
import org.apache.atlas.repository.impexp.ImportService;
import org.apache.atlas.repository.impexp.ZipFileResourceTestUtils;
import org.apache.atlas.repository.impexp.ZipSource;
import org.apache.atlas.runner.LocalSolrRunner;
import org.apache.atlas.store.AtlasTypeDefStore;
import org.apache.atlas.type.AtlasTypeRegistry;
......@@ -37,7 +36,6 @@ import org.testng.annotations.Guice;
import org.testng.annotations.Test;
import javax.inject.Inject;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.time.Clock;
......@@ -49,7 +47,7 @@ import java.util.Map;
import static org.apache.atlas.graph.GraphSandboxUtil.useLocalSolr;
import static org.apache.atlas.model.metrics.AtlasMetrics.*;
import static org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.loadModelFromJson;
import static org.apache.atlas.utils.TestLoadModelUtils.loadModelFromJson;
import static org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.runImportWithNoParameters;
import static org.apache.atlas.services.MetricsService.ENTITY;
import static org.apache.atlas.services.MetricsService.GENERAL;
......
repository/src/test/resources/json/auditSearchParameters/audit-search-parameter-purge.json 0 → 100644
{
"auditFilters": {
"condition": "AND",
"criterion": [
{
"attributeName": "operation",
"operator": "eq",
"attributeValue": "PURGE"
}
]
},
"limit": 10,
"offset": 0
}
\ No newline at end of file
repository/src/test/resources/json/auditSearchParameters/audit-search-parameter-without-filter.json 0 → 100644
{
"auditFilters": null,
"limit": 10,
"offset": 0
}
\ No newline at end of file
server-api/src/main/java/org/apache/atlas/RequestContext.java
......@@ -51,7 +51,6 @@ public class RequestContext {
private final long requestTime = System.currentTimeMillis();
private final Map<String, AtlasEntityHeader> updatedEntities = new HashMap<>();
private final Map<String, AtlasEntityHeader> deletedEntities = new HashMap<>();
private final Map<String, AtlasEntityHeader> purgedEntities = new HashMap<>();
private final Map<String, AtlasEntity> entityCache = new HashMap<>();
private final Map<String, AtlasEntityWithExtInfo> entityExtInfoCache = new HashMap<>();
private final Map<String, List<AtlasClassification>> addedPropagations = new HashMap<>();
......@@ -110,7 +109,6 @@ public class RequestContext {
public void clearCache() {
this.updatedEntities.clear();
this.deletedEntities.clear();
this.purgedEntities.clear();
this.entityCache.clear();
this.entityExtInfoCache.clear();
this.addedPropagations.clear();
......@@ -228,12 +226,6 @@ public class RequestContext {
}
}
public void recordEntityPurge(AtlasEntityHeader entity) {
if (entity != null && entity.getGuid() != null) {
purgedEntities.put(entity.getGuid(), entity);
}
}
public void recordAddedPropagation(String guid, AtlasClassification classification) {
if (StringUtils.isNotEmpty(guid) && classification != null) {
List<AtlasClassification> classifications = addedPropagations.get(guid);
......@@ -314,10 +306,6 @@ public class RequestContext {
return deletedEntities.values();
}
public Collection<AtlasEntityHeader> getPurgedEntities() {
return purgedEntities.values();
}
/**
* Checks if an instance with the given guid is in the cache for this request. Either returns the instance
* or null if it is not in the cache.
......@@ -345,10 +333,6 @@ public class RequestContext {
return deletedEntities.containsKey(guid);
}
public boolean isPurgedEntity(String guid) {
return purgedEntities.containsKey(guid);
}
public MetricRecorder startMetricRecord(String name) { return metrics != null ? metrics.getMetricRecorder(name) : null; }
public void endMetricRecord(MetricRecorder recorder) {
......
webapp/src/main/java/org/apache/atlas/web/resources/AdminResource.java
......@@ -22,12 +22,16 @@ import com.sun.jersey.multipart.FormDataParam;
import org.apache.atlas.ApplicationProperties;
import org.apache.atlas.AtlasClient;
import org.apache.atlas.AtlasErrorCode;
import org.apache.atlas.RequestContext;
import org.apache.atlas.authorize.AtlasAdminAccessRequest;
import org.apache.atlas.authorize.AtlasAuthorizationUtils;
import org.apache.atlas.authorize.AtlasEntityAccessRequest;
import org.apache.atlas.authorize.AtlasPrivilege;
import org.apache.atlas.discovery.SearchContext;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.audit.AtlasAuditEntry;
import org.apache.atlas.model.audit.AtlasAuditEntry.AuditOperation;
import org.apache.atlas.model.audit.AuditSearchParameters;
import org.apache.atlas.model.impexp.AtlasExportRequest;
import org.apache.atlas.model.impexp.AtlasExportResult;
import org.apache.atlas.model.impexp.AtlasImportRequest;
......@@ -37,9 +41,11 @@ import org.apache.atlas.model.impexp.ExportImportAuditEntry;
import org.apache.atlas.model.impexp.MigrationStatus;
import org.apache.atlas.model.instance.AtlasCheckStateRequest;
import org.apache.atlas.model.instance.AtlasCheckStateResult;
import org.apache.atlas.model.instance.AtlasEntityHeader;
import org.apache.atlas.model.instance.EntityMutationResponse;
import org.apache.atlas.model.metrics.AtlasMetrics;
import org.apache.atlas.model.patches.AtlasPatch.AtlasPatches;
import org.apache.atlas.repository.audit.AtlasAuditService;
import org.apache.atlas.repository.impexp.AtlasServerService;
import org.apache.atlas.repository.impexp.ExportImportAuditService;
import org.apache.atlas.repository.impexp.ExportService;
......@@ -90,6 +96,7 @@ import java.io.IOException;
import java.io.InputStream;
import java.util.Arrays;
import java.util.Collection;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
......@@ -140,6 +147,7 @@ public class AdminResource {
private final AtlasServerService atlasServerService;
private final AtlasEntityStore entityStore;
private final AtlasPatchManager patchManager;
private final AtlasAuditService auditService;
static {
try {
......@@ -155,7 +163,7 @@ public class AdminResource {
MigrationProgressService migrationProgressService,
AtlasServerService serverService,
ExportImportAuditService exportImportAuditService, AtlasEntityStore entityStore,
AtlasPatchManager patchManager) {
AtlasPatchManager patchManager, AtlasAuditService auditService) {
this.serviceState = serviceState;
this.metricsService = metricsService;
this.exportService = exportService;
......@@ -168,6 +176,7 @@ public class AdminResource {
this.exportImportAuditService = exportImportAuditService;
this.importExportOperationLock = new ReentrantLock();
this.patchManager = patchManager;
this.auditService = auditService;
}
/**
......@@ -448,10 +457,23 @@ public class AdminResource {
try {
if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) {
perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "AdminResource.purgeByGuids(" + guids + ")");
perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "AdminResource.purgeByIds(" + guids + ")");
}
return entityStore.purgeByIds(guids);
EntityMutationResponse resp = entityStore.purgeByIds(guids);
final List<AtlasEntityHeader> purgedEntities = resp.getPurgedEntities();
if(purgedEntities != null && purgedEntities.size() > 0) {
final String clientId = RequestContext.get().getClientIPAddress();
final Date startTime = new Date(RequestContext.get().getRequestTime());
final Date endTime = new Date();
auditService.add(AtlasAuthorizationUtils.getCurrentUserName(), AuditOperation.PURGE,
clientId != null ? clientId : "", startTime, endTime, guids.toString(),
resp.getPurgedEntitiesIds());
}
return resp;
} finally {
AtlasPerfTracer.log(perf);
}
......@@ -548,6 +570,24 @@ public class AdminResource {
}
}
@POST
@Path("/audits")
@Consumes(Servlets.JSON_MEDIA_TYPE)
@Produces(Servlets.JSON_MEDIA_TYPE)
public List<AtlasAuditEntry> getAtlasAudits(AuditSearchParameters auditSearchParameters) throws AtlasBaseException {
AtlasPerfTracer perf = null;
try {
if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) {
perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "getAtlasAudit(" + auditSearchParameters + ")");
}
return auditService.get(auditSearchParameters);
} finally {
AtlasPerfTracer.log(perf);
}
}
@GET
@Path("activeSearches")
@Produces(Servlets.JSON_MEDIA_TYPE)
......
webapp/src/test/java/org/apache/atlas/web/adapters/TestEntitiesREST.java
......@@ -31,7 +31,7 @@ import static org.apache.atlas.repository.Constants.MODIFICATION_TIMESTAMP_PROPE
import static org.apache.atlas.repository.Constants.STATE_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.TIMESTAMP_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.TYPE_NAME_PROPERTY_KEY;
import static org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.createTypesAsNeeded;
import static org.apache.atlas.utils.TestLoadModelUtils.createTypesAsNeeded;
import org.apache.atlas.AtlasClient;
import org.apache.atlas.RequestContext;
......
webapp/src/test/java/org/apache/atlas/web/integration/EntityV2JerseyResourceIT.java
......@@ -26,18 +26,16 @@ import org.apache.atlas.AtlasClient;
import org.apache.atlas.AtlasServiceException;
import org.apache.atlas.EntityAuditEvent;
import org.apache.atlas.model.TimeBoundary;
import org.apache.atlas.model.instance.AtlasClassification;
import org.apache.atlas.model.audit.AtlasAuditEntry;
import org.apache.atlas.model.audit.AuditSearchParameters;
import org.apache.atlas.model.instance.*;
import org.apache.atlas.model.instance.AtlasClassification.AtlasClassifications;
import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.model.instance.AtlasEntity.AtlasEntityWithExtInfo;
import org.apache.atlas.model.instance.AtlasEntityHeader;
import org.apache.atlas.model.instance.AtlasObjectId;
import org.apache.atlas.model.instance.EntityMutationResponse;
import org.apache.atlas.model.instance.EntityMutations;
import org.apache.atlas.model.typedef.AtlasClassificationDef;
import org.apache.atlas.model.typedef.AtlasEntityDef;
import org.apache.atlas.model.typedef.AtlasTypesDef;
import org.apache.atlas.type.AtlasTypeUtil;
import org.apache.atlas.utils.TestResourceFileUtils;
import org.apache.atlas.v1.typesystem.types.utils.TypesUtil;
import org.apache.commons.lang.RandomStringUtils;
import org.apache.hadoop.util.StringUtils;
......@@ -55,7 +53,6 @@ import java.util.stream.Stream;
import static org.testng.Assert.*;
/**
* Integration tests for Entity Jersey Resource.
*/
......@@ -790,7 +787,8 @@ public class EntityV2JerseyResourceIT extends BaseResourceIT {
assertNotNull(deleteResponse.getEntitiesByOperation(EntityMutations.EntityOperation.DELETE));
assertEquals(deleteResponse.getEntitiesByOperation(EntityMutations.EntityOperation.DELETE).size(), 2);
Thread.sleep(1000);
//Wait for delete operation
Thread.sleep(5000);
// Purge the database entities
Set<String> guids = Stream.of(entity1Header.getGuid(), entity2Header.getGuid()).collect(Collectors.toSet());
......@@ -800,6 +798,12 @@ public class EntityV2JerseyResourceIT extends BaseResourceIT {
assertNotNull(purgeResponse);
assertNotNull(purgeResponse.getEntitiesByOperation(EntityMutations.EntityOperation.PURGE));
assertEquals(purgeResponse.getEntitiesByOperation(EntityMutations.EntityOperation.PURGE).size(), 2);
AuditSearchParameters auditSearchParameters = TestResourceFileUtils.readObjectFromJson("audit-search-parameter-purge",
AuditSearchParameters.class);
List<AtlasAuditEntry> res = atlasClientV2.getAtlasAuditByOperation(auditSearchParameters);
// Verify that the audit entry is set
assertNotNull(res);
}
@Test
......
webapp/src/test/java/org/apache/atlas/web/resources/AdminResourceTest.java
......@@ -51,7 +51,7 @@ public class AdminResourceTest {
when(serviceState.getState()).thenReturn(ServiceState.ServiceStateValue.ACTIVE);
AdminResource adminResource = new AdminResource(serviceState, null, null, null, null, null, null, null, null, null, null);
AdminResource adminResource = new AdminResource(serviceState, null, null, null, null, null, null, null, null, null, null, null);
Response response = adminResource.getStatus();
assertEquals(response.getStatus(), HttpServletResponse.SC_OK);
JsonNode entity = AtlasJson.parseToV1JsonNode((String) response.getEntity());
......@@ -62,7 +62,7 @@ public class AdminResourceTest {
public void testResourceGetsValueFromServiceState() throws IOException {
when(serviceState.getState()).thenReturn(ServiceState.ServiceStateValue.PASSIVE);
AdminResource adminResource = new AdminResource(serviceState, null, null, null, null, null, null, null, null, null, null);
AdminResource adminResource = new AdminResource(serviceState, null, null, null, null, null, null, null, null, null, null, null);
Response response = adminResource.getStatus();
verify(serviceState).getState();
......
webapp/src/test/resources/json/audit-search-parameter-purge.json 0 → 100644
{
"auditFilters": {
"condition": "AND",
"criterion": [
{
"attributeName": "operation",
"operator": "like",
"attributeValue": "PURGE"
},
{
"attributeName": "startTime",
"operator": "gte",
"attributeValue": "1575958152162"
},
{
"attributeName": "endTime",
"operator": "gte",
"attributeValue": "1575958152184"
}
]
},
"limit": 10,
"offset": 0
}
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment