Skip to content

A
atlas
Commit fa2e5b49 by mayanknj Committed by Sarath Subramanian
Options

ATLAS-3679 : Bulk import Business Metadata attribute.

Signed-off-by: 's avatarSarath Subramanian <sarath@apache.org>
parent 1e8fa7e5
Showing with 420 additions and 9 deletions
common/src/main/java/org/apache/atlas/repository/Constants.java
......@@ -202,9 +202,9 @@ public final class Constants {
public static final Integer INCOMPLETE_ENTITY_VALUE = Integer.valueOf(1);
/*
* All supported file-format extensions for AtlasGlossaryTerms file upload
* All supported file-format extensions for Bulk Imports through file upload
*/
public enum GlossaryImportSupportedFileExtensions { XLSX, XLS, CSV }
public enum SupportedFileExtensions { XLSX, XLS, CSV }
private Constants() {
}
......
intg/src/main/java/org/apache/atlas/AtlasErrorCode.java
......@@ -168,6 +168,7 @@ public enum AtlasErrorCode {
BUSINESS_METADATA_ATTRIBUTE_DOES_NOT_EXIST(400, "ATLAS-400-00-096", "Business-metadata attribute does not exist in entity: {0}"),
BUSINESS_METADATA_ATTRIBUTE_ALREADY_EXISTS(400, "ATLAS-400-00-097", "Business-metadata attribute already exists in entity: {0}"),
INVALID_FILE_TYPE(400, "ATLAS-400-00-98", "The provided file type {0} is not supported."),
INVALID_BUSINESS_ATTRIBUTES_IMPORT_DATA(400, "ATLAS-400-00-99","The uploaded file was not processed due to following errors : {0}"),
UNAUTHORIZED_ACCESS(403, "ATLAS-403-00-001", "{0} is not authorized to perform {1}"),
......@@ -192,6 +193,7 @@ public enum AtlasErrorCode {
INSTANCE_GUID_DELETED(404, "ATLAS-404-00-012", "Given instance guid {0} has been deleted"),
NO_PROPAGATED_CLASSIFICATIONS_FOUND_FOR_ENTITY(404, "ATLAS-404-00-013", "No propagated classifications associated with entity: {0}"),
NO_DATA_FOUND(404, "ATLAS-404-00-014", "No data found in the uploaded file"),
FILE_NAME_NOT_FOUND(404, "ATLAS-404-00-015", "File name should not be blank"),
// All data conflict errors go here
TYPE_ALREADY_EXISTS(409, "ATLAS-409-00-001", "Given type {0} already exists"),
......
intg/src/main/java/org/apache/atlas/bulkimport/BulkImportResponse.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.bulkimport;
import java.util.ArrayList;
import java.util.List;
public class BulkImportResponse {
private List<ImportInfo> failedImportInfoList = new ArrayList<ImportInfo>();
private List<ImportInfo> successImportInfoList = new ArrayList<ImportInfo>();
public BulkImportResponse() {}
public List<ImportInfo> getFailedImportInfoList() {
return failedImportInfoList;
}
public void setFailedImportInfoList(List<ImportInfo> failedImportInfoList){
this.failedImportInfoList = failedImportInfoList;
}
public void setFailedImportInfoList(ImportInfo importInfo){
List<ImportInfo> failedImportInfoList = this.failedImportInfoList;
if (failedImportInfoList == null) {
failedImportInfoList = new ArrayList<>();
}
failedImportInfoList.add(importInfo);
setFailedImportInfoList(failedImportInfoList);
}
public List<ImportInfo> getSuccessImportInfoList() {
return successImportInfoList;
}
public void setSuccessImportInfoList(List<ImportInfo> successImportInfoList){
this.successImportInfoList = successImportInfoList;
}
public void setSuccessImportInfoList(ImportInfo importInfo){
List<ImportInfo> successImportInfoList = this.successImportInfoList;
if (successImportInfoList == null) {
successImportInfoList = new ArrayList<>();
}
successImportInfoList.add(importInfo);
setSuccessImportInfoList(successImportInfoList);
}
public enum ImportStatus {
SUCCESS, FAILED
}
@Override
public String toString() {
return "BulkImportResponse{" +
"failedImportInfoList=" + failedImportInfoList +
", successImportInfoList=" + successImportInfoList +
'}';
}
static public class ImportInfo {
private String parentObjectName;
private String childObjectName;
private ImportStatus importStatus;
private String remarks;
private Integer rowNumber;
public ImportInfo(String parentObjectName, String childObjectName, ImportStatus importStatus, String remarks, Integer rowNumber) {
this.parentObjectName = parentObjectName;
this.childObjectName = childObjectName;
this.importStatus = importStatus;
this.remarks = remarks;
this.rowNumber = rowNumber;
}
public ImportInfo(String parentObjectName, String childObjectName, ImportStatus importStatus) {
this(parentObjectName, childObjectName, importStatus, "",-1);
}
public ImportInfo( ImportStatus importStatus, String remarks, Integer rowNumber) {
this("","", importStatus, remarks, rowNumber);
}
public ImportInfo(String parentObjectName, String childObjectName) {
this(parentObjectName,childObjectName, ImportStatus.SUCCESS, "", -1);
}
public ImportInfo(String parentObjectName, String childObjectName, ImportStatus importStatus, String remarks) {
this(parentObjectName, childObjectName, importStatus, remarks, -1);
}
public String getParentObjectName() {
return parentObjectName;
}
public void setParentObjectName(String parentObjectName) {
this.parentObjectName = parentObjectName;
}
public String getChildObjectName() {
return childObjectName;
}
public void setChildObjectName(String childObjectName) {
this.childObjectName = childObjectName;
}
public String getRemarks() {
return remarks;
}
public void setRemarks(String remarks) {
this.remarks = remarks;
}
public ImportStatus getImportStatus() {
return importStatus;
}
public void setImportStatus(ImportStatus importStatus) {
this.importStatus = importStatus;
}
@Override
public String toString() {
return "ImportInfo{" +
"parentObjectName='" + parentObjectName + '\'' +
", childObjectName='" + childObjectName + '\'' +
", remarks='" + remarks + '\'' +
", importStatus=" + importStatus +
", rowNumber=" + rowNumber +
'}';
}
}
}
\ No newline at end of file
intg/src/test/java/org/apache/atlas/TestUtilsV2.java
......@@ -35,8 +35,17 @@ import org.apache.atlas.model.typedef.AtlasStructDef.AtlasConstraintDef;
import org.apache.atlas.model.typedef.AtlasTypesDef;
import org.apache.atlas.type.AtlasTypeUtil;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.RandomStringUtils;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.util.ArrayList;
......@@ -66,7 +75,7 @@ import static org.apache.atlas.type.AtlasTypeUtil.createBusinessMetadataDef;
* Test utility class.
*/
public final class TestUtilsV2 {
private static final Logger LOG = LoggerFactory.getLogger(TestUtilsV2.class);
public static final long TEST_DATE_IN_LONG = 1418265358440L;
public static final String TEST_USER = "testUser";
......@@ -1545,4 +1554,35 @@ public final class TestUtilsV2 {
typeDef.setCreatedBy(TestUtilsV2.TEST_USER);
typeDef.setUpdatedBy(TestUtilsV2.TEST_USER);
}
public static InputStream getFile(String subDir, String fileName){
final String userDir = System.getProperty("user.dir");
String filePath = getTestFilePath(userDir, subDir, fileName);
File file = new File(filePath);
InputStream fs = null;
try {
fs = new FileInputStream(file);
} catch (FileNotFoundException e) {
LOG.error("File could not be found at: {}", filePath, e);
}
return fs;
}
public static String getFileData(String subDir, String fileName)throws IOException {
final String userDir = System.getProperty("user.dir");
String filePath = getTestFilePath(userDir, subDir, fileName);
File f = new File(filePath);
String ret = FileUtils.readFileToString(f, "UTF-8");
return ret;
}
private static String getTestFilePath(String startPath, String subDir, String fileName) {
if (StringUtils.isNotEmpty(subDir)) {
return startPath + "/src/test/resources/" + subDir + "/" + fileName;
} else {
return startPath + "/src/test/resources/" + fileName;
}
}
}
repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasEntityStore.java
......@@ -29,7 +29,9 @@ import org.apache.atlas.model.instance.AtlasObjectId;
import org.apache.atlas.model.instance.EntityMutationResponse;
import org.apache.atlas.repository.store.graph.v2.EntityStream;
import org.apache.atlas.type.AtlasEntityType;
import org.apache.atlas.bulkimport.BulkImportResponse;
import java.io.InputStream;
import java.util.List;
import java.util.Map;
import java.util.Set;
......@@ -282,4 +284,13 @@ public interface AtlasEntityStore {
* Add given labels to the given entity, if labels is null/empty, no labels will be added.
*/
void addLabels(String guid, Set<String> labels) throws AtlasBaseException;
/**
*
* @param inputStream
* @param fileName
* @throws AtlasBaseException
*
*/
BulkImportResponse bulkCreateOrUpdateBusinessAttributes(InputStream inputStream, String fileName) throws AtlasBaseException;
}
repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasGraphUtilsV2.java
......@@ -29,6 +29,7 @@ import org.apache.atlas.model.TypeCategory;
import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.model.instance.AtlasEntity.Status;
import org.apache.atlas.model.typedef.AtlasBaseTypeDef;
import org.apache.atlas.model.typedef.AtlasEnumDef;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.graph.GraphHelper;
import org.apache.atlas.repository.graphdb.AtlasEdge;
......@@ -39,9 +40,11 @@ import org.apache.atlas.repository.graphdb.AtlasIndexQuery;
import org.apache.atlas.repository.graphdb.AtlasIndexQuery.Result;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.type.AtlasEntityType;
import org.apache.atlas.type.AtlasEnumType;
import org.apache.atlas.type.AtlasStructType;
import org.apache.atlas.type.AtlasStructType.AtlasAttribute;
import org.apache.atlas.type.AtlasType;
import org.apache.atlas.util.FileUtils;
import org.apache.atlas.utils.AtlasPerfMetrics.MetricRecorder;
import org.apache.commons.collections.MapUtils;
import org.apache.commons.configuration.Configuration;
......@@ -49,6 +52,7 @@ import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
......@@ -62,11 +66,10 @@ import static org.apache.atlas.repository.Constants.CLASSIFICATION_NAMES_KEY;
import static org.apache.atlas.repository.Constants.ENTITY_TYPE_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.INDEX_SEARCH_VERTEX_PREFIX_DEFAULT;
import static org.apache.atlas.repository.Constants.INDEX_SEARCH_VERTEX_PREFIX_PROPERTY;
import static org.apache.atlas.repository.Constants.LABELS_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.PROPAGATED_CLASSIFICATION_NAMES_KEY;
import static org.apache.atlas.repository.Constants.STATE_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.TYPE_NAME_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.TYPENAME_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.TYPE_NAME_PROPERTY_KEY;
import static org.apache.atlas.repository.graph.AtlasGraphProvider.getGraphInstance;
import static org.apache.atlas.repository.graphdb.AtlasGraphQuery.SortOrder.ASC;
import static org.apache.atlas.repository.graphdb.AtlasGraphQuery.SortOrder.DESC;
......@@ -674,4 +677,128 @@ public class AtlasGraphUtilsV2 {
}
return classificationNames;
}
public static List<Date> dateParser(String[] arr, List failedTermMsgList, int lineIndex) {
List<Date> ret = new ArrayList();
for (String s : arr) {
try{
SimpleDateFormat formatter = new SimpleDateFormat("MM/dd/yyyy");
Date date = formatter.parse(s);
ret.add(date);
}
catch(Exception e){
LOG.error("Provided value "+s+" is not of Date type at line #"+lineIndex);
failedTermMsgList.add("Provided value "+s+" is not of Date type at line #"+lineIndex);
}
}
return ret;
}
public static List<Boolean> booleanParser(String[] arr, List failedTermMsgList, int lineIndex) {
List<Boolean> ret = new ArrayList();
for (String s : arr) {
try{
ret.add(Boolean.parseBoolean(s));
}
catch(Exception e){
LOG.error("Provided value "+s+" is not of Boolean type at line #"+lineIndex);
failedTermMsgList.add("Provided value "+s+" is not of Boolean type at line #"+lineIndex);
}
}
return ret;
}
public static List<Double> doubleParser(String[] arr, List failedTermMsgList, int lineIndex) {
List<Double> ret = new ArrayList();
for (String s : arr) {
try{
ret.add(Double.parseDouble(s));
}
catch(Exception e){
LOG.error("Provided value "+s+" is not of Double type at line #"+lineIndex);
failedTermMsgList.add("Provided value "+s+" is not of Double type at line #"+lineIndex);
}
}
return ret;
}
public static List<Short> shortParser(String[] arr, List failedTermMsgList, int lineIndex) {
List<Short> ret = new ArrayList();
for (String s : arr) {
try{
ret.add(Short.parseShort(s));
}
catch(Exception e){
LOG.error("Provided value "+s+" is not of Short type at line #"+lineIndex);
failedTermMsgList.add("Provided value "+s+" is not of Short type at line #"+lineIndex);
}
}
return ret;
}
public static List<Long> longParser(String[] arr, List failedTermMsgList, int lineIndex) {
List<Long> ret = new ArrayList();
for (String s : arr) {
try{
ret.add(Long.parseLong(s));
}
catch(Exception e){
LOG.error("Provided value "+s+" is not of Long type at line #"+lineIndex);
failedTermMsgList.add("Provided value "+s+" is not of Long type at line #"+lineIndex);
}
}
return ret;
}
public static List<Integer> intParser(String[] arr, List failedTermMsgList, int lineIndex) {
List<Integer> ret = new ArrayList();
for (String s : arr) {
try{
ret.add(Integer.parseInt(s));
}
catch(Exception e){
LOG.error("Provided value "+s+" is not of Integer type at line #"+lineIndex);
failedTermMsgList.add("Provided value "+s+" is Integer of Long type at line #"+lineIndex);
}
}
return ret;
}
public static List<Float> floatParser(String[] arr, List failedTermMsgList, int lineIndex) {
List<Float> ret = new ArrayList();
for (String s : arr) {
try{
ret.add(Float.parseFloat(s));
}
catch(Exception e){
LOG.error("Provided value "+s+" is Float of Long type at line #"+lineIndex);
failedTermMsgList.add("Provided value "+s+" is Float of Long type at line #"+lineIndex);
}
}
return ret;
}
public static List assignEnumValues(String bmAttributeValues, AtlasEnumType enumType, List<String> failedTermMsgList, int lineIndex) {
List<String> ret = new ArrayList<>();
String[] arr = bmAttributeValues.split(FileUtils.ESCAPE_CHARACTER + FileUtils.PIPE_CHARACTER);
AtlasEnumDef.AtlasEnumElementDef atlasEnumDef;
for(String s : arr){
atlasEnumDef = enumType.getEnumElementDef(s);
if(atlasEnumDef==null){
LOG.error("Provided value "+s+" is Enumeration of Long type at line #"+lineIndex);
failedTermMsgList.add("Provided value "+s+" is Enumeration of Long type at line #"+lineIndex);
}else{
ret.add(s);
}
}
return ret;
}
}
\ No newline at end of file
repository/src/main/java/org/apache/atlas/util/FileUtils.java
......@@ -22,6 +22,7 @@ import org.apache.atlas.AtlasErrorCode;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.io.FilenameUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.poi.hssf.usermodel.HSSFWorkbook;
import org.apache.poi.ss.usermodel.Cell;
import org.apache.poi.ss.usermodel.Row;
......@@ -36,13 +37,20 @@ import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import static org.apache.atlas.repository.Constants.GlossaryImportSupportedFileExtensions.*;
import static org.apache.atlas.repository.Constants.SupportedFileExtensions.*;
public class FileUtils {
public static final String PIPE_CHARACTER = "|";
public static final String COLON_CHARACTER = ":";
public static final String ESCAPE_CHARACTER = "\\";
//BusinessMetadata attributes association uploads
public static final int TYPENAME_COLUMN_INDEX = 0;
public static final int UNIQUE_ATTR_VALUE_COLUMN_INDEX = 1;
public static final int BM_ATTR_NAME_COLUMN_INDEX = 2;
public static final int BM_ATTR_VALUE_COLUMN_INDEX = 3;
public static final int UNIQUE_ATTR_NAME_COLUMN_INDEX = 4;
public static List<String[]> readFileData(String fileName, InputStream inputStream) throws IOException, AtlasBaseException {
List<String[]> ret;
String extension = FilenameUtils.getExtension(fileName);
......@@ -123,4 +131,16 @@ public class FileUtils {
return true;
}
public static String getBusinessMetadataHeaders() {
List<String> bMHeader = new ArrayList<>();
bMHeader.add("EntityType");
bMHeader.add("EntityUniqueAttributeValue");
bMHeader.add("BusinessAttributeName");
bMHeader.add("BusinessAttributeValue");
bMHeader.add("EntityUniqueAttributeName[optional]");
return StringUtils.join(bMHeader, ",");
}
}
\ No newline at end of file
repository/src/test/resources/csvFiles/incorrectFile_2.csv 0 → 100644
TypeName,UniqueAttributeValue,BusinessAttributeName,BusinessAttributeValue,UniqueAttributeName[optional]
incorrectEntityType,hive_db_1,bmWithAllTypes.attr8,"Awesome Attribute 1",name
repository/src/test/resources/csvFiles/template_1.csv
GlossaryName, TermName, ShortDescription, LongDescription, Examples, Abbreviation, Usage, AdditionalAttributes, TranslationTerms, ValidValuesFor, Synonyms, ReplacedBy, ValidValues, ReplacementTerms, SeeAlso, TranslatedTerms, IsA, Antonyms, Classifies, PreferredToTerms, PreferredTerms
testBankingGlossary,BankBranch,SD4,LD4,"EXAMPLE","ABBREVIATION","USAGE",,,,,,,,,,,,,,
testBankingGlossary,BankBranch,SD4,LD4,"EXAMPLE","ABBREVIATION","USAGE",,,,,,,,,,,,,,
\ No newline at end of file
repository/src/test/resources/csvFiles/template_2.csv 0 → 100644
TypeName,UniqueAttributeValue,BusinessAttributeName,BusinessAttributeValue,UniqueAttributeName[optional]
hive_database,hive_db_1,bmWithAllTypes.attr8,"Awesome Attribute 1",name
webapp/src/main/java/org/apache/atlas/web/rest/EntityREST.java
......@@ -17,8 +17,11 @@
*/
package org.apache.atlas.web.rest;
import com.sun.jersey.core.header.FormDataContentDisposition;
import com.sun.jersey.multipart.FormDataParam;
import org.apache.atlas.AtlasErrorCode;
import org.apache.atlas.EntityAuditEvent;
import org.apache.atlas.bulkimport.BulkImportResponse;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.TypeCategory;
import org.apache.atlas.model.audit.EntityAuditEventV2;
......@@ -31,14 +34,15 @@ import org.apache.atlas.model.instance.ClassificationAssociateRequest;
import org.apache.atlas.model.instance.EntityMutationResponse;
import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef;
import org.apache.atlas.repository.audit.EntityAuditRepository;
import org.apache.atlas.repository.store.graph.v2.ClassificationAssociator;
import org.apache.atlas.repository.converters.AtlasInstanceConverter;
import org.apache.atlas.repository.store.graph.AtlasEntityStore;
import org.apache.atlas.repository.store.graph.v2.AtlasEntityStream;
import org.apache.atlas.repository.store.graph.v2.ClassificationAssociator;
import org.apache.atlas.repository.store.graph.v2.EntityStream;
import org.apache.atlas.type.AtlasClassificationType;
import org.apache.atlas.type.AtlasEntityType;
import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.atlas.util.FileUtils;
import org.apache.atlas.utils.AtlasPerfTracer;
import org.apache.atlas.web.util.Servlets;
import org.apache.commons.collections.CollectionUtils;
......@@ -61,8 +65,14 @@ import javax.ws.rs.Path;
import javax.ws.rs.PathParam;
import javax.ws.rs.Produces;
import javax.ws.rs.QueryParam;
import javax.ws.rs.WebApplicationException;
import javax.ws.rs.core.Context;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import javax.ws.rs.core.StreamingOutput;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
......@@ -1173,4 +1183,43 @@ public class EntityREST {
}
}
}
/**
* Get the sample Template for uploading/creating bulk BusinessMetaData
*
* @return Template File
* @throws AtlasBaseException
* @HTTP 400 If the provided fileType is not supported
*/
@GET
@Path("/businessmetadata/import/template")
@Produces(MediaType.APPLICATION_OCTET_STREAM)
public Response produceTemplate() {
return Response.ok(new StreamingOutput() {
@Override
public void write(OutputStream outputStream) throws IOException, WebApplicationException {
outputStream.write(FileUtils.getBusinessMetadataHeaders().getBytes());
}
}).header("Content-Disposition", "attachment; filename=\"template_business_metadata\"").build();
}
/**
* Upload the file for creating Business Metadata in BULK
*
* @param uploadedInputStream InputStream of file
* @param fileDetail FormDataContentDisposition metadata of file
* @return
* @throws AtlasBaseException
* @HTTP 200 If Business Metadata creation was successful
* @HTTP 400 If Business Metadata definition has invalid or missing information
* @HTTP 409 If Business Metadata already exists (duplicate qualifiedName)
*/
@POST
@Path("/businessmetadata/import")
@Consumes(MediaType.MULTIPART_FORM_DATA)
public BulkImportResponse importBMAttributes(@FormDataParam("file") InputStream uploadedInputStream,
@FormDataParam("file") FormDataContentDisposition fileDetail) throws AtlasBaseException {
return entitiesStore.bulkCreateOrUpdateBusinessAttributes(uploadedInputStream, fileDetail.getFileName());
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment