Skip to content

A
atlas
Commit 5f956110 by Nikhil Bonte Committed by Madhan Neethiraj
Options

ATLAS-3879: Ozone: ozone_key entity is directly created under ozone_bucket

Signed-off-by: 's avatarMadhan Neethiraj <madhan@apache.org>
parent cea851c5
Showing with 145 additions and 40 deletions
addons/models/3000-Cloud/3050-ozone-typedefs.json
...@@ -90,6 +90,26 @@ ...@@ -90,6 +90,26 @@
"classificationDefs": [], "classificationDefs": [],
"entityDefs": [ "entityDefs": [
{ {
"name": "ozone_parent",
"description": "Atlas entity-type representing parent types (bucket, key) in Ozone",
"superTypes": [
],
"serviceType": "ozone",
"typeVersion": "1.0",
"attributeDefs": [
]
},
{
"name": "ozone_child",
"description": "Atlas entity-type representing child types (key) in Ozone",
"superTypes": [
],
"serviceType": "ozone",
"typeVersion": "1.0",
"attributeDefs": [
]
},
{
"name": "ozone_volume", "name": "ozone_volume",
"description": "Atlas Type representing an volume in an Ozone Object Store", "description": "Atlas Type representing an volume in an Ozone Object Store",
"superTypes": [ "superTypes": [
...@@ -128,7 +148,8 @@ ...@@ -128,7 +148,8 @@
"name": "ozone_bucket", "name": "ozone_bucket",
"description": "Atlas Type representing a bucket in an Ozone Object Store Volume", "description": "Atlas Type representing a bucket in an Ozone Object Store Volume",
"superTypes": [ "superTypes": [
"DataSet" "DataSet",
"ozone_parent"
], ],
"serviceType": "ozone", "serviceType": "ozone",
"typeVersion": "1.0", "typeVersion": "1.0",
...@@ -171,7 +192,9 @@ ...@@ -171,7 +192,9 @@
"name": "ozone_key", "name": "ozone_key",
"description": "Atlas Type representing a key in an Ozone Object Store Bucket", "description": "Atlas Type representing a key in an Ozone Object Store Bucket",
"superTypes": [ "superTypes": [
"DataSet" "DataSet",
"ozone_parent",
"ozone_child"
], ],
"serviceType": "ozone", "serviceType": "ozone",
"typeVersion": "1.0", "typeVersion": "1.0",
...@@ -240,19 +263,19 @@ ...@@ -240,19 +263,19 @@
"propagateTags": "NONE" "propagateTags": "NONE"
}, },
{ {
"name": "ozone_bucket_keys", "name": "ozone_parent_children",
"serviceType": "ozone", "serviceType": "ozone",
"typeVersion": "1.0", "typeVersion": "1.0",
"relationshipCategory": "COMPOSITION", "relationshipCategory": "COMPOSITION",
"endDef1": { "endDef1": {
"type": "ozone_bucket", "type": "ozone_parent",
"name": "keys", "name": "children",
"isContainer": true, "isContainer": true,
"cardinality": "SET" "cardinality": "SET"
}, },
"endDef2": { "endDef2": {
"type": "ozone_key", "type": "ozone_child",
"name": "bucket", "name": "parent",
"isContainer": false, "isContainer": false,
"cardinality": "SINGLE" "cardinality": "SINGLE"
}, },
......
common/src/main/java/org/apache/atlas/utils/AtlasPathExtractorUtil.java
...@@ -80,7 +80,7 @@ public class AtlasPathExtractorUtil { ...@@ -80,7 +80,7 @@ public class AtlasPathExtractorUtil {
public static final String OZONE_3_SCHEME = "o3fs" + SCHEME_SEPARATOR; public static final String OZONE_3_SCHEME = "o3fs" + SCHEME_SEPARATOR;
public static final String ATTRIBUTE_VOLUME = "volume"; public static final String ATTRIBUTE_VOLUME = "volume";
public static final String RELATIONSHIP_OZONE_VOLUME_BUCKET = "ozone_volume_buckets"; public static final String RELATIONSHIP_OZONE_VOLUME_BUCKET = "ozone_volume_buckets";
public static final String RELATIONSHIP_OZONE_BUCKET_KEY = "ozone_bucket_keys"; public static final String RELATIONSHIP_OZONE_PARENT_CHILDREN = "ozone_parent_children";
public static AtlasEntityWithExtInfo getPathEntity(Path path, PathExtractorContext context) { public static AtlasEntityWithExtInfo getPathEntity(Path path, PathExtractorContext context) {
AtlasEntityWithExtInfo entityWithExtInfo = new AtlasEntityWithExtInfo(); AtlasEntityWithExtInfo entityWithExtInfo = new AtlasEntityWithExtInfo();
...@@ -400,13 +400,39 @@ public class AtlasPathExtractorUtil { ...@@ -400,13 +400,39 @@ public class AtlasPathExtractorUtil {
extInfo.addReferredEntity(bucketEntity); extInfo.addReferredEntity(bucketEntity);
AtlasRelatedObjectId parentObjId = AtlasTypeUtil.getAtlasRelatedObjectId(bucketEntity, RELATIONSHIP_OZONE_PARENT_CHILDREN);
String parentPath = Path.SEPARATOR;
String dirPath = path.toUri().getPath();
if (StringUtils.isEmpty(dirPath)) {
dirPath = Path.SEPARATOR;
}
String keyQNamePrefix = ozoneScheme + SCHEME_SEPARATOR + path.toUri().getAuthority();
for (String subDirName : dirPath.split(Path.SEPARATOR)) {
if (StringUtils.isEmpty(subDirName)) {
continue;
}
String subDirPath = parentPath + subDirName;
String subDirQualifiedName = keyQNamePrefix + subDirPath + QNAME_SEP_METADATA_NAMESPACE + metadataNamespace;
ret = new AtlasEntity(OZONE_KEY); ret = new AtlasEntity(OZONE_KEY);
ret.setAttribute(ATTRIBUTE_QUALIFIED_NAME, pathQualifiedName); ret.setRelationshipAttribute(ATTRIBUTE_PARENT, parentObjId);
ret.setAttribute(ATTRIBUTE_NAME, path.toUri().getPath()); ret.setAttribute(ATTRIBUTE_QUALIFIED_NAME, subDirQualifiedName);
ret.setRelationshipAttribute( ATTRIBUTE_BUCKET, AtlasTypeUtil.getAtlasRelatedObjectId(bucketEntity, RELATIONSHIP_OZONE_BUCKET_KEY)); ret.setAttribute(ATTRIBUTE_NAME, subDirName);
context.putEntity(pathQualifiedName, ret); if (LOG.isDebugEnabled()) {
LOG.debug("adding entity: typeName={}, qualifiedName={}", ret.getTypeName(), ret.getAttribute(ATTRIBUTE_QUALIFIED_NAME));
}
context.putEntity(subDirQualifiedName, ret);
parentObjId = AtlasTypeUtil.getAtlasRelatedObjectId(ret, RELATIONSHIP_OZONE_PARENT_CHILDREN);
parentPath = subDirPath + Path.SEPARATOR;;
}
} }
if (LOG.isDebugEnabled()) { if (LOG.isDebugEnabled()) {
......
common/src/test/java/org/apache/atlas/utils/AtlasPathExtractorUtilTest.java
...@@ -29,9 +29,7 @@ import org.apache.hadoop.fs.Path; ...@@ -29,9 +29,7 @@ import org.apache.hadoop.fs.Path;
import java.util.HashMap; import java.util.HashMap;
import java.util.Map; import java.util.Map;
import static org.testng.Assert.assertEquals; import static org.testng.Assert.*;
import static org.testng.Assert.assertNotNull;
import static org.testng.Assert.assertNull;
public class AtlasPathExtractorUtilTest { public class AtlasPathExtractorUtilTest {
private static final Logger LOG = LoggerFactory.getLogger(AtlasPathExtractorUtilTest.class); private static final Logger LOG = LoggerFactory.getLogger(AtlasPathExtractorUtilTest.class);
...@@ -83,35 +81,57 @@ public class AtlasPathExtractorUtilTest { ...@@ -83,35 +81,57 @@ public class AtlasPathExtractorUtilTest {
@DataProvider(name = "ozonePathProvider") @DataProvider(name = "ozonePathProvider")
private Object[][] ozonePathProvider(){ private Object[][] ozonePathProvider(){
return new Object[][]{ return new Object[][]{
{ OZONE_SCHEME, "bucket1.volume1.ozone1/files/file.txt", "/files/file.txt" }, { new OzoneKeyValidator(OZONE_SCHEME, "bucket1.volume1.ozone1.com/files/file.txt",
{ OZONE_SCHEME, "bucket1.volume1.ozone1/file21.txt", "/file21.txt" }, "files", "bucket1.volume1.ozone1.com/files",
{ OZONE_SCHEME, "bucket1.volume1.ozone1/quarter_one/sales", "/quarter_one/sales" }, "file.txt", "bucket1.volume1.ozone1.com/files/file.txt")},
{ OZONE_SCHEME, "bucket1.volume1.ozone1/quarter_one/sales/", "/quarter_one/sales" },
{ OZONE_3_SCHEME, "bucket1.volume1.ozone1/files/file.txt", "/files/file.txt" }, { new OzoneKeyValidator(OZONE_SCHEME, "bucket1.volume1.ozone1:1234/file21.txt",
{ OZONE_3_SCHEME, "bucket1.volume1.ozone1/file21.txt", "/file21.txt"}, "file21.txt", "bucket1.volume1.ozone1:1234/file21.txt") },
{ OZONE_3_SCHEME, "bucket1.volume1.ozone1/quarter_one/sales", "/quarter_one/sales" },
{ OZONE_3_SCHEME, "bucket1.volume1.ozone1/quarter_one/sales/", "/quarter_one/sales" }, { new OzoneKeyValidator(OZONE_SCHEME, "bucket1.volume1.ozone1/quarter_one/sales",
"quarter_one", "bucket1.volume1.ozone1/quarter_one",
"sales", "bucket1.volume1.ozone1/quarter_one/sales") },
{ new OzoneKeyValidator(OZONE_SCHEME, "bucket1.volume1.ozone1/quarter_one/sales/",
"quarter_one", "bucket1.volume1.ozone1/quarter_one",
"sales", "bucket1.volume1.ozone1/quarter_one/sales") },
{ new OzoneKeyValidator(OZONE_3_SCHEME, "bucket1.volume1.ozone1/files/file.txt",
"files", "bucket1.volume1.ozone1/files",
"file.txt", "bucket1.volume1.ozone1/files/file.txt") },
{ new OzoneKeyValidator(OZONE_3_SCHEME, "bucket1.volume1.ozone1/file21.txt",
"file21.txt", "bucket1.volume1.ozone1/file21.txt") },
{ new OzoneKeyValidator(OZONE_3_SCHEME, "bucket1.volume1.ozone1/quarter_one/sales",
"quarter_one", "bucket1.volume1.ozone1/quarter_one",
"sales", "bucket1.volume1.ozone1/quarter_one/sales") },
{ new OzoneKeyValidator(OZONE_3_SCHEME, "bucket1.volume1.ozone1/quarter_one/sales/",
"quarter_one", "bucket1.volume1.ozone1/quarter_one",
"sales", "bucket1.volume1.ozone1/quarter_one/sales") },
}; };
} }
@Test(dataProvider = "ozonePathProvider") @Test(dataProvider = "ozonePathProvider")
public void testGetPathEntityOzone3Path(String scheme, String location, String keyName) { public void testGetPathEntityOzone3Path(OzoneKeyValidator validator) {
String ozonePath = scheme + location; String scheme = validator.scheme;
PathExtractorContext extractorContext = new PathExtractorContext(METADATA_NAMESPACE); String ozonePath = scheme + validator.location;
PathExtractorContext extractorContext = new PathExtractorContext(METADATA_NAMESPACE);
Path path = new Path(ozonePath); Path path = new Path(ozonePath);
AtlasEntityWithExtInfo entityWithExtInfo = AtlasPathExtractorUtil.getPathEntity(path, extractorContext); AtlasEntityWithExtInfo entityWithExtInfo = AtlasPathExtractorUtil.getPathEntity(path, extractorContext);
AtlasEntity entity = entityWithExtInfo.getEntity(); AtlasEntity entity = entityWithExtInfo.getEntity();
assertNotNull(entity); assertNotNull(entity);
assertEquals(entity.getTypeName(), OZONE_KEY); verifyOzoneKeyEntity(entity, validator);
verifyOzoneKeyEntity(ozonePath, keyName, entity);
assertEquals(entityWithExtInfo.getReferredEntities().size(), 2); assertEquals(entityWithExtInfo.getReferredEntities().size(), 2);
verifyOzoneEntities(scheme, ozonePath, keyName, entityWithExtInfo.getReferredEntities()); verifyOzoneEntities(entityWithExtInfo.getReferredEntities(), validator);
assertEquals(extractorContext.getKnownEntities().size(), 3); assertEquals(extractorContext.getKnownEntities().size(), validator.knownEntitiesCount);
verifyOzoneEntities(scheme, ozonePath, keyName, extractorContext.getKnownEntities()); verifyOzoneEntities(extractorContext.getKnownEntities(), validator);
} }
@Test @Test
...@@ -244,31 +264,29 @@ public class AtlasPathExtractorUtilTest { ...@@ -244,31 +264,29 @@ public class AtlasPathExtractorUtilTest {
verifyS3KnownEntities(S3A_SCHEME, S3A_PATH, extractorContext.getKnownEntities()); verifyS3KnownEntities(S3A_SCHEME, S3A_PATH, extractorContext.getKnownEntities());
} }
private void verifyOzoneEntities(String scheme, String path, String keyName, Map<String, AtlasEntity> knownEntities) { private void verifyOzoneEntities(Map<String, AtlasEntity> knownEntities, OzoneKeyValidator validator) {
for (AtlasEntity knownEntity : knownEntities.values()) { for (AtlasEntity knownEntity : knownEntities.values()) {
switch (knownEntity.getTypeName()){ switch (knownEntity.getTypeName()){
case OZONE_KEY: case OZONE_KEY:
verifyOzoneKeyEntity(path, keyName, knownEntity); verifyOzoneKeyEntity(knownEntity, validator);
break; break;
case OZONE_VOLUME: case OZONE_VOLUME:
assertEquals(knownEntity.getAttribute(ATTRIBUTE_QUALIFIED_NAME), scheme + "volume1" + QNAME_METADATA_NAMESPACE); assertEquals(knownEntity.getAttribute(ATTRIBUTE_QUALIFIED_NAME), validator.scheme + "volume1" + QNAME_METADATA_NAMESPACE);
assertEquals(knownEntity.getAttribute(ATTRIBUTE_NAME), "volume1"); assertEquals(knownEntity.getAttribute(ATTRIBUTE_NAME), "volume1");
break; break;
case OZONE_BUCKET: case OZONE_BUCKET:
assertEquals(knownEntity.getAttribute(ATTRIBUTE_QUALIFIED_NAME), scheme + "volume1.bucket1" + QNAME_METADATA_NAMESPACE); assertEquals(knownEntity.getAttribute(ATTRIBUTE_QUALIFIED_NAME), validator.scheme + "volume1.bucket1" + QNAME_METADATA_NAMESPACE);
assertEquals(knownEntity.getAttribute(ATTRIBUTE_NAME), "bucket1"); assertEquals(knownEntity.getAttribute(ATTRIBUTE_NAME), "bucket1");
break; break;
} }
} }
} }
private void verifyOzoneKeyEntity(String path, String name, AtlasEntity entity) { private void verifyOzoneKeyEntity(AtlasEntity entity, OzoneKeyValidator validator) {
//remove trailing "/" if present from path assertEquals(entity.getTypeName(), OZONE_KEY);
path = (path.charAt(path.length()-1) == '/') ? path.substring(0, path.length()-1) : path; assertTrue(validator.validateNameQName(entity));
assertEquals(entity.getAttribute(ATTRIBUTE_QUALIFIED_NAME), path + QNAME_METADATA_NAMESPACE);
assertEquals(entity.getAttribute(ATTRIBUTE_NAME), name);
} }
private void verifyHDFSEntity(AtlasEntity entity, boolean toLowerCase) { private void verifyHDFSEntity(AtlasEntity entity, boolean toLowerCase) {
...@@ -392,4 +410,42 @@ public class AtlasPathExtractorUtilTest { ...@@ -392,4 +410,42 @@ public class AtlasPathExtractorUtilTest {
assertEquals(entity.getAttribute(ATTRIBUTE_QUALIFIED_NAME), scheme + "aws_my_bucket1" + QNAME_METADATA_NAMESPACE); assertEquals(entity.getAttribute(ATTRIBUTE_QUALIFIED_NAME), scheme + "aws_my_bucket1" + QNAME_METADATA_NAMESPACE);
assertEquals(entity.getAttribute(ATTRIBUTE_NAME), "aws_my_bucket1"); assertEquals(entity.getAttribute(ATTRIBUTE_NAME), "aws_my_bucket1");
} }
private class OzoneKeyValidator {
private final String scheme;
private final String location;
private final int knownEntitiesCount;
private final Map<String, String> nameQNamePairs;
public OzoneKeyValidator(String scheme, String location, String... pairs) {
this.scheme = scheme;
this.location = location;
this.nameQNamePairs = getPairMap(scheme, pairs);
this.knownEntitiesCount = nameQNamePairs.size() + 2;
}
public boolean validateNameQName(AtlasEntity entity){
String name = (String) entity.getAttribute(ATTRIBUTE_NAME);
if (this.nameQNamePairs.containsKey(name)){
String qName = (String) entity.getAttribute(ATTRIBUTE_QUALIFIED_NAME);
if (qName.equals(this.nameQNamePairs.get(name))) {
return true;
}
}
return false;
}
private Map<String, String> getPairMap(String scheme, String... pairs){
Map< String, String > ret = new HashMap<>();
for (int i = 0; i < pairs.length; i += 2) {
ret.put(pairs[i], scheme + pairs[i+1] + QNAME_METADATA_NAMESPACE);
}
return ret;
}
}
} }
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment