Skip to content

A
atlas
Commit 19d67a13 by rmani Committed by Madhan Neethiraj
Options

ATLAS-2446: updated HBase Model to capture more attributes of table and column-family

Signed-off-by: 's avatarMadhan Neethiraj <madhan@apache.org>
parent a2ccdf12
Showing with 240 additions and 0 deletions
addons/hbase-bridge/src/main/java/org/apache/atlas/hbase/bridge/HBaseAtlasHook.java
......@@ -92,6 +92,30 @@ public class HBaseAtlasHook extends AtlasHook {
public static final String ATTR_OWNER = "owner";
public static final String ATTR_NAME = "name";
// column addition metadata
public static final String ATTR_TABLE_MAX_FILESIZE = "maxFileSize";
public static final String ATTR_TABLE_ISREADONLY = "isReadOnly";
public static final String ATTR_TABLE_ISCOMPACTION_ENABLED = "isCompactionEnabled";
public static final String ATTR_TABLE_REPLICATION_PER_REGION = "replicasPerRegion";
public static final String ATTR_TABLE_DURABLILITY = "durability";
// column family additional metadata
public static final String ATTR_CF_BLOOMFILTER_TYPE = "bloomFilterType";
public static final String ATTR_CF_COMPRESSION_TYPE = "compressionType";
public static final String ATTR_CF_COMPACTION_COMPRESSION_TYPE = "compactionCompressionType";
public static final String ATTR_CF_ENCRYPTION_TYPE = "encryptionType";
public static final String ATTR_CF_KEEP_DELETE_CELLS = "keepDeletedCells";
public static final String ATTR_CF_MAX_VERSIONS = "maxVersions";
public static final String ATTR_CF_MIN_VERSIONS = "minVersions";
public static final String ATTR_CF_DATA_BLOCK_ENCODING = "dataBlockEncoding";
public static final String ATTR_CF_TTL = "ttl";
public static final String ATTR_CF_BLOCK_CACHE_ENABLED = "blockCacheEnabled";
public static final String ATTR_CF_CACHED_BLOOM_ON_WRITE = "cacheBloomsOnWrite";
public static final String ATTR_CF_CACHED_DATA_ON_WRITE = "cacheDataOnWrite";
public static final String ATTR_CF_CACHED_INDEXES_ON_WRITE = "cacheIndexesOnWrite";
public static final String ATTR_CF_EVICT_BLOCK_ONCLOSE = "evictBlocksOnClose";
public static final String ATTR_CF_PREFETCH_BLOCK_ONOPEN = "prefetchBlocksOnOpen";
private static final String REFERENCEABLE_ATTRIBUTE_NAME = "qualifiedName";
private String clusterName = null;
......@@ -422,12 +446,25 @@ public class HBaseAtlasHook extends AtlasHook {
table.setAttribute(ATTR_PARAMETERS, hbaseOperationContext.getHbaseConf());
table.setAttribute(ATTR_NAMESPACE, AtlasTypeUtil.getAtlasObjectId(nameSpace));
HTableDescriptor htableDescriptor = hbaseOperationContext.gethTableDescriptor();
if (htableDescriptor != null) {
table.setAttribute(ATTR_TABLE_MAX_FILESIZE, htableDescriptor.getMaxFileSize());
table.setAttribute(ATTR_TABLE_REPLICATION_PER_REGION, htableDescriptor.getRegionReplication());
table.setAttribute(ATTR_TABLE_ISREADONLY, htableDescriptor.getMaxFileSize());
table.setAttribute(ATTR_TABLE_ISCOMPACTION_ENABLED, htableDescriptor.isCompactionEnabled());
table.setAttribute(ATTR_TABLE_DURABLILITY, (htableDescriptor.getDurability() != null ? htableDescriptor.getDurability().name() : null));
}
switch (operation) {
case CREATE_TABLE:
table.setAttribute(ATTR_CREATE_TIME, now);
table.setAttribute(ATTR_MODIFIED_TIME, now);
break;
case CREATE_COLUMN_FAMILY:
table.setAttribute(ATTR_MODIFIED_TIME, now);
break;
case ALTER_TABLE:
case ALTER_COLUMN_FAMILY:
table.setAttribute(ATTR_MODIFIED_TIME, now);
break;
default:
......@@ -466,8 +503,27 @@ public class HBaseAtlasHook extends AtlasHook {
columnFamily.setAttribute(ATTR_OWNER, hbaseOperationContext.getOwner());
columnFamily.setAttribute(ATTR_TABLE, AtlasTypeUtil.getAtlasObjectId(table));
if (hColumnDescriptor!= null) {
columnFamily.setAttribute(ATTR_CF_BLOCK_CACHE_ENABLED, hColumnDescriptor.isBlockCacheEnabled());
columnFamily.setAttribute(ATTR_CF_BLOOMFILTER_TYPE, (hColumnDescriptor.getBloomFilterType() != null ? hColumnDescriptor.getBloomFilterType().name():null));
columnFamily.setAttribute(ATTR_CF_CACHED_BLOOM_ON_WRITE, hColumnDescriptor.isCacheBloomsOnWrite());
columnFamily.setAttribute(ATTR_CF_CACHED_DATA_ON_WRITE, hColumnDescriptor.isCacheBloomsOnWrite());
columnFamily.setAttribute(ATTR_CF_CACHED_INDEXES_ON_WRITE, hColumnDescriptor.isCacheIndexesOnWrite());
columnFamily.setAttribute(ATTR_CF_COMPACTION_COMPRESSION_TYPE, (hColumnDescriptor.getCompactionCompressionType() != null ? hColumnDescriptor.getCompactionCompressionType().name():null));
columnFamily.setAttribute(ATTR_CF_COMPRESSION_TYPE, (hColumnDescriptor.getCompressionType() != null ? hColumnDescriptor.getCompressionType().name():null));
columnFamily.setAttribute(ATTR_CF_DATA_BLOCK_ENCODING, (hColumnDescriptor.getDataBlockEncoding() != null ? hColumnDescriptor.getDataBlockEncoding().name():null));
columnFamily.setAttribute(ATTR_CF_ENCRYPTION_TYPE, hColumnDescriptor.getEncryptionType());
columnFamily.setAttribute(ATTR_CF_EVICT_BLOCK_ONCLOSE, hColumnDescriptor.isEvictBlocksOnClose());
columnFamily.setAttribute(ATTR_CF_KEEP_DELETE_CELLS, ( hColumnDescriptor.getKeepDeletedCells() != null ? hColumnDescriptor.getKeepDeletedCells().name():null));
columnFamily.setAttribute(ATTR_CF_MAX_VERSIONS, hColumnDescriptor.getMaxVersions());
columnFamily.setAttribute(ATTR_CF_MIN_VERSIONS, hColumnDescriptor.getMinVersions());
columnFamily.setAttribute(ATTR_CF_PREFETCH_BLOCK_ONOPEN, hColumnDescriptor.isPrefetchBlocksOnOpen());
columnFamily.setAttribute(ATTR_CF_TTL, hColumnDescriptor.getTimeToLive());
}
switch (hbaseOperationContext.getOperation()) {
case CREATE_COLUMN_FAMILY:
case CREATE_TABLE:
columnFamily.setAttribute(ATTR_CREATE_TIME, now);
columnFamily.setAttribute(ATTR_MODIFIED_TIME, now);
break;
......
addons/models/1000-Hadoop/patches/005-hbase_table_column_family_add_additional_attribute.json 0 → 100644
{
"patches": [
{
"action": "ADD_ATTRIBUTE",
"typeName": "hbase_table",
"applyToVersion": "1.2",
"updateToVersion": "1.3",
"params": null,
"attributeDefs": [
{
"name": "maxFileSize",
"typeName": "int",
"cardinality": "SINGLE",
"isIndexable": false,
"isOptional": true,
"isUnique": false
},
{
"name": "isReadOnly",
"typeName": "boolean",
"cardinality": "SINGLE",
"isIndexable": false,
"isOptional": true,
"isUnique": false
},
{
"name": "isCompactionEnabled",
"typeName": "boolean",
"cardinality": "SINGLE",
"isIndexable": false,
"isOptional": true,
"isUnique": false
},
{
"name": "replicasPerRegion",
"typeName": "int",
"cardinality": "SINGLE",
"isIndexable": false,
"isOptional": true,
"isUnique": false
},
{
"name": "durability",
"typeName": "string",
"cardinality": "SINGLE",
"isIndexable": false,
"isOptional": true,
"isUnique": false
}
]
},
{
"action": "ADD_ATTRIBUTE",
"typeName": "hbase_column_family",
"applyToVersion": "1.1",
"updateToVersion": "1.2",
"params": null,
"attributeDefs": [
{
"name": "bloomFilterType",
"typeName": "string",
"cardinality": "SINGLE",
"isIndexable": false,
"isOptional": true,
"isUnique": false
},
{
"name": "compressionType",
"typeName": "string",
"cardinality": "SINGLE",
"isIndexable": false,
"isOptional": true,
"isUnique": false
},
{
"name": "compactionCompressionType",
"typeName": "string",
"cardinality": "SINGLE",
"isIndexable": false,
"isOptional": true,
"isUnique": false
},
{
"name": "encryptionType",
"typeName": "string",
"cardinality": "SINGLE",
"isIndexable": false,
"isOptional": true,
"isUnique": false
},
{
"name": "keepDeletedCells",
"typeName": "boolean",
"cardinality": "SINGLE",
"isIndexable": false,
"isOptional": true,
"isUnique": false
},
{
"name": "maxVersions",
"typeName": "int",
"cardinality": "SINGLE",
"isIndexable": false,
"isOptional": true,
"isUnique": false
},
{
"name": "minVersions",
"typeName": "int",
"cardinality": "SINGLE",
"isIndexable": false,
"isOptional": true,
"isUnique": false
},
{
"name": "dataBlockEncoding",
"typeName": "string",
"cardinality": "SINGLE",
"isIndexable": false,
"isOptional": true,
"isUnique": false
},
{
"name": "ttl",
"typeName": "int",
"cardinality": "SINGLE",
"isIndexable": false,
"isOptional": true,
"isUnique": false
},
{
"name": "blockCacheEnabled",
"typeName": "boolean",
"cardinality": "SINGLE",
"isIndexable": false,
"isOptional": true,
"isUnique": false
},
{
"name": "cacheBloomsOnWrite",
"typeName": "boolean",
"cardinality": "SINGLE",
"isIndexable": false,
"isOptional": true,
"isUnique": false
},
{
"name": "cacheDataOnWrite",
"typeName": "boolean",
"cardinality": "SINGLE",
"isIndexable": false,
"isOptional": true,
"isUnique": false
},
{
"name": "cacheIndexesOnWrite",
"typeName": "boolean",
"cardinality": "SINGLE",
"isIndexable": false,
"isOptional": true,
"isUnique": false
},
{
"name": "evictBlocksOnClose",
"typeName": "boolean",
"cardinality": "SINGLE",
"isIndexable": false,
"isOptional": true,
"isUnique": false
},
{
"name": "prefetchBlocksOnOpen",
"typeName": "boolean",
"cardinality": "SINGLE",
"isIndexable": false,
"isOptional": true,
"isUnique": false
}
]
}
]
}
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment