Skip to content

A
atlas
Commit 4ce1e370 by Le Ma Committed by Madhan Neethiraj
Options

ATLAS-3292: remove legacy attributes from hbase/kafka/storm/falcon/avro models

Signed-off-by: 's avatarMadhan Neethiraj <madhan@apache.org>
parent 37e441c1
Showing with 224 additions and 222 deletions
addons/models/1000-Hadoop/1050-falcon_model.json
......@@ -55,14 +55,6 @@
"isUnique": false
},
{
"name": "stored-in",
"typeName": "falcon_cluster",
"cardinality": "SINGLE",
"isIndexable": true,
"isOptional": false,
"isUnique": false
},
{
"name": "groups",
"typeName": "string",
"cardinality": "SINGLE",
......@@ -97,14 +89,6 @@
"isUnique": false
},
{
"name": "runs-on",
"typeName": "falcon_cluster",
"cardinality": "SINGLE",
"isIndexable": true,
"isOptional": false,
"isUnique": false
},
{
"name": "tags",
"typeName": "map<string,string>",
"cardinality": "SINGLE",
......@@ -137,16 +121,7 @@
],
"serviceType": "falcon",
"typeVersion": "1.0",
"attributeDefs": [
{
"name": "stored-in",
"typeName": "falcon_cluster",
"cardinality": "SINGLE",
"isIndexable": true,
"isOptional": false,
"isUnique": false
}
]
"attributeDefs": []
}
],
"relationshipDefs": [
......@@ -155,6 +130,7 @@
"serviceType": "falcon",
"typeVersion": "1.0",
"relationshipCategory": "COMPOSITION",
"relationshipLabel": "__falcon_feed.stored-in",
"endDef1": {
"type": "falcon_feed",
"name": "stored-in",
......@@ -175,6 +151,7 @@
"serviceType": "falcon",
"typeVersion": "1.0",
"relationshipCategory": "COMPOSITION",
"relationshipLabel": "__falcon_cluster.processes",
"endDef1": {
"type": "falcon_cluster",
"name": "processes",
......@@ -195,6 +172,7 @@
"serviceType": "falcon",
"typeVersion": "1.0",
"relationshipCategory": "COMPOSITION",
"relationshipLabel": "__falcon_cluster.feedCreations",
"endDef1": {
"type": "falcon_cluster",
"name": "feedCreations",
......
addons/models/1000-Hadoop/1060-hbase_model.json
......@@ -55,33 +55,12 @@
"typeVersion": "1.1",
"attributeDefs": [
{
"name": "namespace",
"typeName": "hbase_namespace",
"cardinality": "SINGLE",
"isIndexable": false,
"isOptional": true,
"isUnique": false
},
{
"name": "uri",
"typeName": "string",
"cardinality": "SINGLE",
"isIndexable": false,
"isOptional": false,
"isUnique": false
},
{
"name": "column_families",
"typeName": "array<hbase_column_family>",
"cardinality": "SET",
"constraints": [
{
"type": "ownedRef"
}
],
"isIndexable": false,
"isOptional": true,
"isUnique": false
}
]
},
......@@ -90,65 +69,18 @@
"superTypes": [
"DataSet"
],
"attributeDefs": [
{
"name": "table",
"typeName": "hbase_table",
"cardinality": "SINGLE",
"isIndexable": false,
"isOptional": false,
"isUnique": false,
"constraints": [
{
"type": "inverseRef",
"params": {
"attribute": "column_families"
}
}
]
},
{
"name": "columns",
"typeName": "array<hbase_column>",
"cardinality": "SET",
"constraints": [
{
"type": "ownedRef"
}
],
"isIndexable": false,
"isOptional": true,
"isUnique": false
}
],
"serviceType": "hbase",
"typeVersion": "1.0"
"typeVersion": "1.0",
"attributeDefs": []
},
{
"name": "hbase_column",
"superTypes": [
"DataSet"
],
"attributeDefs": [
{
"name": "column_family",
"typeName": "hbase_column_family",
"cardinality": "SINGLE",
"isIndexable": false,
"isOptional": false,
"isUnique": false,
"constraints": [
{
"type": "inverseRef",
"params": {
"attribute": "columns"
}
}
]
}
],
"serviceType": "hbase",
"typeVersion": "1.0"
"typeVersion": "1.0",
"attributeDefs": []
}
],
"relationshipDefs": [
......@@ -157,6 +89,7 @@
"serviceType": "hbase",
"typeVersion": "1.0",
"relationshipCategory": "COMPOSITION",
"relationshipLabel": "__hbase_table.namespace",
"endDef1": {
"type": "hbase_table",
"name": "namespace",
......@@ -177,6 +110,7 @@
"serviceType": "hbase",
"typeVersion": "1.0",
"relationshipCategory": "COMPOSITION",
"relationshipLabel": "__hbase_table.column_families",
"endDef1": {
"type": "hbase_table",
"name": "column_families",
......@@ -198,6 +132,7 @@
"serviceType": "hbase",
"typeVersion": "1.0",
"relationshipCategory": "COMPOSITION",
"relationshipLabel": "__hbase_column_family.columns",
"endDef1": {
"type": "hbase_column_family",
"name": "columns",
......
addons/models/1000-Hadoop/1065-avro_model.json
......@@ -24,14 +24,6 @@
"typeVersion": "1.0",
"attributeDefs": [
{
"name": "type",
"typeName": "array<avro_type>",
"cardinality": "LIST",
"isIndexable": true,
"isOptional": false,
"isUnique": false
},
{
"name": "doc",
"typeName": "string",
"cardinality": "SINGLE",
......@@ -75,19 +67,6 @@
"isUnique": false
},
{
"name": "fields",
"typeName": "array<avro_field>",
"cardinality": "LIST",
"isIndexable": true,
"isOptional": true,
"isUnique": false,
"constraints": [
{
"type": "ownedRef"
}
]
},
{
"name": "avro_notation",
"typeName": "string",
"cardinality": "SINGLE",
......@@ -137,14 +116,6 @@
"isIndexable": true,
"isOptional": false,
"isUnique": false
},
{
"name": "associatedEntities",
"typeName": "array<DataSet>",
"cardinality": "LIST",
"isIndexable": false,
"isOptional": true,
"isUnique": false
}
]
},
......@@ -156,8 +127,7 @@
],
"serviceType": "avro",
"typeVersion": "1.0",
"attributeDefs": [
]
"attributeDefs": []
},
{
"name": "avro_fixed",
......@@ -167,8 +137,7 @@
],
"serviceType": "avro",
"typeVersion": "1.0",
"attributeDefs": [
]
"attributeDefs": []
},
{
"name": "avro_enum",
......@@ -231,6 +200,7 @@
"serviceType": "avro",
"typeVersion": "1.0",
"relationshipCategory": "ASSOCIATION",
"relationshipLabel": "__avro_schema.associatedEntities",
"endDef1": {
"type": "avro_schema",
"name": "associatedEntities",
......@@ -251,6 +221,7 @@
"serviceType": "avro",
"typeVersion": "1.0",
"relationshipCategory": "AGGREGATION",
"relationshipLabel": "__avro_record.fields",
"endDef1": {
"type": "avro_record",
"name": "fields",
......@@ -271,6 +242,7 @@
"serviceType": "avro",
"typeVersion": "1.0",
"relationshipCategory": "ASSOCIATION",
"relationshipLabel": "__avro_field.type",
"endDef1": {
"type": "avro_field",
"name": "type",
......
addons/models/1000-Hadoop/1070-kafka_model.json
......@@ -29,14 +29,6 @@
"isUnique": false
},
{
"name": "avroSchema",
"typeName": "array<avro_type>",
"isOptional": true,
"cardinality": "LIST",
"isUnique": false,
"isIndexable": true
},
{
"name": "replicationFactorNational",
"typeName": "int",
"isOptional": true,
......@@ -207,6 +199,7 @@
"serviceType": "kafka",
"typeVersion": "1.0",
"relationshipCategory": "ASSOCIATION",
"relationshipLabel": "__kafka_topic.avroSchema",
"endDef1": {
"type": "kafka_topic",
"name": "avroSchema",
......
addons/models/1000-Hadoop/1080-storm_model.json
......@@ -51,15 +51,6 @@
"isOptional": true,
"includeInNotification": true,
"isUnique": false
},
{
"name": "nodes",
"typeName": "array<storm_node>",
"cardinality": "LIST",
"constraints": [],
"isIndexable": false,
"isOptional": false,
"isUnique": false
}
]
},
......@@ -154,6 +145,7 @@
"serviceType": "storm",
"typeVersion": "1.0",
"relationshipCategory": "ASSOCIATION",
"relationshipLabel": "__storm_topology.nodes",
"endDef1": {
"type": "storm_topology",
"name": "nodes",
......
addons/models/1000-Hadoop/patches/010-hbase_remove_legacy_attributes.json 0 → 100644
{
"patches": [
{
"id": "TYPEDEF_PATCH_1000_010_001",
"description": "Remove legacy reference attribute 'namespace' from hbase_table",
"action": "REMOVE_LEGACY_REF_ATTRIBUTES",
"typeName": "hbase_table_namespace",
"applyToVersion": "1.1",
"updateToVersion": "1.2",
"params": {
"relationshipLabel": "__hbase_table.namespace",
"relationshipCategory": "COMPOSITION"
}
},
{
"id": "TYPEDEF_PATCH_1000_010_002",
"description": "Remove legacy reference attribute 'column_families' from hbase_table",
"action": "REMOVE_LEGACY_REF_ATTRIBUTES",
"typeName": "hbase_table_column_families",
"applyToVersion": "1.1",
"updateToVersion": "1.2",
"params": {
"relationshipLabel": "__hbase_table.column_families"
}
},
{
"id": "TYPEDEF_PATCH_1000_010_003",
"description": "Remove legacy reference attribute 'columns' from hbase_column_family",
"action": "REMOVE_LEGACY_REF_ATTRIBUTES",
"typeName": "hbase_column_family_columns",
"applyToVersion": "1.1",
"updateToVersion": "1.2",
"params": {
"relationshipLabel": "__hbase_column_family.columns",
"relationshipCategory": "COMPOSITION"
}
}
]
}
addons/models/1000-Hadoop/patches/011-falcon_remove_legacy_atrributes.json 0 → 100644
{
"patches": [
{
"id": "TYPEDEF_PATCH_1000_011_001",
"description": "Remove legacy reference attribute 'stored-in' from falcon_feed",
"action": "REMOVE_LEGACY_REF_ATTRIBUTES",
"typeName": "falcon_feed_cluster",
"applyToVersion": "1.1",
"updateToVersion": "1.2",
"params": {
"relationshipLabel": "__falcon_feed.stored-in",
"relationshipCategory": "COMPOSITION"
}
},
{
"id": "TYPEDEF_PATCH_1000_011_002",
"description": "Remove legacy reference attribute 'runs-on' from falcon_process",
"action": "REMOVE_LEGACY_REF_ATTRIBUTES",
"typeName": "falcon_cluster_process",
"applyToVersion": "1.1",
"updateToVersion": "1.2",
"params": {
"relationshipLabel": "__falcon_cluster.processes"
}
},
{
"id": "TYPEDEF_PATCH_1000_011_003",
"description": "Remove legacy reference attribute 'stored-in' from falcon_feed_creation",
"action": "REMOVE_LEGACY_REF_ATTRIBUTES",
"typeName": "falcon_cluster_feed_creation",
"applyToVersion": "1.1",
"updateToVersion": "1.2",
"params": {
"relationshipLabel": "__falcon_cluster.feedCreations"
}
}
]
}
\ No newline at end of file
addons/models/1000-Hadoop/patches/012-avro_remove_legacy_attributs.json 0 → 100644
{
"patches": [
{
"id": "TYPEDEF_PATCH_1000_012_001",
"description": "Remove legacy reference attribute 'associatedEntities' from avro_schema",
"action": "REMOVE_LEGACY_REF_ATTRIBUTES",
"typeName": "avro_schema_associatedEntities",
"applyToVersion": "1.1",
"updateToVersion": "1.2",
"params": {
"relationshipLabel": "__avro_schema.associatedEntities"
}
},
{
"id": "TYPEDEF_PATCH_1000_012_002",
"description": "Remove legacy reference attribute 'fields' from avro_record",
"action": "REMOVE_LEGACY_REF_ATTRIBUTES",
"typeName": "avro_record_fields",
"applyToVersion": "1.1",
"updateToVersion": "1.2",
"params": {
"relationshipLabel": "__avro_record.fields"
}
},
{
"id": "TYPEDEF_PATCH_1000_012_003",
"description": "Remove legacy reference attribute 'type' from avro_field",
"action": "REMOVE_LEGACY_REF_ATTRIBUTES",
"typeName": "avro_field_types",
"applyToVersion": "1.1",
"updateToVersion": "1.2",
"params": {
"relationshipLabel": "__avro_field.type"
}
}
]
}
\ No newline at end of file
addons/models/1000-Hadoop/patches/013-kafka_remove_legacy_attributes.json 0 → 100644
{
"patches": [
{
"id": "YPEDEF_PATCH_1000_013_001",
"description": "Remove legacy reference attribute 'avroSchema' from kafka_topic",
"action": "REMOVE_LEGACY_REF_ATTRIBUTES",
"typeName": "kafka_topic_avroSchema",
"applyToVersion": "1.1",
"updateToVersion": "1.2",
"params": {
"relationshipLabel": "__kafka_topic.avroSchema",
"relationshipCategory": "ASSOCIATION"
}
}
]
}
\ No newline at end of file
addons/models/1000-Hadoop/patches/014-storm_remove_legacy_attributes.json 0 → 100644
{
"patches": [
{
"id": "TYPEDEF_PATCH_1000_014_001",
"description": "Remove legacy reference attribute 'nodes' from storm_topology",
"action": "REMOVE_LEGACY_REF_ATTRIBUTES",
"typeName": "storm_topology_nodes",
"applyToVersion": "1.1",
"updateToVersion": "1.2",
"params": {
"relationshipLabel": "__storm_topology.nodes"
}
}
]
}
\ No newline at end of file
addons/models/3000-Cloud/3020-aws_s3_typedefs.json
......@@ -60,22 +60,6 @@
"typeVersion": "1.0",
"attributeDefs": [
{
"name": "pseudoDirectory",
"typeName": "aws_s3_pseudo_dir",
"cardinality": "SINGLE",
"isIndexable": false,
"isOptional": false,
"isUnique": false,
"constraints": [
{
"type": "inverseRef",
"params": {
"attribute": "s3Objects"
}
}
]
},
{
"name": "dataType",
"typeName": "string",
"cardinality": "SINGLE",
......@@ -84,14 +68,6 @@
"isUnique": false
},
{
"name": "avroSchema",
"typeName": "avro_schema",
"cardinality": "SINGLE",
"isIndexable": true,
"isOptional": true,
"isUnique": false
},
{
"name": "compressionType",
"typeName": "string",
"cardinality": "SINGLE",
......@@ -133,43 +109,6 @@
"isUnique": true
},
{
"name": "bucket",
"typeName": "aws_s3_bucket",
"cardinality": "SINGLE",
"isIndexable": false,
"isOptional": false,
"isUnique": false,
"constraints": [
{
"type": "inverseRef",
"params": {
"attribute": "pseudoDirectories"
}
}
]
},
{
"name": "s3Objects",
"typeName": "array<aws_s3_object>",
"cardinality": "SINGLE",
"isIndexable": false,
"isOptional": true,
"isUnique": false,
"constraints": [
{
"type": "ownedRef"
}
]
},
{
"name": "avroSchemas",
"typeName": "array<avro_schema>",
"cardinality": "SINGLE",
"isIndexable": true,
"isOptional": true,
"isUnique": false
},
{
"name": "subdomain",
"typeName": "string",
"cardinality": "SINGLE",
......@@ -243,19 +182,6 @@
"isUnique": false
},
{
"name": "pseudoDirectories",
"typeName": "array<aws_s3_pseudo_dir>",
"cardinality": "SINGLE",
"isIndexable": false,
"isOptional": true,
"isUnique": false,
"constraints": [
{
"type": "ownedRef"
}
]
},
{
"name": "region",
"typeName": "string",
"cardinality": "SINGLE",
......@@ -328,6 +254,7 @@
"serviceType": "aws",
"typeVersion": "1.0",
"relationshipCategory": "COMPOSITION",
"relationshipLabel": "__aws_s3_bucket.pseudoDirectories",
"endDef1": {
"type": "aws_s3_bucket",
"name": "pseudoDirectories",
......@@ -349,6 +276,7 @@
"serviceType": "aws",
"typeVersion": "1.0",
"relationshipCategory": "COMPOSITION",
"relationshipLabel": "__aws_s3_pseudo_dir.s3Objects",
"endDef1": {
"type": "aws_s3_pseudo_dir",
"name": "s3Objects",
......@@ -370,6 +298,7 @@
"serviceType": "aws",
"typeVersion": "1.0",
"relationshipCategory": "ASSOCIATION",
"relationshipLabel": "__aws_s3_object.avroSchema",
"endDef1": {
"type": "aws_s3_object",
"name": "avroSchema",
......@@ -390,6 +319,7 @@
"serviceType": "aws",
"typeVersion": "1.0",
"relationshipCategory": "ASSOCIATION",
"relationshipLabel": "__aws_s3_pseudo_dir.avroSchemas",
"endDef1": {
"type": "aws_s3_pseudo_dir",
"name": "avroSchemas",
......
addons/models/3000-Cloud/patches/003-cloud_model_remove_legacy_attributes.json 0 → 100644
{
"patches": [
{
"id": "TYPEDEF_PATCH_3000_003_001",
"description": "Remove legacy reference attribute 's3Objects' from aws_s3_pseudo_dir",
"action": "REMOVE_LEGACY_REF_ATTRIBUTES",
"typeName": "aws_s3_pseudo_dir_aws_objects",
"applyToVersion": "1.1",
"updateToVersion": "1.2",
"params": {
"relationshipLabel": "__aws_s3_pseudo_dir.s3Objects",
"relationshipCategory": "COMPOSITION"
}
},
{
"id": "TYPEDEF_PATCH_3000_003_002",
"description": "Remove legacy reference attribute 'pseudoDirectories' from aws_s3_bucket",
"action": "REMOVE_LEGACY_REF_ATTRIBUTES",
"typeName": "aws_s3_bucket_aws_s3_pseudo_dirs",
"applyToVersion": "1.1",
"updateToVersion": "1.2",
"params": {
"relationshipLabel": "__aws_s3_bucket.pseudoDirectories",
"relationshipCategory": "COMPOSITION"
}
},
{
"id": "TYPEDEF_PATCH_3000_003_003",
"description": "Remove legacy reference attribute 'avroSchema from aws_s3_object",
"action": "REMOVE_LEGACY_REF_ATTRIBUTES",
"typeName": "aws_s3_object_avro_schema",
"applyToVersion": "1.1",
"updateToVersion": "1.2",
"params": {
"relationshipLabel": "__aws_s3_object.avroSchema",
"relationshipCategory": "ASSOCIATION"
}
},
{
"id": "TYPEDEF_PATCH_3000_003_004",
"description": "Remove legacy reference attribute 'avroSchemas' from aws_s3_pseudo_dir",
"action": "REMOVE_LEGACY_REF_ATTRIBUTES",
"typeName": "aws_s3_pseudo_dir_avro_schema",
"applyToVersion": "1.1",
"updateToVersion": "1.2",
"params": {
"relationshipLabel": "__aws_s3_pseudo_dir.avroSchemas",
"relationshipCategory": "ASSOCIATION"
}
}
]
}
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment