ATLAS-3214: Create Spark models in atlas to support spark lineage

Signed-off-by: Sarath Subramanian <ssubramanian@cloudera.com>

ATLAS-3214: Create Spark models in atlas to support spark lineage
fce87e33 · Aadarsh Jajodia · Sarath Subramanian · 8d30f75b · fce87e33
Commit fce87e33 authored 6 years ago by Aadarsh Jajodia Committed by Sarath Subramanian 6 years ago
Hide whitespace changes
Inline Side-by-side

Showing with 478 additions and 0 deletions

1100-spark_model.json addons/models/1000-Hadoop/1100-spark_model.json +478 -0

No files found.
--- a/addons/models/1000-Hadoop/1100-spark_model.json
+++ b/addons/models/1000-Hadoop/1100-spark_model.json
+{
+  "enumDefs": [],
+  "structDefs": [],
+  "classificationDefs": [],
+  "entityDefs": [
+    {
+      "name": "spark_db",
+      "superTypes": [
+        "DataSet"
+      ],
+      "serviceType": "spark",
+      "typeVersion": "1.0",
+      "attributeDefs": [
+        {
+          "name": "location",
+          "typeName": "string",
+          "isOptional": true,
+          "cardinality": "SINGLE",
+          "isUnique": false,
+          "isIndexable": false,
+          "searchWeight": 5
+        },
+        {
+          "name": "clusterName",
+          "typeName": "string",
+          "isOptional": true,
+          "cardinality": "SINGLE",
+          "isUnique": false,
+          "isIndexable": false,
+          "searchWeight": 8
+        },
+        {
+          "name": "parameters",
+          "typeName": "map<string,string>",
+          "isOptional": true,
+          "cardinality": "SINGLE",
+          "isUnique": false,
+          "isIndexable": false
+        },
+        {
+          "name": "ownerType",
+          "typeName": "string",
+          "isOptional": true,
+          "cardinality": "SINGLE",
+          "isUnique": false,
+          "isIndexable": false
+        }
+      ]
+    },
+    {
+      "name": "spark_table",
+      "superTypes": [
+        "DataSet"
+      ],
+      "serviceType": "spark",
+      "typeVersion": "1.0",
+      "attributeDefs": [
+        {
+          "name": "tableType",
+          "typeName": "string",
+          "isOptional": true,
+          "cardinality": "SINGLE",
+          "isUnique": false,
+          "isIndexable": false
+        },
+        {
+          "name": "provider",
+          "typeName": "string",
+          "isOptional": true,
+          "cardinality": "SINGLE",
+          "isUnique": false,
+          "isIndexable": false,
+          "searchWeight": 5
+        },
+        {
+          "name": "partitionColumnNames",
+          "typeName": "array<string>",
+          "isOptional": true,
+          "cardinality": "SINGLE",
+          "isUnique": false,
+          "isIndexable": false
+        },
+        {
+          "name": "bucketSpec",
+          "typeName": "map<string,string>",
+          "isOptional": true,
+          "cardinality": "SINGLE",
+          "isUnique": false,
+          "isIndexable": false
+        },
+        {
+          "name": "ownerType",
+          "typeName": "string",
+          "isOptional": true,
+          "cardinality": "SINGLE",
+          "isUnique": false,
+          "isIndexable": false
+        },
+        {
+          "name": "createTime",
+          "typeName": "date",
+          "isOptional": true,
+          "cardinality": "SINGLE",
+          "isUnique": false,
+          "isIndexable": false
+        },
+        {
+          "name": "parameters",
+          "typeName": "map<string,string>",
+          "isOptional": true,
+          "cardinality": "SINGLE",
+          "isUnique": false,
+          "isIndexable": false
+        },
+        {
+          "name": "comment",
+          "typeName": "string",
+          "isOptional": true,
+          "cardinality": "SINGLE",
+          "isUnique": false,
+          "isIndexable": false,
+          "searchWeight": 9
+        },
+        {
+          "name": "unsupportedFeatures",
+          "typeName": "array<string>",
+          "isOptional": true,
+          "cardinality": "SINGLE",
+          "isUnique": false,
+          "isIndexable": false
+        },
+        {
+          "name": "viewOriginalText",
+          "typeName": "string",
+          "isOptional": true,
+          "cardinality": "SINGLE",
+          "isUnique": false,
+          "isIndexable": false,
+          "searchWeight": 9
+        },
+        {
+          "name": "schemaDesc",
+          "typeName": "string",
+          "isOptional": true,
+          "cardinality": "SINGLE",
+          "isUnique": false,
+          "isIndexable": false,
+          "searchWeight": 5
+        },
+        {
+          "name": "partitionProvider",
+          "typeName": "string",
+          "isOptional": true,
+          "cardinality": "SINGLE",
+          "isUnique": false,
+          "isIndexable": false
+        }
+      ]
+    },
+    {
+      "name": "spark_column",
+      "superTypes": [
+        "DataSet"
+      ],
+      "serviceType": "spark",
+      "typeVersion": "1.0",
+      "attributeDefs": [
+        {
+          "name": "type",
+          "typeName": "string",
+          "isOptional": false,
+          "cardinality": "SINGLE",
+          "isUnique": false,
+          "isIndexable": true
+        },
+        {
+          "name": "nullable",
+          "typeName": "boolean",
+          "isOptional": true,
+          "cardinality": "SINGLE",
+          "isUnique": false,
+          "isIndexable": false
+        },
+        {
+          "name": "metadata",
+          "typeName": "string",
+          "isOptional": true,
+          "cardinality": "SINGLE",
+          "isUnique": false,
+          "isIndexable": false
+        },
+        {
+          "name": "comment",
+          "typeName": "string",
+          "isOptional": true,
+          "cardinality": "SINGLE",
+          "isUnique": false,
+          "isIndexable": false,
+          "searchWeight": 9
+        }
+      ]
+    },
+    {
+      "name": "spark_ml_directory",
+      "superTypes": [
+        "DataSet"
+      ],
+      "serviceType": "spark",
+      "typeVersion": "1.0",
+      "attributeDefs": [
+        {
+          "name": "uri",
+          "typeName": "string",
+          "isOptional": false,
+          "cardinality": "SINGLE",
+          "isUnique": false,
+          "isIndexable": true,
+          "searchWeight": 10
+        },
+        {
+          "name": "directory",
+          "typeName": "string",
+          "isOptional": false,
+          "cardinality": "SINGLE",
+          "isUnique": false,
+          "isIndexable": true,
+          "searchWeight": 10
+        }
+      ]
+    },
+    {
+      "name": "spark_storagedesc",
+      "superTypes": [
+        "Referenceable"
+      ],
+      "serviceType": "spark",
+      "typeVersion": "1.0",
+      "attributeDefs": [
+        {
+          "name": "location",
+          "typeName": "string",
+          "isOptional": true,
+          "cardinality": "SINGLE",
+          "isUnique": false,
+          "isIndexable": false,
+          "searchWeight": 10
+        },
+        {
+          "name": "inputFormat",
+          "typeName": "string",
+          "isOptional": true,
+          "cardinality": "SINGLE",
+          "isUnique": false,
+          "isIndexable": false
+        },
+        {
+          "name": "outputFormat",
+          "typeName": "string",
+          "isOptional": true,
+          "cardinality": "SINGLE",
+          "isUnique": false,
+          "isIndexable": false
+        },
+        {
+          "name": "serde",
+          "typeName": "string",
+          "isOptional": true,
+          "cardinality": "SINGLE",
+          "isUnique": false,
+          "isIndexable": false
+        },
+        {
+          "name": "compressed",
+          "typeName": "boolean",
+          "isOptional": false,
+          "cardinality": "SINGLE",
+          "isUnique": false,
+          "isIndexable": true
+        },
+        {
+          "name": "parameters",
+          "typeName": "map<string,string>",
+          "isOptional": true,
+          "cardinality": "SINGLE",
+          "isUnique": false,
+          "isIndexable": false
+        }
+      ]
+    },
+    {
+      "name": "spark_ml_model",
+      "superTypes": [
+        "DataSet"
+      ],
+      "serviceType": "spark",
+      "typeVersion": "1.0",
+      "attributeDefs": [
+        {
+          "name": "extra",
+          "typeName": "string",
+          "isOptional": true,
+          "cardinality": "SINGLE",
+          "isUnique": false,
+          "isIndexable": false
+        }
+      ]
+    },
+    {
+      "name": "spark_process",
+      "superTypes": [
+        "Process"
+      ],
+      "serviceType": "spark",
+      "typeVersion": "1.0",
+      "attributeDefs": [
+        {
+          "name": "executionId",
+          "typeName": "long",
+          "isOptional": true,
+          "cardinality": "SINGLE",
+          "isUnique": false,
+          "isIndexable": false,
+          "searchWeight": 10
+        },
+        {
+          "name": "currUser",
+          "typeName": "string",
+          "isOptional": true,
+          "cardinality": "SINGLE",
+          "isUnique": false,
+          "isIndexable": false,
+          "searchWeight": 10
+        },
+        {
+          "name": "remoteUser",
+          "typeName": "string",
+          "isOptional": true,
+          "cardinality": "SINGLE",
+          "isUnique": false,
+          "isIndexable": false,
+          "searchWeight": 10
+        },
+        {
+          "name": "details",
+          "typeName": "string",
+          "isOptional": true,
+          "cardinality": "SINGLE",
+          "isUnique": false,
+          "isIndexable": false
+        },
+        {
+          "name": "sparkPlanDescription",
+          "typeName": "string",
+          "isOptional": true,
+          "cardinality": "SINGLE",
+          "isUnique": false,
+          "isIndexable": false
+        }
+      ]
+    },
+    {
+      "name": "spark_ml_pipeline",
+      "superTypes": [
+        "DataSet"
+      ],
+      "serviceType": "spark",
+      "typeVersion": "1.0",
+      "attributeDefs": [
+        {
+          "name": "extra",
+          "typeName": "string",
+          "isOptional": true,
+          "cardinality": "SINGLE",
+          "isUnique": false,
+          "isIndexable": false
+        }
+      ]
+    }
+  ],
+  "relationshipDefs": [
+    {
+      "name": "spark_table_db",
+      "serviceType": "spark",
+      "typeVersion": "1.0",
+      "relationshipCategory": "AGGREGATION",
+      "endDef1": {
+        "type": "spark_table",
+        "name": "db",
+        "isContainer": false,
+        "cardinality": "SINGLE"
+      },
+      "endDef2": {
+        "type": "spark_db",
+        "name": "tables",
+        "isContainer": true,
+        "cardinality": "SET"
+      },
+      "propagateTags": "NONE"
+    },
+    {
+      "name": "spark_table_columns",
+      "serviceType": "spark",
+      "typeVersion": "1.0",
+      "relationshipCategory": "COMPOSITION",
+      "endDef1": {
+        "type": "spark_table",
+        "name": "columns",
+        "isContainer": true,
+        "cardinality": "SET"
+      },
+      "endDef2": {
+        "type": "spark_column",
+        "name": "table",
+        "isContainer": false,
+        "cardinality": "SINGLE"
+      },
+      "propagateTags": "NONE"
+    },
+    {
+      "name": "spark_table_storagedesc",
+      "serviceType": "spark",
+      "typeVersion": "1.0",
+      "relationshipCategory": "COMPOSITION",
+      "endDef1": {
+        "type": "spark_table",
+        "name": "sd",
+        "isContainer": true,
+        "cardinality": "SINGLE"
+      },
+      "endDef2": {
+        "type": "spark_storagedesc",
+        "name": "table",
+        "isContainer": false,
+        "cardinality": "SINGLE"
+      },
+      "propagateTags": "NONE"
+    },
+    {
+      "name": "spark_ml_model_ml_directory",
+      "serviceType": "spark",
+      "typeVersion": "1.0",
+      "relationshipCategory": "COMPOSITION",
+      "endDef1": {
+        "type": "spark_ml_model",
+        "name": "directory",
+        "isContainer": true,
+        "cardinality": "SINGLE"
+      },
+      "endDef2": {
+        "type": "spark_ml_directory",
+        "name": "model",
+        "isContainer": false,
+        "cardinality": "SINGLE"
+      },
+      "propagateTags": "NONE"
+    },
+    {
+      "name": "spark_ml_pipeline_ml_directory",
+      "serviceType": "spark",
+      "typeVersion": "1.0",
+      "relationshipCategory": "COMPOSITION",
+      "endDef1": {
+        "type": "spark_ml_pipeline",
+        "name": "directory",
+        "isContainer": true,
+        "cardinality": "SINGLE"
+      },
+      "endDef2": {
+        "type": "spark_ml_directory",
+        "name": "pipeline",
+        "isContainer": false,
+        "cardinality": "SINGLE"
+      },
+      "propagateTags": "NONE"
+    }
+  ]
+}
\ No newline at end of file