Skip to content

A
atlas
Commit 77943a94 by Madhan Neethiraj
Options

ATLAS-4077: updated Python client to support Python 2.7

parent cc333af6
Showing with 877 additions and 721 deletions
atlas-examples/sample-app/src/main/python/discovery_example.py
......@@ -16,6 +16,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import logging
from utils import TABLE_TYPE
......
atlas-examples/sample-app/src/main/python/glossary_example.py
......@@ -18,6 +18,7 @@
# limitations under the License.
import logging
from apache_atlas.model.glossary import AtlasGlossary, AtlasGlossaryCategory, AtlasGlossaryTerm, AtlasGlossaryHeader
LOG = logging.getLogger('glossary-example')
......@@ -33,7 +34,7 @@ class GlossaryExample:
self.emp_company_category = None
def create_glossary(self):
glossary = AtlasGlossary(None, None, GlossaryExample.glossaryName, "This is a test Glossary")
glossary = AtlasGlossary({ 'name': GlossaryExample.glossaryName, 'shortDescription': 'This is a test Glossary' })
self.emp_glossary = self.client.glossary.create_glossary(glossary)
LOG.info("Created glossary with name: %s and guid: %s", self.emp_glossary.name, self.emp_glossary.guid)
......@@ -47,16 +48,18 @@ class GlossaryExample:
LOG.info("Glossary extended info: %s; name: %s; language: %s", ext_info.guid, ext_info.name, ext_info.language)
def create_glossary_term(self):
header = AtlasGlossaryHeader(self.emp_glossary.guid, None, self.emp_glossary.name)
term = AtlasGlossaryTerm(None, None, "EmpSalaryTerm", None, None, None, None, None, None, None, header)
header = AtlasGlossaryHeader({ 'glossaryGuid': self.emp_glossary.guid, 'displayText': self.emp_glossary.name })
term = AtlasGlossaryTerm({ 'name': 'EmpSalaryTerm', 'anchor': header })
self.emp_salary_term = self.client.glossary.create_glossary_term(term)
if self.emp_salary_term:
LOG.info("Created Term for Employee Salary: %s with guid: %s", self.emp_salary_term.name, self.emp_salary_term.guid)
def create_glossary_category(self):
header = AtlasGlossaryHeader(self.emp_glossary.guid, None, self.emp_glossary.name)
category = AtlasGlossaryCategory(None, None, "EmpSalaryCategory", None, None, None, None, header)
header = AtlasGlossaryHeader({ 'glossaryGuid': self.emp_glossary.guid, 'displayText': self.emp_glossary.name })
category = AtlasGlossaryCategory({ 'name': 'EmpSalaryCategory', 'anchor': header })
self.emp_company_category = self.client.glossary.create_glossary_category(category)
if self.emp_company_category:
......
atlas-examples/sample-app/src/main/python/lineage_example.py
......@@ -16,20 +16,20 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import logging
from apache_atlas.model.lineage import AtlasLineageInfo
from apache_atlas.model.enums import LineageDirection
LOG = logging.getLogger('lineage-example')
class LineageExample:
def __init__(self, client):
self.client = client
def lineage(self, guid):
direction = AtlasLineageInfo.lineageDirection_enum.BOTH.name
direction = LineageDirection.BOTH.name
lineage_info = self.client.lineage.get_lineage_info(guid, direction, 0)
if not lineage_info:
......@@ -40,7 +40,7 @@ class LineageExample:
guid_entity_map = lineage_info.guidEntityMap
for relation in relations:
from_entity = guid_entity_map[relation['fromEntityId']]
to_entity = guid_entity_map[relation['toEntityId']]
from_entity = guid_entity_map[relation.fromEntityId]
to_entity = guid_entity_map[relation.toEntityId]
LOG.info("%s (%s) -> %s (%s)", from_entity['displayText'], from_entity['typeName'], to_entity['displayText'], to_entity['typeName'])
\ No newline at end of file
LOG.info("%s (%s) -> %s (%s)", from_entity.displayText, from_entity.typeName, to_entity.displayText, to_entity.typeName)
\ No newline at end of file
atlas-examples/sample-app/src/main/python/request_json/entity_create_db.json
{
"entity": {
"typeName": "sample_db_type",
"guid": "-1",
"typeName": "sample_db",
"attributes": {
"owner": "user",
"createTime": 1000,
"@cl1": "employeeCluster",
"qualifiedName": "employee_db_entity@cl1",
"name": "employee_db_entity",
"description": "employee database",
"locationUri": "/tmp"
},
"guid": "-222736766326565",
"isIncomplete": false,
"provenanceType": 0,
"version": 0,
"proxy": false
"name": "employee_db",
"qualifiedName": "employee_db@cl1",
"description": "employee database",
"owner": "user",
"clusterName": "cl1",
"locationUri": "/hive/database/employee_db",
"createTime": 1607476058882
}
}
}
\ No newline at end of file
}
atlas-examples/sample-app/src/main/python/request_json/entity_create_process.json
{
"entity": {
"typeName": "sample_process_type",
"typeName": "sample_process",
"attributes": {
"queryGraph": "graph",
"qualifiedName": "employee_process_entity@cl1",
"name": "employee_process_entity",
"queryText": "create table as select ",
"description": "hive query for monthly avg salary",
"startTime": 1596855233685,
"queryPlan": "plan",
"operationType": "testOperation",
"endTime": 1596855243685,
"userName": "user ETL",
"queryId": "id"
"name": "employee_process",
"description": "hive query for monthly avg salary",
"qualifiedName": "employee_process@cl1",
"userName": "user ETL",
"startTime": 1607476549507,
"endTime": 1607476552529,
"queryText": "create table as select ",
"queryId": "<query-id>"
},
"guid": "-222736766326574",
"isIncomplete": false,
"provenanceType": 0,
"version": 0,
"relationshipAttributes": {
"outputs": [{
"guid": "b0619fb9-b025-4348-8b8a-79e8faa0e789",
"typeName": "sample_table_type"
}],
"inputs": [{
"guid": "b49bb0fe-4714-4122-901a-9ac433a89ccd",
"typeName": "sample_table_type"
}]
},
"classifications": [{
"typeName": "classification",
"entityStatus": "ACTIVE"
}],
"proxy": false
"inputs": [ { "typeName": "sample_table", "uniqueAttributes": { "qualifiedName": "employee_db.employees_us@cl1" } } ],
"outputs": [ { "typeName": "sample_table", "uniqueAttributes": { "qualifiedName": "employee_db.employees_canada@cl1" } } ]
}
}
}
\ No newline at end of file
}
atlas-examples/sample-app/src/main/python/request_json/entity_create_table_canada.json
{
"entity": {
"guid": "-1",
"typeName": "sample_table",
"attributes": {
"name": "employees_canada",
"description": "Canada employees",
"qualifiedName": "employee_db.employees_canada@cl1",
"tableType": "Managed",
"serde1": { "typeName": "sample_serdeType", "attributes": { "name": "serde1", "serde": "serde1" } },
"serde2": { "typeName": "sample_serdeType", "attributes": { "name": "serde2", "serde": "serde2" } }
},
"relationshipAttributes": {
"db": { "typeName": "sample_db", "uniqueAttributes": { "qualifiedName": "employee_db@cl1" } },
"columns": [
{ "guid": "-2" },
{ "guid": "-3" },
{ "guid": "-4" }
]
}
},
"referredEntities": {
"-222736766326566": {
"typeName": "sample_column_type",
"-2": {
"guid": "-2",
"typeName": "sample_column",
"attributes": {
"qualifiedName": "time_id@cl1",
"dataType": "int",
"name": "time_id",
"comment": "time id"
},
"guid": "-222736766326566",
"isIncomplete": false,
"provenanceType": 0,
"version": 0,
"relationshipAttributes": {
"table": {
"guid": "-222736766326569",
"typeName": "sample_table_type"
}
},
"classifications": [],
"proxy": false
"table": { "guid": "-1" },
"name": "time_id",
"dataType": "int",
"comment": "time id",
"qualifiedName": "employee_db.employees_canada.time_id@cl1"
}
},
"-222736766326567": {
"typeName": "sample_column_type",
"-3": {
"guid": "-3",
"typeName": "sample_column",
"attributes": {
"qualifiedName": "customer_id@cl1",
"dataType": "int",
"name": "customer_id",
"comment": "customer id"
},
"guid": "-222736766326567",
"isIncomplete": false,
"provenanceType": 0,
"version": 0,
"relationshipAttributes": {
"table": {
"guid": "-222736766326569",
"typeName": "sample_table_type"
}
},
"classifications": [{
"typeName": "sample_pii_Tag",
"entityStatus": "ACTIVE"
}],
"proxy": false
"table": { "guid": "-1" },
"name": "customer_id",
"dataType": "int",
"comment": "customer id",
"qualifiedName": "employee_db.employees_canada.customer_id@cl1"
}
},
"-222736766326568": {
"typeName": "sample_column_type",
"-4": {
"guid": "-4",
"typeName": "sample_column",
"attributes": {
"qualifiedName": "company_id@cl1",
"dataType": "double",
"name": "company_id",
"comment": "company id"
},
"guid": "-222736766326568",
"isIncomplete": false,
"provenanceType": 0,
"version": 0,
"relationshipAttributes": {
"table": {
"guid": "-222736766326569",
"typeName": "sample_table_type"
}
},
"classifications": [{
"typeName": "sample_finance_Tag",
"entityStatus": "ACTIVE"
}],
"proxy": false
}
},
"entity": {
"typeName": "sample_table_type",
"attributes": {
"serde2": {
"typeName": "serdeType",
"attributes": {
"serde": "serde2",
"name": "serde2"
}
},
"tableType": "Managed",
"serde1": {
"typeName": "serdeType",
"attributes": {
"serde": "serde1",
"name": "serde1"
}
},
"lastAccessTime": "2014-07-11T08:00:00.000Z",
"level": 2,
"qualifiedName": "employee_table_entity_CANADA@cl1",
"name": "employee_table_entity_CANADA",
"description": "emp table",
"compressed": false
},
"guid": "-222736766326569",
"isIncomplete": false,
"provenanceType": 0,
"version": 0,
"relationshipAttributes": {
"columns": [{
"guid": "-222736766326566",
"typeName": "sample_column_type"
}, {
"guid": "-222736766326567",
"typeName": "sample_column_type"
}, {
"guid": "-222736766326568",
"typeName": "sample_column_type"
}],
"db": {
"guid": "c1e152c4-e40a-4bc9-a8a1-3c09d8d85f17",
"typeName": "sample_db_type"
"table": { "guid": "-1" },
"name": "company_id",
"dataType": "double",
"comment": "company id",
"qualifiedName": "employee_db.employees_canada.company_id@cl1"
}
},
"classifications": [{
"typeName": "Metric",
"entityStatus": "ACTIVE"
}],
"proxy": false
}
}
}
\ No newline at end of file
}
atlas-examples/sample-app/src/main/python/request_json/entity_create_table_us.json
{
"entity": {
"guid": "-1",
"typeName": "sample_table",
"attributes": {
"name": "employees_us",
"description": "US employees",
"qualifiedName": "employee_db.employees_us@cl1",
"tableType": "Managed",
"serde1": { "typeName": "sample_serdeType", "attributes": { "name": "serde1", "serde": "serde1" } },
"serde2": { "typeName": "sample_serdeType", "attributes": { "name": "serde2", "serde": "serde2" } }
},
"relationshipAttributes": {
"db": { "typeName": "sample_db", "uniqueAttributes": { "qualifiedName": "employee_db@cl1" } },
"columns": [
{ "guid": "-2" },
{ "guid": "-3" },
{ "guid": "-4" }
]
}
},
"referredEntities": {
"-222736766326570": {
"typeName": "sample_column_type",
"-2": {
"guid": "-2",
"typeName": "sample_column",
"attributes": {
"qualifiedName": "time_id@cl1",
"dataType": "int",
"name": "time_id",
"comment": "time id"
},
"guid": "-222736766326570",
"isIncomplete": false,
"provenanceType": 0,
"version": 0,
"relationshipAttributes": {
"table": {
"guid": "-222736766326573",
"typeName": "sample_table_type"
}
},
"classifications": [],
"proxy": false
"name": "time_id",
"dataType": "int",
"comment": "time id",
"qualifiedName": "employee_db.employees_us.time_id@cl1",
"table": { "guid": "-1" }
}
},
"-222736766326571": {
"typeName": "sample_column_type",
"-3": {
"guid": "-3",
"typeName": "sample_column",
"attributes": {
"qualifiedName": "customer_id@cl1",
"dataType": "int",
"name": "customer_id",
"comment": "customer id"
},
"guid": "-222736766326571",
"isIncomplete": false,
"provenanceType": 0,
"version": 0,
"relationshipAttributes": {
"table": {
"guid": "-222736766326573",
"typeName": "sample_table_type"
}
},
"classifications": [{
"typeName": "sample_pii_Tag",
"entityStatus": "ACTIVE"
}],
"proxy": false
"dataType": "int",
"comment": "customer id",
"qualifiedName": "employee_db.employees_us.customer_id@cl1",
"table": { "guid": "-1" }
}
},
"-222736766326572": {
"typeName": "sample_column_type",
"-4": {
"guid": "-4",
"typeName": "sample_column",
"attributes": {
"qualifiedName": "company_id@cl1",
"dataType": "double",
"name": "company_id",
"comment": "company id"
},
"guid": "-222736766326572",
"isIncomplete": false,
"provenanceType": 0,
"version": 0,
"relationshipAttributes": {
"table": {
"guid": "-222736766326573",
"typeName": "sample_table_type"
}
},
"classifications": [{
"typeName": "sample_finance_Tag",
"entityStatus": "ACTIVE"
}],
"proxy": false
}
},
"entity": {
"typeName": "sample_table_type",
"attributes": {
"serde2": {
"typeName": "serdeType",
"attributes": {
"serde": "serde2",
"name": "serde2"
}
},
"tableType": "Managed",
"serde1": {
"typeName": "serdeType",
"attributes": {
"serde": "serde1",
"name": "serde1"
}
},
"lastAccessTime": "2014-07-11T08:00:00.000Z",
"level": 2,
"qualifiedName": "employee_table_entity_US@cl1",
"name": "employee_table_entity_US",
"description": "emp table",
"compressed": false
},
"guid": "-222736766326573",
"isIncomplete": false,
"provenanceType": 0,
"version": 0,
"relationshipAttributes": {
"columns": [{
"guid": "-222736766326570",
"typeName": "sample_column_type"
}, {
"guid": "-222736766326571",
"typeName": "sample_column_type"
}, {
"guid": "-222736766326572",
"typeName": "sample_column_type"
}],
"db": {
"guid": "c1e152c4-e40a-4bc9-a8a1-3c09d8d85f17",
"typeName": "sample_db_type"
"name": "company_id",
"dataType": "double",
"comment": "company id",
"qualifiedName": "employee_db.employees_us.company_id@cl1",
"table": { "guid": "-1" }
}
},
"classifications": [{
"typeName": "Metric",
"entityStatus": "ACTIVE"
}],
"proxy": false
}
}
}
\ No newline at end of file
}
atlas-examples/sample-app/src/main/python/sample_client.py
......@@ -16,17 +16,18 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import logging
import getpass
from apache_atlas.base_client import AtlasClient
from apache_atlas.client.base_client import AtlasClient
from typedef_example import TypeDefExample
from entity_example import EntityExample
from lineage_example import LineageExample
from glossary_example import GlossaryExample
from discovery_example import DiscoveryExample
from utils import METRIC_CLASSIFICATION, NAME
from typedef_example import TypeDefExample
from entity_example import EntityExample
from lineage_example import LineageExample
from glossary_example import GlossaryExample
from discovery_example import DiscoveryExample
from utils import METRIC_CLASSIFICATION, NAME
import getpass
LOG = logging.getLogger('sample-example')
......@@ -36,70 +37,62 @@ class SampleApp:
self.created_entity = None
def main(self):
url = input("Enter Atlas URL: ")
username = input("Enter username: ")
password = getpass.getpass('Enter password: ')
client = AtlasClient(url, username, password)
# Python3
global input
try: input = raw_input
except NameError: pass
# Typedef examples
LOG.info("\n")
LOG.info("---------- Creating Sample Types -----------")
typedef = TypeDefExample(client)
typedef.create_type_def()
typedef.print_typedefs()
url = input('Enter Atlas URL: ')
username = input('Enter username: ')
password = getpass.getpass('Enter password: ')
# Entity example
LOG.info("\n")
LOG.info("---------- Creating Sample Entities -----------")
entity = EntityExample(client)
entity.create_entities()
client = AtlasClient(url, (username, password))
self.created_entity = entity.get_table_entity()
self.__entity_example(client)
if self.created_entity and self.created_entity['guid']:
entity.get_entity_by_guid(self.created_entity['guid'])
self.__typedef_example(client)
# Lineage Examples
LOG.info("\n")
LOG.info("---------- Lineage example -----------")
self.__lineage_example(client)
# Discovery Example
LOG.info("\n")
LOG.info("---------- Search example -----------")
self.__discovery_example(client)
# Glossary Examples
LOG.info("\n")
LOG.info("---------- Glossary Example -----------")
self.__glossary_example(client)
LOG.info("\n")
LOG.info("---------- Deleting Entities -----------")
entity.remove_entities()
self.__entity_cleanup()
def __glossary_example(self, client):
glossary = GlossaryExample(client)
glossary_obj = glossary.create_glossary()
if not glossary_obj:
LOG.info("Create glossary first")
return
def __typedef_example(self, client):
LOG.info("\n---------- Creating Sample Types -----------")
glossary.create_glossary_term()
glossary.get_glossary_detail()
glossary.create_glossary_category()
glossary.delete_glossary()
typedefExample = TypeDefExample(client)
typedefExample.create_type_def()
def __entity_example(self, client):
LOG.info("\n---------- Creating Sample Entities -----------")
self.entityExample = EntityExample(client)
self.entityExample.create_entities()
self.created_entity = self.entityExample.get_table_entity()
if self.created_entity and self.created_entity.guid:
self.entityExample.get_entity_by_guid(self.created_entity.guid)
def __lineage_example(self, client):
LOG.info("\n---------- Lineage example -----------")
lineage = LineageExample(client)
if self.created_entity:
lineage.lineage(self.created_entity['guid'])
lineage.lineage(self.created_entity.guid)
else:
LOG.info("Create entity first to get lineage info")
def __discovery_example(self, client):
LOG.info("\n---------- Search example -----------")
discovery = DiscoveryExample(client)
discovery.dsl_search()
......@@ -108,10 +101,30 @@ class SampleApp:
LOG.info("Create entity first to get search info")
return
discovery.quick_search(self.created_entity['typeName'])
discovery.quick_search(self.created_entity.typeName)
discovery.basic_search(self.created_entity.typeName, METRIC_CLASSIFICATION, self.created_entity.attributes[NAME])
def __glossary_example(self, client):
LOG.info("\n---------- Glossary Example -----------")
glossary = GlossaryExample(client)
glossary_obj = glossary.create_glossary()
if not glossary_obj:
LOG.info("Create glossary first")
return
glossary.create_glossary_term()
glossary.get_glossary_detail()
glossary.create_glossary_category()
glossary.delete_glossary()
def __entity_cleanup(self):
LOG.info("\n---------- Deleting Entities -----------")
discovery.basic_search(self.created_entity['typeName'], METRIC_CLASSIFICATION, self.created_entity['attributes'][NAME])
self.entityExample.remove_entities()
if __name__ == "__main__":
SampleApp().main()
\ No newline at end of file
SampleApp().main()
atlas-examples/sample-app/src/main/python/typedef_example.py
......@@ -17,26 +17,28 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import json
import logging
import utils
from apache_atlas.utils import type_coerce
from apache_atlas.model.discovery import SearchFilter
from apache_atlas.model.typedef import AtlasTypesDef
from apache_atlas.model.typedef import AtlasTypesDef
import json
LOG = logging.getLogger('sample-example')
class TypeDefExample:
SAMPLE_APP_TYPES = {
utils.PROCESS_TYPE,
utils.COLUMN_TYPE,
utils.TABLE_TYPE,
SAMPLE_APP_TYPES = [
utils.DATABASE_TYPE,
utils.TABLE_TYPE,
utils.COLUMN_TYPE,
utils.PROCESS_TYPE,
utils.PII_TAG,
utils.CLASSIFICATION,
utils.FINANCE_TAG,
utils.METRIC_CLASSIFICATION}
utils.METRIC_CLASSIFICATION
]
def __init__(self, client):
self.typesDef = None
......@@ -46,10 +48,10 @@ class TypeDefExample:
try:
if not self.typesDef:
with open('request_json/typedef_create.json') as f:
typedef = json.load(f)
typedef = type_coerce(json.load(f), AtlasTypesDef)
self.typesDef = self.__create(typedef)
except Exception as e:
LOG.exception("Error in creating typeDef.")
LOG.exception("Error in creating typeDef", exc_info=e)
def print_typedefs(self):
for type_name in TypeDefExample.SAMPLE_APP_TYPES:
......@@ -63,52 +65,58 @@ class TypeDefExample:
def remove_typedefs(self):
if not self.typesDef:
LOG.info("There is no typeDef to delete.")
return
for type_name in TypeDefExample.SAMPLE_APP_TYPES:
self.client.typedef.delete_type_by_name(type_name)
else:
for type_name in TypeDefExample.SAMPLE_APP_TYPES:
self.client.typedef.delete_type_by_name(type_name)
self.typesDef = None
self.typesDef = None
LOG.info("Deleted typeDef successfully!")
def __create(self, type_def):
types_to_create = AtlasTypesDef().__dict__
for enum_def in type_def['enumDefs']:
if self.client.typedef.type_with_name_exists(enum_def['name']):
LOG.info("Type with name %s already exists. Skipping.", enum_def['name'])
types_to_create = AtlasTypesDef()
types_to_create.enumDefs = []
types_to_create.structDefs = []
types_to_create.classificationDefs = []
types_to_create.entityDefs = []
types_to_create.relationshipDefs = []
types_to_create.businessMetadataDefs = []
for enum_def in type_def.enumDefs:
if self.client.typedef.type_with_name_exists(enum_def.name):
LOG.info("Type with name %s already exists. Skipping.", enum_def.name)
else:
types_to_create['enumDefs'].append(enum_def)
types_to_create.enumDefs.append(enum_def)
for struct_def in type_def['structDefs']:
if self.client.typedef.type_with_name_exists(struct_def['name']):
LOG.info("Type with name %s already exists. Skipping.", struct_def['name'])
for struct_def in type_def.structDefs:
if self.client.typedef.type_with_name_exists(struct_def.name):
LOG.info("Type with name %s already exists. Skipping.", struct_def.name)
else:
types_to_create['structDefs'].append(struct_def)
types_to_create.structDefs.append(struct_def)
for classification_def in type_def['classificationDefs']:
if self.client.typedef.type_with_name_exists(classification_def['name']):
LOG.info("Type with name %s already exists. Skipping.", classification_def['name'])
for classification_def in type_def.classificationDefs:
if self.client.typedef.type_with_name_exists(classification_def.name):
LOG.info("Type with name %s already exists. Skipping.", classification_def.name)
else:
types_to_create['classificationDefs'].append(classification_def)
types_to_create.classificationDefs.append(classification_def)
for entity_def in type_def['entityDefs']:
if self.client.typedef.type_with_name_exists(entity_def['name']):
LOG.info("Type with name %s already exists. Skipping.", entity_def['name'])
for entity_def in type_def.entityDefs:
if self.client.typedef.type_with_name_exists(entity_def.name):
LOG.info("Type with name %s already exists. Skipping.", entity_def.name)
else:
types_to_create['entityDefs'].append(entity_def)
types_to_create.entityDefs.append(entity_def)
for relationship_def in type_def['relationshipDefs']:
if self.client.typedef.type_with_name_exists(relationship_def['name']):
LOG.info("Type with name %s already exists. Skipping.", relationship_def['name'])
for relationship_def in type_def.relationshipDefs:
if self.client.typedef.type_with_name_exists(relationship_def.name):
LOG.info("Type with name %s already exists. Skipping.", relationship_def.name)
else:
types_to_create['relationshipDefs'].append(relationship_def)
types_to_create.relationshipDefs.append(relationship_def)
for business_metadata_def in type_def['businessMetadataDefs']:
if self.client.typedef.type_with_name_exists(business_metadata_def['name']):
LOG.info("Type with name %s already exists. Skipping.", business_metadata_def['name'])
for business_metadata_def in type_def.businessMetadataDefs:
if self.client.typedef.type_with_name_exists(business_metadata_def.name):
LOG.info("Type with name %s already exists. Skipping.", business_metadata_def.name)
else:
types_to_create['businessMetadataDefs'].append(business_metadata_def)
types_to_create.businessMetadataDefs.append(business_metadata_def)
return self.client.typedef.create_atlas_typedefs(types_to_create)
\ No newline at end of file
return self.client.typedef.create_atlas_typedefs(types_to_create)
atlas-examples/sample-app/src/main/python/utils.py
......@@ -20,19 +20,18 @@
NAME = "name"
DESCRIPTION = "description"
PII_TAG = "sample_pii_Tag"
FINANCE_TAG = "sample_finance_Tag"
CLASSIFICATION = "classification"
METRIC_CLASSIFICATION = "Metric"
PII_TAG = "sample_pii"
FINANCE_TAG = "sample_finance"
METRIC_CLASSIFICATION = "sample_metric"
DATABASE_TYPE = "sample_db_type"
PROCESS_TYPE = "sample_process_type"
TABLE_TYPE = "sample_table_type"
COLUMN_TYPE = "sample_column_type"
DATABASE_TYPE = "sample_db"
PROCESS_TYPE = "sample_process"
TABLE_TYPE = "sample_table"
COLUMN_TYPE = "sample_column"
TABLE_DATABASE_TYPE = "sample_Table_DB"
TABLE_COLUMNS_TYPE = "sample_Table_Columns"
ENUM_TABLE_TYPE = "tableType"
BUSINESS_METADATA_TYPE = "bmWithAllTypes"
BUSINESS_METADATA_TYPE_MV = "bmWithAllTypesMV"
STRUCT_TYPE_SERDE = "serdeType"
\ No newline at end of file
TABLE_DATABASE_TYPE = "sample_db_tables"
TABLE_COLUMNS_TYPE = "sample_table_columns"
ENUM_TABLE_TYPE = "sample_tableType"
BUSINESS_METADATA_TYPE = "sample_bm"
BUSINESS_METADATA_TYPE_MV = "sample_bm_mv"
STRUCT_TYPE_SERDE = "sample_serdeType"
intg/src/main/python/README.md
# Atlas Python Client
# Apache Atlas Python Client
This is a python library for Atlas. Users can integrate with Atlas using the python client.
Currently, compatible with Python 3.5+
Python library for Apache Atlas.
## Installation
Use the package manager [pip](https://pip.pypa.io/en/stable/) to install python client for Atlas.
Use the package manager [pip](https://pip.pypa.io/en/stable/) to install Python client for Apache Atlas.
```bash
# After publishing apache-atlas use
> pip install apache-atlas
```
......@@ -19,22 +16,146 @@ Verify if apache-atlas client is installed:
Package Version
------------ ---------
apache-atlas 0.0.1
apache-atlas 0.0.2
```
## Usage
```python create_glossary.py```
```python atlas_example.py```
```python
# create_glossary.py
# atlas_example.py
import time
from apache_atlas.client.base_client import AtlasClient
from apache_atlas.model.instance import *
## Step 1: create a client to connect to Apache Atlas server
client = AtlasClient('http://localhost:21000', ('admin', 'atlasR0cks!'))
# For Kerberos authentication, use HTTPKerberosAuth as shown below
#
# from requests_kerberos import HTTPKerberosAuth
#
# client = AtlasClient('http://localhost:21000', HTTPKerberosAuth())
# to disable SSL certificate validation (not recommended for production use!)
#
# client.session.verify = False
## Step 2: Let's create a database entity
test_db = AtlasEntity({ 'typeName': 'hive_db' })
test_db.attributes = { 'name': 'test_db', 'clusterName': 'prod', 'qualifiedName': 'test_db@prod' }
entity_info = AtlasEntityWithExtInfo()
entity_info.entity = test_db
print('Creating test_db')
resp = client.entity.create_entity(entity_info)
guid_db = resp.get_assigned_guid(test_db.guid)
print(' created test_db: guid=' + guid_db)
## Step 3: Let's create a table entity, and two column entities - in one call
test_tbl = AtlasEntity({ 'typeName': 'hive_table' })
test_tbl.attributes = { 'name': 'test_tbl', 'qualifiedName': 'test_db.test_tbl@prod' }
test_tbl.relationshipAttributes = { 'db': AtlasRelatedObjectId({ 'guid': guid_db }) }
test_col1 = AtlasEntity({ 'typeName': 'hive_column' })
test_col1.attributes = { 'name': 'test_col1', 'type': 'string', 'qualifiedName': 'test_db.test_tbl.test_col1@prod' }
test_col1.relationshipAttributes = { 'table': AtlasRelatedObjectId({ 'guid': test_tbl.guid }) }
test_col2 = AtlasEntity({ 'typeName': 'hive_column' })
test_col2.attributes = { 'name': 'test_col2', 'type': 'string', 'qualifiedName': 'test_db.test_tbl.test_col2@prod' }
test_col2.relationshipAttributes = { 'table': AtlasRelatedObjectId({ 'guid': test_tbl.guid }) }
entities_info = AtlasEntitiesWithExtInfo()
entities_info.entities = [ test_tbl, test_col1, test_col2 ]
print('Creating test_tbl')
resp = client.entity.create_entities(entities_info)
guid_tbl = resp.get_assigned_guid(test_tbl.guid)
guid_col1 = resp.get_assigned_guid(test_col1.guid)
guid_col2 = resp.get_assigned_guid(test_col2.guid)
print(' created test_tbl: guid=' + guid_tbl)
print(' created test_tbl.test_col1: guid=' + guid_col1)
print(' created test_tbl.test_col2: guid=' + guid_col2)
## Step 4: Let's create a view entity that feeds from the table created earlier
# Also create a lineage between the table and the view, and lineages between their columns as well
test_view = AtlasEntity({ 'typeName': 'hive_table' })
test_view.attributes = { 'name': 'test_view', 'qualifiedName': 'test_db.test_view@prod' }
test_view.relationshipAttributes = { 'db': AtlasRelatedObjectId({ 'guid': guid_db }) }
test_view_col1 = AtlasEntity({ 'typeName': 'hive_column' })
test_view_col1.attributes = { 'name': 'test_col1', 'type': 'string', 'qualifiedName': 'test_db.test_view.test_col1@prod' }
test_view_col1.relationshipAttributes = { 'table': AtlasRelatedObjectId({ 'guid': test_view.guid }) }
test_view_col2 = AtlasEntity({ 'typeName': 'hive_column' })
test_view_col2.attributes = { 'name': 'test_col2', 'type': 'string', 'qualifiedName': 'test_db.test_view.test_col2@prod' }
test_view_col2.relationshipAttributes = { 'table': AtlasRelatedObjectId({ 'guid': test_view.guid }) }
test_process = AtlasEntity({ 'typeName': 'hive_process' })
test_process.attributes = { 'name': 'create_test_view', 'userName': 'admin', 'operationType': 'CREATE', 'qualifiedName': 'create_test_view@prod' }
test_process.attributes['queryText'] = 'create view test_view as select * from test_tbl'
test_process.attributes['queryPlan'] = '<queryPlan>'
test_process.attributes['queryId'] = '<queryId>'
test_process.attributes['startTime'] = int(time.time() * 1000)
test_process.attributes['endTime'] = int(time.time() * 1000)
test_process.relationshipAttributes = { 'inputs': [ AtlasRelatedObjectId({ 'guid': guid_tbl }) ], 'outputs': [ AtlasRelatedObjectId({ 'guid': test_view.guid }) ] }
test_col1_lineage = AtlasEntity({ 'typeName': 'hive_column_lineage' })
test_col1_lineage.attributes = { 'name': 'test_view.test_col1 lineage', 'depenendencyType': 'read', 'qualifiedName': 'test_db.test_view.test_col1@prod' }
test_col1_lineage.attributes['query'] = { 'guid': test_process.guid }
test_col1_lineage.relationshipAttributes = { 'inputs': [ AtlasRelatedObjectId({ 'guid': guid_col1 }) ], 'outputs': [ AtlasRelatedObjectId({ 'guid': test_view_col1.guid }) ] }
test_col2_lineage = AtlasEntity({ 'typeName': 'hive_column_lineage' })
test_col2_lineage.attributes = { 'name': 'test_view.test_col2 lineage', 'depenendencyType': 'read', 'qualifiedName': 'test_db.test_view.test_col2@prod' }
test_col2_lineage.attributes['query'] = { 'guid': test_process.guid }
test_col2_lineage.relationshipAttributes = { 'inputs': [ AtlasRelatedObjectId({ 'guid': guid_col2 }) ], 'outputs': [ AtlasRelatedObjectId({ 'guid': test_view_col2.guid }) ] }
entities_info = AtlasEntitiesWithExtInfo()
entities_info.entities = [ test_process, test_col1_lineage, test_col2_lineage ]
entities_info.add_referenced_entity(test_view)
entities_info.add_referenced_entity(test_view_col1)
entities_info.add_referenced_entity(test_view_col2)
print('Creating test_view')
resp = client.entity.create_entities(entities_info)
guid_view = resp.get_assigned_guid(test_view.guid)
guid_view_col1 = resp.get_assigned_guid(test_view_col1.guid)
guid_view_col2 = resp.get_assigned_guid(test_view_col2.guid)
guid_process = resp.get_assigned_guid(test_process.guid)
guid_col1_lineage = resp.get_assigned_guid(test_col1_lineage.guid)
guid_col2_lineage = resp.get_assigned_guid(test_col2_lineage.guid)
print(' created test_view: guid=' + guid_view)
print(' created test_view.test_col1: guid=' + guid_view_col1)
print(' created test_view.test_col2: guid=' + guid_view_col1)
print(' created test_view lineage: guid=' + guid_process)
print(' created test_col1 lineage: guid=' + guid_col1_lineage)
print(' created test_col2 lineage: guid=' + guid_col2_lineage)
## Step 5: Finally, cleanup by deleting entities created above
print('Deleting entities')
from apache_atlas.base_client import AtlasClient
from apache_atlas.model.glossary import AtlasGlossary
resp = client.entity.delete_entities_by_guids([ guid_col1_lineage, guid_col2_lineage, guid_process, guid_view, guid_tbl, guid_db ])
client = AtlasClient("http://localhost:31000", "admin", "admin123")
glossary = AtlasGlossary(None, None, "Glossary_Test", "This is a test Glossary")
test_glossary = client.glossary.create_glossary(glossary)
deleted_count = len(resp.mutatedEntities[EntityOperation.DELETE.name]) if resp and resp.mutatedEntities and EntityOperation.DELETE.name in resp.mutatedEntities else 0
print('Created Test Glossary with guid: ' + test_glossary.guid)
print(' ' + str(deleted_count) + ' entities deleted')
```
For more examples, checkout `sample-app` python project in atlas-examples module.
\ No newline at end of file
For more examples, checkout `sample-app` python project in [atlas-examples](https://github.com/apache/atlas/blob/master/atlas-examples/sample-app/src/main/python/sample_client.py) module.
intg/src/main/python/apache_atlas/base_client.py → intg/src/main/python/apache_atlas/client/base_client.py
......@@ -19,31 +19,27 @@
import copy
import os
from http import HTTPStatus
from requests import Session
import json
import logging
import logging.config
from apache_atlas.utils import CustomEncoder, HttpMethod
from apache_atlas.client.typedef import TypeDefClient
from apache_atlas.client.discovery import DiscoveryClient
from apache_atlas.client.entity import EntityClient
from apache_atlas.client.glossary import GlossaryClient
from apache_atlas.client.lineage import LineageClient
from requests import Session
from apache_atlas.client.discovery import DiscoveryClient
from apache_atlas.client.entity import EntityClient
from apache_atlas.client.glossary import GlossaryClient
from apache_atlas.client.lineage import LineageClient
from apache_atlas.client.relationship import RelationshipClient
from apache_atlas.client.typedef import TypeDefClient
from apache_atlas.exceptions import AtlasServiceException
from apache_atlas.utils import HttpMethod, HTTPStatus, type_coerce
from apache_atlas.exceptions import AtlasServiceException
LOG = logging.getLogger('apache_atlas')
class AtlasClient:
def __init__(self, host, username, password):
def __init__(self, host, auth):
session = Session()
session.auth = (username, password)
session.auth = auth
self.host = host
self.session = session
......@@ -55,6 +51,9 @@ class AtlasClient:
self.glossary = GlossaryClient(self)
self.relationship = RelationshipClient(self)
logging.getLogger("requests").setLevel(logging.WARNING)
def call_api(self, api, response_type=None, query_params=None, request_obj=None):
params = copy.deepcopy(self.request_params)
path = os.path.join(self.host, api.path)
......@@ -62,13 +61,11 @@ class AtlasClient:
params['headers']['Accept'] = api.consumes
params['headers']['Content-type'] = api.produces
print(path)
if query_params:
params['params'] = query_params
if request_obj:
params['data'] = json.dumps(request_obj, indent=4, cls=CustomEncoder)
params['data'] = json.dumps(request_obj)
if LOG.isEnabledFor(logging.DEBUG):
LOG.debug("------------------------------------------------------")
......@@ -92,7 +89,6 @@ class AtlasClient:
if response is None:
return None
elif response.status_code == api.expected_status:
if not response_type:
return None
......@@ -106,7 +102,7 @@ class AtlasClient:
if response_type == str:
return json.dumps(response.json())
return response_type(**response.json())
return type_coerce(response.json(), response_type)
else:
return None
except Exception as e:
......@@ -115,11 +111,9 @@ class AtlasClient:
LOG.exception("Exception occurred while parsing response with msg: %s", e)
raise AtlasServiceException(api, response)
elif response.status_code == HTTPStatus.SERVICE_UNAVAILABLE:
LOG.error("Atlas Service unavailable. HTTP Status: %s", HTTPStatus.SERVICE_UNAVAILABLE)
return None
else:
raise AtlasServiceException(api, response)
\ No newline at end of file
raise AtlasServiceException(api, response)
intg/src/main/python/apache_atlas/client/discovery.py
......@@ -17,10 +17,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from http import HTTPStatus
from apache_atlas.model.discovery import AtlasSearchResult, AtlasUserSavedSearch, AtlasSuggestionsResult, AtlasQuickSearchResult
from apache_atlas.utils import API, HttpMethod, BASE_URI
from apache_atlas.model.discovery import *
from apache_atlas.utils import API, HttpMethod, HTTPStatus
class DiscoveryClient:
......@@ -145,14 +143,14 @@ class DiscoveryClient:
return self.client.call_api(DiscoveryClient.UPDATE_SAVED_SEARCH, AtlasUserSavedSearch, None, saved_search)
def delete_saved_search(self, guid):
return self.client.call_api(DiscoveryClient.DELETE_SAVED_SEARCH.format_map({'guid': guid}))
return self.client.call_api(DiscoveryClient.DELETE_SAVED_SEARCH.format_path({'guid': guid}))
def execute_saved_search(self, user_name, search_name):
query_params = {"user", user_name}
return self.client.call_api(DiscoveryClient.EXECUTE_SAVED_SEARCH_BY_NAME.format_map({'search_name': search_name}),
return self.client.call_api(DiscoveryClient.EXECUTE_SAVED_SEARCH_BY_NAME.format_path({'search_name': search_name}),
AtlasSearchResult, query_params)
def execute_saved_search(self, search_guid):
return self.client.call_api(DiscoveryClient.EXECUTE_SAVED_SEARCH_BY_GUID.format_map({'search_guid': search_guid}),
return self.client.call_api(DiscoveryClient.EXECUTE_SAVED_SEARCH_BY_GUID.format_path({'search_guid': search_guid}),
AtlasSearchResult)
\ No newline at end of file
intg/src/main/python/apache_atlas/client/entity.py
......@@ -17,13 +17,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from http import HTTPStatus
from apache_atlas.model.entity import AtlasEntityWithExtInfo, AtlasEntitiesWithExtInfo, AtlasEntityHeader, \
AtlasClassifications, AtlasEntityHeaders, EntityMutationResponse
from apache_atlas.utils import API, HttpMethod, BASE_URI, APPLICATION_OCTET_STREAM, APPLICATION_JSON, \
MULTIPART_FORM_DATA, attributes_to_params, list_attributes_to_params
from apache_atlas.model.instance import *
from apache_atlas.utils import *
class EntityClient:
......
intg/src/main/python/apache_atlas/client/glossary.py
......@@ -17,10 +17,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from http import HTTPStatus
from apache_atlas.model.glossary import AtlasGlossaryCategory, AtlasGlossaryTerm, AtlasGlossary, AtlasGlossaryExtInfo
from apache_atlas.utils import BASE_URI, API, HttpMethod, APPLICATION_JSON, APPLICATION_OCTET_STREAM, MULTIPART_FORM_DATA
from apache_atlas.model.glossary import *
from apache_atlas.utils import *
class GlossaryClient:
......@@ -142,7 +140,7 @@ class GlossaryClient:
def get_related_categories(self, category_guid, sort_by_attribute, limit, offset):
query_params = {GlossaryClient.LIMIT: limit, GlossaryClient.OFFSET: offset, "sort": sort_by_attribute}
return self.client.call_api(GlossaryClient.GET_RELATED_CATEGORIES.format_map({'category_guid': category_guid}),
return self.client.call_api(GlossaryClient.GET_RELATED_CATEGORIES.format_path({'category_guid': category_guid}),
dict, query_params)
def create_glossary(self, glossary):
......
intg/src/main/python/apache_atlas/client/lineage.py
......@@ -17,10 +17,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from http import HTTPStatus
from apache_atlas.model.lineage import AtlasLineageInfo
from apache_atlas.utils import BASE_URI, API, HttpMethod, attributes_to_params
from apache_atlas.model.lineage import *
from apache_atlas.utils import *
class LineageClient:
......
intg/src/main/python/apache_atlas/client/relationship.py
......@@ -17,10 +17,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from http import HTTPStatus
from apache_atlas.model.relationship import AtlasRelationshipWithExtInfo, AtlasRelationship
from apache_atlas.utils import BASE_URI, API, HttpMethod
from apache_atlas.model.relationship import *
from apache_atlas.utils import *
class RelationshipClient:
......
intg/src/main/python/apache_atlas/client/typedef.py
......@@ -17,11 +17,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from http import HTTPStatus
from apache_atlas.utils import API, HttpMethod, BASE_URI
from apache_atlas.model.typedef import AtlasEnumDef, AtlasClassificationDef, AtlasEntityDef, AtlasStructDef, \
AtlasRelationshipDef, AtlasBusinessMetadataDef, AtlasTypesDef, AtlasBaseTypeDef
from apache_atlas.model.typedef import *
from apache_atlas.utils import *
class TypeDefClient:
......
intg/src/main/python/apache_atlas/exceptions.py
......@@ -17,6 +17,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
class AtlasServiceException(Exception):
"""Exception raised for errors in API calls.
......@@ -29,14 +30,12 @@ class AtlasServiceException(Exception):
msg = ""
if api:
msg = "Metadata service API {method} : {path} failed".format_map({'method': api.method, 'path': api.path})
msg = "Metadata service API {method} : {path} failed".format(**{'method': api.method, 'path': api.path})
if response.content:
status = response.status_code if response.status_code else -1
msg = "Metadata service API with url {url} and method {method} : failed with status {status} and " \
"Response Body is :{response}". \
format_map({'url': response.url, 'method': api.method, 'status': status, 'response': response.json()})
self.message = msg
format(**{'url': response.url, 'method': api.method, 'status': status, 'response': response.json()})
super().__init__(self.message)
\ No newline at end of file
Exception.__init__(self, msg)
intg/src/main/python/apache_atlas/model/entity.py deleted 100644 → 0
#!/usr/bin/env/python
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import enum
from apache_atlas.utils import s_nextId
class AtlasStruct:
def __init__(self, typeName=None, attributes=None):
self.typeName = typeName
self.attributes = attributes if attributes is not None else {}
class AtlasEntity(AtlasStruct):
status_enum = enum.Enum('status_enum', 'ACTIVE DELETED PURGED', module=__name__)
def __init__(self, typeName=None, attributes=None, guid=None, homeId=None, isIncomplete=None, provenanceType=None,
status=None, createdBy=None, updatedBy=None, createTime=None, updateTime=None, version=None,
relationshipAttributes=None, classifications=None, meanings=None, customAttributes=None,
businessAttributes=None, labels=None, proxy=None):
super().__init__(typeName, attributes)
self.guid = guid if guid is not None else "-" + str(s_nextId)
self.homeId = homeId
self.isIncomplete = isIncomplete
self.provenanceType = provenanceType
self.status = status
self.createdBy = createdBy
self.updatedBy = updatedBy
self.createTime = createTime
self.updateTime = updateTime
self.version = version
self.relationshipAttributes = relationshipAttributes if relationshipAttributes is not None else {}
self.classifications = classifications if classifications is not None else []
self.meanings = meanings if meanings is not None else []
self.customAttributes = customAttributes if customAttributes is not None else {}
self.businessAttributes = businessAttributes if businessAttributes is not None else {}
self.labels = labels if labels is not None else set()
self.proxy = proxy
class AtlasEntityExtInfo:
def __init__(self, referredEntities=None):
self.referredEntities = referredEntities if referredEntities is not None else {}
class AtlasEntityWithExtInfo(AtlasEntityExtInfo):
def __init__(self, referredEntities=None, entity=None):
super().__init__(referredEntities)
self.entity = entity
class AtlasEntitiesWithExtInfo(AtlasEntityExtInfo):
def __init__(self, referredEntities=None, entities=None):
super().__init__(referredEntities)
self.entities = entities if entities is not None else {}
class AtlasEntityHeader(AtlasStruct):
status_enum = enum.Enum('status_enum', 'ACTIVE DELETED PURGED', module=__name__)
def __init__(self, typeName=None, attributes=None, guid=None, status=None, displayText=None, classificationNames=None,
classifications=None, meaningNames=None, meanings=None, isIncomplete=None, labels=None):
super().__init__(typeName, attributes)
self.guid = guid if guid is not None else "-" + str(s_nextId)
self.status = status
self.displayText = displayText
self.classificationNames = classificationNames if classificationNames is not None else []
self.classifications = classifications
self.meaningNames = meaningNames
self.meanings = meanings if meanings is not None else []
self.isIncomplete = isIncomplete
self.labels = labels if labels is not None else set()
class AtlasClassification(AtlasStruct):
entityStatus_enum = enum.Enum('entityStatus_enum', 'ACTIVE DELETED PURGED', module=__name__)
def __init__(self, typeName=None, attributes=None, entityGuid=None, entityStatus=None, propagate=None,
validityPeriods=None, removePropagationsOnEntityDelete=None):
self.typeName = typeName
self.attributes = attributes
self.entityGuid = entityGuid
self.entityStatus = entityStatus
self.propagate = propagate
self.validityPeriods = validityPeriods if validityPeriods is not None else []
self.removePropagationsOnEntityDelete = removePropagationsOnEntityDelete
class TimeBoundary:
def __init__(self, startTime=None, endTime=None, timeZone=None):
self.startTime = startTime
self.endTime = endTime
self.timeZone = timeZone
class Plist:
sortType_enum = enum.Enum('sortType_enum', 'NONE ASC DESC', module=__name__)
def __init__(self, list=None, startIndex=None, pageSize=None, totalCount=None, sortBy=None, sortType=None):
self.list = list
self.startIndex = startIndex
self.pageSize = pageSize
self.totalCount = totalCount
self.sortBy = sortBy
self.sortType = sortType
class AtlasClassifications(Plist):
sortType_enum = enum.Enum('sortType_enum', 'NONE ASC DESC', module=__name__)
def __init__(self, list=None, startIndex=None, pageSize=None, totalCount=None, sortBy=None, sortType=None):
super().__init__(list, startIndex, pageSize, totalCount, sortBy, sortType)
class AtlasEntityHeaders:
def __init__(self, guidHeaderMap=None):
self.guidHeaderMap = guidHeaderMap if guidHeaderMap is not None else {}
class EntityMutationResponse:
def __init__(self, mutatedEntities=None, guidAssignments=None):
self.mutatedEntities = mutatedEntities if mutatedEntities is not None else {}
self.guidAssignments = guidAssignments if guidAssignments is not None else {}
class EntityMutations:
entity_operation_enum = enum.Enum('entity_operation_enum', 'CREATE UPDATE PARTIAL_UPDATE DELETE PURGE', module=__name__)
def __init__(self, entity_mutations):
self.entity_mutations = entity_mutations if entity_mutations is not None else []
class EntityMutation:
def __init__(self, op, entity):
self.op = op
self.entity = entity
\ No newline at end of file
intg/src/main/python/apache_atlas/model/enums.py 0 → 100644
#!/usr/bin/env/python
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicabwle law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import enum
class AtlasTermAssignmentStatus(enum.Enum):
DISCOVERED = 0
PROPOSED = 1
IMPORTED = 2
VALIDATED = 3
DEPRECATED = 4
OBSOLETE = 5
OTHER = 6
class AtlasTermRelationshipStatus(enum.Enum):
DRAFT = 0
ACTIVE = 1
DEPRECATED = 2
OBSOLETE = 3
OTHER = 99
class TypeCategory(enum.Enum):
PRIMITIVE = 0
OBJECT_ID_TYPE = 1
ENUM = 2
STRUCT = 3
CLASSIFICATION = 4
ENTITY = 5
ARRAY = 6
MAP = 7
RELATIONSHIP = 8
BUSINESS_METADATA = 9
class Cardinality(enum.Enum):
SINGLE = 0
LIST = 1
SET = 2
class Condition(enum.Enum):
AND = 0
OR = 1
class EntityOperation(enum.Enum):
CREATE = 0
UPDATE = 1
PARTIAL_UPDATE = 2
DELETE = 3
PURGE = 4
class EntityStatus(enum.Enum):
ACTIVE = 0
DELETED = 1
PURGED = 2
class IndexType(enum.Enum):
DEFAULT = 0
STRING = 1
class LineageDirection(enum.Enum):
INPUT = 0
OUTPUT = 1
BOTH = 2
class Operator(enum.Enum):
LT = ("<", "lt")
GT = ('>', 'gt')
LTE = ('<=', 'lte')
GTE = ('>=', 'gte')
EQ = ('=', 'eq')
NEQ = ('!=', 'neq')
IN = ('in', 'IN')
LIKE = ('like', 'LIKE')
STARTS_WITH = ('startsWith', 'STARTSWITH', 'begins_with', 'BEGINS_WITH')
ENDS_WITH = ('endsWith', 'ENDSWITH', 'ends_with', 'ENDS_WITH')
CONTAINS = ('contains', 'CONTAINS')
NOT_CONTAINS = ('not_contains', 'NOT_CONTAINS')
CONTAINS_ANY = ('containsAny', 'CONTAINSANY', 'contains_any', 'CONTAINS_ANY')
CONTAINS_ALL = ('containsAll', 'CONTAINSALL', 'contains_all', 'CONTAINS_ALL')
IS_NULL = ('isNull', 'ISNULL', 'is_null', 'IS_NULL')
NOT_NULL = ('notNull', 'NOTNULL', 'not_null', 'NOT_NULL')
class PropagateTags(enum.Enum):
NONE = 0
ONE_TO_TWO = 1
TWO_TO_ONE = 2
BOTH = 3
class QueryType(enum.Enum):
DSL = 0
FULL_TEXT = 1
GREMLIN = 2
BASIC = 3
ATTRIBUTE = 4
RELATIONSHIP = 5
class RelationshipCategory(enum.Enum):
ASSOCIATION = 0
AGGREGATION = 1
COMPOSITION = 2
class RelationshipStatus(enum.Enum):
ACTIVE = 0
DELETED = 1
class SavedSearchType(enum.Enum):
BASIC = 0
ADVANCED = 1
class SortOrder(enum.Enum):
ASCENDING = 0
DESCENDING = 1
class SortType(enum.Enum):
NONE = 0
ASC = 1
DESC = 2
intg/src/main/python/apache_atlas/model/lineage.py
......@@ -17,23 +17,32 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import enum
from apache_atlas.model.instance import *
from apache_atlas.utils import *
class AtlasLineageInfo:
lineageDirection_enum = enum.Enum('lineageDirection_enum', 'INPUT OUTPUT BOTH', module=__name__)
class AtlasLineageInfo(AtlasBase):
def __init__(self, attrs={}):
AtlasBase.__init__(self, attrs)
def __init__(self, baseEntityGuid=None, lineageDirection=None, lineageDepth=None, guidEntityMap=None, relations=None):
self.baseEntityGuid = baseEntityGuid
self.lineageDirection = lineageDirection
self.lineageDepth = lineageDepth
self.guidEntityMap = guidEntityMap if guidEntityMap is not None else {}
self.relations = relations if relations is not None else set()
self.baseEntityGuid = attrs.get('baseEntityGuid')
self.lineageDirection = attrs.get('lineageDirection')
self.lineageDepth = attrs.get('lineageDepth')
self.guidEntityMap = attrs.get('guidEntityMap')
self.relations = attrs.get('relations')
class LineageRelation:
def type_coerce_attrs(self):
super(AtlasLineageInfo, self).type_coerce_attrs()
def __init__(self, fromEntityId=None, toEntityId=None, relationshipId=None):
self.fromEntityId = fromEntityId
self.toEntityId = toEntityId
self.relationshipId = relationshipId
\ No newline at end of file
self.guidEntityMap = type_coerce_dict(self.guidEntityMap, AtlasEntityHeader)
self.relations = type_coerce_list(self.relations, LineageRelation)
class LineageRelation(AtlasBase):
def __init__(self, attrs):
AtlasBase.__init__(self, attrs)
self.fromEntityId = attrs.get('fromEntityId')
self.toEntityId = attrs.get('toEntityId')
self.relationshipId = attrs.get('relationshipId')
intg/src/main/python/apache_atlas/model/metrics.py 0 → 100644
#!/usr/bin/env/python
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from apache_atlas.utils import *
class AtlasMetrics(AtlasBase):
def __init__(self, attrs={}):
AtlasBase.__init__(self, attrs)
self.data = attrs.get('data')
intg/src/main/python/apache_atlas/model/misc.py 0 → 100644
#!/usr/bin/env/python
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import json
import sys
from apache_atlas.utils import *
class AtlasBase(dict):
def __init__(self, attrs):
pass
def __getattr__(self, attr):
return self.get(attr)
def __setattr__(self, key, value):
self.__setitem__(key, value)
def __setitem__(self, key, value):
super(AtlasBase, self).__setitem__(key, value)
self.__dict__.update({key: value})
def __delattr__(self, item):
self.__delitem__(item)
def __delitem__(self, key):
super(AtlasBase, self).__delitem__(key)
del self.__dict__[key]
def __repr__(self):
return json.dumps(self)
def type_coerce_attrs(self):
pass
class AtlasBaseModelObject(AtlasBase):
def __init__(self, members):
AtlasBase.__init__(self, members)
self.guid = members.get('guid')
if self.guid is None:
self.guid = next_id()
class TimeBoundary(AtlasBase):
def __init__(self, attrs={}):
AtlasBase.__init__(self, attrs)
self.startTime = attrs.get('startTime')
self.endTime = attrs.get('endTime')
self.timeZone = attrs.get('timeZone')
class Plist(AtlasBase):
def __init__(self, attrs={}):
AtlasBase.__init__(self, attrs)
self.list = non_null(attrs.get('list'), [])
self.startIndex = non_null(attrs.get('startIndex'), 0)
self.pageSize = non_null(attrs.get('pageSize'), 0)
self.totalCount = non_null(attrs.get('totalCount'), 0)
self.sortBy = attrs.get('sortBy')
self.sortType = attrs.get('sortType')
class SearchFilter(AtlasBase):
def __init__(self, attrs={}):
AtlasBase.__init__(self, attrs)
self.startIndex = non_null(attrs.get('startIndex'), 0)
self.maxsize = non_null(attrs.get('maxsize'), sys.maxsize)
self.getCount = non_null(attrs.get('getCount'), True)
intg/src/main/python/apache_atlas/model/profile.py 0 → 100644
#!/usr/bin/env/python
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicabwle law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from apache_atlas.model.discovery import *
from apache_atlas.utils import *
class AtlasUserSavedSearch(AtlasBaseModelObject):
def __init__(self, attrs={}):
AtlasBaseModelObject.__init__(self, attrs)
self.ownerName = attrs.get('ownerName')
self.name = attrs.get('name')
self.searchType = attrs.get('searchType')
self.searchParameters = attrs.get('searchParameters')
self.uiParameters = attrs.get('uiParameters')
def type_coerce_attrs(self):
super(AtlasUserSavedSearch, self).type_coerce_attrs()
self.searchParameters = type_coerce(self.searchParameters, SearchParameters)
intg/src/main/python/apache_atlas/model/relationship.py
......@@ -17,41 +17,48 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import enum
from apache_atlas.model.entity import AtlasStruct
from apache_atlas.model.instance import *
from apache_atlas.utils import *
class AtlasRelationship(AtlasStruct):
def __init__(self, attrs={}):
AtlasStruct.__init__(self, attrs)
self.guid = attrs.get('guid')
self.homeId = attrs.get('homeId')
self.provenanceType = attrs.get('provenanceType')
self.end1 = attrs.get('end1')
self.end2 = attrs.get('end2')
self.label = attrs.get('label')
self.propagateTags = attrs.get('propagateTags')
self.status = attrs.get('status')
self.createdBy = attrs.get('createdBy')
self.updatedBy = attrs.get('updatedBy')
self.createTime = attrs.get('createTime')
self.updateTime = attrs.get('updateTime')
self.version = attrs.get('version')
self.propagatedClassifications = attrs.get('propagatedClassifications')
self.blockedPropagatedClassifications = attrs.get('blockedPropagatedClassifications')
def type_coerce_attrs(self):
super(AtlasRelationship, self).type_coerce_attrs()
self.end1 = type_coerce(self.end1, AtlasObjectId)
self.end2 = type_coerce(self.end2, AtlasObjectId)
self.propagatedClassifications = type_coerce_list(self.propagatedClassifications, AtlasClassification)
self.blockedPropagatedClassifications = type_coerce_list(self.blockedPropagatedClassifications, AtlasClassification)
class AtlasRelationshipWithExtInfo(AtlasBase):
def __init__(self, attrs={}):
AtlasBase.__init__(self, attrs)
self.relationship = attrs.get('relationship')
self.referredEntities = attrs.get('referredEntities')
def type_coerce_attrs(self):
super(AtlasBase, self).type_coerce_attrs()
propagateTags_enum = enum.Enum('propagateTags_enum', 'NONE ONE_TO_TWO TWO_TO_ONE BOTH', module=__name__)
status_enum = enum.Enum('status_enum', 'ACTIVE DELETED', module=__name__)
def __init__(self, typeName=None, attributes=None, guid=None, homeId=None, provenanceType=None, end1=None, end2=None,
label=None, propagateTags=None, status=None, createdBy=None, updatedBy=None, createTime=None, updateTime=None,
version=0, propagatedClassifications=None, blockedPropagatedClassifications=None):
super().__init__(typeName, attributes)
self.guid = guid
self.homeId = homeId
self.provenanceType = provenanceType
self.end1 = end1
self.end2 = end2
self.label = label
self.propagateTags = propagateTags if propagateTags is not None else AtlasRelationship.propagateTags_enum.NONE.name
self.status = status if status is not None else AtlasRelationship.status_enum.ACTIVE.name
self.createdBy = createdBy
self.updatedBy = updatedBy
self.createTime = createTime
self.updateTime = updateTime
self.version = version
self.propagatedClassifications = propagatedClassifications if propagatedClassifications is not None else set()
self.blockedPropagatedClassifications = blockedPropagatedClassifications if blockedPropagatedClassifications is not None else set()
class AtlasRelationshipWithExtInfo:
def __init__(self, relationship=None, referredEntities=None):
self.relationship = relationship
self.referredEntities = referredEntities if referredEntities is not None else {}
self.relationship = type_coerce(self.relationship, AtlasRelationship)
self.referredEntities = type_coerce_dict(self.referredEntities, AtlasEntityHeader)
intg/src/main/python/apache_atlas/utils.py
......@@ -18,7 +18,6 @@
# limitations under the License.
import enum
from json import JSONEncoder
import time
BASE_URI = "api/atlas/"
......@@ -30,6 +29,12 @@ PREFIX_ATTR_ = "attr_"
s_nextId = milliseconds = int(round(time.time() * 1000)) + 1
def next_id():
global s_nextId
s_nextId += 1
return "-" + str(s_nextId)
def list_attributes_to_params(attributes_list, query_params=None):
if not query_params:
......@@ -42,7 +47,6 @@ def list_attributes_to_params(attributes_list, query_params=None):
return query_params
def attributes_to_params(attributes, query_params=None):
if not query_params:
query_params = {}
......@@ -54,12 +58,50 @@ def attributes_to_params(attributes, query_params=None):
return query_params
def non_null(obj, defValue):
return obj if obj is not None else defValue
class HttpMethod(enum.Enum):
GET = "GET"
PUT = "PUT"
POST = "POST"
DELETE = "DELETE"
def type_coerce(obj, objType):
if isinstance(obj, objType):
ret = obj
elif isinstance(obj, dict):
ret = objType(obj)
ret.type_coerce_attrs()
else:
ret = None
return ret
def type_coerce_list(obj, objType):
if isinstance(obj, list):
ret = []
for entry in obj:
ret.append(type_coerce(entry, objType))
else:
ret = None
return ret
def type_coerce_dict(obj, objType):
if isinstance(obj, dict):
ret = {}
for k, v in obj.items():
ret[k] = type_coerce(v, objType)
else:
ret = None
return ret
def type_coerce_dict_list(obj, objType):
if isinstance(obj, dict):
ret = {}
for k, v in obj.items():
ret[k] = type_coerce_list(v, objType)
else:
ret = None
return ret
class API:
......@@ -71,7 +113,7 @@ class API:
self.produces = produces
def format_path(self, params):
return API(self.path.format_map(params), self.method, self.expected_status, self.consumes, self.produces)
return API(self.path.format(**params), self.method, self.expected_status, self.consumes, self.produces)
def format_path_with_params(self, *params):
path = self.path
......@@ -82,14 +124,14 @@ class API:
return API(path, self.method, self.expected_status, self.consumes, self.produces)
class CustomEncoder(JSONEncoder):
def default(self, o):
if isinstance(o, set):
return list(o)
return o.__dict__
class HttpMethod(enum.Enum):
GET = "GET"
PUT = "PUT"
POST = "POST"
DELETE = "DELETE"
class AtlasBaseModelObject:
def __init__(self, guid=None):
self.guid = guid if guid is not None else "-" + str(s_nextId)
\ No newline at end of file
class HTTPStatus:
OK = 200
NO_CONTENT = 204
SERVICE_UNAVAILABLE = 503
intg/src/main/python/setup.py
......@@ -22,18 +22,22 @@ from setuptools import setup, find_packages
# External dependencies
requirements = ['requests>=2.24']
long_description = ''
with open("README.md", "r") as fh:
long_description = fh.read()
setup(
name='apache-atlas',
version='0.0.1',
version='0.0.2',
author="Apache Atlas",
author_email='dev@atlas.apache.org',
description="Apache Atlas Python Client",
long_description="Apache Atlas Python client",
long_description=long_description,
long_description_content_type="text/markdown",
url="https://github.com/apache/atlas/tree/master/intg/src/main/python",
license='Apache LICENSE 2.0',
classifiers=[
"Programming Language :: Python :: 3",
"Programming Language :: Python :: 2.7",
"License :: OSI Approved :: Apache Software License",
"Operating System :: OS Independent",
],
......@@ -42,4 +46,5 @@ setup(
include_package_data=True,
zip_safe=False,
keywords='atlas client, apache atlas',
)
\ No newline at end of file
python_requires='>=2.7',
)
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment