Skip to content

A
atlas
Commit cbdbed1d by Venkatesh Seetharam
Options

Add Quick Start Example and minor cleanup

parent 73748b46
Showing with 450 additions and 1112 deletions
dgi-aetna/js/app.js~ deleted 100644 → 0
var DgcApp = angular.module('DgcApp', [
'ngRoute',
'DgcControllers',
'ui.bootstrap'
]);
DgcApp.config(['$routeProvider', function($routeProvider) {
$routeProvider.
when('/Search', {
templateUrl: 'partials/search.html',
controller: 'ListController'
}).
when('/Search/:Id', {
templateUrl: 'partials/wiki.html',
controller: 'DefinitionController'
}).
otherwise({
redirectTo: '/Search'
});
}]);
dgi-aetna/js/attributes.html~ deleted 100644 → 0
<h4>{{key}}:</h4>
<p>{{val}}</p>
dgi-aetna/js/controllers.js~ deleted 100644 → 0
var DgcControllers = angular.module("DgcControllers", []);
DgcControllers.controller("ListController", ['$scope','$http', function($scope, $http)
{
$scope.executeSearch = function executeSearch() {
$scope.SearchQuery=$scope.query;
$scope.iswiki=false;
//$http.get('http://162.249.6.76:21000/api/metadata/entities/list/'+$scope.query)
var searchQry=$scope.query.split(",");
$http.get('http://162.249.6.76:21000/api/metadata/discovery/search/fulltext?depth=1&'+searchQry[0]+'&'+searchQry[1])
.success(function (data) {
$scope.iserror=false;
$scope.entities=angular.fromJson(data.vertices);
var count=0;
angular.forEach(data.vertices, function(v, index) {
count++;
});
$scope.matchingResults=count;
})
.error(function (e) {
$scope.iserror=true;
$scope.error=e;
$scope.matchingResults=0;
});
}
}]
);
DgcControllers.controller("DefinitionController", ['$scope','$http','$routeParams', function($scope, $http, $routeParams)
{
$scope.isString=function isString(value){
return typeof value === 'string';
}
$scope.Guid=$routeParams.Id;
$scope.iswiki=true;
$scope.selectedDefination={
"path":"wiki.html"
};
$http.get('http://162.249.6.76:21000/api/metadata/entities/definition/'+$routeParams.Id)
.success(function (data) {
$scope.iserror1=false;
$scope.details=angular.fromJson(data.definition);
})
.error(function (e) {
$scope.iserror1=true;
$scope.error1=e;
});
}]
);
DgcControllers.controller("LineageController", ['$scope','$http','$routeParams', function($scope, $http, $routeParams)
{
$scope.width = 1110;
$scope.height = 400;
//Initialize a default force layout, using the nodes and edges in dataset
$http.get('http://162.249.6.76:21000/api/metadata/discovery/search/relationships/'+$routeParams.Id+'?depth=3&&edgesToFollow=HiveLineage.sourceTables.0,HiveLineage.sourceTables.1,HiveLineage.tableName')
.success(function (data) {
$scope.iserror1=false;
$scope.lineage=angular.fromJson(data);
$scope.vertices = data.vertices; // response data
$scope.vts = [];
$scope.egs = [];
$scope.ids = [];
angular.forEach(data.vertices, function(v, index) {
if(v["hive_table.name"]==undefined){
$scope.vts.push({"name" :index,"values":v["HiveLineage.query"],"guid":v["guid"],"hasChild":"True"});
$scope.ids.push({"Id" :index,"Name":index,"values":v["HiveLineage.query"],"guid":v["guid"],"hasChild":"True"});
}else{
$scope.vts.push({"name" :v["hive_table.name"],"values":v["hive_table.description"],"guid":v["guid"],"hasChild":"False"});
$scope.ids.push({"Id" :index,"Name":v["hive_table.name"],"values":v["hive_table.description"],"guid":v["guid"],"hasChild":"False"});
}
});
angular.forEach(data.edges, function(e, index) {
$scope.egs.push({"source" :e["head"],"target":e["tail"]});
});
var edges2 = [];
$scope.egs.forEach(function(e) {
var sourceNode = $scope.ids.filter(function(n) { return n.Id === e.source; })[0],
targetNode = $scope.ids.filter(function(n) { return n.Id === e.target; })[0];
if(sourceNode==undefined){
sourceNode=e.source;
$scope.vts.push({"name" :e.source+", Missing Node","values":e.source+", Missing Node","guid":$scope.ids["guid"],"hasChild":"False"});
$scope.ids.push({"Id" :e.source,"Name":e.source+", Missing Node","values":e.source+", Missing Node","guid":$scope.ids["guid"],"hasChild":"False"});
}
if(targetNode==undefined){
targetNode=e.target;
$scope.vts.push({"name" :e.target+", Missing Node","values":e.target+", Missing Node","guid":$scope.ids["guid"],"hasChild":"False"});
$scope.ids.push({"Id" :e.target,"Name":e.target+", Missing Node","values":e.target+", Missing Node","guid":$scope.ids["guid"],"hasChild":"False"});
}
edges2.push({source: sourceNode, target: targetNode});
});
var edges1 = [];
$scope.egs.forEach(function(e) {
var sourceNode = $scope.ids.filter(function(n) { return n.Id === e.source; })[0],
targetNode = $scope.ids.filter(function(n) { return n.Id === e.target; })[0];
edges1.push({source: sourceNode, target: targetNode});
});
//Width and height
var w = 1110;
var h = 400;
var force = d3.layout.force()
.nodes($scope.ids)
.links(edges1)
.size([w, h])
.linkDistance([180])
.charge([-250])
.start();
var colors = d3.scale.category10();
//Create SVG element
var svg = d3.select("svg")
.attr("width", w)
.attr("height", h);
var tip = d3.tip()
.attr('class', 'd3-tip')
.offset([-10, 0])
.html(function(d) {
return "<pre class='alert alert-success' style='max-width:400px;'>" + d.values + "</pre>";
});
svg.call(tip);
//Create edges as lines
var edges = svg.selectAll("line")
.data(edges1)
.enter()
.append("line")
.style("stroke", "#23A410")
.style("stroke-width", 3);
var node = svg.selectAll(".node")
.data($scope.ids)
.enter().append("g")
.attr("class", "node")
.on("mouseover", tip.show)
.on("mouseout", tip.hide)
.on("click", function(d){
tip.hide();
if(d.guid==undefined){
}
else
{
location.href="#/Search/"+d.guid;
}
})
.call(force.drag);
svg.append("svg:pattern").attr("id","processICO").attr("width",1).attr("height",1)
.append("svg:image").attr("xlink:href","img/process.png").attr("x",-5.5).attr("y",-4).attr("width",41).attr("height",42);
svg.append("svg:pattern").attr("id","textICO").attr("width",1).attr("height",1)
.append("svg:image").attr("xlink:href","img/text.ico").attr("x",2).attr("y",2).attr("width",25).attr("height",25);
// define arrow markers for graph links
svg.append('svg:defs').append('svg:marker')
.attr('id', 'end-arrow')
.attr('viewBox', '0 -5 10 10')
.attr('refX', 10)
.attr('markerWidth', 5)
.attr('markerHeight', 5)
.attr('orient', 'auto')
.append('svg:path')
.attr('d', 'M0,-5L10,0L0,5')
.attr('fill', '#7B7A7A');
svg.append('svg:defs').append('svg:marker')
.attr('id', 'start-arrow')
.attr('viewBox', '0 -5 10 10')
.attr('refX', 4)
.attr('markerWidth', 3)
.attr('markerHeight', 3)
.attr('orient', 'auto')
.append('svg:path')
.attr('d', 'M10,-5L0,0L10,5')
.attr('fill', '#000');
// handles to link and node element groups
var path = svg.append('svg:g').selectAll('path')
.data(force.links())
.enter().append("svg:path")
.attr("class", "link")
.attr('marker-end','url(#end-arrow)');
//Create nodes as circles
//var nodes = svg.selectAll("circle")
//.data(dataset.nodes)
// .enter()
node.append("circle")
.attr("r", function(d, i) {
if(d.hasChild=="True"){
return 15;
}else{
return 15;
}
return 10;
})
.attr("cursor","pointer")
.style("fill", function(d, i) {
if(d.hasChild=="True"){
return "url('#processICO')";
}else{
return "url('#textICO')";
}
return colors(i);
})
.attr("class","circle");
//.call(force.drag);
//Add text
node.append("text")
.attr("x", 12)
.attr("dy", ".35em")
.text(function(d) { return d.Name; });
//Every time the simulation "ticks", this will be called
force.on("tick", function() {
edges.attr("x1", function(d) { return d.source.x; })
.attr("y1", function(d) { return d.source.y; })
.attr("x2", function(d) { return d.target.x; })
.attr("y2", function(d) { return d.target.y; });
path.attr('d', function(d) {
var deltaX = d.target.x - d.source.x,
deltaY = d.target.y - d.source.y,
dist = Math.sqrt(deltaX * deltaX + deltaY * deltaY),
normX = deltaX / dist,
normY = deltaY / dist,
sourcePadding = d.left ? 17 : 12,
targetPadding = d.right ? 17 : 12,
sourceX = d.source.x + (sourcePadding * normX),
sourceY = d.source.y + (sourcePadding * normY),
targetX = d.target.x - (targetPadding * normX),
targetY = d.target.y - (targetPadding * normY);
return 'M' + sourceX + ',' + sourceY + 'L' + targetX + ',' + targetY;
});
//node.attr("cx", function(d) { return d.x; })
//.attr("cy", function(d) { return d.y; });
node
.attr("transform", function(d) { return "translate(" + d.x + "," + d.y + ")"; });
})
function mouseover(d) {
d3.select(this).select("circle").transition()
.duration(750)
.attr("r", 16);
}
function mouseout() {
d3.select(this).select("circle").transition()
.duration(750)
.attr("r", 10);
}
});
}]
);
dgi-aetna/partials/attribute.html~ deleted 100644 → 0
<h4 class="titletxt2">{{key}}:</h4>
<p class="sm-txt2">{{value | date:'medium'}}</p>
dgi-aetna/partials/lineage.html~ deleted 100644 → 0
<div class="container" ng-controller="LineageController">
<div class="lineage"></div>
<svg ng-attr-width="{{width}}"
ng-attr-height="{{height}}">
</svg>
</div>
dgi-aetna/partials/search.html~ deleted 100644 → 0
<!-- Page Content -->
<div style="margin-top:-20px;padding-top:20px; background-color:#eeeded;padding-bottom:10px;border-bottom:solid 1px #d9d9d8;">
<form name=form" class="container">
<div class="col-lg-7">
<div class="row input-group">
<input type="text" ng-model="query" ng-keyup="$event.keyCode == 13 && executeSearch()" class="form-control search_table" placeholder="Search">
<span class="input-group-btn">
<button class="btn btn-success" ng-click="executeSearch()" type="submit"><i class="glyphicon glyphicon-search white "></i></button>
</span>
</div>
<div class="row">
<small style="color:#999999; margin-top:2px;">property=HiveLineage.executionEngine&text=tez</small><br/>
<small style="color:#999999; margin-top:2px;">property=type&text=HiveLineage</small><br/>
<small style="color:#999999; margin-top:2px;">property=type&text=hive_table</small>
</div>
</div>
</form>
</div>
<div class="container" style="min-height:330px;">
<div class="row">
<div class="col-lg-11">
<div ng-hide="iswiki">
<input type="hidden" ng-model="iserror">
<h4 class="txt1" ng-hide="!matchingResults">{{matchingResults}} results matching your query "{{SearchQuery}}" were found"</h4>
<ul class="list-unstyled">
<li ng-hide="iserror" class="sm-txt1" ng-repeat="entity in entities"><u><a href="#Search/{{entity.guid}}" ng-click="StoreJson(entity)" style="line-height: 2.5;">{{ entity.guid}}</a></u>
</li>
<li ng-show="iserror" class="sm-txt1"></li>
</ul>
</ul>
</div>
<div ng-show="iswiki" data-ng-include="selectedDefination.path"></div>
</div>
</div>
</div>
dgi-aetna/partials/wiki.html~ deleted 100644 → 0
<div class="content">
<div class="container">
<div class="col-lg-12">
<h4 class="txt1">{{Guid}}</h4>
<br/>
<div ng-controller="LineageController" style="border-bottom: 2px solid #060;margin-bottom: 30px;">
<div class="lineage"></div>
<svg ng-attr-width="{{width}}"
ng-attr-height="{{height}}">
</svg>
</div>
<div ng-repeat="(key,value) in details" ng-if="isString(value)" ng-include="'partials/attribute.html'"></div>
<!--<tab heading="Raw"><pre>{{details}}</pre></tab>-->
</div>
</div>
</div>
src/bin/bootstrap-demo-data.sh deleted 100644 → 0
#!/bin/bash
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License. See accompanying LICENSE file.
#
# resolve links - $0 may be a softlink
PRG="${0}"
while [ -h "${PRG}" ]; do
ls=`ls -ld "${PRG}"`
link=`expr "$ls" : '.*-> \(.*\)$'`
if expr "$link" : '/.*' > /dev/null; then
PRG="$link"
else
PRG=`dirname "${PRG}"`/"$link"
fi
done
BASEDIR=`dirname ${PRG}`
BASEDIR=`cd ${BASEDIR}/..;pwd`
if [ -z "$METADATA_CONF" ]; then
METADATA_CONF=${BASEDIR}/conf
fi
export METADATA_CONF
if [ -f "${METADATA_CONF}/metadata-env.sh" ]; then
. "${METADATA_CONF}/metadata-env.sh"
fi
if test -z ${JAVA_HOME}
then
JAVA_BIN=`which java`
JAR_BIN=`which jar`
else
JAVA_BIN=${JAVA_HOME}/bin/java
JAR_BIN=${JAVA_HOME}/bin/jar
fi
export JAVA_BIN
if [ ! -e $JAVA_BIN ] || [ ! -e $JAR_BIN ]; then
echo "$JAVA_BIN and/or $JAR_BIN not found on the system. Please make sure java and jar commands are available."
exit 1
fi
# default the heap size to 1GB
DEFAULT_JAVA_HEAP_MAX=-Xmx1024m
METADATA_OPTS="$DEFAULT_JAVA_HEAP_MAX $METADATA_OPTS"
METADATACPPATH="$METADATA_CONF"
METADATA_EXPANDED_WEBAPP_DIR=${METADATA_EXPANDED_WEBAPP_DIR:-${BASEDIR}/server/webapp}
export METADATA_EXPANDED_WEBAPP_DIR
METADATACPPATH="${METADATACPPATH}:${METADATA_EXPANDED_WEBAPP_DIR}/metadata/WEB-INF/classes"
METADATACPPATH="${METADATACPPATH}:${METADATA_EXPANDED_WEBAPP_DIR}/metadata/WEB-INF/lib/*:${BASEDIR}/libext/*"
# log and pid dirs for applications
METADATA_LOG_DIR="${METADATA_LOG_DIR:-$BASEDIR/logs}"
export METADATA_LOG_DIR
METADATA_HOME_DIR="${METADATA_HOME_DIR:-$BASEDIR}"
export METADATA_HOME_DIR
JAVA_PROPERTIES="$METADATA_OPTS $METADATA_PROPERTIES -Dmetadata.log.dir=$METADATA_LOG_DIR -Dmetadata.home=${METADATA_HOME_DIR}"
${JAVA_BIN} ${JAVA_PROPERTIES} -cp ${METADATACPPATH} org.apache.hadoop.metadata.DemoDataDriver
echo Test data added to Metadata Server!!!
src/bin/bootstrap-data.sh → src/bin/quick-start.sh
......@@ -74,6 +74,6 @@ export METADATA_HOME_DIR
JAVA_PROPERTIES="$METADATA_OPTS $METADATA_PROPERTIES -Dmetadata.log.dir=$METADATA_LOG_DIR -Dmetadata.home=${METADATA_HOME_DIR}"
${JAVA_BIN} ${JAVA_PROPERTIES} -cp ${METADATACPPATH} org.apache.hadoop.metadata.TestDataDriver
${JAVA_BIN} ${JAVA_PROPERTIES} -cp ${METADATACPPATH} org.apache.hadoop.metadata.examples.QuickStart
echo Test data added to Metadata Server!!!
echo Example data added to Metadata Server!!!
webapp/src/main/java/org/apache/hadoop/metadata/DemoDataDriver.java deleted 100644 → 0
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.metadata;
import com.google.common.collect.ImmutableList;
import com.google.gson.JsonArray;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import com.google.gson.JsonParser;
import com.sun.jersey.api.client.Client;
import com.sun.jersey.api.client.ClientResponse;
import com.sun.jersey.api.client.WebResource;
import com.sun.jersey.api.client.config.DefaultClientConfig;
import org.apache.hadoop.metadata.typesystem.json.Serialization$;
import org.apache.hadoop.metadata.typesystem.json.TypesSerialization;
import org.apache.hadoop.metadata.typesystem.ITypedReferenceableInstance;
import org.apache.hadoop.metadata.typesystem.Referenceable;
import org.apache.hadoop.metadata.typesystem.Struct;
import org.apache.hadoop.metadata.typesystem.types.AttributeDefinition;
import org.apache.hadoop.metadata.typesystem.types.ClassType;
import org.apache.hadoop.metadata.typesystem.types.DataTypes;
import org.apache.hadoop.metadata.typesystem.types.HierarchicalTypeDefinition;
import org.apache.hadoop.metadata.typesystem.types.Multiplicity;
import org.apache.hadoop.metadata.typesystem.types.StructTypeDefinition;
import org.apache.hadoop.metadata.typesystem.types.TraitType;
import org.apache.hadoop.metadata.typesystem.types.TypeSystem;
import org.codehaus.jettison.json.JSONArray;
import org.codehaus.jettison.json.JSONException;
import org.codehaus.jettison.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.ws.rs.HttpMethod;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import javax.ws.rs.core.UriBuilder;
import java.util.ArrayList;
import java.util.Arrays;
import static org.apache.hadoop.metadata.typesystem.types.utils.TypesUtil.createClassTypeDef;
import static org.apache.hadoop.metadata.typesystem.types.utils.TypesUtil.createRequiredAttrDef;
import static org.apache.hadoop.metadata.typesystem.types.utils.TypesUtil.createTraitTypeDef;
public class DemoDataDriver {
private static final Logger LOG = LoggerFactory
.getLogger(DemoDataDriver.class);
private static final String DATABASE_TYPE = "hive_database";
private static final String TABLE_TYPE = "hive_table";
private static final String LINEAGE_TYPE = "HiveLineage";
private static ArrayList<ITypedReferenceableInstance> initTableArray;
private static ArrayList<ITypedReferenceableInstance> postSaveTableArray;
private static ArrayList<ITypedReferenceableInstance> lineageArray;
protected TypeSystem typeSystem;
protected WebResource service;
public static void main(String[] args) throws Exception {
DemoDataDriver driver = new DemoDataDriver();
driver.setUp();
driver.createTypes();
driver.submitTypes();
DemoDataDriver.initTableArray = new ArrayList<ITypedReferenceableInstance>();
DemoDataDriver.postSaveTableArray = new ArrayList<ITypedReferenceableInstance>();
DemoDataDriver.lineageArray = new ArrayList<ITypedReferenceableInstance>();
String[][] tableData = getTestTableData();
//Create Table Objects
for (String[] row : tableData) {
ITypedReferenceableInstance tableInstance = driver
.createHiveTableInstance(row[0], row[1], row[2], row[3],
row[4]);
initTableArray.add(tableInstance);
}
//Save Table Objects
for (ITypedReferenceableInstance i : initTableArray) {
driver.submitEntity(i);
}
//Returned the Saved Table Objects
JsonParser jp = new JsonParser();
JsonObject jo = (JsonObject) jp.parse(driver.getEntityReturnList());
JsonArray ja = jo.getAsJsonArray("results");
for (JsonElement e : ja) {
JsonObject joInner = (JsonObject) jp
.parse(driver.getTableEntityByGUID(e.getAsString()));
ITypedReferenceableInstance tabRef = Serialization$.MODULE$
.fromJson(joInner.get("definition").getAsString().toString());
postSaveTableArray.add(tabRef);
}
//Create Lineage Objects
String[][] lineageData = getTestLineageData();
for (String[] row : lineageData) {
ITypedReferenceableInstance lineageInstance = driver
.createLingeageInstance(row[0], row[1], row[2], row[3],
row[4], row[5], row[6], row[7], row[8], row[9]);
lineageArray.add(lineageInstance);
}
//Save Lineage Objects
for (ITypedReferenceableInstance i : lineageArray) {
driver.submitEntity(i);
}
System.out.println("###############DATABASES ADDED##############################");
driver.getEntityList(DemoDataDriver.DATABASE_TYPE);
System.out.println("##################TABLES ADDED##############################");
driver.getEntityList(DemoDataDriver.TABLE_TYPE);
System.out.println("#################LINEAGE ADDED##############################");
driver.getEntityList(DemoDataDriver.LINEAGE_TYPE);
System.out.println("############################################################");
System.out.println("DEMO DATA ADDED SUCCESSFULLY");
}
private static String[][] getTestLineageData() {
return new String[][]{
{
"s123456_20150106120303_036186d5-a991-4dfc-9ff2-05b072c7e711",
"90797386-3933-4ab0-ae68-a7baa7e155d4",
"Service User 02",
"1420563838114",
"1420563853806",
"CREATE TABLE providerCharges AS SELECT providerMasterList.*, claimPayments.* FROM providerMasterList LEFT JOIN claimPayments ON providerMasterList.providerID = claimPayments.providerId WHERE claimPayments.paidStatus = \"true\";",
"providerCharges", "true", "tez",
"providerMasterList,claimPayments"},
{
"s123456_20150106120304_036125d5-a991-4dfc-9ff2-05b665c7e711",
"90797386-3933-4ab0-ae68-a7baa72435d4",
"Service User 02",
"1420563838314",
"1420563853906",
"CREATE TABLE providerComparativeModel AS SELECT providerCharges.*, LocationsOfThings.* FROM providerCharges LEFT JOIN LocationsOfThings ON providerCharges.providerName = LocationsOfThings.peopleName WHERE LocationsOfThings.isDr = \"true\";",
"providerComparativeModel", "true", "mapred",
"providerCharges,LocationsOfThings"}};
}
private static String[][] getTestTableData() {
return new String[][]{
{
"provider_db",
"providerMasterList",
"Providers Addresses and Locations of performed procedures",
"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe",
"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"},
{"charges_db", "claimPayments", "Claims paid",
"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe",
"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"},
{"model_db", "providerCharges",
"Combined Claims and Providers Mapping",
"org.apache.hadoop.hive.ql.io.orc.OrcSerde",
"org.apache.hadoop.hive.ql.io.orc.OrcSerde"},
{"ds_db", "LocationsOfThings",
"DS imported dataset from internet of ideas",
"org.apache.hadoop.hive.ql.io.orc.OrcSerde",
"org.apache.hadoop.hive.ql.io.orc.OrcSerde"},
{"ds_db",
"providerComparativeModel",
"DS created Table for comparing charges findings to dataset from internet",
"org.apache.hadoop.hive.ql.io.orc.OrcSerde",
"org.apache.hadoop.hive.ql.io.orc.OrcSerde"}
};
}
public void setUp() throws Exception {
typeSystem = TypeSystem.getInstance();
typeSystem.reset();
String baseUrl = "http://localhost:21000/";
DefaultClientConfig config = new DefaultClientConfig();
Client client = Client.create(config);
client.resource(UriBuilder.fromUri(baseUrl).build());
service = client.resource(UriBuilder.fromUri(baseUrl).build());
}
public void submitEntity(ITypedReferenceableInstance tableInstance)
throws Exception {
String tableInstanceAsJSON = Serialization$.MODULE$
.toJson(tableInstance);
LOG.debug("tableInstance = " + tableInstanceAsJSON);
WebResource resource = service.path("api/metadata/entities/submit")
.path(TABLE_TYPE);
ClientResponse clientResponse = resource
.accept(MediaType.APPLICATION_JSON)
.type(MediaType.APPLICATION_JSON)
.method(HttpMethod.POST, ClientResponse.class,
tableInstanceAsJSON);
assert clientResponse.getStatus() == Response.Status.OK.getStatusCode();
}
public void getEntityList(String s) throws Exception {
ClientResponse clientResponse = service
.path("api/metadata/entities/list/").path(s)
.accept(MediaType.APPLICATION_JSON)
.type(MediaType.APPLICATION_JSON)
.method(HttpMethod.GET, ClientResponse.class);
assert clientResponse.getStatus() == Response.Status.OK.getStatusCode();
String responseAsString = clientResponse.getEntity(String.class);
JSONObject response = new JSONObject(responseAsString);
final JSONArray list = response.getJSONArray("list");
System.out.println("list = " + list);
assert list != null;
assert list.length() > 0;
}
public String getEntityReturnList() {
ClientResponse clientResponse = service
.path("api/metadata/entities/list/").path(TABLE_TYPE)
.accept(MediaType.APPLICATION_JSON)
.type(MediaType.APPLICATION_JSON)
.method(HttpMethod.GET, ClientResponse.class);
assert clientResponse.getStatus() == Response.Status.OK.getStatusCode();
String responseAsString = clientResponse.getEntity(String.class);
return responseAsString;
}
public String getTableEntityByGUID(String guid) throws Exception {
ClientResponse clientResponse = service
.path("api/metadata/entities/definition/" + guid)
.accept(MediaType.APPLICATION_JSON)
.type(MediaType.APPLICATION_JSON)
.method(HttpMethod.GET, ClientResponse.class);
assert clientResponse.getStatus() == Response.Status.OK.getStatusCode();
String responseAsString = clientResponse.getEntity(String.class);
return responseAsString;
}
private void createTypes() throws Exception {
HierarchicalTypeDefinition<ClassType> databaseTypeDefinition = createClassTypeDef(
DATABASE_TYPE, ImmutableList.<String>of(),
createRequiredAttrDef("name", DataTypes.STRING_TYPE),
createRequiredAttrDef("description", DataTypes.STRING_TYPE));
StructTypeDefinition structTypeDefinition = new StructTypeDefinition(
"serdeType", new AttributeDefinition[]{
createRequiredAttrDef("name", DataTypes.STRING_TYPE),
createRequiredAttrDef("serde", DataTypes.STRING_TYPE)});
HierarchicalTypeDefinition<ClassType> tableTypeDefinition = createClassTypeDef(
TABLE_TYPE, ImmutableList.<String>of(),
createRequiredAttrDef("name", DataTypes.STRING_TYPE),
createRequiredAttrDef("description", DataTypes.STRING_TYPE),
createRequiredAttrDef("type", DataTypes.STRING_TYPE),
new AttributeDefinition("serde1", "serdeType",
Multiplicity.REQUIRED, false, null),
new AttributeDefinition("serde2", "serdeType",
Multiplicity.REQUIRED, false, null),
new AttributeDefinition("database", DATABASE_TYPE,
Multiplicity.REQUIRED, true, null));
HierarchicalTypeDefinition<TraitType> classificationTypeDefinition = createTraitTypeDef(
"classification", ImmutableList.<String>of(),
createRequiredAttrDef("tag", DataTypes.STRING_TYPE));
HierarchicalTypeDefinition<ClassType> lineageTypeDefinition = createClassTypeDef(
"HiveLineage",
ImmutableList.<String>of(),
createRequiredAttrDef("queryId", DataTypes.STRING_TYPE),
createRequiredAttrDef("hiveId", DataTypes.STRING_TYPE),
createRequiredAttrDef("user", DataTypes.STRING_TYPE),
createRequiredAttrDef("queryStartTime", DataTypes.STRING_TYPE),
createRequiredAttrDef("queryEndTime", DataTypes.STRING_TYPE),
createRequiredAttrDef("query", DataTypes.STRING_TYPE),
new AttributeDefinition("tableName", TABLE_TYPE,
Multiplicity.REQUIRED, true, null),
createRequiredAttrDef("success", DataTypes.STRING_TYPE),
createRequiredAttrDef("executionEngine", DataTypes.STRING_TYPE),
new AttributeDefinition("sourceTables", DataTypes
.arrayTypeName(TABLE_TYPE), Multiplicity.COLLECTION,
true, "forwardLineage"));
typeSystem.defineTypes(ImmutableList.of(structTypeDefinition),
ImmutableList.of(classificationTypeDefinition), ImmutableList
.of(databaseTypeDefinition, tableTypeDefinition,
lineageTypeDefinition));
}
private void submitTypes() throws Exception {
String tableTypesAsJSON = TypesSerialization.toJson(
typeSystem,
Arrays.asList(new String[]{DATABASE_TYPE, TABLE_TYPE,
"serdeType", "classification"}));
String lineageTypesAsJSON = TypesSerialization.toJson(typeSystem,
Arrays.asList(new String[]{"HiveLineage"}));
sumbitType(tableTypesAsJSON, TABLE_TYPE);
sumbitType(lineageTypesAsJSON, "HiveLineage");
}
private void sumbitType(String typesAsJSON, String type)
throws JSONException {
WebResource resource = service.path("api/metadata/types/submit");
ClientResponse clientResponse = resource
.accept(MediaType.APPLICATION_JSON)
.type(MediaType.APPLICATION_JSON)
.method(HttpMethod.POST, ClientResponse.class, typesAsJSON);
assert clientResponse.getStatus() == Response.Status.OK.getStatusCode();
String responseAsString = clientResponse.getEntity(String.class);
JSONObject response = new JSONObject(responseAsString);
assert response.get("typeName").equals(type);
assert response.get("types") != null;
}
private ITypedReferenceableInstance createHiveTableInstance(String db,
String table, String trait,
String serde1, String serde2)
throws Exception {
Referenceable databaseInstance = new Referenceable(DATABASE_TYPE);
databaseInstance.set("name", db);
databaseInstance.set("description", db + " database");
Referenceable tableInstance = new Referenceable(TABLE_TYPE,
"classification");
tableInstance.set("name", table);
tableInstance.set("description", table + " table");
tableInstance.set("type", "managed");
tableInstance.set("database", databaseInstance);
Struct traitInstance = (Struct) tableInstance
.getTrait("classification");
traitInstance.set("tag", trait);
Struct serde1Instance = new Struct("serdeType");
serde1Instance.set("name", serde1);
serde1Instance.set("serde", serde1);
tableInstance.set("serde1", serde1Instance);
Struct serde2Instance = new Struct("serdeType");
serde2Instance.set("name", serde2);
serde2Instance.set("serde", serde2);
tableInstance.set("serde2", serde2Instance);
ClassType tableType = typeSystem.getDataType(ClassType.class,
TABLE_TYPE);
return tableType.convert(tableInstance, Multiplicity.REQUIRED);
}
private ITypedReferenceableInstance createLingeageInstance(String queryId,
String hiveId, String user,
String queryStartTime,
String queryEndTime, String query,
String tableName,
String success,
String executionEngine,
String sourceTables)
throws Exception {
Referenceable lineageInstance = new Referenceable("HiveLineage");
lineageInstance.set("queryId", queryId);
lineageInstance.set("hiveId", hiveId);
lineageInstance.set("user", user);
lineageInstance.set("queryStartTime", queryStartTime);
lineageInstance.set("queryEndTime", queryEndTime);
lineageInstance.set("query", query);
lineageInstance.set("success", success);
lineageInstance.set("executionEngine", executionEngine);
for (ITypedReferenceableInstance table : postSaveTableArray) {
if (table.get("name").equals(tableName)) {
lineageInstance.set("tableName", table);
break;
}
}
ArrayList<ITypedReferenceableInstance> sourceTablesRefArr
= new ArrayList<ITypedReferenceableInstance>();
for (String s : sourceTables.split(",")) {
System.out.println("search for table " + s);
for (ITypedReferenceableInstance table : postSaveTableArray) {
if (table.get("name").equals(s)) {
sourceTablesRefArr.add(table);
}
}
}
lineageInstance.set("sourceTables",
ImmutableList.copyOf(sourceTablesRefArr));
ClassType lineageType = typeSystem.getDataType(ClassType.class,
"HiveLineage");
return lineageType.convert(lineageInstance, Multiplicity.REQUIRED);
}
}
webapp/src/main/java/org/apache/hadoop/metadata/TestDataDriver.java deleted 100644 → 0
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.metadata;
import com.google.common.collect.ImmutableList;
import com.sun.jersey.api.client.Client;
import com.sun.jersey.api.client.ClientResponse;
import com.sun.jersey.api.client.WebResource;
import com.sun.jersey.api.client.config.DefaultClientConfig;
import org.apache.hadoop.metadata.typesystem.json.Serialization$;
import org.apache.hadoop.metadata.typesystem.json.TypesSerialization;
import org.apache.hadoop.metadata.typesystem.ITypedReferenceableInstance;
import org.apache.hadoop.metadata.typesystem.Referenceable;
import org.apache.hadoop.metadata.typesystem.Struct;
import org.apache.hadoop.metadata.typesystem.types.AttributeDefinition;
import org.apache.hadoop.metadata.typesystem.types.ClassType;
import org.apache.hadoop.metadata.typesystem.types.DataTypes;
import org.apache.hadoop.metadata.typesystem.types.HierarchicalTypeDefinition;
import org.apache.hadoop.metadata.typesystem.types.Multiplicity;
import org.apache.hadoop.metadata.typesystem.types.StructTypeDefinition;
import org.apache.hadoop.metadata.typesystem.types.TraitType;
import org.apache.hadoop.metadata.typesystem.types.TypeSystem;
import org.codehaus.jettison.json.JSONArray;
import org.codehaus.jettison.json.JSONException;
import org.codehaus.jettison.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.ws.rs.HttpMethod;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import javax.ws.rs.core.UriBuilder;
import java.util.Arrays;
import static org.apache.hadoop.metadata.typesystem.types.utils.TypesUtil.createClassTypeDef;
import static org.apache.hadoop.metadata.typesystem.types.utils.TypesUtil.createRequiredAttrDef;
import static org.apache.hadoop.metadata.typesystem.types.utils.TypesUtil.createTraitTypeDef;
public class TestDataDriver {
private static final Logger LOG = LoggerFactory.getLogger(TestDataDriver.class);
private static final String DATABASE_TYPE = "hive_database";
private static final String TABLE_TYPE = "hive_table";
protected TypeSystem typeSystem;
protected WebResource service;
public static void main(String[] args) throws Exception {
TestDataDriver driver = new TestDataDriver();
driver.setUp();
driver.createHiveTypes();
driver.submitTypes();
String[][] data = getTestData();
for (String[] row : data) {
ITypedReferenceableInstance tableInstance = driver.createHiveTableInstance(
row[0], row[1], row[2], row[3], row[4]);
driver.submitEntity(tableInstance);
}
driver.getEntityList();
}
private static String[][] getTestData() {
return new String[][]{
{"sales_db", "customer_fact", "pii", "serde1", "serde2"},
{"sales_db", "sales_dim", "dim", "serde1", "serde2"},
{"sales_db", "product_dim", "dim", "serde1", "serde2"},
{"sales_db", "time_dim", "dim", "serde1", "serde2"},
{"reporting_db", "weekly_sales_summary", "summary", "serde1", "serde2"},
{"reporting_db", "daily_sales_summary", "summary", "serde1", "serde2"},
{"reporting_db", "monthly_sales_summary", "summary", "serde1", "serde2"},
{"reporting_db", "quarterly_sales_summary", "summary", "serde1", "serde2"},
{"reporting_db", "yearly_sales_summary", "summary", "serde1", "serde2"},
};
}
public void setUp() throws Exception {
typeSystem = TypeSystem.getInstance();
typeSystem.reset();
String baseUrl = "http://localhost:21000/";
DefaultClientConfig config = new DefaultClientConfig();
Client client = Client.create(config);
client.resource(UriBuilder.fromUri(baseUrl).build());
service = client.resource(UriBuilder.fromUri(baseUrl).build());
}
public void submitEntity(ITypedReferenceableInstance tableInstance) throws Exception {
String tableInstanceAsJSON = Serialization$.MODULE$.toJson(tableInstance);
LOG.debug("tableInstance = " + tableInstanceAsJSON);
WebResource resource = service
.path("api/metadata/entities/submit")
.path(TABLE_TYPE);
ClientResponse clientResponse = resource
.accept(MediaType.APPLICATION_JSON)
.type(MediaType.APPLICATION_JSON)
.method(HttpMethod.POST, ClientResponse.class, tableInstanceAsJSON);
assert clientResponse.getStatus() == Response.Status.OK.getStatusCode();
}
public void getEntityList() throws Exception {
ClientResponse clientResponse = service
.path("api/metadata/entities/list/")
.path(TABLE_TYPE)
.accept(MediaType.APPLICATION_JSON)
.type(MediaType.APPLICATION_JSON)
.method(HttpMethod.GET, ClientResponse.class);
assert clientResponse.getStatus() == Response.Status.OK.getStatusCode();
String responseAsString = clientResponse.getEntity(String.class);
JSONObject response = new JSONObject(responseAsString);
final JSONArray list = response.getJSONArray("list");
System.out.println("list = " + list);
assert list != null;
assert list.length() > 0;
}
private void createHiveTypes() throws Exception {
HierarchicalTypeDefinition<ClassType> databaseTypeDefinition =
createClassTypeDef(DATABASE_TYPE,
ImmutableList.<String>of(),
createRequiredAttrDef("name", DataTypes.STRING_TYPE),
createRequiredAttrDef("description", DataTypes.STRING_TYPE));
StructTypeDefinition structTypeDefinition =
new StructTypeDefinition("serdeType",
new AttributeDefinition[]{
createRequiredAttrDef("name", DataTypes.STRING_TYPE),
createRequiredAttrDef("serde", DataTypes.STRING_TYPE)
});
HierarchicalTypeDefinition<ClassType> tableTypeDefinition =
createClassTypeDef(TABLE_TYPE,
ImmutableList.<String>of(),
createRequiredAttrDef("name", DataTypes.STRING_TYPE),
createRequiredAttrDef("description", DataTypes.STRING_TYPE),
createRequiredAttrDef("type", DataTypes.STRING_TYPE),
new AttributeDefinition("serde1",
"serdeType", Multiplicity.REQUIRED, false, null),
new AttributeDefinition("serde2",
"serdeType", Multiplicity.REQUIRED, false, null),
new AttributeDefinition("database",
DATABASE_TYPE, Multiplicity.REQUIRED, true, null));
HierarchicalTypeDefinition<TraitType> classificationTypeDefinition =
createTraitTypeDef("classification",
ImmutableList.<String>of(),
createRequiredAttrDef("tag", DataTypes.STRING_TYPE));
typeSystem.defineTypes(
ImmutableList.of(structTypeDefinition),
ImmutableList.of(classificationTypeDefinition),
ImmutableList.of(databaseTypeDefinition, tableTypeDefinition));
}
private void submitTypes() throws Exception {
String typesAsJSON = TypesSerialization.toJson(typeSystem,
Arrays.asList(
new String[]{DATABASE_TYPE, TABLE_TYPE, "serdeType", "classification"}));
sumbitType(typesAsJSON, TABLE_TYPE);
}
private void sumbitType(String typesAsJSON, String type) throws JSONException {
WebResource resource = service
.path("api/metadata/types/submit");
ClientResponse clientResponse = resource
.accept(MediaType.APPLICATION_JSON)
.type(MediaType.APPLICATION_JSON)
.method(HttpMethod.POST, ClientResponse.class, typesAsJSON);
assert clientResponse.getStatus() == Response.Status.OK.getStatusCode();
String responseAsString = clientResponse.getEntity(String.class);
JSONObject response = new JSONObject(responseAsString);
assert response.get("typeName").equals(type);
assert response.get("types") != null;
}
private ITypedReferenceableInstance createHiveTableInstance(String db,
String table,
String trait,
String serde1,
String serde2) throws Exception {
Referenceable databaseInstance = new Referenceable(DATABASE_TYPE);
databaseInstance.set("name", db);
databaseInstance.set("description", db + " database");
Referenceable tableInstance = new Referenceable(TABLE_TYPE, "classification");
tableInstance.set("name", table);
tableInstance.set("description", table + " table");
tableInstance.set("type", "managed");
tableInstance.set("database", databaseInstance);
Struct traitInstance = (Struct) tableInstance.getTrait("classification");
traitInstance.set("tag", trait);
Struct serde1Instance = new Struct("serdeType");
serde1Instance.set("name", serde1);
serde1Instance.set("serde", serde1);
tableInstance.set("serde1", serde1Instance);
Struct serde2Instance = new Struct("serdeType");
serde2Instance.set("name", serde2);
serde2Instance.set("serde", serde2);
tableInstance.set("serde2", serde2Instance);
ClassType tableType = typeSystem.getDataType(ClassType.class, TABLE_TYPE);
return tableType.convert(tableInstance, Multiplicity.REQUIRED);
}
}
webapp/src/main/java/org/apache/hadoop/metadata/examples/QuickStart.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.metadata.examples;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import org.apache.hadoop.metadata.MetadataServiceClient;
import org.apache.hadoop.metadata.typesystem.Referenceable;
import org.apache.hadoop.metadata.typesystem.TypesDef;
import org.apache.hadoop.metadata.typesystem.json.InstanceSerialization;
import org.apache.hadoop.metadata.typesystem.json.TypesSerialization;
import org.apache.hadoop.metadata.typesystem.types.AttributeDefinition;
import org.apache.hadoop.metadata.typesystem.types.ClassType;
import org.apache.hadoop.metadata.typesystem.types.DataTypes;
import org.apache.hadoop.metadata.typesystem.types.EnumTypeDefinition;
import org.apache.hadoop.metadata.typesystem.types.HierarchicalTypeDefinition;
import org.apache.hadoop.metadata.typesystem.types.IDataType;
import org.apache.hadoop.metadata.typesystem.types.Multiplicity;
import org.apache.hadoop.metadata.typesystem.types.StructTypeDefinition;
import org.apache.hadoop.metadata.typesystem.types.TraitType;
import org.apache.hadoop.metadata.typesystem.types.TypeUtils;
import org.apache.hadoop.metadata.typesystem.types.utils.TypesUtil;
import org.codehaus.jettison.json.JSONObject;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
/**
* A driver that sets up sample types and data for testing purposes.
* todo - move this to examples module. Fix collections as well.
*/
public class QuickStart {
public static void main(String[] args) throws Exception {
String baseUrl = getServerUrl(args);
QuickStart quickStart = new QuickStart(baseUrl);
quickStart.createTypes();
// verify types created
quickStart.verifyTypesCreated();
quickStart.createEntities();
// verify entity created
quickStart.verifyEntityCreated();
}
static String getServerUrl(String[] args) {
String baseUrl = "http://localhost:21000";
if (args.length > 0) {
baseUrl = args[0];
}
return baseUrl;
}
private static final String DATABASE_TYPE = "DB";
private static final String COLUMN_TYPE = "Column";
private static final String TABLE_TYPE = "Table";
private static final String VIEW_TYPE = "View";
private static final String LOAD_PROCESS_TYPE = "LoadProcess";
private static final String STORAGE_DESC_TYPE = "StorageDesc";
private static final String[] TYPES = {
DATABASE_TYPE, TABLE_TYPE, STORAGE_DESC_TYPE, COLUMN_TYPE, LOAD_PROCESS_TYPE, VIEW_TYPE,
"JdbcAccess", "ETL", "Metric", "PII", "Fact", "Dimension"
};
private final MetadataServiceClient metadataServiceClient;
QuickStart(String baseUrl) {
metadataServiceClient = new MetadataServiceClient(baseUrl);
}
void createTypes() throws Exception {
TypesDef typesDef = setupTypes();
String typesAsJSON = TypesSerialization.toJson(typesDef);
metadataServiceClient.createType(typesAsJSON);
}
TypesDef setupTypes() throws Exception {
HierarchicalTypeDefinition<ClassType> dbClsDef
= TypesUtil.createClassTypeDef(DATABASE_TYPE, null,
attrDef("name", DataTypes.STRING_TYPE),
attrDef("description", DataTypes.STRING_TYPE),
attrDef("locationUri", DataTypes.STRING_TYPE),
attrDef("owner", DataTypes.STRING_TYPE),
attrDef("createTime", DataTypes.INT_TYPE)
);
HierarchicalTypeDefinition<ClassType> storageDescClsDef =
TypesUtil.createClassTypeDef(STORAGE_DESC_TYPE, null,
attrDef("location", DataTypes.STRING_TYPE),
attrDef("inputFormat", DataTypes.STRING_TYPE),
attrDef("outputFormat", DataTypes.STRING_TYPE),
attrDef("compressed", DataTypes.STRING_TYPE,
Multiplicity.REQUIRED, false, null)
);
HierarchicalTypeDefinition<ClassType> columnClsDef =
TypesUtil.createClassTypeDef(COLUMN_TYPE, null,
attrDef("name", DataTypes.STRING_TYPE),
attrDef("dataType", DataTypes.STRING_TYPE),
attrDef("comment", DataTypes.STRING_TYPE),
new AttributeDefinition("sd", STORAGE_DESC_TYPE,
Multiplicity.OPTIONAL, false, null),
new AttributeDefinition("table", "Table",
Multiplicity.OPTIONAL, false, null)
);
HierarchicalTypeDefinition<ClassType> tblClsDef =
TypesUtil.createClassTypeDef(TABLE_TYPE, null,
attrDef("name", DataTypes.STRING_TYPE),
attrDef("description", DataTypes.STRING_TYPE),
new AttributeDefinition("db", DATABASE_TYPE,
Multiplicity.REQUIRED, false, null),
new AttributeDefinition("sd", STORAGE_DESC_TYPE,
Multiplicity.REQUIRED, false, null),
attrDef("owner", DataTypes.STRING_TYPE),
attrDef("createTime", DataTypes.INT_TYPE),
attrDef("lastAccessTime", DataTypes.INT_TYPE),
attrDef("retention", DataTypes.INT_TYPE),
attrDef("viewOriginalText", DataTypes.STRING_TYPE),
attrDef("viewExpandedText", DataTypes.STRING_TYPE),
attrDef("tableType", DataTypes.STRING_TYPE),
attrDef("temporary", DataTypes.BOOLEAN_TYPE),
new AttributeDefinition("columns",
DataTypes.arrayTypeName(DataTypes.STRING_TYPE.getName()),
Multiplicity.COLLECTION, false, null)
// todo - fix this post serialization support for collections
// new AttributeDefinition("columns", DataTypes.arrayTypeName(COLUMN_TYPE),
// Multiplicity.COLLECTION, true, null)
);
HierarchicalTypeDefinition<ClassType> loadProcessClsDef =
TypesUtil.createClassTypeDef(LOAD_PROCESS_TYPE, null,
attrDef("name", DataTypes.STRING_TYPE),
attrDef("userName", DataTypes.STRING_TYPE),
attrDef("startTime", DataTypes.INT_TYPE),
attrDef("endTime", DataTypes.INT_TYPE),
// todo - fix this post serialization support for collections
// new AttributeDefinition("inputTables", DataTypes.arrayTypeName(TABLE_TYPE),
// Multiplicity.COLLECTION, false, null),
// new AttributeDefinition("outputTable", TABLE_TYPE,
// Multiplicity.REQUIRED, false, null),
new AttributeDefinition("inputTables", DataTypes.STRING_TYPE.getName(),
Multiplicity.COLLECTION, false, null),
new AttributeDefinition("outputTable", DataTypes.STRING_TYPE.getName(),
Multiplicity.REQUIRED, false, null),
attrDef("queryText", DataTypes.STRING_TYPE, Multiplicity.REQUIRED),
attrDef("queryPlan", DataTypes.STRING_TYPE, Multiplicity.REQUIRED),
attrDef("queryId", DataTypes.STRING_TYPE, Multiplicity.REQUIRED),
attrDef("queryGraph", DataTypes.STRING_TYPE, Multiplicity.REQUIRED)
);
HierarchicalTypeDefinition<ClassType> viewClsDef =
TypesUtil.createClassTypeDef(VIEW_TYPE, null,
attrDef("name", DataTypes.STRING_TYPE),
new AttributeDefinition("db", DATABASE_TYPE,
Multiplicity.REQUIRED, false, null),
// todo - fix this post serialization support for collections
// new AttributeDefinition("inputTables", TABLE_TYPE, Multiplicity.COLLECTION,
// false, null)
new AttributeDefinition("inputTables", DataTypes.STRING_TYPE.getName(),
Multiplicity.COLLECTION, false, null)
);
HierarchicalTypeDefinition<TraitType> dimTraitDef =
TypesUtil.createTraitTypeDef("Dimension", null);
HierarchicalTypeDefinition<TraitType> factTraitDef =
TypesUtil.createTraitTypeDef("Fact", null);
HierarchicalTypeDefinition<TraitType> piiTraitDef =
TypesUtil.createTraitTypeDef("PII", null);
HierarchicalTypeDefinition<TraitType> metricTraitDef =
TypesUtil.createTraitTypeDef("Metric", null);
HierarchicalTypeDefinition<TraitType> etlTraitDef =
TypesUtil.createTraitTypeDef("ETL", null);
HierarchicalTypeDefinition<TraitType> jdbcTraitDef =
TypesUtil.createTraitTypeDef("JdbcAccess", null);
return TypeUtils.getTypesDef(
ImmutableList.<EnumTypeDefinition>of(),
ImmutableList.<StructTypeDefinition>of(),
ImmutableList.of(dimTraitDef, factTraitDef,
piiTraitDef, metricTraitDef, etlTraitDef, jdbcTraitDef),
ImmutableList.of(dbClsDef, storageDescClsDef, columnClsDef,
tblClsDef, loadProcessClsDef, viewClsDef)
);
}
AttributeDefinition attrDef(String name, IDataType dT) {
return attrDef(name, dT, Multiplicity.OPTIONAL, false, null);
}
AttributeDefinition attrDef(String name, IDataType dT, Multiplicity m) {
return attrDef(name, dT, m, false, null);
}
AttributeDefinition attrDef(String name, IDataType dT,
Multiplicity m, boolean isComposite, String reverseAttributeName) {
Preconditions.checkNotNull(name);
Preconditions.checkNotNull(dT);
return new AttributeDefinition(name, dT.getName(), m, isComposite, reverseAttributeName);
}
void createEntities() throws Exception {
Referenceable salesDB = database(
"Sales", "Sales Database", "John ETL", "hdfs://host:8000/apps/warehouse/sales");
Referenceable sd = storageDescriptor("hdfs://host:8000/apps/warehouse/sales",
"TextInputFormat", "TextOutputFormat", true);
ArrayList<Referenceable> salesFactColumns = new ArrayList<>();
Referenceable column = column("time_id", "int", "time id", null);
salesFactColumns.add(column);
column = column("product_id", "int", "product id", null);
salesFactColumns.add(column);
column = column("customer_id", "int", "customer id", null, "PII");
salesFactColumns.add(column);
column = column("sales", "double", "product id", null, "Metric");
salesFactColumns.add(column);
Referenceable salesFact = table("sales_fact", "sales fact table",
salesDB, sd, "Joe", "Managed", salesFactColumns, "Fact");
ArrayList<Referenceable> productDimColumns = new ArrayList<>();
column = column("product_id", "int", "product id", null);
productDimColumns.add(column);
column = column("product_name", "string", "product name", null);
productDimColumns.add(column);
column = column("brand_name", "int", "brand name", null);
productDimColumns.add(column);
Referenceable productDim = table("product_dim", "product dimension table",
salesDB, sd, "John Doe", "Managed", productDimColumns, "Dimension");
ArrayList<Referenceable> timeDimColumns = new ArrayList<>();
column = column("time_id", "int", "time id", null);
timeDimColumns.add(column);
column = column("dayOfYear", "int", "day Of Year", null);
timeDimColumns.add(column);
column = column("weekDay", "int", "week Day", null);
timeDimColumns.add(column);
Referenceable timeDim = table("time_dim", "time dimension table",
salesDB, sd, "John Doe", "External", timeDimColumns, "Dimension");
ArrayList<Referenceable> customerDimColumns = new ArrayList<>();
column = column("customer_id", "int", "customer id", null, "PII");
customerDimColumns.add(column);
column = column("name", "string", "customer name", null, "PII");
customerDimColumns.add(column);
column = column("address", "string", "customer address", null, "PII");
customerDimColumns.add(column);
Referenceable customerDim = table("customer_dim", "customer dimension table",
salesDB, sd, "fetl", "External", customerDimColumns, "Dimension");
Referenceable reportingDB = database("Reporting", "reporting database", "Jane BI",
"hdfs://host:8000/apps/warehouse/reporting");
Referenceable salesFactDaily = table("sales_fact_daily_mv",
"sales fact daily materialized view", reportingDB, sd,
"Joe BI", "Managed", salesFactColumns, "Metric");
Referenceable loadSalesFactDaily = loadProcess("loadSalesDaily", "John ETL",
Arrays.asList(salesFact, timeDim), salesFactDaily,
"create table as select ", "plan", "id", "graph",
"ETL");
System.out.println("added loadSalesFactDaily = " + loadSalesFactDaily);
Referenceable productDimView = view("product_dim_view", reportingDB,
Arrays.asList(productDim), "Dimension", "JdbcAccess");
System.out.println("added productDimView = " + productDimView);
Referenceable customerDimView = view("customer_dim_view", reportingDB,
Arrays.asList(customerDim), "Dimension", "JdbcAccess");
System.out.println("added customerDimView = " + customerDimView);
Referenceable salesFactMonthly = table("sales_fact_monthly_mv",
"sales fact monthly materialized view",
reportingDB, sd, "Jane BI", "Managed", salesFactColumns, "Metric");
Referenceable loadSalesFactMonthly = loadProcess("loadSalesMonthly", "John ETL",
Arrays.asList(salesFactDaily), salesFactMonthly,
"create table as select ", "plan", "id", "graph",
"ETL");
System.out.println("added loadSalesFactMonthly = " + loadSalesFactMonthly);
}
private Referenceable createInstance(Referenceable referenceable) throws Exception {
String typeName = referenceable.getTypeName();
System.out.println("creating instance of type " + typeName);
String entityJSON = InstanceSerialization.toJson(referenceable, true);
System.out.println("Submitting new entity= " + entityJSON);
JSONObject jsonObject = metadataServiceClient.createEntity(entityJSON);
String guid = jsonObject.getString(MetadataServiceClient.RESULTS);
System.out.println("created instance for type " + typeName + ", guid: " + guid);
// return the reference to created instance with guid
return new Referenceable(guid, referenceable.getTypeName(), referenceable.getValuesMap());
}
Referenceable database(String name, String description,
String owner, String locationUri,
String... traitNames) throws Exception {
Referenceable referenceable = new Referenceable(DATABASE_TYPE, traitNames);
referenceable.set("name", name);
referenceable.set("description", description);
referenceable.set("owner", owner);
referenceable.set("locationUri", locationUri);
referenceable.set("createTime", System.currentTimeMillis());
return createInstance(referenceable);
}
Referenceable storageDescriptor(String location, String inputFormat,
String outputFormat,
boolean compressed) throws Exception {
Referenceable referenceable = new Referenceable(STORAGE_DESC_TYPE);
referenceable.set("location", location);
referenceable.set("inputFormat", inputFormat);
referenceable.set("outputFormat", outputFormat);
referenceable.set("compressed", compressed);
return createInstance(referenceable);
}
Referenceable column(String name, String dataType,
String comment, Referenceable sd,
String... traitNames) throws Exception {
Referenceable referenceable = new Referenceable(COLUMN_TYPE, traitNames);
referenceable.set("name", name);
referenceable.set("dataType", dataType);
referenceable.set("comment", comment);
referenceable.set("sd", sd);
return createInstance(referenceable);
}
Referenceable table(String name, String description,
Referenceable db, Referenceable sd,
String owner, String tableType,
List<Referenceable> columns,
String... traitNames) throws Exception {
Referenceable referenceable = new Referenceable(TABLE_TYPE, traitNames);
referenceable.set("name", name);
referenceable.set("description", description);
referenceable.set("owner", owner);
referenceable.set("tableType", tableType);
referenceable.set("createTime", System.currentTimeMillis());
referenceable.set("lastAccessTime", System.currentTimeMillis());
referenceable.set("retention", System.currentTimeMillis());
referenceable.set("db", db);
referenceable.set("sd", sd);
// todo - fix this post serialization support for collections
ArrayList<String> columnNames = new ArrayList<>(columns.size());
for (Referenceable column : columns) {
columnNames.add(String.valueOf(column.get("name")));
}
referenceable.set("columns", columnNames);
return createInstance(referenceable);
}
Referenceable loadProcess(String name, String user,
List<Referenceable> inputTables,
Referenceable outputTable,
String queryText, String queryPlan,
String queryId, String queryGraph,
String... traitNames) throws Exception {
Referenceable referenceable = new Referenceable(LOAD_PROCESS_TYPE, traitNames);
referenceable.set("name", name);
referenceable.set("user", user);
referenceable.set("startTime", System.currentTimeMillis());
referenceable.set("endTime", System.currentTimeMillis() + 10000);
// todo - fix this post serialization support for collections
ArrayList<String> inputTableNames = new ArrayList<>(inputTables.size());
for (Referenceable inputTable : inputTables) {
inputTableNames.add(String.valueOf(inputTable.get("name")));
}
referenceable.set("inputTables", inputTableNames);
referenceable.set("outputTable", outputTable.get("name"));
referenceable.set("queryText", queryText);
referenceable.set("queryPlan", queryPlan);
referenceable.set("queryId", queryId);
referenceable.set("queryGraph", queryGraph);
return createInstance(referenceable);
}
Referenceable view(String name, Referenceable db,
List<Referenceable> inputTables,
String... traitNames) throws Exception {
Referenceable referenceable = new Referenceable(VIEW_TYPE, traitNames);
referenceable.set("name", name);
referenceable.set("db", db);
// todo - fix this post serialization support for collections
// referenceable.set("inputTables", inputTables);
ArrayList<String> inputTableNames = new ArrayList<>(inputTables.size());
for (Referenceable inputTable : inputTables) {
inputTableNames.add(String.valueOf(inputTable.get("name")));
}
referenceable.set("inputTables", inputTableNames);
return createInstance(referenceable);
}
private void verifyTypesCreated() throws Exception {
List<String> types = metadataServiceClient.listTypes();
for (String type : TYPES) {
assert types.contains(type);
}
}
private void verifyEntityCreated() {
// todo
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment