Added more DSL tests, Docs and resolved RAT issues

addons/hive-bridge/src/test/resources/application.properties

@@ -19,11 +19,11 @@
 #########  Graph Database Configs  #########
 # Graph Storage
 metadata.graph.storage.backend=berkeleyje
-metadata.graph.storage.directory=./data/berkeley
+metadata.graph.storage.directory=./target/data/berkeley
 
 # Graph Search Index
 metadata.graph.index.search.backend=elasticsearch
-metadata.graph.index.search.directory=./data/es
+metadata.graph.index.search.directory=./target/data/es
 metadata.graph.index.search.elasticsearch.client-only=false
 metadata.graph.index.search.elasticsearch.local-mode=true
 

docs/src/site/twiki/QuickStart.twiki

+---+ Quick Start Guide
+
+---++ Introduction
+This quick start user guide is a simple client that adds a few sample type definitions modeled
+after the example as shown below. It also adds example entities along with traits as shown in the
+instance graph below.
+
+
+---+++ Example Type Definitions
+
+<img src="guide-class-diagram.png"/>
+
+
+---+++ Example Instance Graph
+
+<img src="guide-instance-graph.png"/>
+
+
+---++ Running the example
+
+This will add sample types and instances along with traits as shown in the instance graph above.
+
+   * bin/quick-start.sh
+
+---++ Dashboard
+A simple dashboard with search is available.
+
+   * http://localhost:21000/dashboard

docs/src/site/twiki/index.twiki

----+ Data Governance and Metadata platform for Hadoop
+---+ Data Governance and Metadata framework for Hadoop
 
 
 ---++ Overview
 
+DGI is a scalable and extensible set of core foundational governance services – enabling
+enterprises to effectively and efficiently meet their compliance requirements within Hadoop and
+allows integration with the whole enterprise data ecosystem.
 
----++ Use Cases
+---++ Features
 
-   * Enables modeling
+---+++ Data Classification
+   * Import or define taxonomy business-oriented annotations for data
+   * Define, annotate, and automate capture of relationships between data sets and underlying
+   elements including source, target, and derivation processes
+   * Export metadata to third-party systems
 
-   * Captures Lineage information for data sets and processes
+---+++ Centralized Auditing
+   * Capture security access information for every application, process, and interaction with data
+   * Capture the operational information for execution, steps, and activities
+
+---+++ Search & Lineage (Browse)
+   * Pre-defined navigation paths to explore the data classification and audit information
+   * Text-based search features locates relevant data and audit event across Data Lake quickly
+   and accurately
+   * Browse visualization of data set lineage allowing users to drill-down into operational,
+   security, and provenance related information
+
+---+++ Security & Policy Engine
+   * Rationalize compliance policy at runtime based on data classification schemes, attributes
+   and roles.
+   * Advanced definition of policies for preventing data derivation based on classification
+   (i.e. re-identification) – Prohibitions
+   * Column and Row level masking based on cell values and attibutes.
 
 
 ---++ Getting Started
 
+   * [[QuickStart][Quick Start Guide]]
+
 
 ---++ Documentation
 
    * [[Architecture][High Level Architecture]]
    * [[TypeSystem][Type System]]
    * [[Repository][Metadata Repository]]
-   * [[Discovery][Metadata Discovery]]
+   * [[Discovery][Search]]
 
 
 ---++ API Documentation

repository/src/main/java/org/apache/hadoop/metadata/discovery/graph/GraphBackedDiscoveryService.java

@@ -82,7 +82,7 @@ public class GraphBackedDiscoveryService implements DiscoveryService {
                 return queryResult.toJson();
             }
         } catch (Exception e) { // unable to catch ExpressionException
-            throw new DiscoveryException("Invalid expression : " + dslQuery);
+            throw new DiscoveryException("Invalid expression : " + dslQuery, e);
         }
 
         throw new DiscoveryException("Invalid expression : " + dslQuery);

repository/src/main/java/org/apache/hadoop/metadata/repository/graph/GraphBackedSearchIndexer.java

@@ -36,7 +36,6 @@ import org.apache.hadoop.metadata.typesystem.types.AttributeInfo;
 import org.apache.hadoop.metadata.typesystem.types.ClassType;
 import org.apache.hadoop.metadata.typesystem.types.DataTypes;
 import org.apache.hadoop.metadata.typesystem.types.IDataType;
-import org.apache.hadoop.metadata.typesystem.types.Multiplicity;
 import org.apache.hadoop.metadata.typesystem.types.StructType;
 import org.apache.hadoop.metadata.typesystem.types.TraitType;
 import org.slf4j.Logger;
@@ -162,20 +161,17 @@ public class GraphBackedSearchIndexer implements SearchIndexer {
         final String propertyName = typeName + "." + field.name;
         switch (field.dataType().getTypeCategory()) {
             case PRIMITIVE:
-                createVertexMixedIndex(propertyName,
-                        getPrimitiveClass(field.dataType()), getCardinality(field.multiplicity));
+                createVertexMixedIndex(propertyName, getPrimitiveClass(field.dataType()));
                 break;
 
             case ENUM:
-                createVertexMixedIndex(
-                        propertyName, Integer.class, getCardinality(field.multiplicity));
+                createVertexMixedIndex(propertyName, Integer.class);
                 break;
 
             case ARRAY:
             case MAP:
-                // index the property holder for element names
-                createVertexMixedIndex(
-                        propertyName, String.class, getCardinality(field.multiplicity));
+                // todo - how do we overcome this limitation?
+                // IGNORE: Can only index single-valued property keys on vertices in Mixed Index
                 break;
 
             case STRUCT:
@@ -223,6 +219,7 @@ public class GraphBackedSearchIndexer implements SearchIndexer {
         throw new IllegalArgumentException("unknown data type " + dataType);
     }
 
+/*
     private Cardinality getCardinality(Multiplicity multiplicity) {
         if (multiplicity == Multiplicity.OPTIONAL || multiplicity == Multiplicity.REQUIRED) {
             return Cardinality.SINGLE;
@@ -235,12 +232,13 @@ public class GraphBackedSearchIndexer implements SearchIndexer {
         // todo - default to LIST as this is the most forgiving
         return Cardinality.LIST;
     }
+*/
 
     private void createCompositeAndMixedIndex(String indexName,
                                               String propertyName, Class propertyClass,
                                               boolean isUnique, Cardinality cardinality) {
         createCompositeIndex(indexName, propertyName, propertyClass, isUnique, cardinality);
-        createVertexMixedIndex(propertyName, propertyClass, cardinality);
+        createVertexMixedIndex(propertyName, propertyClass);
     }
 
     private PropertyKey createCompositeIndex(String indexName,
@@ -272,15 +270,14 @@ public class GraphBackedSearchIndexer implements SearchIndexer {
         return propertyKey;
     }
 
-    private PropertyKey createVertexMixedIndex(String propertyName, Class propertyClass,
-                                               Cardinality cardinality) {
+    private PropertyKey createVertexMixedIndex(String propertyName, Class propertyClass) {
         TitanManagement management = titanGraph.getManagementSystem();
         PropertyKey propertyKey = management.getPropertyKey(propertyName);
         if (propertyKey == null) {
+            // ignored cardinality as Can only index single-valued property keys on vertices
             propertyKey = management
                     .makePropertyKey(propertyName)
                     .dataType(propertyClass)
-                    .cardinality(cardinality)
                     .make();
 
             TitanGraphIndex vertexIndex = management.getGraphIndex(Constants.VERTEX_INDEX);

repository/src/test/java/org/apache/hadoop/metadata/discovery/GraphBackedDiscoveryServiceTest.java

@@ -216,6 +216,7 @@ public class GraphBackedDiscoveryServiceTest {
             {"Table loop (LoadProcess outputTable)"},
             {"Table as _loop0 loop (LoadProcess outputTable) withPath"},
             {"Table as src loop (LoadProcess outputTable) as dest select src.name as srcTable, dest.name as destTable withPath"},
+            {"Table as t, sd, Column as c where t.name=\"sales_fact\" select c.name as colName, c.dataType as colType"},
         };
     }
 

repository/src/test/resources/application.properties

@@ -22,7 +22,7 @@ metadata.graph.storage.backend=inmemory
 
 # Graph Search Index
 metadata.graph.index.search.backend=elasticsearch
-metadata.graph.index.search.directory=./data/es
+metadata.graph.index.search.directory=./target/data/es
 metadata.graph.index.search.elasticsearch.client-only=false
 metadata.graph.index.search.elasticsearch.local-mode=true