BUG-32825 Add tests for Ranger Integration. Contributed by Venkatesh Seetharam

pom.xml

@@ -905,11 +905,9 @@
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-surefire-plugin</artifactId>
                 <version>2.7.2</version>
-                <!--
                 <configuration>
                     <skipTests>true</skipTests>
                 </configuration>
-                -->
             </plugin>
 
             <plugin>

repository/src/main/java/org/apache/hadoop/metadata/repository/graph/Constants.java

@@ -20,22 +20,42 @@ package org.apache.hadoop.metadata.repository.graph;
 
 public final class Constants {
 
+    /**
+     * Globally Unique identifier property key.
+     */
     public static final String GUID_PROPERTY_KEY = "guid";
     public static final String GUID_INDEX = "guid_index";
-    public static final String ENTITY_TYPE_PROPERTY_KEY = "type";
+
+    /**
+     * Entity type name property key.
+     */
+    public static final String ENTITY_TYPE_PROPERTY_KEY = "typeName";
     public static final String ENTITY_TYPE_INDEX = "type_index";
+
+    /**
+     * Data type property key.
+     */
+    // public static final String DATA_TYPE_PROPERTY_KEY = "dataType";
+
+    /**
+     * Trait names property key.
+     */
+    public static final String TRAIT_NAMES_PROPERTY_KEY = "traitNames";
+
     public static final String VERSION_PROPERTY_KEY = "version";
     public static final String TIMESTAMP_PROPERTY_KEY = "timestamp";
-    public static final String TRAIT_NAMES_PROPERTY_KEY = "traits";
+
     /**
      * search backing index name.
      */
     public static final String BACKING_INDEX = "search";
     public static final String INDEX_NAME = "metadata";
+
     /**
      * search backing index name for vertex keys.
      */
     public static final String VERTEX_INDEX = "vertex_index";
+
     /**
      * search backing index name for edge labels.
      */

repository/src/test/java/org/apache/hadoop/metadata/discovery/GraphBackedDiscoveryServiceTest.java

@@ -18,10 +18,19 @@
 
 package org.apache.hadoop.metadata.discovery;
 
+import com.thinkaurelius.titan.core.TitanGraph;
+import com.tinkerpop.blueprints.Edge;
+import com.tinkerpop.blueprints.Vertex;
+import org.apache.commons.io.FileUtils;
 import org.apache.hadoop.metadata.RepositoryMetadataModule;
 import org.apache.hadoop.metadata.TestUtils;
 import org.apache.hadoop.metadata.discovery.graph.GraphBackedDiscoveryService;
+import org.apache.hadoop.metadata.query.HiveTitanSample;
+import org.apache.hadoop.metadata.query.QueryTestsUtils;
 import org.apache.hadoop.metadata.repository.graph.GraphBackedMetadataRepository;
+import org.apache.hadoop.metadata.repository.graph.GraphHelper;
+import org.apache.hadoop.metadata.repository.graph.GraphService;
+import org.apache.hadoop.metadata.repository.graph.TitanGraphService;
 import org.apache.hadoop.metadata.typesystem.ITypedReferenceableInstance;
 import org.apache.hadoop.metadata.typesystem.Referenceable;
 import org.apache.hadoop.metadata.typesystem.types.ClassType;
@@ -32,15 +41,24 @@ import org.codehaus.jettison.json.JSONObject;
 import org.testng.Assert;
 import org.testng.annotations.AfterClass;
 import org.testng.annotations.BeforeClass;
+import org.testng.annotations.DataProvider;
 import org.testng.annotations.Guice;
 import org.testng.annotations.Test;
 
 import javax.inject.Inject;
+import javax.script.Bindings;
+import javax.script.ScriptEngine;
+import javax.script.ScriptEngineManager;
+import javax.script.ScriptException;
+import java.io.File;
 
 @Guice(modules = RepositoryMetadataModule.class)
 public class GraphBackedDiscoveryServiceTest {
 
     @Inject
+    private GraphService graphService;
+
+    @Inject
     private GraphBackedMetadataRepository repositoryService;
 
     @Inject
@@ -51,6 +69,9 @@ public class GraphBackedDiscoveryServiceTest {
         TypeSystem typeSystem = TypeSystem.getInstance();
         typeSystem.reset();
 
+        QueryTestsUtils.setupTypes();
+        setupSampleData();
+
         TestUtils.defineDeptEmployeeTypes(typeSystem);
 
         Referenceable hrDept = TestUtils.createDeptEg1(typeSystem);
@@ -60,6 +81,34 @@ public class GraphBackedDiscoveryServiceTest {
         repositoryService.createEntity(hrDept2, "Department");
     }
 
+    private void setupSampleData() throws ScriptException {
+        TitanGraph titanGraph = ((TitanGraphService) graphService).getTitanGraph();
+
+        ScriptEngineManager manager = new ScriptEngineManager();
+        ScriptEngine engine = manager.getEngineByName("gremlin-groovy");
+        Bindings bindings = engine.createBindings();
+        bindings.put("g", titanGraph);
+
+        String hiveGraphFile = FileUtils.getTempDirectory().getPath()
+                + File.separator + System.nanoTime() + ".gson";
+        System.out.println("hiveGraphFile = " + hiveGraphFile);
+        HiveTitanSample.writeGson(hiveGraphFile);
+        bindings.put("hiveGraphFile", hiveGraphFile);
+
+        engine.eval("g.loadGraphSON(hiveGraphFile)", bindings);
+        titanGraph.commit();
+
+        System.out.println("*******************Graph Dump****************************");
+        for (Vertex vertex : titanGraph.getVertices()) {
+            System.out.println(GraphHelper.vertexString(vertex));
+        }
+
+        for (Edge edge : titanGraph.getEdges()) {
+            System.out.println(GraphHelper.edgeString(edge));
+        }
+        System.out.println("*******************Graph Dump****************************");
+    }
+
     @AfterClass
     public void tearDown() throws Exception {
         TypeSystem.getInstance().reset();
@@ -113,4 +162,79 @@ public class GraphBackedDiscoveryServiceTest {
                 .searchByGremlin("g.V.filter{it.typeName == 'Person'}.'Person.name'.toList()");
         System.out.println("search result = " + r);
     }
+
+    @DataProvider(name = "dslQueriesProvider")
+    private Object[][] createDSLQueries() {
+        return new String[][] {
+                {"from DB"},
+                {"DB"},
+                {"from Table"},
+                {"Table"},
+                {"DB, Table"},
+                /*{"DB as db1 Table where db1.name = \"Reporting\""},*/
+                {"DB name = \"Reporting\""},
+                {"Column as PII"},
+                {"Table as Dimension"},
+                {"View as Dimension"},
+                {"Column as PII select Column.name"},
+                {"Column select Column.name"},
+                {"from Table select Table.name"},
+        };
+    }
+
+    @Test (dataProvider = "dslQueriesProvider")
+    public void testSearchByDSLQueries(String dslQuery) throws Exception {
+        System.out.println("Executing dslQuery = " + dslQuery);
+        String jsonResults = discoveryService.searchByDSL(dslQuery);
+        Assert.assertNotNull(jsonResults);
+
+        JSONObject results = new JSONObject(jsonResults);
+        Assert.assertEquals(results.length(), 3);
+        System.out.println("results = " + results);
+
+        Object query = results.get("query");
+        Assert.assertNotNull(query);
+
+        JSONObject dataType = results.getJSONObject("dataType");
+        Assert.assertNotNull(dataType);
+        String typeName = dataType.getString("typeName");
+        Assert.assertNotNull(typeName);
+
+        JSONArray rows = results.getJSONArray("rows");
+        Assert.assertNotNull(rows);
+        Assert.assertTrue(rows.length() > 0);
+    }
+
+    @Test
+    public void testSearchByDSLQuery() throws Exception {
+        String dslQuery = "Column as PII";
+        System.out.println("Executing dslQuery = " + dslQuery);
+        String jsonResults = discoveryService.searchByDSL(dslQuery);
+        Assert.assertNotNull(jsonResults);
+
+        JSONObject results = new JSONObject(jsonResults);
+        Assert.assertEquals(results.length(), 3);
+        System.out.println("results = " + results);
+
+        Object query = results.get("query");
+        Assert.assertNotNull(query);
+
+        JSONObject dataType = results.getJSONObject("dataType");
+        Assert.assertNotNull(dataType);
+        String typeName = dataType.getString("typeName");
+        Assert.assertNotNull(typeName);
+
+        JSONArray rows = results.getJSONArray("rows");
+        Assert.assertNotNull(rows);
+        Assert.assertTrue(rows.length() > 0);
+
+        for (int index = 0; index < rows.length(); index++) {
+            JSONObject row = rows.getJSONObject(index);
+            String type = row.getString("$typeName$");
+            Assert.assertEquals(type, "Column");
+
+            String name = row.getString("name");
+            Assert.assertNotEquals(name, "null");
+        }
+    }
 }
\ No newline at end of file

repository/src/test/java/org/apache/hadoop/metadata/repository/graph/GraphRepoMapperTest.java

@@ -74,6 +74,7 @@ public class GraphRepoMapperTest {
         new GraphBackedSearchIndexer(titanGraphService);
 
         typeSystem = TypeSystem.getInstance();
+        typeSystem.reset();
 
         createHiveTypes();
     }
@@ -109,7 +110,7 @@ public class GraphRepoMapperTest {
     public void testGetEntityDefinition() throws Exception {
         TitanGraph graph = titanGraphService.getTitanGraph();
         GraphQuery query = graph.query()
-                .has("type", Compare.EQUAL, TABLE_TYPE);
+                .has(Constants.ENTITY_TYPE_PROPERTY_KEY, Compare.EQUAL, TABLE_TYPE);
         Iterator<Vertex> results = query.vertices().iterator();
         // returning one since guid should be unique
         Vertex tableVertex = results.hasNext() ? results.next() : null;

repository/src/test/scala/org/apache/hadoop/metadata/query/HiveTitanSample.scala

@@ -19,6 +19,7 @@
 package org.apache.hadoop.metadata.query
 
 import java.io.File
+import java.util.UUID
 import java.util.concurrent.atomic.AtomicInteger
 import javax.script.{Bindings, ScriptEngine, ScriptEngineManager}
 
@@ -37,6 +38,8 @@ object HiveTitanSample {
         val _id: String
 
         def id = _id
+        val version = 0
+        val guid = s"""${UUID.randomUUID()}""".stripMargin
 
         def addEdge(to: Vertex, label: String, edges: ArrayBuffer[String]): Unit = {
             edges +=
@@ -62,6 +65,7 @@ object HiveTitanSample {
                     case l: List[_] => l.foreach(x => addEdge(x.asInstanceOf[Vertex],
                         s"${this.getClass.getSimpleName}.${f.getName}", edges))
                     case _ => sb.append( s""", "${f.getName}" : $fV""")
+                        sb.append( s""", "${this.getClass.getSimpleName}.${f.getName}" : $fV""")
                 }
             }
 
@@ -322,7 +326,7 @@ object TestApp extends App with GraphUtils {
     val bindings: Bindings = engine.createBindings
     bindings.put("g", g)
 
-    val hiveGraphFile = FileUtils.getTempDirectory().getPath.toString + File.separator + System.nanoTime() + ".gson"
+    val hiveGraphFile = FileUtils.getTempDirectory().getPath + File.separator + System.nanoTime() + ".gson"
     HiveTitanSample.writeGson(hiveGraphFile)
     bindings.put("hiveGraphFile", hiveGraphFile)
 

webapp/src/test/java/org/apache/hadoop/metadata/web/resources/EntityJerseyResourceIT.java

@@ -21,13 +21,13 @@ package org.apache.hadoop.metadata.web.resources;
 import com.google.common.collect.ImmutableList;
 import com.sun.jersey.api.client.ClientResponse;
 import com.sun.jersey.api.client.WebResource;
-import org.apache.hadoop.metadata.typesystem.json.Serialization$;
-import org.apache.hadoop.metadata.typesystem.json.TypesSerialization;
 import org.apache.hadoop.metadata.typesystem.ITypedInstance;
 import org.apache.hadoop.metadata.typesystem.ITypedReferenceableInstance;
 import org.apache.hadoop.metadata.typesystem.ITypedStruct;
 import org.apache.hadoop.metadata.typesystem.Referenceable;
 import org.apache.hadoop.metadata.typesystem.Struct;
+import org.apache.hadoop.metadata.typesystem.json.Serialization$;
+import org.apache.hadoop.metadata.typesystem.json.TypesSerialization;
 import org.apache.hadoop.metadata.typesystem.types.AttributeDefinition;
 import org.apache.hadoop.metadata.typesystem.types.AttributeInfo;
 import org.apache.hadoop.metadata.typesystem.types.ClassType;
@@ -46,11 +46,12 @@ import org.slf4j.LoggerFactory;
 import org.testng.Assert;
 import org.testng.annotations.BeforeClass;
 import org.testng.annotations.Test;
-import scala.actors.threadpool.Arrays;
 
 import javax.ws.rs.HttpMethod;
 import javax.ws.rs.core.MediaType;
 import javax.ws.rs.core.Response;
+import java.util.Arrays;
+import java.util.List;
 import java.util.UUID;
 
 /**
@@ -131,17 +132,17 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
         LOG.debug("tableInstanceAfterGet = " + definition);
 
         // todo - this fails with type error, strange
-        ITypedReferenceableInstance tableInstanceAfterGet = Serialization$.MODULE$
-                .fromJson(definition);
+        ITypedReferenceableInstance tableInstanceAfterGet =
+                Serialization$.MODULE$.fromJson(definition);
         Assert.assertTrue(areEqual(tableInstance, tableInstanceAfterGet));
     }
 
     private boolean areEqual(ITypedInstance actual,
                              ITypedInstance expected) throws Exception {
-/*
+        /*
         Assert.assertEquals(Serialization$.MODULE$.toJson(actual),
                 Serialization$.MODULE$.toJson(expected));
-*/
+        */
 
         for (AttributeInfo attributeInfo : actual.fieldMapping().fields.values()) {
             final DataTypes.TypeCategory typeCategory = attributeInfo.dataType().getTypeCategory();
@@ -283,14 +284,27 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
                         new AttributeDefinition("database",
                                 DATABASE_TYPE, Multiplicity.REQUIRED, true, null));
 
-        HierarchicalTypeDefinition<TraitType> classificationTypeDefinition =
+        HierarchicalTypeDefinition<TraitType> classificationTraitDefinition =
                 TypesUtil.createTraitTypeDef("classification",
                         ImmutableList.<String>of(),
                         TypesUtil.createRequiredAttrDef("tag", DataTypes.STRING_TYPE));
+        HierarchicalTypeDefinition<TraitType> piiTrait =
+                TypesUtil.createTraitTypeDef("pii", ImmutableList.<String>of());
+        HierarchicalTypeDefinition<TraitType> phiTrait =
+                TypesUtil.createTraitTypeDef("phi", ImmutableList.<String>of());
+        HierarchicalTypeDefinition<TraitType> pciTrait =
+                TypesUtil.createTraitTypeDef("pci", ImmutableList.<String>of());
+        HierarchicalTypeDefinition<TraitType> soxTrait =
+                TypesUtil.createTraitTypeDef("sox", ImmutableList.<String>of());
+        HierarchicalTypeDefinition<TraitType> secTrait =
+                TypesUtil.createTraitTypeDef("sec", ImmutableList.<String>of());
+        HierarchicalTypeDefinition<TraitType> financeTrait =
+                TypesUtil.createTraitTypeDef("finance", ImmutableList.<String>of());
 
         typeSystem.defineTypes(
                 ImmutableList.of(structTypeDefinition),
-                ImmutableList.of(classificationTypeDefinition),
+                ImmutableList.of(classificationTraitDefinition, piiTrait, phiTrait, pciTrait,
+                        soxTrait, secTrait, financeTrait),
                 ImmutableList.of(databaseTypeDefinition, tableTypeDefinition));
     }
 
@@ -302,7 +316,14 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
                         DATABASE_TYPE,
                         TABLE_TYPE,
                         "serdeType",
-                        "classification"}));
+                        "classification",
+                        "pii",
+                        "phi",
+                        "pci",
+                        "sox",
+                        "sec",
+                        "finance",
+                }));
         sumbitType(typesAsJSON, TABLE_TYPE);
     }
 
@@ -311,7 +332,8 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
         databaseInstance.set("name", DATABASE_NAME);
         databaseInstance.set("description", "foo database");
 
-        Referenceable tableInstance = new Referenceable(TABLE_TYPE, "classification");
+        Referenceable tableInstance = new Referenceable(TABLE_TYPE,
+                "classification", "pii", "phi", "pci", "sox", "sec", "finance");
         tableInstance.set("name", TABLE_NAME);
         tableInstance.set("description", "bar table");
         tableInstance.set("type", "managed");
@@ -331,6 +353,9 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
         serde2Instance.set("serde", "serde2");
         tableInstance.set("serde2", serde2Instance);
 
+        List<String> traits = tableInstance.getTraits();
+        Assert.assertEquals(traits.size(), 7);
+
         ClassType tableType = typeSystem.getDataType(ClassType.class, TABLE_TYPE);
         return tableType.convert(tableInstance, Multiplicity.REQUIRED);
     }

webapp/src/test/java/org/apache/hadoop/metadata/web/resources/MetadataDiscoveryResourceIT.java

@@ -21,6 +21,7 @@ package org.apache.hadoop.metadata.web.resources;
 import com.google.common.collect.ImmutableList;
 import com.sun.jersey.api.client.ClientResponse;
 import com.sun.jersey.api.client.WebResource;
+import org.apache.hadoop.metadata.typesystem.Struct;
 import org.apache.hadoop.metadata.typesystem.json.Serialization$;
 import org.apache.hadoop.metadata.typesystem.json.TypesSerialization;
 import org.apache.hadoop.metadata.typesystem.ITypedReferenceableInstance;
@@ -43,6 +44,7 @@ import scala.actors.threadpool.Arrays;
 import javax.ws.rs.HttpMethod;
 import javax.ws.rs.core.MediaType;
 import javax.ws.rs.core.Response;
+import java.util.List;
 
 
 public class MetadataDiscoveryResourceIT extends BaseResourceIT {
@@ -263,9 +265,27 @@ public class MetadataDiscoveryResourceIT extends BaseResourceIT {
                         TypesUtil.createUniqueRequiredAttrDef("name", DataTypes.STRING_TYPE),
                         TypesUtil.createRequiredAttrDef("description", DataTypes.STRING_TYPE));
 
+        HierarchicalTypeDefinition<TraitType> classificationTraitDefinition =
+                TypesUtil.createTraitTypeDef("Classification",
+                        ImmutableList.<String>of(),
+                        TypesUtil.createRequiredAttrDef("tag", DataTypes.STRING_TYPE));
+        HierarchicalTypeDefinition<TraitType> piiTrait =
+                TypesUtil.createTraitTypeDef("PII", ImmutableList.<String>of());
+        HierarchicalTypeDefinition<TraitType> phiTrait =
+                TypesUtil.createTraitTypeDef("PHI", ImmutableList.<String>of());
+        HierarchicalTypeDefinition<TraitType> pciTrait =
+                TypesUtil.createTraitTypeDef("PCI", ImmutableList.<String>of());
+        HierarchicalTypeDefinition<TraitType> soxTrait =
+                TypesUtil.createTraitTypeDef("SOX", ImmutableList.<String>of());
+        HierarchicalTypeDefinition<TraitType> secTrait =
+                TypesUtil.createTraitTypeDef("SEC", ImmutableList.<String>of());
+        HierarchicalTypeDefinition<TraitType> financeTrait =
+                TypesUtil.createTraitTypeDef("Finance", ImmutableList.<String>of());
+
         typeSystem.defineTypes(
                 ImmutableList.<StructTypeDefinition>of(),
-                ImmutableList.<HierarchicalTypeDefinition<TraitType>>of(),
+                ImmutableList.of(classificationTraitDefinition, piiTrait, phiTrait, pciTrait,
+                        soxTrait, secTrait, financeTrait),
                 ImmutableList.of(dslTestTypeDefinition));
     }
 
@@ -274,15 +294,29 @@ public class MetadataDiscoveryResourceIT extends BaseResourceIT {
         String typesAsJSON = TypesSerialization.toJson(typeSystem,
                 Arrays.asList(new String[]{
                         "dsl_test_type",
+                        "Classification",
+                        "PII",
+                        "PHI",
+                        "PCI",
+                        "SOX",
+                        "SEC",
+                        "Finance",
                 }));
         sumbitType(typesAsJSON, "dsl_test_type");
     }
 
     private ITypedReferenceableInstance createInstance() throws Exception {
-        Referenceable entityInstance = new Referenceable("dsl_test_type");
+        Referenceable entityInstance = new Referenceable("dsl_test_type",
+                "Classification", "PII", "PHI", "PCI", "SOX", "SEC", "Finance");
         entityInstance.set("name", "foo name");
         entityInstance.set("description", "bar description");
 
+        Struct traitInstance = (Struct) entityInstance.getTrait("Classification");
+        traitInstance.set("tag", "foundation_etl");
+
+        List<String> traits = entityInstance.getTraits();
+        Assert.assertEquals(traits.size(), 7);
+
         ClassType tableType = typeSystem.getDataType(ClassType.class, "dsl_test_type");
         return tableType.convert(entityInstance, Multiplicity.REQUIRED);
     }