Skip to content

A
atlas
Commit e79ca3ca by Venkatesh Seetharam
Options

BUG-32825 Add tests for Ranger Integration. Contributed by Venkatesh Seetharam

parent d3a275c3
Showing with 226 additions and 19 deletions
pom.xml
......@@ -905,11 +905,9 @@
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<version>2.7.2</version>
<!--
<configuration>
<skipTests>true</skipTests>
</configuration>
-->
</plugin>
<plugin>
......
repository/src/main/java/org/apache/hadoop/metadata/repository/graph/Constants.java
......@@ -20,22 +20,42 @@ package org.apache.hadoop.metadata.repository.graph;
public final class Constants {
/**
* Globally Unique identifier property key.
*/
public static final String GUID_PROPERTY_KEY = "guid";
public static final String GUID_INDEX = "guid_index";
public static final String ENTITY_TYPE_PROPERTY_KEY = "type";
/**
* Entity type name property key.
*/
public static final String ENTITY_TYPE_PROPERTY_KEY = "typeName";
public static final String ENTITY_TYPE_INDEX = "type_index";
/**
* Data type property key.
*/
// public static final String DATA_TYPE_PROPERTY_KEY = "dataType";
/**
* Trait names property key.
*/
public static final String TRAIT_NAMES_PROPERTY_KEY = "traitNames";
public static final String VERSION_PROPERTY_KEY = "version";
public static final String TIMESTAMP_PROPERTY_KEY = "timestamp";
public static final String TRAIT_NAMES_PROPERTY_KEY = "traits";
/**
* search backing index name.
*/
public static final String BACKING_INDEX = "search";
public static final String INDEX_NAME = "metadata";
/**
* search backing index name for vertex keys.
*/
public static final String VERTEX_INDEX = "vertex_index";
/**
* search backing index name for edge labels.
*/
......
repository/src/test/java/org/apache/hadoop/metadata/discovery/GraphBackedDiscoveryServiceTest.java
......@@ -18,10 +18,19 @@
package org.apache.hadoop.metadata.discovery;
import com.thinkaurelius.titan.core.TitanGraph;
import com.tinkerpop.blueprints.Edge;
import com.tinkerpop.blueprints.Vertex;
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.metadata.RepositoryMetadataModule;
import org.apache.hadoop.metadata.TestUtils;
import org.apache.hadoop.metadata.discovery.graph.GraphBackedDiscoveryService;
import org.apache.hadoop.metadata.query.HiveTitanSample;
import org.apache.hadoop.metadata.query.QueryTestsUtils;
import org.apache.hadoop.metadata.repository.graph.GraphBackedMetadataRepository;
import org.apache.hadoop.metadata.repository.graph.GraphHelper;
import org.apache.hadoop.metadata.repository.graph.GraphService;
import org.apache.hadoop.metadata.repository.graph.TitanGraphService;
import org.apache.hadoop.metadata.typesystem.ITypedReferenceableInstance;
import org.apache.hadoop.metadata.typesystem.Referenceable;
import org.apache.hadoop.metadata.typesystem.types.ClassType;
......@@ -32,15 +41,24 @@ import org.codehaus.jettison.json.JSONObject;
import org.testng.Assert;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.DataProvider;
import org.testng.annotations.Guice;
import org.testng.annotations.Test;
import javax.inject.Inject;
import javax.script.Bindings;
import javax.script.ScriptEngine;
import javax.script.ScriptEngineManager;
import javax.script.ScriptException;
import java.io.File;
@Guice(modules = RepositoryMetadataModule.class)
public class GraphBackedDiscoveryServiceTest {
@Inject
private GraphService graphService;
@Inject
private GraphBackedMetadataRepository repositoryService;
@Inject
......@@ -51,6 +69,9 @@ public class GraphBackedDiscoveryServiceTest {
TypeSystem typeSystem = TypeSystem.getInstance();
typeSystem.reset();
QueryTestsUtils.setupTypes();
setupSampleData();
TestUtils.defineDeptEmployeeTypes(typeSystem);
Referenceable hrDept = TestUtils.createDeptEg1(typeSystem);
......@@ -60,6 +81,34 @@ public class GraphBackedDiscoveryServiceTest {
repositoryService.createEntity(hrDept2, "Department");
}
private void setupSampleData() throws ScriptException {
TitanGraph titanGraph = ((TitanGraphService) graphService).getTitanGraph();
ScriptEngineManager manager = new ScriptEngineManager();
ScriptEngine engine = manager.getEngineByName("gremlin-groovy");
Bindings bindings = engine.createBindings();
bindings.put("g", titanGraph);
String hiveGraphFile = FileUtils.getTempDirectory().getPath()
+ File.separator + System.nanoTime() + ".gson";
System.out.println("hiveGraphFile = " + hiveGraphFile);
HiveTitanSample.writeGson(hiveGraphFile);
bindings.put("hiveGraphFile", hiveGraphFile);
engine.eval("g.loadGraphSON(hiveGraphFile)", bindings);
titanGraph.commit();
System.out.println("*******************Graph Dump****************************");
for (Vertex vertex : titanGraph.getVertices()) {
System.out.println(GraphHelper.vertexString(vertex));
}
for (Edge edge : titanGraph.getEdges()) {
System.out.println(GraphHelper.edgeString(edge));
}
System.out.println("*******************Graph Dump****************************");
}
@AfterClass
public void tearDown() throws Exception {
TypeSystem.getInstance().reset();
......@@ -113,4 +162,79 @@ public class GraphBackedDiscoveryServiceTest {
.searchByGremlin("g.V.filter{it.typeName == 'Person'}.'Person.name'.toList()");
System.out.println("search result = " + r);
}
@DataProvider(name = "dslQueriesProvider")
private Object[][] createDSLQueries() {
return new String[][] {
{"from DB"},
{"DB"},
{"from Table"},
{"Table"},
{"DB, Table"},
/*{"DB as db1 Table where db1.name = \"Reporting\""},*/
{"DB name = \"Reporting\""},
{"Column as PII"},
{"Table as Dimension"},
{"View as Dimension"},
{"Column as PII select Column.name"},
{"Column select Column.name"},
{"from Table select Table.name"},
};
}
@Test (dataProvider = "dslQueriesProvider")
public void testSearchByDSLQueries(String dslQuery) throws Exception {
System.out.println("Executing dslQuery = " + dslQuery);
String jsonResults = discoveryService.searchByDSL(dslQuery);
Assert.assertNotNull(jsonResults);
JSONObject results = new JSONObject(jsonResults);
Assert.assertEquals(results.length(), 3);
System.out.println("results = " + results);
Object query = results.get("query");
Assert.assertNotNull(query);
JSONObject dataType = results.getJSONObject("dataType");
Assert.assertNotNull(dataType);
String typeName = dataType.getString("typeName");
Assert.assertNotNull(typeName);
JSONArray rows = results.getJSONArray("rows");
Assert.assertNotNull(rows);
Assert.assertTrue(rows.length() > 0);
}
@Test
public void testSearchByDSLQuery() throws Exception {
String dslQuery = "Column as PII";
System.out.println("Executing dslQuery = " + dslQuery);
String jsonResults = discoveryService.searchByDSL(dslQuery);
Assert.assertNotNull(jsonResults);
JSONObject results = new JSONObject(jsonResults);
Assert.assertEquals(results.length(), 3);
System.out.println("results = " + results);
Object query = results.get("query");
Assert.assertNotNull(query);
JSONObject dataType = results.getJSONObject("dataType");
Assert.assertNotNull(dataType);
String typeName = dataType.getString("typeName");
Assert.assertNotNull(typeName);
JSONArray rows = results.getJSONArray("rows");
Assert.assertNotNull(rows);
Assert.assertTrue(rows.length() > 0);
for (int index = 0; index < rows.length(); index++) {
JSONObject row = rows.getJSONObject(index);
String type = row.getString("$typeName$");
Assert.assertEquals(type, "Column");
String name = row.getString("name");
Assert.assertNotEquals(name, "null");
}
}
}
\ No newline at end of file
repository/src/test/java/org/apache/hadoop/metadata/repository/graph/GraphRepoMapperTest.java
......@@ -74,6 +74,7 @@ public class GraphRepoMapperTest {
new GraphBackedSearchIndexer(titanGraphService);
typeSystem = TypeSystem.getInstance();
typeSystem.reset();
createHiveTypes();
}
......@@ -109,7 +110,7 @@ public class GraphRepoMapperTest {
public void testGetEntityDefinition() throws Exception {
TitanGraph graph = titanGraphService.getTitanGraph();
GraphQuery query = graph.query()
.has("type", Compare.EQUAL, TABLE_TYPE);
.has(Constants.ENTITY_TYPE_PROPERTY_KEY, Compare.EQUAL, TABLE_TYPE);
Iterator<Vertex> results = query.vertices().iterator();
// returning one since guid should be unique
Vertex tableVertex = results.hasNext() ? results.next() : null;
......
repository/src/test/scala/org/apache/hadoop/metadata/query/HiveTitanSample.scala
......@@ -19,6 +19,7 @@
package org.apache.hadoop.metadata.query
import java.io.File
import java.util.UUID
import java.util.concurrent.atomic.AtomicInteger
import javax.script.{Bindings, ScriptEngine, ScriptEngineManager}
......@@ -37,6 +38,8 @@ object HiveTitanSample {
val _id: String
def id = _id
val version = 0
val guid = s"""${UUID.randomUUID()}""".stripMargin
def addEdge(to: Vertex, label: String, edges: ArrayBuffer[String]): Unit = {
edges +=
......@@ -62,6 +65,7 @@ object HiveTitanSample {
case l: List[_] => l.foreach(x => addEdge(x.asInstanceOf[Vertex],
s"${this.getClass.getSimpleName}.${f.getName}", edges))
case _ => sb.append( s""", "${f.getName}" : $fV""")
sb.append( s""", "${this.getClass.getSimpleName}.${f.getName}" : $fV""")
}
}
......@@ -322,7 +326,7 @@ object TestApp extends App with GraphUtils {
val bindings: Bindings = engine.createBindings
bindings.put("g", g)
val hiveGraphFile = FileUtils.getTempDirectory().getPath.toString + File.separator + System.nanoTime() + ".gson"
val hiveGraphFile = FileUtils.getTempDirectory().getPath + File.separator + System.nanoTime() + ".gson"
HiveTitanSample.writeGson(hiveGraphFile)
bindings.put("hiveGraphFile", hiveGraphFile)
......
webapp/src/test/java/org/apache/hadoop/metadata/web/resources/EntityJerseyResourceIT.java
......@@ -21,13 +21,13 @@ package org.apache.hadoop.metadata.web.resources;
import com.google.common.collect.ImmutableList;
import com.sun.jersey.api.client.ClientResponse;
import com.sun.jersey.api.client.WebResource;
import org.apache.hadoop.metadata.typesystem.json.Serialization$;
import org.apache.hadoop.metadata.typesystem.json.TypesSerialization;
import org.apache.hadoop.metadata.typesystem.ITypedInstance;
import org.apache.hadoop.metadata.typesystem.ITypedReferenceableInstance;
import org.apache.hadoop.metadata.typesystem.ITypedStruct;
import org.apache.hadoop.metadata.typesystem.Referenceable;
import org.apache.hadoop.metadata.typesystem.Struct;
import org.apache.hadoop.metadata.typesystem.json.Serialization$;
import org.apache.hadoop.metadata.typesystem.json.TypesSerialization;
import org.apache.hadoop.metadata.typesystem.types.AttributeDefinition;
import org.apache.hadoop.metadata.typesystem.types.AttributeInfo;
import org.apache.hadoop.metadata.typesystem.types.ClassType;
......@@ -46,11 +46,12 @@ import org.slf4j.LoggerFactory;
import org.testng.Assert;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
import scala.actors.threadpool.Arrays;
import javax.ws.rs.HttpMethod;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import java.util.Arrays;
import java.util.List;
import java.util.UUID;
/**
......@@ -131,17 +132,17 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
LOG.debug("tableInstanceAfterGet = " + definition);
// todo - this fails with type error, strange
ITypedReferenceableInstance tableInstanceAfterGet = Serialization$.MODULE$
.fromJson(definition);
ITypedReferenceableInstance tableInstanceAfterGet =
Serialization$.MODULE$.fromJson(definition);
Assert.assertTrue(areEqual(tableInstance, tableInstanceAfterGet));
}
private boolean areEqual(ITypedInstance actual,
ITypedInstance expected) throws Exception {
/*
/*
Assert.assertEquals(Serialization$.MODULE$.toJson(actual),
Serialization$.MODULE$.toJson(expected));
*/
*/
for (AttributeInfo attributeInfo : actual.fieldMapping().fields.values()) {
final DataTypes.TypeCategory typeCategory = attributeInfo.dataType().getTypeCategory();
......@@ -283,14 +284,27 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
new AttributeDefinition("database",
DATABASE_TYPE, Multiplicity.REQUIRED, true, null));
HierarchicalTypeDefinition<TraitType> classificationTypeDefinition =
HierarchicalTypeDefinition<TraitType> classificationTraitDefinition =
TypesUtil.createTraitTypeDef("classification",
ImmutableList.<String>of(),
TypesUtil.createRequiredAttrDef("tag", DataTypes.STRING_TYPE));
HierarchicalTypeDefinition<TraitType> piiTrait =
TypesUtil.createTraitTypeDef("pii", ImmutableList.<String>of());
HierarchicalTypeDefinition<TraitType> phiTrait =
TypesUtil.createTraitTypeDef("phi", ImmutableList.<String>of());
HierarchicalTypeDefinition<TraitType> pciTrait =
TypesUtil.createTraitTypeDef("pci", ImmutableList.<String>of());
HierarchicalTypeDefinition<TraitType> soxTrait =
TypesUtil.createTraitTypeDef("sox", ImmutableList.<String>of());
HierarchicalTypeDefinition<TraitType> secTrait =
TypesUtil.createTraitTypeDef("sec", ImmutableList.<String>of());
HierarchicalTypeDefinition<TraitType> financeTrait =
TypesUtil.createTraitTypeDef("finance", ImmutableList.<String>of());
typeSystem.defineTypes(
ImmutableList.of(structTypeDefinition),
ImmutableList.of(classificationTypeDefinition),
ImmutableList.of(classificationTraitDefinition, piiTrait, phiTrait, pciTrait,
soxTrait, secTrait, financeTrait),
ImmutableList.of(databaseTypeDefinition, tableTypeDefinition));
}
......@@ -302,7 +316,14 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
DATABASE_TYPE,
TABLE_TYPE,
"serdeType",
"classification"}));
"classification",
"pii",
"phi",
"pci",
"sox",
"sec",
"finance",
}));
sumbitType(typesAsJSON, TABLE_TYPE);
}
......@@ -311,7 +332,8 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
databaseInstance.set("name", DATABASE_NAME);
databaseInstance.set("description", "foo database");
Referenceable tableInstance = new Referenceable(TABLE_TYPE, "classification");
Referenceable tableInstance = new Referenceable(TABLE_TYPE,
"classification", "pii", "phi", "pci", "sox", "sec", "finance");
tableInstance.set("name", TABLE_NAME);
tableInstance.set("description", "bar table");
tableInstance.set("type", "managed");
......@@ -331,6 +353,9 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
serde2Instance.set("serde", "serde2");
tableInstance.set("serde2", serde2Instance);
List<String> traits = tableInstance.getTraits();
Assert.assertEquals(traits.size(), 7);
ClassType tableType = typeSystem.getDataType(ClassType.class, TABLE_TYPE);
return tableType.convert(tableInstance, Multiplicity.REQUIRED);
}
......
webapp/src/test/java/org/apache/hadoop/metadata/web/resources/MetadataDiscoveryResourceIT.java
......@@ -21,6 +21,7 @@ package org.apache.hadoop.metadata.web.resources;
import com.google.common.collect.ImmutableList;
import com.sun.jersey.api.client.ClientResponse;
import com.sun.jersey.api.client.WebResource;
import org.apache.hadoop.metadata.typesystem.Struct;
import org.apache.hadoop.metadata.typesystem.json.Serialization$;
import org.apache.hadoop.metadata.typesystem.json.TypesSerialization;
import org.apache.hadoop.metadata.typesystem.ITypedReferenceableInstance;
......@@ -43,6 +44,7 @@ import scala.actors.threadpool.Arrays;
import javax.ws.rs.HttpMethod;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import java.util.List;
public class MetadataDiscoveryResourceIT extends BaseResourceIT {
......@@ -263,9 +265,27 @@ public class MetadataDiscoveryResourceIT extends BaseResourceIT {
TypesUtil.createUniqueRequiredAttrDef("name", DataTypes.STRING_TYPE),
TypesUtil.createRequiredAttrDef("description", DataTypes.STRING_TYPE));
HierarchicalTypeDefinition<TraitType> classificationTraitDefinition =
TypesUtil.createTraitTypeDef("Classification",
ImmutableList.<String>of(),
TypesUtil.createRequiredAttrDef("tag", DataTypes.STRING_TYPE));
HierarchicalTypeDefinition<TraitType> piiTrait =
TypesUtil.createTraitTypeDef("PII", ImmutableList.<String>of());
HierarchicalTypeDefinition<TraitType> phiTrait =
TypesUtil.createTraitTypeDef("PHI", ImmutableList.<String>of());
HierarchicalTypeDefinition<TraitType> pciTrait =
TypesUtil.createTraitTypeDef("PCI", ImmutableList.<String>of());
HierarchicalTypeDefinition<TraitType> soxTrait =
TypesUtil.createTraitTypeDef("SOX", ImmutableList.<String>of());
HierarchicalTypeDefinition<TraitType> secTrait =
TypesUtil.createTraitTypeDef("SEC", ImmutableList.<String>of());
HierarchicalTypeDefinition<TraitType> financeTrait =
TypesUtil.createTraitTypeDef("Finance", ImmutableList.<String>of());
typeSystem.defineTypes(
ImmutableList.<StructTypeDefinition>of(),
ImmutableList.<HierarchicalTypeDefinition<TraitType>>of(),
ImmutableList.of(classificationTraitDefinition, piiTrait, phiTrait, pciTrait,
soxTrait, secTrait, financeTrait),
ImmutableList.of(dslTestTypeDefinition));
}
......@@ -274,15 +294,29 @@ public class MetadataDiscoveryResourceIT extends BaseResourceIT {
String typesAsJSON = TypesSerialization.toJson(typeSystem,
Arrays.asList(new String[]{
"dsl_test_type",
"Classification",
"PII",
"PHI",
"PCI",
"SOX",
"SEC",
"Finance",
}));
sumbitType(typesAsJSON, "dsl_test_type");
}
private ITypedReferenceableInstance createInstance() throws Exception {
Referenceable entityInstance = new Referenceable("dsl_test_type");
Referenceable entityInstance = new Referenceable("dsl_test_type",
"Classification", "PII", "PHI", "PCI", "SOX", "SEC", "Finance");
entityInstance.set("name", "foo name");
entityInstance.set("description", "bar description");
Struct traitInstance = (Struct) entityInstance.getTrait("Classification");
traitInstance.set("tag", "foundation_etl");
List<String> traits = entityInstance.getTraits();
Assert.assertEquals(traits.size(), 7);
ClassType tableType = typeSystem.getDataType(ClassType.class, "dsl_test_type");
return tableType.convert(entityInstance, Multiplicity.REQUIRED);
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment