Commit 09134770 by Sarath Subramanian

ATLAS-2240: Update JanusGraph to version 0.2.0 - fix/enable test testConcurrentCalls()

parent 8e40ab15
...@@ -129,8 +129,7 @@ public class GraphBackedMetadataRepositoryTest { ...@@ -129,8 +129,7 @@ public class GraphBackedMetadataRepositoryTest {
// AtlasGraphProvider.cleanup(); // AtlasGraphProvider.cleanup();
} }
// Disabling this test as it fails with janus profile, will enable it once fixed. @Test
@Test(enabled = false)
//In some cases of parallel APIs, the edge is added, but get edge by label doesn't return the edge. ATLAS-1104 //In some cases of parallel APIs, the edge is added, but get edge by label doesn't return the edge. ATLAS-1104
public void testConcurrentCalls() throws Exception { public void testConcurrentCalls() throws Exception {
final HierarchicalTypeDefinition<ClassType> refType = final HierarchicalTypeDefinition<ClassType> refType =
...@@ -798,9 +797,10 @@ public class GraphBackedMetadataRepositoryTest { ...@@ -798,9 +797,10 @@ public class GraphBackedMetadataRepositoryTest {
private boolean assertEdge(String id, String typeName) throws Exception { private boolean assertEdge(String id, String typeName) throws Exception {
AtlasGraph graph = TestUtils.getGraph(); AtlasGraph graph = TestUtils.getGraph();
Iterable<AtlasVertex> vertices = graph.query().has(Constants.GUID_PROPERTY_KEY, id).vertices(); Iterable<AtlasVertex> vertices = graph.query().has(Constants.GUID_PROPERTY_KEY, id).vertices();
AtlasVertex AtlasVertex = vertices.iterator().next(); AtlasVertex vertex = vertices.iterator().next();
Iterable<AtlasEdge> edges = AtlasVertex.getEdges(AtlasEdgeDirection.OUT, Constants.INTERNAL_PROPERTY_KEY_PREFIX + typeName + ".ref"); Iterable<AtlasEdge> edges = vertex.getEdges(AtlasEdgeDirection.OUT, Constants.INTERNAL_PROPERTY_KEY_PREFIX + typeName + ".ref");
if (!edges.iterator().hasNext()) {
if (edges.iterator().hasNext()) {
ITypedReferenceableInstance entity = repositoryService.getEntityDefinition(id); ITypedReferenceableInstance entity = repositoryService.getEntityDefinition(id);
assertNotNull(entity.get("ref")); assertNotNull(entity.get("ref"));
return true; return true;
......
...@@ -86,6 +86,13 @@ public abstract class BaseResourceIT { ...@@ -86,6 +86,13 @@ public abstract class BaseResourceIT {
public static final String QUALIFIED_NAME = "qualifiedName"; public static final String QUALIFIED_NAME = "qualifiedName";
public static final String CLUSTER_NAME = "clusterName"; public static final String CLUSTER_NAME = "clusterName";
public static final String DESCRIPTION = "description"; public static final String DESCRIPTION = "description";
public static final String PII_TAG = "pii_Tag";
public static final String PHI_TAG = "phi_Tag";
public static final String PCI_TAG = "pci_Tag";
public static final String SOX_TAG = "sox_Tag";
public static final String SEC_TAG = "sec_Tag";
public static final String FINANCE_TAG = "finance_Tag";
public static final String CLASSIFICATION = "classification";
// All service clients // All service clients
protected AtlasClient atlasClientV1; protected AtlasClient atlasClientV1;
...@@ -334,17 +341,17 @@ public abstract class BaseResourceIT { ...@@ -334,17 +341,17 @@ public abstract class BaseResourceIT {
.createTraitTypeDef("classification", ImmutableSet.<String>of(), .createTraitTypeDef("classification", ImmutableSet.<String>of(),
TypesUtil.createRequiredAttrDef("tag", DataTypes.STRING_TYPE)); TypesUtil.createRequiredAttrDef("tag", DataTypes.STRING_TYPE));
HierarchicalTypeDefinition<TraitType> piiTrait = HierarchicalTypeDefinition<TraitType> piiTrait =
TypesUtil.createTraitTypeDef("pii", ImmutableSet.<String>of()); TypesUtil.createTraitTypeDef(PII_TAG, ImmutableSet.<String>of());
HierarchicalTypeDefinition<TraitType> phiTrait = HierarchicalTypeDefinition<TraitType> phiTrait =
TypesUtil.createTraitTypeDef("phi", ImmutableSet.<String>of()); TypesUtil.createTraitTypeDef(PHI_TAG, ImmutableSet.<String>of());
HierarchicalTypeDefinition<TraitType> pciTrait = HierarchicalTypeDefinition<TraitType> pciTrait =
TypesUtil.createTraitTypeDef("pci", ImmutableSet.<String>of()); TypesUtil.createTraitTypeDef(PCI_TAG, ImmutableSet.<String>of());
HierarchicalTypeDefinition<TraitType> soxTrait = HierarchicalTypeDefinition<TraitType> soxTrait =
TypesUtil.createTraitTypeDef("sox", ImmutableSet.<String>of()); TypesUtil.createTraitTypeDef(SOX_TAG, ImmutableSet.<String>of());
HierarchicalTypeDefinition<TraitType> secTrait = HierarchicalTypeDefinition<TraitType> secTrait =
TypesUtil.createTraitTypeDef("sec", ImmutableSet.<String>of()); TypesUtil.createTraitTypeDef(SEC_TAG, ImmutableSet.<String>of());
HierarchicalTypeDefinition<TraitType> financeTrait = HierarchicalTypeDefinition<TraitType> financeTrait =
TypesUtil.createTraitTypeDef("finance", ImmutableSet.<String>of()); TypesUtil.createTraitTypeDef(FINANCE_TAG, ImmutableSet.<String>of());
HierarchicalTypeDefinition<TraitType> factTrait = HierarchicalTypeDefinition<TraitType> factTrait =
TypesUtil.createTraitTypeDef("Fact" + randomString(), ImmutableSet.<String>of()); TypesUtil.createTraitTypeDef("Fact" + randomString(), ImmutableSet.<String>of());
HierarchicalTypeDefinition<TraitType> etlTrait = HierarchicalTypeDefinition<TraitType> etlTrait =
...@@ -434,17 +441,17 @@ public abstract class BaseResourceIT { ...@@ -434,17 +441,17 @@ public abstract class BaseResourceIT {
.createTraitTypeDef("classification",ImmutableSet.<String>of(), .createTraitTypeDef("classification",ImmutableSet.<String>of(),
AtlasTypeUtil.createRequiredAttrDef("tag", "string")); AtlasTypeUtil.createRequiredAttrDef("tag", "string"));
AtlasClassificationDef piiTrait = AtlasClassificationDef piiTrait =
AtlasTypeUtil.createTraitTypeDef("pii", ImmutableSet.<String>of()); AtlasTypeUtil.createTraitTypeDef(PII_TAG, ImmutableSet.<String>of());
AtlasClassificationDef phiTrait = AtlasClassificationDef phiTrait =
AtlasTypeUtil.createTraitTypeDef("phi", ImmutableSet.<String>of()); AtlasTypeUtil.createTraitTypeDef(PHI_TAG, ImmutableSet.<String>of());
AtlasClassificationDef pciTrait = AtlasClassificationDef pciTrait =
AtlasTypeUtil.createTraitTypeDef("pci", ImmutableSet.<String>of()); AtlasTypeUtil.createTraitTypeDef(PCI_TAG, ImmutableSet.<String>of());
AtlasClassificationDef soxTrait = AtlasClassificationDef soxTrait =
AtlasTypeUtil.createTraitTypeDef("sox", ImmutableSet.<String>of()); AtlasTypeUtil.createTraitTypeDef(SOX_TAG, ImmutableSet.<String>of());
AtlasClassificationDef secTrait = AtlasClassificationDef secTrait =
AtlasTypeUtil.createTraitTypeDef("sec", ImmutableSet.<String>of()); AtlasTypeUtil.createTraitTypeDef(SEC_TAG, ImmutableSet.<String>of());
AtlasClassificationDef financeTrait = AtlasClassificationDef financeTrait =
AtlasTypeUtil.createTraitTypeDef("finance", ImmutableSet.<String>of()); AtlasTypeUtil.createTraitTypeDef(FINANCE_TAG, ImmutableSet.<String>of());
AtlasTypesDef typesDef = new AtlasTypesDef(ImmutableList.of(enumDef), AtlasTypesDef typesDef = new AtlasTypesDef(ImmutableList.of(enumDef),
ImmutableList.of(structTypeDef), ImmutableList.of(structTypeDef),
...@@ -485,7 +492,7 @@ public abstract class BaseResourceIT { ...@@ -485,7 +492,7 @@ public abstract class BaseResourceIT {
values.put("location", "/tmp"); values.put("location", "/tmp");
Referenceable databaseInstance = new Referenceable(dbId._getId(), dbId.getTypeName(), values); Referenceable databaseInstance = new Referenceable(dbId._getId(), dbId.getTypeName(), values);
Referenceable tableInstance = Referenceable tableInstance =
new Referenceable(HIVE_TABLE_TYPE_BUILTIN, "classification", "pii", "phi", "pci", "sox", "sec", "finance"); new Referenceable(HIVE_TABLE_TYPE_BUILTIN, CLASSIFICATION, PII_TAG, PHI_TAG, PCI_TAG, SOX_TAG, SEC_TAG, FINANCE_TAG);
tableInstance.set(NAME, tableName); tableInstance.set(NAME, tableName);
tableInstance.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, tableName); tableInstance.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, tableName);
tableInstance.set("db", databaseInstance); tableInstance.set("db", databaseInstance);
...@@ -518,13 +525,13 @@ public abstract class BaseResourceIT { ...@@ -518,13 +525,13 @@ public abstract class BaseResourceIT {
protected AtlasEntity createHiveTableInstanceV2(AtlasEntity databaseInstance, String tableName) throws Exception { protected AtlasEntity createHiveTableInstanceV2(AtlasEntity databaseInstance, String tableName) throws Exception {
AtlasEntity tableInstance = new AtlasEntity(HIVE_TABLE_TYPE_V2); AtlasEntity tableInstance = new AtlasEntity(HIVE_TABLE_TYPE_V2);
tableInstance.setClassifications( tableInstance.setClassifications(
Arrays.asList(new AtlasClassification("classification"), Arrays.asList(new AtlasClassification(CLASSIFICATION),
new AtlasClassification("pii"), new AtlasClassification(PII_TAG),
new AtlasClassification("phi"), new AtlasClassification(PHI_TAG),
new AtlasClassification("pci"), new AtlasClassification(PCI_TAG),
new AtlasClassification("sox"), new AtlasClassification(SOX_TAG),
new AtlasClassification("sec"), new AtlasClassification(SEC_TAG),
new AtlasClassification("finance")) new AtlasClassification(FINANCE_TAG))
); );
tableInstance.setAttribute(NAME, tableName); tableInstance.setAttribute(NAME, tableName);
......
...@@ -48,6 +48,11 @@ public class DataSetLineageJerseyResourceIT extends BaseResourceIT { ...@@ -48,6 +48,11 @@ public class DataSetLineageJerseyResourceIT extends BaseResourceIT {
private String salesFactTable; private String salesFactTable;
private String salesMonthlyTable; private String salesMonthlyTable;
private String salesDBName; private String salesDBName;
private static String FACT = "Fact_Tag";
private static String ETL = "ETL_Tag";
private static String DIMENSION = "Dimension_Tag";
private static String METRIC = "Metric_Tag";
private static String PII = "pii_Tag";
@BeforeClass @BeforeClass
public void setUp() throws Exception { public void setUp() throws Exception {
...@@ -183,13 +188,13 @@ public class DataSetLineageJerseyResourceIT extends BaseResourceIT { ...@@ -183,13 +188,13 @@ public class DataSetLineageJerseyResourceIT extends BaseResourceIT {
private void setupInstances() throws Exception { private void setupInstances() throws Exception {
HierarchicalTypeDefinition<TraitType> factTrait = HierarchicalTypeDefinition<TraitType> factTrait =
TypesUtil.createTraitTypeDef("Fact", ImmutableSet.<String>of()); TypesUtil.createTraitTypeDef(FACT, ImmutableSet.<String>of());
HierarchicalTypeDefinition<TraitType> etlTrait = HierarchicalTypeDefinition<TraitType> etlTrait =
TypesUtil.createTraitTypeDef("ETL", ImmutableSet.<String>of()); TypesUtil.createTraitTypeDef(ETL, ImmutableSet.<String>of());
HierarchicalTypeDefinition<TraitType> dimensionTrait = HierarchicalTypeDefinition<TraitType> dimensionTrait =
TypesUtil.createTraitTypeDef("Dimension", ImmutableSet.<String>of()); TypesUtil.createTraitTypeDef(DIMENSION, ImmutableSet.<String>of());
HierarchicalTypeDefinition<TraitType> metricTrait = HierarchicalTypeDefinition<TraitType> metricTrait =
TypesUtil.createTraitTypeDef("Metric", ImmutableSet.<String>of()); TypesUtil.createTraitTypeDef(METRIC, ImmutableSet.<String>of());
createType(getTypesDef(null, null, createType(getTypesDef(null, null,
ImmutableList.of(factTrait, etlTrait, dimensionTrait, metricTrait), null)); ImmutableList.of(factTrait, etlTrait, dimensionTrait, metricTrait), null));
...@@ -199,11 +204,11 @@ public class DataSetLineageJerseyResourceIT extends BaseResourceIT { ...@@ -199,11 +204,11 @@ public class DataSetLineageJerseyResourceIT extends BaseResourceIT {
List<Referenceable> salesFactColumns = ImmutableList List<Referenceable> salesFactColumns = ImmutableList
.of(column("time_id", "int", "time id"), column("product_id", "int", "product id"), .of(column("time_id", "int", "time id"), column("product_id", "int", "product id"),
column("customer_id", "int", "customer id", "pii"), column("customer_id", "int", "customer id", PII),
column("sales", "double", "product id", "Metric")); column("sales", "double", "product id", METRIC));
salesFactTable = "sales_fact" + randomString(); salesFactTable = "sales_fact" + randomString();
Id salesFact = table(salesFactTable, "sales fact table", salesDB, "Joe", "MANAGED", salesFactColumns, "Fact"); Id salesFact = table(salesFactTable, "sales fact table", salesDB, "Joe", "MANAGED", salesFactColumns, FACT);
List<Referenceable> timeDimColumns = ImmutableList List<Referenceable> timeDimColumns = ImmutableList
.of(column("time_id", "int", "time id"), column("dayOfYear", "int", "day Of Year"), .of(column("time_id", "int", "time id"), column("dayOfYear", "int", "day Of Year"),
...@@ -211,7 +216,7 @@ public class DataSetLineageJerseyResourceIT extends BaseResourceIT { ...@@ -211,7 +216,7 @@ public class DataSetLineageJerseyResourceIT extends BaseResourceIT {
Id timeDim = Id timeDim =
table("time_dim" + randomString(), "time dimension table", salesDB, "John Doe", "EXTERNAL", table("time_dim" + randomString(), "time dimension table", salesDB, "John Doe", "EXTERNAL",
timeDimColumns, "Dimension"); timeDimColumns, DIMENSION);
Id reportingDB = Id reportingDB =
database("Reporting" + randomString(), "reporting database", "Jane BI", database("Reporting" + randomString(), "reporting database", "Jane BI",
...@@ -219,18 +224,18 @@ public class DataSetLineageJerseyResourceIT extends BaseResourceIT { ...@@ -219,18 +224,18 @@ public class DataSetLineageJerseyResourceIT extends BaseResourceIT {
Id salesFactDaily = Id salesFactDaily =
table("sales_fact_daily_mv" + randomString(), "sales fact daily materialized view", reportingDB, table("sales_fact_daily_mv" + randomString(), "sales fact daily materialized view", reportingDB,
"Joe BI", "MANAGED", salesFactColumns, "Metric"); "Joe BI", "MANAGED", salesFactColumns, METRIC);
loadProcess("loadSalesDaily" + randomString(), "John ETL", ImmutableList.of(salesFact, timeDim), loadProcess("loadSalesDaily" + randomString(), "John ETL", ImmutableList.of(salesFact, timeDim),
ImmutableList.of(salesFactDaily), "create table as select ", "plan", "id", "graph", "ETL"); ImmutableList.of(salesFactDaily), "create table as select ", "plan", "id", "graph", ETL);
salesMonthlyTable = "sales_fact_monthly_mv" + randomString(); salesMonthlyTable = "sales_fact_monthly_mv" + randomString();
Id salesFactMonthly = Id salesFactMonthly =
table(salesMonthlyTable, "sales fact monthly materialized view", reportingDB, "Jane BI", table(salesMonthlyTable, "sales fact monthly materialized view", reportingDB, "Jane BI",
"MANAGED", salesFactColumns, "Metric"); "MANAGED", salesFactColumns, METRIC);
loadProcess("loadSalesMonthly" + randomString(), "John ETL", ImmutableList.of(salesFactDaily), loadProcess("loadSalesMonthly" + randomString(), "John ETL", ImmutableList.of(salesFactDaily),
ImmutableList.of(salesFactMonthly), "create table as select ", "plan", "id", "graph", "ETL"); ImmutableList.of(salesFactMonthly), "create table as select ", "plan", "id", "graph", ETL);
} }
Id database(String name, String description, String owner, String locationUri, String... traitNames) Id database(String name, String description, String owner, String locationUri, String... traitNames)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment