Skip to content

A
atlas
Commit d6d349e5 by apoorvnaik Committed by Madhan Neethiraj
Options

ATLAS-2092: Create edgeLabel while updating the index

Signed-off-by: 's avatarMadhan Neethiraj <madhan@apache.org>
parent 20f2fba7
Showing with 375 additions and 27 deletions
graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasEdgeLabel.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.repository.graphdb;
/**
* Represent an edge label.
*
*/
public interface AtlasEdgeLabel {
String getName();
}
graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraphManagement.java
......@@ -63,6 +63,12 @@ public interface AtlasGraphManagement {
AtlasPropertyKey makePropertyKey(String propertyName, Class propertyClass, AtlasCardinality cardinality);
/**
*
* @param label edge label to be created
*/
AtlasEdgeLabel makeEdgeLabel(String label);
/**
* @param propertyKey
*
*/
......@@ -75,6 +81,12 @@ public interface AtlasGraphManagement {
AtlasPropertyKey getPropertyKey(String propertyName);
/**
* @param label
* @return
*/
AtlasEdgeLabel getEdgeLabel(String label);
/**
* Creates a composite index for the graph.
*
* @param propertyName
......
graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/GraphDbObjectFactory.java
......@@ -18,6 +18,7 @@
package org.apache.atlas.repository.graphdb.titan0;
import com.thinkaurelius.titan.core.EdgeLabel;
import org.apache.atlas.repository.graphdb.AtlasCardinality;
import org.apache.atlas.repository.graphdb.AtlasGraphIndex;
import org.apache.atlas.repository.graphdb.titan0.query.Titan0GraphQuery;
......@@ -88,6 +89,17 @@ public final class GraphDbObjectFactory {
}
/**
* @param label The label.
*
*/
public static Titan0EdgeLabel createEdgeLabel(EdgeLabel label) {
if (label == null) {
return null;
}
return new Titan0EdgeLabel(label);
}
/**
* @param index The gremlin index.
* @return
*/
......
graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/Titan0EdgeLabel.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.repository.graphdb.titan0;
import com.thinkaurelius.titan.core.EdgeLabel;
import org.apache.atlas.repository.graphdb.AtlasEdgeLabel;
/**
* Titan 0.5.4 implementaiton of AtlasEdgeLabel.
*/
public class Titan0EdgeLabel implements AtlasEdgeLabel {
private final EdgeLabel wrappedEdgeLabel;
public Titan0EdgeLabel(EdgeLabel toWrap) {
wrappedEdgeLabel = toWrap;
}
/*
* (non-Javadoc)
*
* @see org.apache.atlas.repository.graphdb.AtlasEdgeLabel#getName()
*/
@Override
public String getName() {
return wrappedEdgeLabel.getName();
}
@Override
public boolean equals(Object other) {
if (!(other instanceof Titan0EdgeLabel)) {
return false;
}
Titan0EdgeLabel otherLabel = (Titan0EdgeLabel) other;
return wrappedEdgeLabel.equals(otherLabel.wrappedEdgeLabel);
}
@Override
public int hashCode() {
int result = 17;
result = 37 * result + wrappedEdgeLabel.hashCode();
return result;
}
@Override
public String toString() {
return wrappedEdgeLabel.getName();
}
}
graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/Titan0GraphManagement.java
......@@ -18,6 +18,7 @@
package org.apache.atlas.repository.graphdb.titan0;
import com.thinkaurelius.titan.core.Cardinality;
import com.thinkaurelius.titan.core.EdgeLabel;
import com.thinkaurelius.titan.core.PropertyKey;
import com.thinkaurelius.titan.core.schema.Mapping;
import com.thinkaurelius.titan.core.schema.PropertyKeyMaker;
......@@ -27,6 +28,7 @@ import com.tinkerpop.blueprints.Edge;
import com.tinkerpop.blueprints.Element;
import com.tinkerpop.blueprints.Vertex;
import org.apache.atlas.repository.graphdb.AtlasCardinality;
import org.apache.atlas.repository.graphdb.AtlasEdgeLabel;
import org.apache.atlas.repository.graphdb.AtlasGraphIndex;
import org.apache.atlas.repository.graphdb.AtlasGraphManagement;
import org.apache.atlas.repository.graphdb.AtlasPropertyKey;
......@@ -111,6 +113,13 @@ public class Titan0GraphManagement implements AtlasGraphManagement {
}
@Override
public AtlasEdgeLabel makeEdgeLabel(String label) {
EdgeLabel edgeLabel = management.makeEdgeLabel(label).make();
return GraphDbObjectFactory.createEdgeLabel(edgeLabel);
}
@Override
public void deletePropertyKey(String propertyKey) {
PropertyKey titanPropertyKey = management.getPropertyKey(propertyKey);
......@@ -132,6 +141,11 @@ public class Titan0GraphManagement implements AtlasGraphManagement {
}
@Override
public AtlasEdgeLabel getEdgeLabel(String label) {
return GraphDbObjectFactory.createEdgeLabel(management.getEdgeLabel(label));
}
@Override
public void createExactMatchIndex(String propertyName, boolean enforceUniqueness,
List<AtlasPropertyKey> propertyKeys) {
......
graphdb/titan1/src/main/java/org/apache/atlas/repository/graphdb/titan1/GraphDbObjectFactory.java
......@@ -18,6 +18,7 @@
package org.apache.atlas.repository.graphdb.titan1;
import com.thinkaurelius.titan.core.EdgeLabel;
import org.apache.atlas.repository.graphdb.AtlasCardinality;
import org.apache.atlas.repository.graphdb.AtlasGraphIndex;
import org.apache.atlas.repository.graphdb.titan1.query.Titan1GraphQuery;
......@@ -89,6 +90,17 @@ public final class GraphDbObjectFactory {
}
/**
* @param label The Gremlin propertyKey.
*
*/
public static Titan1EdgeLabel createEdgeLabel(EdgeLabel label) {
if (label == null) {
return null;
}
return new Titan1EdgeLabel(label);
}
/**
* @param index The gremlin index.
* @return
*/
......
graphdb/titan1/src/main/java/org/apache/atlas/repository/graphdb/titan1/Titan1EdgeLabel.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.repository.graphdb.titan1;
import com.thinkaurelius.titan.core.EdgeLabel;
import org.apache.atlas.repository.graphdb.AtlasEdgeLabel;
/**
*
*/
public class Titan1EdgeLabel implements AtlasEdgeLabel {
private final EdgeLabel wrapped;
public Titan1EdgeLabel(EdgeLabel toWrap) {
wrapped = toWrap;
}
/* (non-Javadoc)
* @see org.apache.atlas.repository.graphdb.AtlasPropertyKey#getName()
*/
@Override
public String getName() {
return wrapped.name();
}
@Override
public int hashCode() {
int result = 17;
result = 37*result + wrapped.hashCode();
return result;
}
@Override
public boolean equals(Object other) {
if (!(other instanceof Titan1EdgeLabel)) {
return false;
}
Titan1EdgeLabel otherKey = (Titan1EdgeLabel)other;
return otherKey.wrapped.equals(wrapped);
}
}
graphdb/titan1/src/main/java/org/apache/atlas/repository/graphdb/titan1/Titan1GraphManagement.java
......@@ -19,6 +19,7 @@ package org.apache.atlas.repository.graphdb.titan1;
import com.google.common.base.Preconditions;
import com.thinkaurelius.titan.core.Cardinality;
import com.thinkaurelius.titan.core.EdgeLabel;
import com.thinkaurelius.titan.core.PropertyKey;
import com.thinkaurelius.titan.core.schema.Mapping;
import com.thinkaurelius.titan.core.schema.PropertyKeyMaker;
......@@ -26,6 +27,7 @@ import com.thinkaurelius.titan.core.schema.TitanGraphIndex;
import com.thinkaurelius.titan.core.schema.TitanManagement;
import com.thinkaurelius.titan.graphdb.internal.Token;
import org.apache.atlas.repository.graphdb.AtlasCardinality;
import org.apache.atlas.repository.graphdb.AtlasEdgeLabel;
import org.apache.atlas.repository.graphdb.AtlasGraphIndex;
import org.apache.atlas.repository.graphdb.AtlasGraphManagement;
import org.apache.atlas.repository.graphdb.AtlasPropertyKey;
......@@ -135,6 +137,13 @@ public class Titan1GraphManagement implements AtlasGraphManagement {
}
@Override
public AtlasEdgeLabel makeEdgeLabel(String label) {
EdgeLabel edgeLabel = management.makeEdgeLabel(label).make();
return GraphDbObjectFactory.createEdgeLabel(edgeLabel);
}
@Override
public void deletePropertyKey(String propertyKey) {
PropertyKey titanPropertyKey = management.getPropertyKey(propertyKey);
......@@ -155,6 +164,11 @@ public class Titan1GraphManagement implements AtlasGraphManagement {
return GraphDbObjectFactory.createPropertyKey(management.getPropertyKey(propertyName));
}
@Override
public AtlasEdgeLabel getEdgeLabel(String label) {
return GraphDbObjectFactory.createEdgeLabel(management.getEdgeLabel(label));
}
public void createExactMatchVertexIndex(String propertyName, boolean enforceUniqueness,
List<AtlasPropertyKey> propertyKeys) {
......
repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java
......@@ -191,7 +191,6 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
//Indexes for graph backed type system store
createTypeStoreIndexes(management);
commit(management);
LOG.info("Index creation for global keys complete.");
} catch (Throwable t) {
......@@ -319,8 +318,12 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
try {
AtlasType atlasType = typeRegistry.getType(attribTypeName);
if (isMapType || isArrayType || isClassificationType(atlasType) || isEntityType(atlasType)) {
if (isMapType || isClassificationType(atlasType)) {
LOG.warn("Ignoring non-indexable attribute {}", attribTypeName);
} if (isArrayType) {
createLabelIfNeeded(management, propertyName, attribTypeName);
} if (isEntityType(atlasType)) {
createEdgeLabel(management, propertyName);
} else if (isBuiltInType) {
createIndexes(management, propertyName, getPrimitiveClass(attribTypeName), isUnique, cardinality, false, isIndexable);
} else if (isEnumType(atlasType)) {
......@@ -334,6 +337,16 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
}
}
private void createLabelIfNeeded(final AtlasGraphManagement management, final String propertyName, final String attribTypeName) {
// If any of the referenced typename is of type Entity or Struct then the edge label needs to be created
for (String typeName : AtlasTypeUtil.getReferencedTypeNames(attribTypeName)) {
if (typeRegistry.getEntityDefByName(typeName) != null || typeRegistry.getStructDefByName(typeName) != null) {
// Create the edge label upfront to avoid running into concurrent call issue (ATLAS-2092)
createEdgeLabel(management, propertyName);
}
}
}
private boolean isEntityType(AtlasType type) {
return type instanceof AtlasEntityType;
}
......@@ -442,6 +455,8 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
break;
case ARRAY:
createLabelIfNeeded(management, propertyName, field.dataType().getName());
break;
case MAP:
// todo - how do we overcome this limitation?
// IGNORE: Can only index single-valued property keys on vertices in Mixed Index
......@@ -457,9 +472,7 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
break;
case CLASS:
// this is only A reference, index the attribute for edge
// Commenting this out since we do not need an index for edge here
//createEdgeMixedIndex(propertyName);
createEdgeLabel(management, propertyName);
break;
default:
......@@ -467,6 +480,22 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
}
}
private void createEdgeLabel(final AtlasGraphManagement management, final String propertyName) {
// Create the edge label upfront to avoid running into concurrent call issue (ATLAS-2092)
// ATLAS-2092 addresses this problem by creating the edge label upfront while type creation
// which resolves the race condition during the entity creation
String label = Constants.INTERNAL_PROPERTY_KEY_PREFIX + propertyName;
org.apache.atlas.repository.graphdb.AtlasEdgeLabel edgeLabel = management.getEdgeLabel(label);
if (edgeLabel == null) {
management.makeEdgeLabel(label);
LOG.info("Created edge label {} ", label);
}
}
private Class getPrimitiveClass(IDataType dataType) {
if (dataType == DataTypes.STRING_TYPE) {
return String.class;
......
repository/src/test/java/org/apache/atlas/TestUtils.java
......@@ -481,7 +481,13 @@ public final class TestUtils {
}
public static final String randomString() {
return RandomStringUtils.randomAlphanumeric(10);
return randomString(10);
}
public static final String randomString(int count) {
final String prefix = "r";
return prefix + RandomStringUtils.randomAlphanumeric(count - prefix.length()); // ensure that the string starts with a letter
}
public static Referenceable createDBEntity() {
......
repository/src/test/java/org/apache/atlas/discovery/DataSetLineageServiceTest.java
......@@ -22,6 +22,7 @@ import com.google.common.collect.ImmutableList;
import org.apache.atlas.AtlasException;
import org.apache.atlas.BaseRepositoryTest;
import org.apache.atlas.TestModules;
import org.apache.atlas.TestUtils;
import org.apache.atlas.model.legacy.EntityResult;
import org.apache.atlas.query.QueryParams;
import org.apache.atlas.typesystem.ITypedReferenceableInstance;
......@@ -31,7 +32,6 @@ import org.apache.atlas.typesystem.exception.EntityNotFoundException;
import org.apache.atlas.typesystem.json.InstanceSerialization;
import org.apache.atlas.typesystem.persistence.Id;
import org.apache.commons.collections.ArrayStack;
import org.apache.commons.lang.RandomStringUtils;
import org.codehaus.jettison.json.JSONArray;
import org.codehaus.jettison.json.JSONException;
import org.codehaus.jettison.json.JSONObject;
......@@ -488,7 +488,7 @@ public class DataSetLineageServiceTest extends BaseRepositoryTest {
}
private String random() {
return RandomStringUtils.randomAlphanumeric(5);
return TestUtils.randomString(5);
}
private String getEntityId(String typeName, String attributeName, String attributeValue) throws Exception {
......
repository/src/test/java/org/apache/atlas/lineage/EntityLineageServiceTest.java
......@@ -34,7 +34,6 @@ import org.apache.atlas.model.lineage.AtlasLineageInfo.LineageRelation;
import org.apache.atlas.typesystem.Referenceable;
import org.apache.atlas.typesystem.persistence.Id;
import org.apache.commons.collections.ArrayStack;
import org.apache.commons.lang.RandomStringUtils;
import org.testng.Assert;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
......@@ -347,7 +346,7 @@ public class EntityLineageServiceTest extends BaseRepositoryTest {
}
private String random() {
return RandomStringUtils.randomAlphanumeric(5);
return TestUtils.randomString(5);
}
private String getEntityId(String typeName, String attributeName, String attributeValue) throws Exception {
......
repository/src/test/java/org/apache/atlas/repository/audit/AuditRepositoryTestBase.java
......@@ -21,7 +21,6 @@ package org.apache.atlas.repository.audit;
import org.apache.atlas.EntityAuditEvent;
import org.apache.atlas.TestUtils;
import org.apache.atlas.typesystem.Referenceable;
import org.apache.commons.lang.RandomStringUtils;
import org.testng.annotations.BeforeTest;
import org.testng.annotations.Test;
......@@ -35,7 +34,7 @@ public class AuditRepositoryTestBase {
protected EntityAuditRepository eventRepository;
private String rand() {
return RandomStringUtils.randomAlphanumeric(10);
return TestUtils.randomString(10);
}
@BeforeTest
......
repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryTest.java
......@@ -885,11 +885,11 @@ public class GraphBackedMetadataRepositoryTest {
}
private String randomUTF() {
return RandomStringUtils.random(10);
RandomStringUtils.random(10);
}
private String randomString() {
return RandomStringUtils.randomAlphanumeric(10);
return TestUtils.randomString(10);
}
@Test
......
repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexerTest.java
......@@ -34,7 +34,6 @@ import org.apache.atlas.typesystem.types.EnumValue;
import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition;
import org.apache.atlas.typesystem.types.TypeSystem;
import org.apache.atlas.typesystem.types.utils.TypesUtil;
import org.apache.commons.lang.RandomStringUtils;
import org.testng.annotations.Guice;
import org.testng.annotations.Test;
......@@ -136,7 +135,7 @@ public class GraphBackedSearchIndexerTest {
try {
TypeSystem typeSystem = TypeSystem.getInstance();
String enumName = "randomEnum" + RandomStringUtils.randomAlphanumeric(10);
String enumName = "randomEnum" + TestUtils.randomString(10);
EnumType managedType = typeSystem.defineEnumType(enumName, new EnumValue("randomEnumValue", 0));
HierarchicalTypeDefinition<ClassType> databaseTypeDefinition =
......
repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityStoreV1Test.java
......@@ -55,7 +55,6 @@ import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.atlas.type.AtlasTypeUtil;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.MapUtils;
import org.apache.commons.lang.RandomStringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.testng.Assert;
......@@ -629,7 +628,7 @@ public class AtlasEntityStoreV1Test {
//TODO : Failing in typedef creation
public void testSpecialCharacters() throws Exception {
//Verify that type can be created with reserved characters in typename, attribute name
final String typeName = "test_type_"+ RandomStringUtils.randomAlphanumeric(10);
final String typeName = TestUtils.randomString(10);
String strAttrName = randomStrWithReservedChars();
String arrayAttrName = randomStrWithReservedChars();
String mapAttrName = randomStrWithReservedChars();
......@@ -685,7 +684,7 @@ public class AtlasEntityStoreV1Test {
init();
AtlasEntity dbEntity = new AtlasEntity(TestUtilsV2.DATABASE_TYPE);
dbEntity.setAttribute("name", RandomStringUtils.randomAlphanumeric(10));
dbEntity.setAttribute("name", TestUtils.randomString(10));
dbEntity.setAttribute("description", "us db");
dbEntity.setAttribute("isReplicated", false);
dbEntity.setAttribute("created", "09081988");
......@@ -731,7 +730,7 @@ public class AtlasEntityStoreV1Test {
// create a new table type
AtlasEntity tblEntity = new AtlasEntity(TABLE_TYPE);
tblEntity.setAttribute("name", RandomStringUtils.randomAlphanumeric(10));
tblEntity.setAttribute("name", TestUtils.randomString(10));
tblEntity.setAttribute("type", "type");
tblEntity.setAttribute("tableType", "MANAGED");
tblEntity.setAttribute("database", AtlasTypeUtil.getAtlasObjectId(updatedDbEntity));
......
repository/src/test/java/org/apache/atlas/service/DefaultMetadataServiceTest.java
......@@ -23,9 +23,10 @@ import com.google.common.collect.ImmutableSet;
import com.google.inject.Inject;
import org.apache.atlas.AtlasClient;
import org.apache.atlas.AtlasException;
import org.apache.atlas.CreateUpdateEntitiesResult;
import org.apache.atlas.EntityAuditEvent;
import org.apache.atlas.TestModules;
import org.apache.atlas.RequestContext;
import org.apache.atlas.TestModules;
import org.apache.atlas.TestUtils;
import org.apache.atlas.discovery.graph.GraphBackedDiscoveryService;
import org.apache.atlas.exception.AtlasBaseException;
......@@ -34,10 +35,15 @@ import org.apache.atlas.listener.EntityChangeListener;
import org.apache.atlas.listener.TypeDefChangeListener;
import org.apache.atlas.model.legacy.EntityResult;
import org.apache.atlas.query.QueryParams;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.audit.EntityAuditRepository;
import org.apache.atlas.repository.audit.HBaseBasedAuditRepository;
import org.apache.atlas.repository.audit.HBaseTestUtils;
import org.apache.atlas.repository.graph.AtlasGraphProvider;
import org.apache.atlas.repository.graphdb.AtlasEdge;
import org.apache.atlas.repository.graphdb.AtlasEdgeDirection;
import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.services.DefaultMetadataService;
import org.apache.atlas.services.MetadataService;
import org.apache.atlas.type.AtlasTypeUtil;
......@@ -63,7 +69,6 @@ import org.apache.atlas.typesystem.types.ValueConversionException;
import org.apache.atlas.typesystem.types.cache.TypeCache;
import org.apache.atlas.typesystem.types.utils.TypesUtil;
import org.apache.atlas.utils.ParamChecker;
import org.apache.commons.lang.RandomStringUtils;
import org.codehaus.jettison.json.JSONArray;
import org.codehaus.jettison.json.JSONException;
import org.codehaus.jettison.json.JSONObject;
......@@ -80,6 +85,11 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import static org.apache.atlas.TestUtils.*;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createClassTypeDef;
......@@ -164,7 +174,7 @@ public class DefaultMetadataServiceTest {
@Test(expectedExceptions = TypeNotFoundException.class)
public void testCreateEntityWithUnknownDatatype() throws Exception {
Referenceable entity = new Referenceable("Unknown datatype");
String dbName = RandomStringUtils.randomAlphanumeric(10);
String dbName = TestUtils.randomString(10);
entity.set(NAME, dbName);
entity.set("description", "us db");
TestUtils.createInstance(metadataService, entity);
......@@ -197,7 +207,7 @@ public class DefaultMetadataServiceTest {
String arrayAttrName = randomStrWithReservedChars();
String mapAttrName = randomStrWithReservedChars();
HierarchicalTypeDefinition<ClassType> typeDefinition =
createClassTypeDef("test_type_"+ RandomStringUtils.randomAlphanumeric(10), ImmutableSet.<String>of(),
createClassTypeDef("test_type_"+ TestUtils.randomString(10), ImmutableSet.<String>of(),
createOptionalAttrDef(strAttrName, DataTypes.STRING_TYPE),
new AttributeDefinition(arrayAttrName, DataTypes.arrayTypeName(DataTypes.STRING_TYPE.getName()),
Multiplicity.OPTIONAL, false, null),
......@@ -1112,7 +1122,7 @@ public class DefaultMetadataServiceTest {
@Test
public void testTypeUpdateFailureShouldRollBack() throws AtlasException, JSONException {
String typeName = "test_type_"+ RandomStringUtils.randomAlphanumeric(10);
String typeName = TestUtils.randomString(10);
HierarchicalTypeDefinition<ClassType> typeDef = TypesUtil.createClassTypeDef(
typeName, ImmutableSet.<String>of(),
TypesUtil.createUniqueRequiredAttrDef("test_type_attribute", DataTypes.STRING_TYPE));
......@@ -1227,6 +1237,97 @@ public class DefaultMetadataServiceTest {
}
}
@Test
// ATLAS-2092: Concurrent edge label creation leads to inconsistency
// This test tries entity creation in parallel and ensures that the edges with the same label actually get created
public void testConcurrentCalls() {
final HierarchicalTypeDefinition<ClassType> refType =
createClassTypeDef(randomString(), ImmutableSet.<String>of());
HierarchicalTypeDefinition<ClassType> type =
createClassTypeDef(randomString(), ImmutableSet.<String>of(),
new AttributeDefinition("ref", refType.typeName, Multiplicity.OPTIONAL, true, null));
try {
metadataService.createType(TypesSerialization.toJson(refType, false));
metadataService.createType(TypesSerialization.toJson(type, false));
String refId1 = createBasicEntity(refType);
String refId2 = createBasicEntity(refType);
// Add referenced entity for edge creation
final Referenceable instance1 = new Referenceable(type.typeName);
instance1.set("ref", new Referenceable(refId1, refType.typeName, null));
// Add referenced entity for edge creation
final Referenceable instance2 = new Referenceable(type.typeName);
instance2.set("ref", new Referenceable(refId2, refType.typeName, null));
ExecutorService executor = Executors.newFixedThreadPool(3);
List<Future<Object>> futures = new ArrayList<>();
// Try parallel creation of both the entities
futures.add(executor.submit(new Callable<Object>() {
@Override
public Object call() throws Exception {
return createEntity(instance1);
}
}));
futures.add(executor.submit(new Callable<Object>() {
@Override
public Object call() throws Exception {
return createEntity(instance2);
}
}));
futures.add(executor.submit(new Callable<Object>() {
@Override
public Object call() throws Exception {
return discoveryService.searchByDSL(TestUtils.TABLE_TYPE, new QueryParams(10, 0));
}
}));
try {
String id1 = (String) futures.get(0).get();
String id2 = (String) futures.get(1).get();
futures.get(2).get();
executor.shutdown();
assertNotNull(id1);
assertNotNull(id2);
boolean validated1 = assertEdge(id1, type.typeName);
boolean validated2 = assertEdge(id2, type.typeName);
assertTrue(validated1 && validated2);
} catch (InterruptedException | ExecutionException e) {
fail("Parallel entity creation failed", e);
}
} catch (AtlasException e) {
fail("Type/Entity creation failed", e);
}
}
private String createBasicEntity(final HierarchicalTypeDefinition<ClassType> refType) throws AtlasException {
String json = InstanceSerialization.toJson(new Referenceable(refType.typeName), false);
CreateUpdateEntitiesResult entities = metadataService.createEntities("[" + json + "]");
return entities.getCreatedEntities().get(0);
}
private String createEntity(final Referenceable referenceable) throws AtlasException {
String json = InstanceSerialization.toJson(referenceable, false);
CreateUpdateEntitiesResult entities = metadataService.createEntities("[" + json + "]");
return entities.getCreatedEntities().get(0);
}
private boolean assertEdge(String id, String typeName) throws AtlasException {
AtlasGraph graph = TestUtils.getGraph();
Iterable<AtlasVertex> vertices = graph.query().has(Constants.GUID_PROPERTY_KEY, id).vertices();
AtlasVertex AtlasVertex = vertices.iterator().next();
Iterable<AtlasEdge> edges = AtlasVertex.getEdges(AtlasEdgeDirection.OUT, Constants.INTERNAL_PROPERTY_KEY_PREFIX + typeName + ".ref");
if (edges.iterator().hasNext()) {
ITypedReferenceableInstance entity = metadataService.getEntityDefinition(id);
assertNotNull(entity.get("ref"));
return true;
}
return false;
}
private static class EntitiesChangeListener implements EntityChangeListener {
private List<String> deletedEntities = new ArrayList<>();
private List<String> updatedEntities = new ArrayList<>();
......
repository/src/test/scala/org/apache/atlas/query/QueryTestsUtils.scala
......@@ -36,6 +36,8 @@ import org.apache.atlas.repository.MetadataRepository
import org.apache.atlas.repository.graphdb.AtlasGraph
import org.apache.atlas.repository.graph.AtlasGraphProvider
import java.net.URL
import org.apache.atlas.`type`.AtlasTypeRegistry
import org.apache.atlas.repository.graph.GraphBackedSearchIndexer
import org.apache.atlas.typesystem.TypesDef
import org.apache.atlas.typesystem.ITypedReferenceableInstance
......@@ -74,8 +76,7 @@ trait GraphUtils {
object QueryTestsUtils extends GraphUtils {
def setupTypesAndIndices() : Unit = {
// FIXME: Do we need to init the AtlasTypeRegistry here ?
val indexer = new GraphBackedSearchIndexer(null);
val indexer = new GraphBackedSearchIndexer(new AtlasTypeRegistry());
val typesDef : TypesDef = defineTypes;
val newTypes = TypeSystem.getInstance.defineTypes(typesDef);
indexer.onAdd(newTypes.values());
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment