Commit 0b85d5a0 by Suma Shivaprasad

ATLAS-1240 Adding Change listeners to react on changes in TypesDef (apoorvnaik via sumasai)

parent 9dc4cfbc
...@@ -55,6 +55,11 @@ ...@@ -55,6 +55,11 @@
<version>${codehaus.jackson.version}</version> <version>${codehaus.jackson.version}</version>
</dependency> </dependency>
<dependency>
<groupId>javax.inject</groupId>
<artifactId>javax.inject</artifactId>
<version>${javax-inject.version}</version>
</dependency>
<dependency> <dependency>
<groupId>org.testng</groupId> <groupId>org.testng</groupId>
......
...@@ -51,7 +51,10 @@ public enum AtlasErrorCode { ...@@ -51,7 +51,10 @@ public enum AtlasErrorCode {
TYPE_HAS_REFERENCES(409, "ATLAS4092E", "Given type {0} has references"), TYPE_HAS_REFERENCES(409, "ATLAS4092E", "Given type {0} has references"),
TYPE_MATCH_FAILED(409, "ATLAS4093E", "Given type {0} doesn't match {1}"), TYPE_MATCH_FAILED(409, "ATLAS4093E", "Given type {0} doesn't match {1}"),
INTERNAL_ERROR(500, "ATLAS5001E", "Internal server error {0}"); INTERNAL_ERROR(500, "ATLAS5001E", "Internal server error {0}"),
INDEX_CREATION_FAILED(500, "ATLAS5002E", "Index creation failed for {0}"),
INDEX_ROLLBACK_FAILED(500, "ATLAS5003E", "Index rollback failed for {0}")
;
private String errorCode; private String errorCode;
private String errorMessage; private String errorMessage;
......
...@@ -42,6 +42,11 @@ public class AtlasBaseException extends Exception { ...@@ -42,6 +42,11 @@ public class AtlasBaseException extends Exception {
this.atlasErrorCode = AtlasErrorCode.INTERNAL_ERROR; this.atlasErrorCode = AtlasErrorCode.INTERNAL_ERROR;
} }
public AtlasBaseException(AtlasErrorCode errorCode, Throwable cause, String... params) {
super(errorCode.getFormattedErrorMessage(params), cause);
this.atlasErrorCode = errorCode;
}
public AtlasBaseException(String message, Throwable cause) { public AtlasBaseException(String message, Throwable cause) {
super(message, cause); super(message, cause);
this.atlasErrorCode = AtlasErrorCode.INTERNAL_ERROR; this.atlasErrorCode = AtlasErrorCode.INTERNAL_ERROR;
...@@ -52,7 +57,14 @@ public class AtlasBaseException extends Exception { ...@@ -52,7 +57,14 @@ public class AtlasBaseException extends Exception {
this.atlasErrorCode = AtlasErrorCode.INTERNAL_ERROR; this.atlasErrorCode = AtlasErrorCode.INTERNAL_ERROR;
} }
public AtlasBaseException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) { public AtlasBaseException(AtlasErrorCode errorCode, Throwable cause, boolean enableSuppression,
boolean writableStackTrace, String ... params) {
super(errorCode.getFormattedErrorMessage(params), cause, enableSuppression, writableStackTrace);
this.atlasErrorCode = AtlasErrorCode.INTERNAL_ERROR;
}
public AtlasBaseException(String message, Throwable cause, boolean enableSuppression,
boolean writableStackTrace) {
super(message, cause, enableSuppression, writableStackTrace); super(message, cause, enableSuppression, writableStackTrace);
this.atlasErrorCode = AtlasErrorCode.INTERNAL_ERROR; this.atlasErrorCode = AtlasErrorCode.INTERNAL_ERROR;
} }
......
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.listener;
import org.apache.atlas.model.typedef.AtlasBaseTypeDef;
import java.util.ArrayList;
import java.util.List;
public class ChangedTypeDefs {
private List<? extends AtlasBaseTypeDef> createTypeDefs;
private List<? extends AtlasBaseTypeDef> updatedTypeDefs;
private List<? extends AtlasBaseTypeDef> deletedTypeDefs;
public ChangedTypeDefs(List<? extends AtlasBaseTypeDef> createTypeDefs,
List<? extends AtlasBaseTypeDef> updatedTypeDefs,
List<? extends AtlasBaseTypeDef> deletedTypeDefs) {
this.createTypeDefs = createTypeDefs;
this.updatedTypeDefs = updatedTypeDefs;
this.deletedTypeDefs = deletedTypeDefs;
}
public ChangedTypeDefs() {
createTypeDefs = new ArrayList<>();
updatedTypeDefs = new ArrayList<>();
deletedTypeDefs = new ArrayList<>();
}
public List<? extends AtlasBaseTypeDef> getCreateTypeDefs() {
return createTypeDefs;
}
public ChangedTypeDefs setCreateTypeDefs(List<? extends AtlasBaseTypeDef> createTypeDefs) {
this.createTypeDefs = createTypeDefs;
return this;
}
public List<? extends AtlasBaseTypeDef> getUpdatedTypeDefs() {
return updatedTypeDefs;
}
public ChangedTypeDefs setUpdatedTypeDefs(List<? extends AtlasBaseTypeDef> updatedTypeDefs) {
this.updatedTypeDefs = updatedTypeDefs;
return this;
}
public List<? extends AtlasBaseTypeDef> getDeletedTypeDefs() {
return deletedTypeDefs;
}
public ChangedTypeDefs setDeletedTypeDefs(List<? extends AtlasBaseTypeDef> deletedTypeDefs) {
this.deletedTypeDefs = deletedTypeDefs;
return this;
}
}
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.listener;
import org.apache.atlas.exception.AtlasBaseException;
public interface TypeDefChangeListener {
void onChange(ChangedTypeDefs changedTypeDefs) throws AtlasBaseException;
}
...@@ -18,17 +18,19 @@ ...@@ -18,17 +18,19 @@
package org.apache.atlas.type; package org.apache.atlas.type;
import org.apache.atlas.model.typedef.AtlasBaseTypeDef; import org.apache.atlas.model.typedef.AtlasBaseTypeDef;
import static org.apache.atlas.model.typedef.AtlasBaseTypeDef.ATLAS_TYPE_ARRAY_PREFIX;
import static org.apache.atlas.model.typedef.AtlasBaseTypeDef.ATLAS_TYPE_ARRAY_SUFFIX;
import static org.apache.atlas.model.typedef.AtlasBaseTypeDef.ATLAS_TYPE_MAP_PREFIX;
import static org.apache.atlas.model.typedef.AtlasBaseTypeDef.ATLAS_TYPE_MAP_SUFFIX;
import static org.apache.atlas.model.typedef.AtlasBaseTypeDef.ATLAS_TYPE_MAP_KEY_VAL_SEP;
import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.StringUtils;
import java.util.Collections;
import java.util.HashSet; import java.util.HashSet;
import java.util.Map; import java.util.Map;
import java.util.Set; import java.util.Set;
import static org.apache.atlas.model.typedef.AtlasBaseTypeDef.ATLAS_TYPE_ARRAY_PREFIX;
import static org.apache.atlas.model.typedef.AtlasBaseTypeDef.ATLAS_TYPE_ARRAY_SUFFIX;
import static org.apache.atlas.model.typedef.AtlasBaseTypeDef.ATLAS_TYPE_MAP_KEY_VAL_SEP;
import static org.apache.atlas.model.typedef.AtlasBaseTypeDef.ATLAS_TYPE_MAP_PREFIX;
import static org.apache.atlas.model.typedef.AtlasBaseTypeDef.ATLAS_TYPE_MAP_SUFFIX;
/** /**
* Utility methods for AtlasType/AtlasTypeDef. * Utility methods for AtlasType/AtlasTypeDef.
*/ */
...@@ -36,9 +38,7 @@ public class AtlasTypeUtil { ...@@ -36,9 +38,7 @@ public class AtlasTypeUtil {
private static final Set<String> ATLAS_BUILTIN_TYPENAMES = new HashSet<String>(); private static final Set<String> ATLAS_BUILTIN_TYPENAMES = new HashSet<String>();
static { static {
for (String typeName : AtlasBaseTypeDef.ATLAS_BUILTIN_TYPES) { Collections.addAll(ATLAS_BUILTIN_TYPENAMES, AtlasBaseTypeDef.ATLAS_BUILTIN_TYPES);
ATLAS_BUILTIN_TYPENAMES.add(typeName);
}
} }
public static Set<String> getReferencedTypeNames(String typeName) { public static Set<String> getReferencedTypeNames(String typeName) {
...@@ -63,6 +63,7 @@ public class AtlasTypeUtil { ...@@ -63,6 +63,7 @@ public class AtlasTypeUtil {
&& StringUtils.endsWith(typeName, ATLAS_TYPE_MAP_SUFFIX); && StringUtils.endsWith(typeName, ATLAS_TYPE_MAP_SUFFIX);
} }
public static String getStringValue(Map map, Object key) { public static String getStringValue(Map map, Object key) {
Object ret = map != null ? map.get(key) : null; Object ret = map != null ? map.get(key) : null;
...@@ -90,6 +91,5 @@ public class AtlasTypeUtil { ...@@ -90,6 +91,5 @@ public class AtlasTypeUtil {
referencedTypeNames.add(typeName); referencedTypeNames.add(typeName);
} }
} }
} }
} }
...@@ -17,17 +17,16 @@ ...@@ -17,17 +17,16 @@
*/ */
package org.apache.atlas.model.typedef; package org.apache.atlas.model.typedef;
import java.util.List;
import org.apache.atlas.model.ModelTestUtil; import org.apache.atlas.model.ModelTestUtil;
import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef; import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef;
import org.apache.atlas.type.AtlasType; import org.apache.atlas.type.AtlasType;
import org.testng.annotations.Test; import org.testng.annotations.Test;
import java.util.List;
import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertTrue;
import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertFalse;
import static org.testng.Assert.assertTrue;
public class TestAtlasStructDef { public class TestAtlasStructDef {
......
...@@ -9,6 +9,7 @@ ATLAS-1060 Add composite indexes for exact match performance improvements for al ...@@ -9,6 +9,7 @@ ATLAS-1060 Add composite indexes for exact match performance improvements for al
ATLAS-1127 Modify creation and modification timestamps to Date instead of Long(sumasai) ATLAS-1127 Modify creation and modification timestamps to Date instead of Long(sumasai)
ALL CHANGES: ALL CHANGES:
ATLAS-1240 Adding Change listeners to react on changes in TypesDef (apoorvnaik via sumasai)
ATLAS-1239 when stopping Atlas on the command line it should explicitly say when it has stopped (ayubkhan via sumasai) ATLAS-1239 when stopping Atlas on the command line it should explicitly say when it has stopped (ayubkhan via sumasai)
ATLAS-1253 Extract error codes into AtlasErrorCode Enum (apoorvnaik via sumasai) ATLAS-1253 Extract error codes into AtlasErrorCode Enum (apoorvnaik via sumasai)
ATLAS-1195 Clean up DSL Translation (jnhagelb via dkantor) ATLAS-1195 Clean up DSL Translation (jnhagelb via dkantor)
......
...@@ -29,6 +29,7 @@ import org.apache.atlas.discovery.DiscoveryService; ...@@ -29,6 +29,7 @@ import org.apache.atlas.discovery.DiscoveryService;
import org.apache.atlas.discovery.LineageService; import org.apache.atlas.discovery.LineageService;
import org.apache.atlas.discovery.graph.GraphBackedDiscoveryService; import org.apache.atlas.discovery.graph.GraphBackedDiscoveryService;
import org.apache.atlas.listener.EntityChangeListener; import org.apache.atlas.listener.EntityChangeListener;
import org.apache.atlas.listener.TypeDefChangeListener;
import org.apache.atlas.listener.TypesChangeListener; import org.apache.atlas.listener.TypesChangeListener;
import org.apache.atlas.repository.MetadataRepository; import org.apache.atlas.repository.MetadataRepository;
import org.apache.atlas.repository.audit.EntityAuditListener; import org.apache.atlas.repository.audit.EntityAuditListener;
...@@ -76,6 +77,12 @@ public class RepositoryMetadataModule extends com.google.inject.AbstractModule { ...@@ -76,6 +77,12 @@ public class RepositoryMetadataModule extends com.google.inject.AbstractModule {
Multibinder.newSetBinder(binder(), TypesChangeListener.class); Multibinder.newSetBinder(binder(), TypesChangeListener.class);
typesChangeListenerBinder.addBinding().to(GraphBackedSearchIndexer.class).asEagerSingleton(); typesChangeListenerBinder.addBinding().to(GraphBackedSearchIndexer.class).asEagerSingleton();
// New typesdef/instance change listener should also be bound to the corresponding implementation
Multibinder<TypeDefChangeListener> typeDefChangeListenerMultibinder =
Multibinder.newSetBinder(binder(), TypeDefChangeListener.class);
typeDefChangeListenerMultibinder.addBinding().to(DefaultMetadataService.class);
typeDefChangeListenerMultibinder.addBinding().to(GraphBackedSearchIndexer.class).asEagerSingleton();
// bind the MetadataService interface to an implementation // bind the MetadataService interface to an implementation
bind(MetadataService.class).to(DefaultMetadataService.class).asEagerSingleton(); bind(MetadataService.class).to(DefaultMetadataService.class).asEagerSingleton();
......
...@@ -22,6 +22,7 @@ import com.google.inject.Inject; ...@@ -22,6 +22,7 @@ import com.google.inject.Inject;
import org.apache.atlas.AtlasErrorCode; import org.apache.atlas.AtlasErrorCode;
import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.listener.TypeDefChangeListener;
import org.apache.atlas.model.typedef.AtlasBaseTypeDef; import org.apache.atlas.model.typedef.AtlasBaseTypeDef;
import org.apache.atlas.model.typedef.AtlasClassificationDef; import org.apache.atlas.model.typedef.AtlasClassificationDef;
import org.apache.atlas.model.typedef.AtlasEntityDef; import org.apache.atlas.model.typedef.AtlasEntityDef;
...@@ -65,8 +66,9 @@ public class AtlasTypeDefGraphStoreV1 extends AtlasTypeDefGraphStore { ...@@ -65,8 +66,9 @@ public class AtlasTypeDefGraphStoreV1 extends AtlasTypeDefGraphStore {
protected final AtlasGraph atlasGraph = AtlasGraphProvider.getGraphInstance(); protected final AtlasGraph atlasGraph = AtlasGraphProvider.getGraphInstance();
@Inject @Inject
public AtlasTypeDefGraphStoreV1(AtlasTypeRegistry typeRegistry) { public AtlasTypeDefGraphStoreV1(AtlasTypeRegistry typeRegistry,
super(typeRegistry); Set<TypeDefChangeListener> typeDefChangeListeners) {
super(typeRegistry, typeDefChangeListeners);
LOG.info("==> AtlasTypeDefGraphStoreV1()"); LOG.info("==> AtlasTypeDefGraphStoreV1()");
......
...@@ -18,29 +18,25 @@ ...@@ -18,29 +18,25 @@
package org.apache.atlas.services; package org.apache.atlas.services;
import static org.apache.atlas.AtlasClient.PROCESS_ATTRIBUTE_INPUTS; import com.google.common.base.Preconditions;
import static org.apache.atlas.AtlasClient.PROCESS_ATTRIBUTE_OUTPUTS; import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import java.util.ArrayList; import com.google.inject.Provider;
import java.util.Collection;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import javax.inject.Inject;
import javax.inject.Singleton;
import org.apache.atlas.ApplicationProperties; import org.apache.atlas.ApplicationProperties;
import org.apache.atlas.AtlasClient; import org.apache.atlas.AtlasClient;
import org.apache.atlas.AtlasErrorCode;
import org.apache.atlas.AtlasException; import org.apache.atlas.AtlasException;
import org.apache.atlas.EntityAuditEvent; import org.apache.atlas.EntityAuditEvent;
import org.apache.atlas.RequestContext; import org.apache.atlas.RequestContext;
import org.apache.atlas.classification.InterfaceAudience; import org.apache.atlas.classification.InterfaceAudience;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.ha.HAConfiguration; import org.apache.atlas.ha.HAConfiguration;
import org.apache.atlas.listener.ActiveStateChangeHandler; import org.apache.atlas.listener.ActiveStateChangeHandler;
import org.apache.atlas.listener.ChangedTypeDefs;
import org.apache.atlas.listener.EntityChangeListener; import org.apache.atlas.listener.EntityChangeListener;
import org.apache.atlas.listener.TypeDefChangeListener;
import org.apache.atlas.listener.TypesChangeListener; import org.apache.atlas.listener.TypesChangeListener;
import org.apache.atlas.query.QueryParser;
import org.apache.atlas.repository.MetadataRepository; import org.apache.atlas.repository.MetadataRepository;
import org.apache.atlas.repository.RepositoryException; import org.apache.atlas.repository.RepositoryException;
import org.apache.atlas.repository.audit.EntityAuditRepository; import org.apache.atlas.repository.audit.EntityAuditRepository;
...@@ -72,17 +68,23 @@ import org.apache.atlas.typesystem.types.TypeSystem; ...@@ -72,17 +68,23 @@ import org.apache.atlas.typesystem.types.TypeSystem;
import org.apache.atlas.typesystem.types.cache.TypeCache; import org.apache.atlas.typesystem.types.cache.TypeCache;
import org.apache.atlas.typesystem.types.utils.TypesUtil; import org.apache.atlas.typesystem.types.utils.TypesUtil;
import org.apache.atlas.utils.ParamChecker; import org.apache.atlas.utils.ParamChecker;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.configuration.Configuration; import org.apache.commons.configuration.Configuration;
import org.codehaus.jettison.json.JSONException; import org.codehaus.jettison.json.JSONException;
import org.codehaus.jettison.json.JSONObject; import org.codehaus.jettison.json.JSONObject;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import com.google.common.base.Preconditions; import java.util.ArrayList;
import com.google.common.collect.ImmutableList; import java.util.Collection;
import com.google.common.collect.ImmutableSet; import java.util.LinkedHashSet;
import com.google.inject.Provider; import java.util.List;
import java.util.Map;
import javax.inject.Inject;
import javax.inject.Singleton;
import static org.apache.atlas.AtlasClient.PROCESS_ATTRIBUTE_INPUTS;
import static org.apache.atlas.AtlasClient.PROCESS_ATTRIBUTE_OUTPUTS;
...@@ -91,7 +93,7 @@ import com.google.inject.Provider; ...@@ -91,7 +93,7 @@ import com.google.inject.Provider;
* for listening to changes to the repository. * for listening to changes to the repository.
*/ */
@Singleton @Singleton
public class DefaultMetadataService implements MetadataService, ActiveStateChangeHandler { public class DefaultMetadataService implements MetadataService, ActiveStateChangeHandler, TypeDefChangeListener {
private static final Logger LOG = LoggerFactory.getLogger(DefaultMetadataService.class); private static final Logger LOG = LoggerFactory.getLogger(DefaultMetadataService.class);
private final short maxAuditResults; private final short maxAuditResults;
...@@ -774,4 +776,22 @@ public class DefaultMetadataService implements MetadataService, ActiveStateChang ...@@ -774,4 +776,22 @@ public class DefaultMetadataService implements MetadataService, ActiveStateChang
public void instanceIsPassive() { public void instanceIsPassive() {
LOG.info("Reacting to passive state: no action right now"); LOG.info("Reacting to passive state: no action right now");
} }
@Override
public void onChange(ChangedTypeDefs changedTypeDefs) throws AtlasBaseException {
// All we need here is a restore of the type-system
LOG.info("TypeSystem reset invoked by TypeRegistry changes");
try {
TypesDef typesDef = typeStore.restore();
typeSystem.reset();
TypeSystem.TransientTypeSystem transientTypeSystem
= typeSystem.createTransientTypeSystem(typesDef, false);
Map<String, IDataType> typesAdded = transientTypeSystem.getTypesAdded();
LOG.info("Number of types got from transient type system: " + typesAdded.size());
typeSystem.commitTypes(typesAdded);
} catch (AtlasException e) {
LOG.error("Failed to restore type-system after TypeRegistry changes", e);
throw new AtlasBaseException(AtlasErrorCode.INTERNAL_ERROR, e);
}
}
} }
...@@ -20,10 +20,12 @@ package org.apache.atlas; ...@@ -20,10 +20,12 @@ package org.apache.atlas;
import com.google.common.base.Preconditions; import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet; import com.google.common.collect.ImmutableSet;
import org.apache.atlas.repository.MetadataRepository; import org.apache.atlas.repository.MetadataRepository;
import org.apache.atlas.repository.graph.AtlasGraphProvider; import org.apache.atlas.repository.graph.AtlasGraphProvider;
import org.apache.atlas.repository.graph.GraphBackedSearchIndexer; import org.apache.atlas.repository.graph.GraphBackedSearchIndexer;
import org.apache.atlas.services.MetadataService; import org.apache.atlas.services.MetadataService;
import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.atlas.typesystem.ITypedReferenceableInstance; import org.apache.atlas.typesystem.ITypedReferenceableInstance;
import org.apache.atlas.typesystem.Referenceable; import org.apache.atlas.typesystem.Referenceable;
import org.apache.atlas.typesystem.TypesDef; import org.apache.atlas.typesystem.TypesDef;
...@@ -42,11 +44,12 @@ import org.apache.atlas.typesystem.types.TypeSystem; ...@@ -42,11 +44,12 @@ import org.apache.atlas.typesystem.types.TypeSystem;
import org.apache.atlas.typesystem.types.utils.TypesUtil; import org.apache.atlas.typesystem.types.utils.TypesUtil;
import org.testng.annotations.Guice; import org.testng.annotations.Guice;
import javax.inject.Inject;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Date; import java.util.Date;
import java.util.List; import java.util.List;
import javax.inject.Inject;
/** /**
* Base Class to set up hive types and instances for tests * Base Class to set up hive types and instances for tests
*/ */
...@@ -65,7 +68,7 @@ public class BaseRepositoryTest { ...@@ -65,7 +68,7 @@ public class BaseRepositoryTest {
//force graph initialization / built in type registration //force graph initialization / built in type registration
TestUtils.getGraph(); TestUtils.getGraph();
setUpTypes(); setUpTypes();
new GraphBackedSearchIndexer(); new GraphBackedSearchIndexer(new AtlasTypeRegistry());
TestUtils.resetRequestContext(); TestUtils.resetRequestContext();
setupInstances(); setupInstances();
TestUtils.dumpGraph(TestUtils.getGraph()); TestUtils.dumpGraph(TestUtils.getGraph());
......
...@@ -18,23 +18,9 @@ ...@@ -18,23 +18,9 @@
package org.apache.atlas; package org.apache.atlas;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createClassTypeDef; import com.google.common.collect.ImmutableList;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createOptionalAttrDef; import com.google.common.collect.ImmutableSet;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createRequiredAttrDef; import com.google.inject.Provider;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createStructTypeDef;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createTraitTypeDef;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createUniqueRequiredAttrDef;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Date;
import java.util.List;
import org.apache.atlas.listener.EntityChangeListener; import org.apache.atlas.listener.EntityChangeListener;
import org.apache.atlas.listener.TypesChangeListener; import org.apache.atlas.listener.TypesChangeListener;
...@@ -48,6 +34,7 @@ import org.apache.atlas.repository.typestore.ITypeStore; ...@@ -48,6 +34,7 @@ import org.apache.atlas.repository.typestore.ITypeStore;
import org.apache.atlas.services.DefaultMetadataService; import org.apache.atlas.services.DefaultMetadataService;
import org.apache.atlas.services.MetadataService; import org.apache.atlas.services.MetadataService;
import org.apache.atlas.services.ReservedTypesRegistrar; import org.apache.atlas.services.ReservedTypesRegistrar;
import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.atlas.typesystem.ITypedReferenceableInstance; import org.apache.atlas.typesystem.ITypedReferenceableInstance;
import org.apache.atlas.typesystem.Referenceable; import org.apache.atlas.typesystem.Referenceable;
import org.apache.atlas.typesystem.TypesDef; import org.apache.atlas.typesystem.TypesDef;
...@@ -73,9 +60,23 @@ import org.apache.commons.lang.RandomStringUtils; ...@@ -73,9 +60,23 @@ import org.apache.commons.lang.RandomStringUtils;
import org.codehaus.jettison.json.JSONArray; import org.codehaus.jettison.json.JSONArray;
import org.testng.Assert; import org.testng.Assert;
import com.google.common.collect.ImmutableList; import java.io.File;
import com.google.common.collect.ImmutableSet; import java.io.FileOutputStream;
import com.google.inject.Provider; import java.io.IOException;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Date;
import java.util.List;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createClassTypeDef;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createOptionalAttrDef;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createRequiredAttrDef;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createStructTypeDef;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createTraitTypeDef;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createUniqueRequiredAttrDef;
/** /**
* Test utility class. * Test utility class.
...@@ -513,7 +514,7 @@ public final class TestUtils { ...@@ -513,7 +514,7 @@ public final class TestUtils {
catch(Throwable t) { catch(Throwable t) {
typeCache = new DefaultTypeCache(); typeCache = new DefaultTypeCache();
} }
final GraphBackedSearchIndexer indexer = new GraphBackedSearchIndexer(); final GraphBackedSearchIndexer indexer = new GraphBackedSearchIndexer(new AtlasTypeRegistry());
Provider<TypesChangeListener> indexerProvider = new Provider<TypesChangeListener>() { Provider<TypesChangeListener> indexerProvider = new Provider<TypesChangeListener>() {
@Override @Override
......
...@@ -18,22 +18,7 @@ ...@@ -18,22 +18,7 @@
package org.apache.atlas.discovery; package org.apache.atlas.discovery;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createClassTypeDef; import com.google.common.collect.ImmutableSet;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createOptionalAttrDef;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createRequiredAttrDef;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertNotNull;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import javax.inject.Inject;
import org.apache.atlas.AtlasException; import org.apache.atlas.AtlasException;
import org.apache.atlas.BaseRepositoryTest; import org.apache.atlas.BaseRepositoryTest;
...@@ -46,6 +31,7 @@ import org.apache.atlas.repository.Constants; ...@@ -46,6 +31,7 @@ import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.MetadataRepository; import org.apache.atlas.repository.MetadataRepository;
import org.apache.atlas.repository.graph.AtlasGraphProvider; import org.apache.atlas.repository.graph.AtlasGraphProvider;
import org.apache.atlas.repository.graph.GraphBackedSearchIndexer; import org.apache.atlas.repository.graph.GraphBackedSearchIndexer;
import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.atlas.typesystem.ITypedReferenceableInstance; import org.apache.atlas.typesystem.ITypedReferenceableInstance;
import org.apache.atlas.typesystem.Referenceable; import org.apache.atlas.typesystem.Referenceable;
import org.apache.atlas.typesystem.persistence.Id; import org.apache.atlas.typesystem.persistence.Id;
...@@ -65,7 +51,22 @@ import org.testng.annotations.DataProvider; ...@@ -65,7 +51,22 @@ import org.testng.annotations.DataProvider;
import org.testng.annotations.Guice; import org.testng.annotations.Guice;
import org.testng.annotations.Test; import org.testng.annotations.Test;
import com.google.common.collect.ImmutableSet; import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import javax.inject.Inject;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createClassTypeDef;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createOptionalAttrDef;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createRequiredAttrDef;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertNotNull;
@Guice(modules = RepositoryMetadataModule.class) @Guice(modules = RepositoryMetadataModule.class)
public class GraphBackedDiscoveryServiceTest extends BaseRepositoryTest { public class GraphBackedDiscoveryServiceTest extends BaseRepositoryTest {
...@@ -119,7 +120,7 @@ public class GraphBackedDiscoveryServiceTest extends BaseRepositoryTest { ...@@ -119,7 +120,7 @@ public class GraphBackedDiscoveryServiceTest extends BaseRepositoryTest {
//We need to commit the transaction before creating the indices to release the locks held by the transaction. //We need to commit the transaction before creating the indices to release the locks held by the transaction.
//otherwise, the index commit will fail while waiting for the those locks to be released. //otherwise, the index commit will fail while waiting for the those locks to be released.
AtlasGraphProvider.getGraphInstance().commit(); AtlasGraphProvider.getGraphInstance().commit();
GraphBackedSearchIndexer idx = new GraphBackedSearchIndexer(); GraphBackedSearchIndexer idx = new GraphBackedSearchIndexer(new AtlasTypeRegistry());
idx.onAdd(newTypes); idx.onAdd(newTypes);
} }
......
...@@ -18,30 +18,8 @@ ...@@ -18,30 +18,8 @@
package org.apache.atlas.repository.graph; package org.apache.atlas.repository.graph;
import static org.apache.atlas.TestUtils.COLUMNS_ATTR_NAME; import com.google.common.collect.ImmutableList;
import static org.apache.atlas.TestUtils.COLUMN_TYPE; import com.google.common.collect.ImmutableSet;
import static org.apache.atlas.TestUtils.NAME;
import static org.apache.atlas.TestUtils.PII;
import static org.apache.atlas.TestUtils.PROCESS_TYPE;
import static org.apache.atlas.TestUtils.TABLE_TYPE;
import static org.apache.atlas.TestUtils.createColumnEntity;
import static org.apache.atlas.TestUtils.createDBEntity;
import static org.apache.atlas.TestUtils.createTableEntity;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertNotEquals;
import static org.testng.Assert.assertTrue;
import static org.testng.Assert.fail;
import java.lang.reflect.InvocationHandler;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.lang.reflect.Proxy;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.atlas.AtlasClient; import org.apache.atlas.AtlasClient;
import org.apache.atlas.AtlasClient.EntityResult; import org.apache.atlas.AtlasClient.EntityResult;
...@@ -55,6 +33,7 @@ import org.apache.atlas.repository.MetadataRepository; ...@@ -55,6 +33,7 @@ import org.apache.atlas.repository.MetadataRepository;
import org.apache.atlas.repository.RepositoryException; import org.apache.atlas.repository.RepositoryException;
import org.apache.atlas.repository.graphdb.AtlasGraph; import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.repository.graphdb.AtlasVertex; import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.atlas.typesystem.IReferenceableInstance; import org.apache.atlas.typesystem.IReferenceableInstance;
import org.apache.atlas.typesystem.IStruct; import org.apache.atlas.typesystem.IStruct;
import org.apache.atlas.typesystem.ITypedReferenceableInstance; import org.apache.atlas.typesystem.ITypedReferenceableInstance;
...@@ -84,8 +63,30 @@ import org.testng.annotations.BeforeMethod; ...@@ -84,8 +63,30 @@ import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Guice; import org.testng.annotations.Guice;
import org.testng.annotations.Test; import org.testng.annotations.Test;
import com.google.common.collect.ImmutableList; import java.lang.reflect.InvocationHandler;
import com.google.common.collect.ImmutableSet; import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.lang.reflect.Proxy;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import static org.apache.atlas.TestUtils.COLUMNS_ATTR_NAME;
import static org.apache.atlas.TestUtils.COLUMN_TYPE;
import static org.apache.atlas.TestUtils.NAME;
import static org.apache.atlas.TestUtils.PII;
import static org.apache.atlas.TestUtils.PROCESS_TYPE;
import static org.apache.atlas.TestUtils.TABLE_TYPE;
import static org.apache.atlas.TestUtils.createColumnEntity;
import static org.apache.atlas.TestUtils.createDBEntity;
import static org.apache.atlas.TestUtils.createTableEntity;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertNotEquals;
import static org.testng.Assert.assertTrue;
import static org.testng.Assert.fail;
/** /**
* Test for GraphBackedMetadataRepository.deleteEntities * Test for GraphBackedMetadataRepository.deleteEntities
...@@ -110,7 +111,7 @@ public abstract class GraphBackedMetadataRepositoryDeleteTestBase { ...@@ -110,7 +111,7 @@ public abstract class GraphBackedMetadataRepositoryDeleteTestBase {
typeSystem = TypeSystem.getInstance(); typeSystem = TypeSystem.getInstance();
typeSystem.reset(); typeSystem.reset();
new GraphBackedSearchIndexer(); new GraphBackedSearchIndexer(new AtlasTypeRegistry());
final GraphBackedMetadataRepository delegate = new GraphBackedMetadataRepository(getDeleteHandler(typeSystem)); final GraphBackedMetadataRepository delegate = new GraphBackedMetadataRepository(getDeleteHandler(typeSystem));
repositoryService = (MetadataRepository)Proxy.newProxyInstance(Thread.currentThread().getContextClassLoader(), repositoryService = (MetadataRepository)Proxy.newProxyInstance(Thread.currentThread().getContextClassLoader(),
......
...@@ -18,25 +18,8 @@ ...@@ -18,25 +18,8 @@
package org.apache.atlas.repository.graph; package org.apache.atlas.repository.graph;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createClassTypeDef; import com.google.common.collect.ImmutableList;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createUniqueRequiredAttrDef; import com.google.common.collect.ImmutableSet;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertNotEquals;
import static org.testng.Assert.assertNotNull;
import static org.testng.Assert.assertTrue;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.UUID;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import javax.inject.Inject;
import org.apache.atlas.GraphTransaction; import org.apache.atlas.GraphTransaction;
import org.apache.atlas.RepositoryMetadataModule; import org.apache.atlas.RepositoryMetadataModule;
...@@ -52,6 +35,7 @@ import org.apache.atlas.repository.graphdb.AtlasGraph; ...@@ -52,6 +35,7 @@ import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.repository.graphdb.AtlasGraphQuery; import org.apache.atlas.repository.graphdb.AtlasGraphQuery;
import org.apache.atlas.repository.graphdb.AtlasGraphQuery.ComparisionOperator; import org.apache.atlas.repository.graphdb.AtlasGraphQuery.ComparisionOperator;
import org.apache.atlas.repository.graphdb.AtlasVertex; import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.atlas.typesystem.IStruct; import org.apache.atlas.typesystem.IStruct;
import org.apache.atlas.typesystem.ITypedReferenceableInstance; import org.apache.atlas.typesystem.ITypedReferenceableInstance;
import org.apache.atlas.typesystem.ITypedStruct; import org.apache.atlas.typesystem.ITypedStruct;
...@@ -78,11 +62,28 @@ import org.testng.annotations.BeforeMethod; ...@@ -78,11 +62,28 @@ import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Guice; import org.testng.annotations.Guice;
import org.testng.annotations.Test; import org.testng.annotations.Test;
import com.google.common.collect.ImmutableList; import java.util.ArrayList;
import com.google.common.collect.ImmutableSet; import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.UUID;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import javax.inject.Inject;
import scala.actors.threadpool.Arrays; import scala.actors.threadpool.Arrays;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createClassTypeDef;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createUniqueRequiredAttrDef;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertNotEquals;
import static org.testng.Assert.assertNotNull;
import static org.testng.Assert.assertTrue;
/** /**
* GraphBackedMetadataRepository test * GraphBackedMetadataRepository test
* *
...@@ -107,7 +108,7 @@ public class GraphBackedMetadataRepositoryTest { ...@@ -107,7 +108,7 @@ public class GraphBackedMetadataRepositoryTest {
typeSystem = TypeSystem.getInstance(); typeSystem = TypeSystem.getInstance();
typeSystem.reset(); typeSystem.reset();
new GraphBackedSearchIndexer(); new GraphBackedSearchIndexer(new AtlasTypeRegistry());
TestUtils.defineDeptEmployeeTypes(typeSystem); TestUtils.defineDeptEmployeeTypes(typeSystem);
TestUtils.createHiveTypes(typeSystem); TestUtils.createHiveTypes(typeSystem);
......
...@@ -18,10 +18,6 @@ ...@@ -18,10 +18,6 @@
package org.apache.atlas.repository.graph; package org.apache.atlas.repository.graph;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.verifyZeroInteractions;
import static org.mockito.Mockito.when;
import org.apache.atlas.AtlasException; import org.apache.atlas.AtlasException;
import org.apache.atlas.ha.HAConfiguration; import org.apache.atlas.ha.HAConfiguration;
import org.apache.atlas.repository.Constants; import org.apache.atlas.repository.Constants;
...@@ -29,12 +25,17 @@ import org.apache.atlas.repository.IndexException; ...@@ -29,12 +25,17 @@ import org.apache.atlas.repository.IndexException;
import org.apache.atlas.repository.RepositoryException; import org.apache.atlas.repository.RepositoryException;
import org.apache.atlas.repository.graphdb.AtlasGraph; import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.repository.graphdb.AtlasGraphManagement; import org.apache.atlas.repository.graphdb.AtlasGraphManagement;
import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.commons.configuration.Configuration; import org.apache.commons.configuration.Configuration;
import org.mockito.Mock; import org.mockito.Mock;
import org.mockito.MockitoAnnotations; import org.mockito.MockitoAnnotations;
import org.testng.annotations.BeforeMethod; import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test; import org.testng.annotations.Test;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.verifyZeroInteractions;
import static org.mockito.Mockito.when;
public class GraphBackedSearchIndexerMockTest implements IAtlasGraphProvider { public class GraphBackedSearchIndexerMockTest implements IAtlasGraphProvider {
@Mock @Mock
...@@ -46,6 +47,9 @@ public class GraphBackedSearchIndexerMockTest implements IAtlasGraphProvider { ...@@ -46,6 +47,9 @@ public class GraphBackedSearchIndexerMockTest implements IAtlasGraphProvider {
@Mock @Mock
private AtlasGraphManagement management; private AtlasGraphManagement management;
@Mock
private AtlasTypeRegistry typeRegistry;
@BeforeMethod @BeforeMethod
public void setup() { public void setup() {
MockitoAnnotations.initMocks(this); MockitoAnnotations.initMocks(this);
...@@ -57,7 +61,7 @@ public class GraphBackedSearchIndexerMockTest implements IAtlasGraphProvider { ...@@ -57,7 +61,7 @@ public class GraphBackedSearchIndexerMockTest implements IAtlasGraphProvider {
when(graph.getManagementSystem()).thenReturn(management); when(graph.getManagementSystem()).thenReturn(management);
when(management.containsPropertyKey(Constants.VERTEX_TYPE_PROPERTY_KEY)).thenReturn(true); when(management.containsPropertyKey(Constants.VERTEX_TYPE_PROPERTY_KEY)).thenReturn(true);
GraphBackedSearchIndexer graphBackedSearchIndexer = new GraphBackedSearchIndexer(this, configuration); GraphBackedSearchIndexer graphBackedSearchIndexer = new GraphBackedSearchIndexer(this, configuration, typeRegistry);
verify(management).containsPropertyKey(Constants.VERTEX_TYPE_PROPERTY_KEY); verify(management).containsPropertyKey(Constants.VERTEX_TYPE_PROPERTY_KEY);
} }
...@@ -69,7 +73,7 @@ public class GraphBackedSearchIndexerMockTest implements IAtlasGraphProvider { ...@@ -69,7 +73,7 @@ public class GraphBackedSearchIndexerMockTest implements IAtlasGraphProvider {
when(graph.getManagementSystem()).thenReturn(management); when(graph.getManagementSystem()).thenReturn(management);
when(management.containsPropertyKey(Constants.VERTEX_TYPE_PROPERTY_KEY)).thenReturn(true); when(management.containsPropertyKey(Constants.VERTEX_TYPE_PROPERTY_KEY)).thenReturn(true);
new GraphBackedSearchIndexer(this, configuration); new GraphBackedSearchIndexer(this, configuration, typeRegistry);
verifyZeroInteractions(management); verifyZeroInteractions(management);
} }
...@@ -81,7 +85,7 @@ public class GraphBackedSearchIndexerMockTest implements IAtlasGraphProvider { ...@@ -81,7 +85,7 @@ public class GraphBackedSearchIndexerMockTest implements IAtlasGraphProvider {
when(graph.getManagementSystem()).thenReturn(management); when(graph.getManagementSystem()).thenReturn(management);
when(management.containsPropertyKey(Constants.VERTEX_TYPE_PROPERTY_KEY)).thenReturn(true); when(management.containsPropertyKey(Constants.VERTEX_TYPE_PROPERTY_KEY)).thenReturn(true);
GraphBackedSearchIndexer graphBackedSearchIndexer = new GraphBackedSearchIndexer(this, configuration); GraphBackedSearchIndexer graphBackedSearchIndexer = new GraphBackedSearchIndexer(this, configuration, typeRegistry);
graphBackedSearchIndexer.instanceIsActive(); graphBackedSearchIndexer.instanceIsActive();
verify(management).containsPropertyKey(Constants.VERTEX_TYPE_PROPERTY_KEY); verify(management).containsPropertyKey(Constants.VERTEX_TYPE_PROPERTY_KEY);
......
...@@ -18,26 +18,13 @@ ...@@ -18,26 +18,13 @@
package org.apache.atlas.repository.graph; package org.apache.atlas.repository.graph;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertFalse;
import static org.testng.Assert.assertNotNull;
import static org.testng.Assert.assertNull;
import static org.testng.Assert.assertTrue;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
import javax.inject.Inject;
import org.apache.atlas.RepositoryMetadataModule; import org.apache.atlas.RepositoryMetadataModule;
import org.apache.atlas.TestUtils; import org.apache.atlas.TestUtils;
import org.apache.atlas.repository.graph.GraphHelper.VertexInfo; import org.apache.atlas.repository.graph.GraphHelper.VertexInfo;
import org.apache.atlas.repository.graphdb.AtlasEdge; import org.apache.atlas.repository.graphdb.AtlasEdge;
import org.apache.atlas.repository.graphdb.AtlasGraph; import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.repository.graphdb.AtlasVertex; import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.atlas.typesystem.ITypedReferenceableInstance; import org.apache.atlas.typesystem.ITypedReferenceableInstance;
import org.apache.atlas.typesystem.types.TypeSystem; import org.apache.atlas.typesystem.types.TypeSystem;
import org.testng.Assert; import org.testng.Assert;
...@@ -47,6 +34,20 @@ import org.testng.annotations.DataProvider; ...@@ -47,6 +34,20 @@ import org.testng.annotations.DataProvider;
import org.testng.annotations.Guice; import org.testng.annotations.Guice;
import org.testng.annotations.Test; import org.testng.annotations.Test;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
import javax.inject.Inject;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertFalse;
import static org.testng.Assert.assertNotNull;
import static org.testng.Assert.assertNull;
import static org.testng.Assert.assertTrue;
@Guice(modules = RepositoryMetadataModule.class) @Guice(modules = RepositoryMetadataModule.class)
public class GraphHelperTest { public class GraphHelperTest {
...@@ -72,12 +73,15 @@ public class GraphHelperTest { ...@@ -72,12 +73,15 @@ public class GraphHelperTest {
private TypeSystem typeSystem; private TypeSystem typeSystem;
@Inject
private AtlasTypeRegistry typeRegistry;
@BeforeClass @BeforeClass
public void setUp() throws Exception { public void setUp() throws Exception {
typeSystem = TypeSystem.getInstance(); typeSystem = TypeSystem.getInstance();
typeSystem.reset(); typeSystem.reset();
new GraphBackedSearchIndexer(); new GraphBackedSearchIndexer(typeRegistry);
TestUtils.defineDeptEmployeeTypes(typeSystem); TestUtils.defineDeptEmployeeTypes(typeSystem);
} }
......
...@@ -18,13 +18,6 @@ ...@@ -18,13 +18,6 @@
package org.apache.atlas.repository.graph; package org.apache.atlas.repository.graph;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Date;
import java.util.Iterator;
import javax.inject.Inject;
import org.apache.atlas.ApplicationProperties; import org.apache.atlas.ApplicationProperties;
import org.apache.atlas.GraphTransaction; import org.apache.atlas.GraphTransaction;
import org.apache.atlas.RepositoryMetadataModule; import org.apache.atlas.RepositoryMetadataModule;
...@@ -35,6 +28,7 @@ import org.apache.atlas.repository.graphdb.AtlasGraphQuery; ...@@ -35,6 +28,7 @@ import org.apache.atlas.repository.graphdb.AtlasGraphQuery;
import org.apache.atlas.repository.graphdb.AtlasGraphQuery.ComparisionOperator; import org.apache.atlas.repository.graphdb.AtlasGraphQuery.ComparisionOperator;
import org.apache.atlas.repository.graphdb.AtlasIndexQuery; import org.apache.atlas.repository.graphdb.AtlasIndexQuery;
import org.apache.atlas.repository.graphdb.AtlasVertex; import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.atlas.typesystem.ITypedReferenceableInstance; import org.apache.atlas.typesystem.ITypedReferenceableInstance;
import org.apache.atlas.typesystem.Referenceable; import org.apache.atlas.typesystem.Referenceable;
import org.apache.atlas.typesystem.Struct; import org.apache.atlas.typesystem.Struct;
...@@ -50,6 +44,13 @@ import org.testng.annotations.BeforeMethod; ...@@ -50,6 +44,13 @@ import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Guice; import org.testng.annotations.Guice;
import org.testng.annotations.Test; import org.testng.annotations.Test;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Date;
import java.util.Iterator;
import javax.inject.Inject;
@Test @Test
@Guice(modules = RepositoryMetadataModule.class) @Guice(modules = RepositoryMetadataModule.class)
public class GraphRepoMapperScaleTest { public class GraphRepoMapperScaleTest {
...@@ -72,7 +73,7 @@ public class GraphRepoMapperScaleTest { ...@@ -72,7 +73,7 @@ public class GraphRepoMapperScaleTest {
public void setUp() throws Exception { public void setUp() throws Exception {
//force up front graph initialization //force up front graph initialization
TestUtils.getGraph(); TestUtils.getGraph();
searchIndexer = new GraphBackedSearchIndexer(new AtlasGraphProvider(), ApplicationProperties.get()); searchIndexer = new GraphBackedSearchIndexer(new AtlasGraphProvider(), ApplicationProperties.get(), new AtlasTypeRegistry());
//Make sure we can cleanup the index directory //Make sure we can cleanup the index directory
Collection<IDataType> typesAdded = TestUtils.createHiveTypes(typeSystem); Collection<IDataType> typesAdded = TestUtils.createHiveTypes(typeSystem);
searchIndexer.onAdd(typesAdded); searchIndexer.onAdd(typesAdded);
......
...@@ -18,29 +18,9 @@ ...@@ -18,29 +18,9 @@
package org.apache.atlas.service; package org.apache.atlas.service;
import static org.apache.atlas.TestUtils.COLUMNS_ATTR_NAME; import com.google.common.collect.ImmutableList;
import static org.apache.atlas.TestUtils.COLUMN_TYPE; import com.google.common.collect.ImmutableSet;
import static org.apache.atlas.TestUtils.PII; import com.google.inject.Inject;
import static org.apache.atlas.TestUtils.TABLE_TYPE;
import static org.apache.atlas.TestUtils.createColumnEntity;
import static org.apache.atlas.TestUtils.createDBEntity;
import static org.apache.atlas.TestUtils.createInstance;
import static org.apache.atlas.TestUtils.createTableEntity;
import static org.apache.atlas.TestUtils.randomString;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createClassTypeDef;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createOptionalAttrDef;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertNotNull;
import static org.testng.Assert.assertNull;
import static org.testng.Assert.assertTrue;
import static org.testng.Assert.fail;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.atlas.AtlasClient; import org.apache.atlas.AtlasClient;
import org.apache.atlas.AtlasException; import org.apache.atlas.AtlasException;
...@@ -49,12 +29,15 @@ import org.apache.atlas.RepositoryMetadataModule; ...@@ -49,12 +29,15 @@ import org.apache.atlas.RepositoryMetadataModule;
import org.apache.atlas.RequestContext; import org.apache.atlas.RequestContext;
import org.apache.atlas.TestUtils; import org.apache.atlas.TestUtils;
import org.apache.atlas.discovery.graph.GraphBackedDiscoveryService; import org.apache.atlas.discovery.graph.GraphBackedDiscoveryService;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.listener.ChangedTypeDefs;
import org.apache.atlas.listener.EntityChangeListener; import org.apache.atlas.listener.EntityChangeListener;
import org.apache.atlas.query.QueryParams; import org.apache.atlas.query.QueryParams;
import org.apache.atlas.repository.audit.EntityAuditRepository; import org.apache.atlas.repository.audit.EntityAuditRepository;
import org.apache.atlas.repository.audit.HBaseBasedAuditRepository; import org.apache.atlas.repository.audit.HBaseBasedAuditRepository;
import org.apache.atlas.repository.audit.HBaseTestUtils; import org.apache.atlas.repository.audit.HBaseTestUtils;
import org.apache.atlas.repository.graph.AtlasGraphProvider; import org.apache.atlas.repository.graph.AtlasGraphProvider;
import org.apache.atlas.services.DefaultMetadataService;
import org.apache.atlas.services.MetadataService; import org.apache.atlas.services.MetadataService;
import org.apache.atlas.typesystem.IReferenceableInstance; import org.apache.atlas.typesystem.IReferenceableInstance;
import org.apache.atlas.typesystem.IStruct; import org.apache.atlas.typesystem.IStruct;
...@@ -75,6 +58,7 @@ import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition; ...@@ -75,6 +58,7 @@ import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition;
import org.apache.atlas.typesystem.types.Multiplicity; import org.apache.atlas.typesystem.types.Multiplicity;
import org.apache.atlas.typesystem.types.TypeSystem; import org.apache.atlas.typesystem.types.TypeSystem;
import org.apache.atlas.typesystem.types.ValueConversionException; import org.apache.atlas.typesystem.types.ValueConversionException;
import org.apache.atlas.typesystem.types.cache.TypeCache;
import org.apache.atlas.typesystem.types.utils.TypesUtil; import org.apache.atlas.typesystem.types.utils.TypesUtil;
import org.apache.atlas.utils.ParamChecker; import org.apache.atlas.utils.ParamChecker;
import org.apache.commons.lang.RandomStringUtils; import org.apache.commons.lang.RandomStringUtils;
...@@ -87,9 +71,29 @@ import org.testng.annotations.BeforeTest; ...@@ -87,9 +71,29 @@ import org.testng.annotations.BeforeTest;
import org.testng.annotations.Guice; import org.testng.annotations.Guice;
import org.testng.annotations.Test; import org.testng.annotations.Test;
import com.google.common.collect.ImmutableList; import java.util.ArrayList;
import com.google.common.collect.ImmutableSet; import java.util.Arrays;
import com.google.inject.Inject; import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import static org.apache.atlas.TestUtils.COLUMNS_ATTR_NAME;
import static org.apache.atlas.TestUtils.COLUMN_TYPE;
import static org.apache.atlas.TestUtils.PII;
import static org.apache.atlas.TestUtils.TABLE_TYPE;
import static org.apache.atlas.TestUtils.createColumnEntity;
import static org.apache.atlas.TestUtils.createDBEntity;
import static org.apache.atlas.TestUtils.createInstance;
import static org.apache.atlas.TestUtils.createTableEntity;
import static org.apache.atlas.TestUtils.randomString;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createClassTypeDef;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createOptionalAttrDef;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertNotNull;
import static org.testng.Assert.assertNull;
import static org.testng.Assert.assertTrue;
import static org.testng.Assert.fail;
@Guice(modules = RepositoryMetadataModule.class) @Guice(modules = RepositoryMetadataModule.class)
public class DefaultMetadataServiceTest { public class DefaultMetadataServiceTest {
...@@ -1131,6 +1135,22 @@ public class DefaultMetadataServiceTest { ...@@ -1131,6 +1135,22 @@ public class DefaultMetadataServiceTest {
} }
} }
@Test
public void testOnChangeRefresh() {
try {
List<String> beforeChangeTypeNames = metadataService.getTypeNames(new HashMap<TypeCache.TYPE_FILTER, String>());
((DefaultMetadataService)metadataService).onChange(new ChangedTypeDefs());
List<String> afterChangeTypeNames = metadataService.getTypeNames(new HashMap<TypeCache.TYPE_FILTER, String>());
assertEquals(afterChangeTypeNames, beforeChangeTypeNames);
} catch (AtlasBaseException e) {
fail("Should've succeeded", e);
} catch (AtlasException e) {
fail("getTypeNames should've succeeded", e);
}
}
private static class EntitiesChangeListener implements EntityChangeListener { private static class EntitiesChangeListener implements EntityChangeListener {
private List<String> deletedEntities = new ArrayList<>(); private List<String> deletedEntities = new ArrayList<>();
private List<String> updatedEntities = new ArrayList<>(); private List<String> updatedEntities = new ArrayList<>();
......
...@@ -74,7 +74,8 @@ trait GraphUtils { ...@@ -74,7 +74,8 @@ trait GraphUtils {
object QueryTestsUtils extends GraphUtils { object QueryTestsUtils extends GraphUtils {
def setupTypesAndIndices() : Unit = { def setupTypesAndIndices() : Unit = {
val indexer = new GraphBackedSearchIndexer(); // FIXME: Do we need to init the AtlasTypeRegistry here ?
val indexer = new GraphBackedSearchIndexer(null);
val typesDef : TypesDef = defineTypes; val typesDef : TypesDef = defineTypes;
val newTypes = TypeSystem.getInstance.defineTypes(typesDef); val newTypes = TypeSystem.getInstance.defineTypes(typesDef);
indexer.onAdd(newTypes.values()); indexer.onAdd(newTypes.values());
......
...@@ -116,6 +116,10 @@ ...@@ -116,6 +116,10 @@
<groupId>org.apache.commons</groupId> <groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId> <artifactId>commons-lang3</artifactId>
</dependency> </dependency>
<dependency>
<groupId>org.apache.atlas</groupId>
<artifactId>atlas-intg</artifactId>
</dependency>
</dependencies> </dependencies>
<build> <build>
......
...@@ -20,6 +20,7 @@ package org.apache.atlas.typesystem.types; ...@@ -20,6 +20,7 @@ package org.apache.atlas.typesystem.types;
import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet; import com.google.common.collect.ImmutableSet;
import org.apache.atlas.AtlasException; import org.apache.atlas.AtlasException;
import org.apache.atlas.classification.InterfaceAudience; import org.apache.atlas.classification.InterfaceAudience;
import org.apache.atlas.typesystem.TypesDef; import org.apache.atlas.typesystem.TypesDef;
...@@ -30,7 +31,6 @@ import org.apache.atlas.typesystem.types.cache.TypeCache; ...@@ -30,7 +31,6 @@ import org.apache.atlas.typesystem.types.cache.TypeCache;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import javax.inject.Singleton;
import java.lang.reflect.Constructor; import java.lang.reflect.Constructor;
import java.text.SimpleDateFormat; import java.text.SimpleDateFormat;
import java.util.ArrayList; import java.util.ArrayList;
...@@ -41,6 +41,8 @@ import java.util.Set; ...@@ -41,6 +41,8 @@ import java.util.Set;
import java.util.TimeZone; import java.util.TimeZone;
import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentHashMap;
import javax.inject.Singleton;
@Singleton @Singleton
@InterfaceAudience.Private @InterfaceAudience.Private
public class TypeSystem { public class TypeSystem {
......
...@@ -31,7 +31,6 @@ import org.apache.atlas.model.typedef.AtlasStructDef; ...@@ -31,7 +31,6 @@ import org.apache.atlas.model.typedef.AtlasStructDef;
import org.apache.atlas.model.typedef.AtlasStructDef.AtlasStructDefs; import org.apache.atlas.model.typedef.AtlasStructDef.AtlasStructDefs;
import org.apache.atlas.model.typedef.AtlasTypesDef; import org.apache.atlas.model.typedef.AtlasTypesDef;
import org.apache.atlas.store.AtlasTypeDefStore; import org.apache.atlas.store.AtlasTypeDefStore;
import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.atlas.web.util.Servlets; import org.apache.atlas.web.util.Servlets;
import org.apache.http.annotation.Experimental; import org.apache.http.annotation.Experimental;
import org.slf4j.Logger; import org.slf4j.Logger;
......
...@@ -24,6 +24,7 @@ import org.apache.atlas.listener.ActiveStateChangeHandler; ...@@ -24,6 +24,7 @@ import org.apache.atlas.listener.ActiveStateChangeHandler;
import org.apache.atlas.notification.NotificationHookConsumer; import org.apache.atlas.notification.NotificationHookConsumer;
import org.apache.atlas.repository.audit.HBaseBasedAuditRepository; import org.apache.atlas.repository.audit.HBaseBasedAuditRepository;
import org.apache.atlas.repository.graph.GraphBackedSearchIndexer; import org.apache.atlas.repository.graph.GraphBackedSearchIndexer;
import org.apache.atlas.repository.store.graph.v1.AtlasTypeDefGraphStoreV1;
import org.apache.atlas.service.Service; import org.apache.atlas.service.Service;
import org.apache.atlas.services.DefaultMetadataService; import org.apache.atlas.services.DefaultMetadataService;
...@@ -41,6 +42,7 @@ public class ActiveInstanceElectorModule extends AbstractModule { ...@@ -41,6 +42,7 @@ public class ActiveInstanceElectorModule extends AbstractModule {
activeStateChangeHandlerBinder.addBinding().to(DefaultMetadataService.class); activeStateChangeHandlerBinder.addBinding().to(DefaultMetadataService.class);
activeStateChangeHandlerBinder.addBinding().to(NotificationHookConsumer.class); activeStateChangeHandlerBinder.addBinding().to(NotificationHookConsumer.class);
activeStateChangeHandlerBinder.addBinding().to(HBaseBasedAuditRepository.class); activeStateChangeHandlerBinder.addBinding().to(HBaseBasedAuditRepository.class);
activeStateChangeHandlerBinder.addBinding().to(AtlasTypeDefGraphStoreV1.class);
Multibinder<Service> serviceBinder = Multibinder.newSetBinder(binder(), Service.class); Multibinder<Service> serviceBinder = Multibinder.newSetBinder(binder(), Service.class);
serviceBinder.addBinding().to(ActiveInstanceElectorService.class); serviceBinder.addBinding().to(ActiveInstanceElectorService.class);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment