Skip to content

A
atlas
Commit 98fdc6d8 by Suma Shivaprasad
Options

ATLAS-512 Decouple currently integrating components from availability of Atlas…

ATLAS-512 Decouple currently integrating components from availability of Atlas service for raising metadata events ( yhemanth via sumasai)
parent 16107915
Showing with 544 additions and 22 deletions
addons/falcon-bridge/pom.xml
......@@ -269,6 +269,27 @@
<generateReports>false</generateReports>
</configuration>
</plugin>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>exec-maven-plugin</artifactId>
<version>1.2.1</version>
<inherited>false</inherited>
<executions>
<execution>
<configuration>
<mainClass>org.apache.atlas.falcon.model.FalconDataModelGenerator</mainClass>
<arguments>
<argument>${project.build.directory}/models/falcon_model.json</argument>
</arguments>
</configuration>
<phase>package</phase>
<goals>
<goal>java</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>
addons/falcon-bridge/src/main/java/org/apache/atlas/falcon/hook/FalconHook.java
......@@ -166,11 +166,6 @@ public class FalconHook extends FalconEventPublisher {
private void fireAndForget(FalconEvent event) throws Exception {
LOG.info("Entered Atlas hook for Falcon hook operation {}", event.getOperation());
if (!typesRegistered) {
registerFalconDataModel();
typesRegistered = true;
}
notifyEntity(createEntities(event));
}
......
addons/falcon-bridge/src/main/java/org/apache/atlas/falcon/model/FalconDataModelGenerator.java
......@@ -21,6 +21,7 @@ package org.apache.atlas.falcon.model;
import com.google.common.collect.ImmutableList;
import org.apache.atlas.AtlasClient;
import org.apache.atlas.AtlasException;
import org.apache.atlas.addons.ModelDefinitionDump;
import org.apache.atlas.typesystem.TypesDef;
import org.apache.atlas.typesystem.json.TypesSerialization;
import org.apache.atlas.typesystem.types.AttributeDefinition;
......@@ -130,7 +131,14 @@ public class FalconDataModelGenerator {
public static void main(String[] args) throws Exception {
FalconDataModelGenerator falconDataModelGenerator = new FalconDataModelGenerator();
System.out.println("falconDataModelAsJSON = " + falconDataModelGenerator.getModelAsJson());
String modelAsJson = falconDataModelGenerator.getModelAsJson();
if (args.length == 1) {
ModelDefinitionDump.dumpModelToFile(args[0], modelAsJson);
return;
}
System.out.println("falconDataModelAsJSON = " + modelAsJson);
TypesDef typesDef = falconDataModelGenerator.getTypesDef();
for (EnumTypeDefinition enumType : typesDef.enumTypesAsJavaList()) {
......
addons/falcon-bridge/src/test/java/org/apache/atlas/falcon/hook/FalconHookIT.java
......@@ -65,6 +65,7 @@ public class FalconHookIT {
AtlasService service = new AtlasService();
service.init();
STORE.registerListener(service);
new FalconHook().registerFalconDataModel();
CurrentUser.authenticate(System.getProperty("user.name"));
}
......
addons/hive-bridge/pom.xml
......@@ -347,6 +347,27 @@
<generateReports>false</generateReports>
</configuration>
</plugin>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>exec-maven-plugin</artifactId>
<version>1.2.1</version>
<inherited>false</inherited>
<executions>
<execution>
<configuration>
<mainClass>org.apache.atlas.hive.model.HiveDataModelGenerator</mainClass>
<arguments>
<argument>${project.build.directory}/models/hive_model.json</argument>
</arguments>
</configuration>
<phase>package</phase>
<goals>
<goal>java</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>
addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java
......@@ -201,11 +201,6 @@ public class HiveHook extends AtlasHook implements ExecuteWithHookContext {
HiveMetaStoreBridge dgiBridge = new HiveMetaStoreBridge(hiveConf, atlasProperties, event.user, event.ugi);
if (!typesRegistered) {
dgiBridge.registerHiveDataModel();
typesRegistered = true;
}
switch (event.operation) {
case CREATEDATABASE:
handleEventOutputs(dgiBridge, event, Type.DATABASE);
......
addons/hive-bridge/src/main/java/org/apache/atlas/hive/model/HiveDataModelGenerator.java
......@@ -21,6 +21,7 @@ package org.apache.atlas.hive.model;
import com.google.common.collect.ImmutableList;
import org.apache.atlas.AtlasClient;
import org.apache.atlas.AtlasException;
import org.apache.atlas.addons.ModelDefinitionDump;
import org.apache.atlas.typesystem.TypesDef;
import org.apache.atlas.typesystem.json.TypesSerialization;
import org.apache.atlas.typesystem.types.AttributeDefinition;
......@@ -39,6 +40,11 @@ import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.io.PrintWriter;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
......@@ -370,7 +376,14 @@ public class HiveDataModelGenerator {
public static void main(String[] args) throws Exception {
HiveDataModelGenerator hiveDataModelGenerator = new HiveDataModelGenerator();
System.out.println("hiveDataModelAsJSON = " + hiveDataModelGenerator.getModelAsJson());
String modelAsJson = hiveDataModelGenerator.getModelAsJson();
if (args.length==1) {
ModelDefinitionDump.dumpModelToFile(args[0], modelAsJson);
return;
}
System.out.println("hiveDataModelAsJSON = " + modelAsJson);
TypesDef typesDef = hiveDataModelGenerator.getTypesDef();
for (EnumTypeDefinition enumType : typesDef.enumTypesAsJavaList()) {
......@@ -390,4 +403,5 @@ public class HiveDataModelGenerator {
Arrays.toString(traitType.attributeDefinitions)));
}
}
}
addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java
......@@ -68,6 +68,8 @@ public class HiveHookIT {
SessionState.setCurrentSessionState(ss);
Configuration configuration = ApplicationProperties.get();
HiveMetaStoreBridge hiveMetaStoreBridge = new HiveMetaStoreBridge(conf, configuration);
hiveMetaStoreBridge.registerHiveDataModel();
dgiCLient = new AtlasClient(configuration.getString(HiveMetaStoreBridge.ATLAS_ENDPOINT, DGI_URL));
}
......
addons/sqoop-bridge/pom.xml
......@@ -352,6 +352,27 @@
<generateReports>false</generateReports>
</configuration>
</plugin>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>exec-maven-plugin</artifactId>
<version>1.2.1</version>
<inherited>false</inherited>
<executions>
<execution>
<configuration>
<mainClass>org.apache.atlas.sqoop.model.SqoopDataModelGenerator</mainClass>
<arguments>
<argument>${project.build.directory}/models/sqoop_model.json</argument>
</arguments>
</configuration>
<phase>package</phase>
<goals>
<goal>java</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>
addons/sqoop-bridge/src/main/java/org/apache/atlas/sqoop/hook/SqoopHook.java
......@@ -70,7 +70,7 @@ public class SqoopHook extends SqoopJobDataPublisher {
org.apache.hadoop.conf.Configuration.addDefaultResource("sqoop-site.xml");
}
private synchronized void registerDataModels(AtlasClient client, Configuration atlasConf) throws Exception {
synchronized void registerDataModels(AtlasClient client, Configuration atlasConf) throws Exception {
// Make sure hive model exists
HiveMetaStoreBridge hiveMetaStoreBridge = new HiveMetaStoreBridge(new HiveConf(), atlasConf,
UserGroupInformation.getCurrentUser().getShortUserName(), UserGroupInformation.getCurrentUser());
......@@ -190,7 +190,6 @@ public class SqoopHook extends SqoopJobDataPublisher {
UserGroupInformation.getCurrentUser(), UserGroupInformation.getCurrentUser().getShortUserName());
org.apache.hadoop.conf.Configuration sqoopConf = new org.apache.hadoop.conf.Configuration();
String clusterName = sqoopConf.get(ATLAS_CLUSTER_NAME, DEFAULT_CLUSTER_NAME);
registerDataModels(atlasClient, atlasProperties);
Referenceable dbStoreRef = createDBStoreInstance(data);
Referenceable dbRef = createHiveDatabaseInstance(clusterName, data.getHiveDB());
......
addons/sqoop-bridge/src/main/java/org/apache/atlas/sqoop/model/SqoopDataModelGenerator.java
......@@ -21,6 +21,7 @@ package org.apache.atlas.sqoop.model;
import com.google.common.collect.ImmutableList;
import org.apache.atlas.AtlasClient;
import org.apache.atlas.AtlasException;
import org.apache.atlas.addons.ModelDefinitionDump;
import org.apache.atlas.typesystem.TypesDef;
import org.apache.atlas.typesystem.json.TypesSerialization;
import org.apache.atlas.typesystem.types.AttributeDefinition;
......@@ -161,7 +162,14 @@ public class SqoopDataModelGenerator {
public static void main(String[] args) throws Exception {
SqoopDataModelGenerator dataModelGenerator = new SqoopDataModelGenerator();
System.out.println("sqoopDataModelAsJSON = " + dataModelGenerator.getModelAsJson());
String modelAsJson = dataModelGenerator.getModelAsJson();
if (args.length == 1) {
ModelDefinitionDump.dumpModelToFile(args[0], modelAsJson);
return;
}
System.out.println("sqoopDataModelAsJSON = " + modelAsJson);
TypesDef typesDef = dataModelGenerator.getTypesDef();
for (EnumTypeDefinition enumType : typesDef.enumTypesAsJavaList()) {
......
addons/sqoop-bridge/src/test/java/org/apache/atlas/sqoop/hook/SqoopHookIT.java
......@@ -22,6 +22,7 @@ import org.apache.atlas.ApplicationProperties;
import org.apache.atlas.AtlasClient;
import org.apache.atlas.hive.model.HiveDataTypes;
import org.apache.atlas.sqoop.model.SqoopDataTypes;
import org.apache.commons.configuration.Configuration;
import org.apache.sqoop.SqoopJobDataPublisher;
import org.codehaus.jettison.json.JSONArray;
import org.codehaus.jettison.json.JSONObject;
......@@ -41,7 +42,9 @@ public class SqoopHookIT {
@BeforeClass
public void setUp() throws Exception {
//Set-up sqoop session
dgiCLient = new AtlasClient(ApplicationProperties.get().getString("atlas.rest.address"));
Configuration configuration = ApplicationProperties.get();
dgiCLient = new AtlasClient(configuration.getString("atlas.rest.address"));
new SqoopHook().registerDataModels(dgiCLient, configuration);
}
@Test
......
addons/storm-bridge/pom.xml
......@@ -383,6 +383,28 @@
</configuration>
</plugin>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>exec-maven-plugin</artifactId>
<version>1.2.1</version>
<inherited>false</inherited>
<executions>
<execution>
<configuration>
<mainClass>org.apache.atlas.storm.model.StormDataModelGenerator</mainClass>
<arguments>
<argument>${project.build.directory}/models/storm_model.json</argument>
</arguments>
</configuration>
<phase>package</phase>
<goals>
<goal>java</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>net.alchim31.maven</groupId>
<artifactId>scala-maven-plugin</artifactId>
......
addons/storm-bridge/src/main/java/org/apache/atlas/storm/hook/StormAtlasHook.java
......@@ -98,10 +98,6 @@ public class StormAtlasHook extends AtlasHook implements ISubmitterHook {
LOG.info("Collecting metadata for a new storm topology: {}", topologyInfo.get_name());
try {
if( ! typesRegistered ) {
registerDataModel(new HiveDataModelGenerator());
}
ArrayList<Referenceable> entities = new ArrayList<>();
Referenceable topologyReferenceable = createTopologyInstance(topologyInfo, stormConf);
List<Referenceable> dependentEntities = addTopologyDataSets(stormTopology, topologyReferenceable,
......
addons/storm-bridge/src/main/java/org/apache/atlas/storm/model/StormDataModelGenerator.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.storm.model;
import org.apache.atlas.addons.ModelDefinitionDump;
import org.apache.atlas.typesystem.TypesDef;
import org.apache.atlas.typesystem.json.TypesSerialization;
import java.io.IOException;
public class StormDataModelGenerator {
public static void main(String[] args) throws IOException {
StormDataModel.main(new String[]{});
TypesDef typesDef = StormDataModel.typesDef();
String stormTypesAsJSON = TypesSerialization.toJson(typesDef);
if (args.length == 1) {
ModelDefinitionDump.dumpModelToFile(args[0], stormTypesAsJSON);
return;
}
System.out.println("stormTypesAsJSON = " + stormTypesAsJSON);
}
}
addons/storm-bridge/src/test/java/org/apache/atlas/storm/hook/StormAtlasHookIT.java
......@@ -57,6 +57,7 @@ public class StormAtlasHookIT {
Configuration configuration = ApplicationProperties.get();
atlasClient = new AtlasClient(configuration.getString("atlas.rest.address", ATLAS_URL));
new StormAtlasHook().registerDataModel(new HiveDataModelGenerator());
}
@AfterClass
......
common/src/main/java/org/apache/atlas/addons/ModelDefinitionDump.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.addons;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.io.PrintWriter;
/**
* A class to write a model to a file.
*/
public final class ModelDefinitionDump {
private ModelDefinitionDump() {
}
/**
* Write given model as JSON to given file.
* @param outputFileName file name to write model to
* @param modelAsJson model serialized as JSON
* @throws IOException
*/
public static void dumpModelToFile(String outputFileName, String modelAsJson) throws IOException {
File dir = new File(outputFileName).getParentFile();
if (!dir.exists()) {
dir.mkdirs();
}
PrintWriter printWriter = new PrintWriter(new BufferedWriter(new FileWriter(outputFileName)));
printWriter.write(modelAsJson);
printWriter.close();
}
}
distro/src/main/assemblies/standalone-package.xml
......@@ -99,23 +99,43 @@
<outputDirectory>hook</outputDirectory>
</fileSet>
<fileSet>
<directory>../addons/hive-bridge/target/models</directory>
<outputDirectory>models</outputDirectory>
</fileSet>
<!-- addons/falcon -->
<fileSet>
<directory>../addons/falcon-bridge/target/dependency/hook</directory>
<outputDirectory>hook</outputDirectory>
</fileSet>
<fileSet>
<directory>../addons/falcon-bridge/target/models</directory>
<outputDirectory>models</outputDirectory>
</fileSet>
<!-- addons/sqoop -->
<fileSet>
<directory>../addons/sqoop-bridge/target/dependency/hook</directory>
<outputDirectory>hook</outputDirectory>
</fileSet>
<fileSet>
<directory>../addons/sqoop-bridge/target/models</directory>
<outputDirectory>models</outputDirectory>
</fileSet>
<!-- addons/storm -->
<fileSet>
<directory>../addons/storm-bridge/target/dependency/hook</directory>
<outputDirectory>hook</outputDirectory>
</fileSet>
<fileSet>
<directory>../addons/storm-bridge/target/models</directory>
<outputDirectory>models</outputDirectory>
</fileSet>
</fileSets>
<files>
......
release-log.txt
......@@ -11,6 +11,7 @@ ATLAS-409 Atlas will not import avro tables with schema read from a file (dosset
ATLAS-379 Create sqoop and falcon metadata addons (venkatnrangan,bvellanki,sowmyaramesh via shwethags)
ALL CHANGES:
ATLAS-512 Decouple currently integrating components from availability of Atlas service for raising metadata events ( yhemanth via sumasai)
ATLAS-537 Falcon hook failing when tried to submit a process which creates a hive table ( shwethags via sumasai)
ATLAS-476 Update type attribute with Reserved characters updated the original type as unknown (yhemanth via shwethags)
ATLAS-463 Disconnect inverse references ( dkantor via sumasai)
......
repository/src/main/java/org/apache/atlas/RepositoryMetadataModule.java
......@@ -38,7 +38,9 @@ import org.apache.atlas.repository.graph.TitanGraphProvider;
import org.apache.atlas.repository.typestore.GraphBackedTypeStore;
import org.apache.atlas.repository.typestore.ITypeStore;
import org.apache.atlas.services.DefaultMetadataService;
import org.apache.atlas.services.IBootstrapTypesRegistrar;
import org.apache.atlas.services.MetadataService;
import org.apache.atlas.services.ReservedTypesRegistrar;
import org.apache.atlas.typesystem.types.TypeSystem;
/**
......@@ -71,6 +73,8 @@ public class RepositoryMetadataModule extends com.google.inject.AbstractModule {
// bind the MetadataService interface to an implementation
bind(MetadataService.class).to(DefaultMetadataService.class).asEagerSingleton();
bind(IBootstrapTypesRegistrar.class).to(ReservedTypesRegistrar.class);
// bind the DiscoveryService interface to an implementation
bind(DiscoveryService.class).to(GraphBackedDiscoveryService.class).asEagerSingleton();
......
repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java
......@@ -68,6 +68,7 @@ import org.slf4j.LoggerFactory;
import javax.inject.Inject;
import javax.inject.Singleton;
import java.io.File;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
......@@ -89,18 +90,28 @@ public class DefaultMetadataService implements MetadataService {
private final TypeSystem typeSystem;
private final MetadataRepository repository;
private final ITypeStore typeStore;
private IBootstrapTypesRegistrar typesRegistrar;
private final Collection<Provider<TypesChangeListener>> typeChangeListeners;
@Inject
DefaultMetadataService(final MetadataRepository repository, final ITypeStore typeStore,
final IBootstrapTypesRegistrar typesRegistrar,
final Collection<Provider<TypesChangeListener>> typeChangeListeners) throws AtlasException {
this(repository, typeStore, typesRegistrar, typeChangeListeners, TypeSystem.getInstance());
}
DefaultMetadataService(final MetadataRepository repository, final ITypeStore typeStore,
final IBootstrapTypesRegistrar typesRegistrar,
final Collection<Provider<TypesChangeListener>> typeChangeListeners,
final TypeSystem typeSystem) throws AtlasException {
this.typeStore = typeStore;
this.typeSystem = TypeSystem.getInstance();
this.typesRegistrar = typesRegistrar;
this.typeSystem = typeSystem;
this.repository = repository;
this.typeChangeListeners = typeChangeListeners;
restoreTypeSystem();
typesRegistrar.registerTypes(ReservedTypesRegistrar.getTypesDir(), typeSystem, this);
}
private void restoreTypeSystem() {
......@@ -111,7 +122,6 @@ public class DefaultMetadataService implements MetadataService {
// restore types before creating super types
createSuperTypes();
} catch (AtlasException e) {
throw new RuntimeException(e);
}
......
repository/src/main/java/org/apache/atlas/services/IBootstrapTypesRegistrar.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.services;
import org.apache.atlas.AtlasException;
import org.apache.atlas.typesystem.types.TypeSystem;
public interface IBootstrapTypesRegistrar {
void registerTypes(String typesDirName, TypeSystem typeSystem, MetadataService metadataService)
throws AtlasException;
}
repository/src/main/java/org/apache/atlas/services/ReservedTypesRegistrar.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.services;
import org.apache.atlas.AtlasException;
import org.apache.atlas.typesystem.TypesDef;
import org.apache.atlas.typesystem.json.TypesSerialization;
import org.apache.atlas.typesystem.types.ClassType;
import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition;
import org.apache.atlas.typesystem.types.TypeSystem;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
public class ReservedTypesRegistrar implements IBootstrapTypesRegistrar {
private static final Logger LOG = LoggerFactory.getLogger(ReservedTypesRegistrar.class);
static String getTypesDir() {
return System.getProperty("atlas.home")+ File.separator+"models";
}
@Override
public void registerTypes(String typesDirName, TypeSystem typeSystem, MetadataService metadataService)
throws AtlasException {
File typesDir = new File(typesDirName);
if (!typesDir.exists()) {
LOG.info("No types directory {} found - not registering any reserved types", typesDirName);
return;
}
File[] typeDefFiles = typesDir.listFiles();
for (File typeDefFile : typeDefFiles) {
try {
String typeDefJSON = new String(Files.readAllBytes(typeDefFile.toPath()), StandardCharsets.UTF_8);
registerType(typeSystem, metadataService, typeDefFile.getAbsolutePath(), typeDefJSON);
} catch (IOException e) {
e.printStackTrace();
}
}
}
void registerType(TypeSystem typeSystem, MetadataService metadataService, String typeDefName, String typeDefJSON)
throws AtlasException {
TypesDef typesDef = null;
try {
typesDef = TypesSerialization.fromJson(typeDefJSON);
} catch (Exception e) {
LOG.error("Error while deserializing JSON in {}", typeDefName);
throw new ReservedTypesRegistrationException("Error while deserializing JSON in " + typeDefName, e);
}
HierarchicalTypeDefinition<ClassType> classDef = typesDef.classTypesAsJavaList().get(0);
if (!typeSystem.isRegistered(classDef.typeName)) {
metadataService.createType(typeDefJSON);
LOG.info("Registered types in {}", typeDefName);
} else {
LOG.warn("class {} already registered, ignoring types in {}", classDef.typeName,
typeDefName);
}
}
}
repository/src/main/java/org/apache/atlas/services/ReservedTypesRegistrationException.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.services;
public class ReservedTypesRegistrationException extends RuntimeException {
public ReservedTypesRegistrationException(String message, Exception e) {
super(message, e);
}
}
repository/src/test/java/org/apache/atlas/services/DefaultMetadataServiceMockTest.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.services;
import com.google.inject.Provider;
import org.apache.atlas.AtlasException;
import org.apache.atlas.listener.TypesChangeListener;
import org.apache.atlas.repository.MetadataRepository;
import org.apache.atlas.repository.typestore.ITypeStore;
import org.apache.atlas.typesystem.TypesDef;
import org.apache.atlas.typesystem.types.TypeSystem;
import org.mockito.Matchers;
import org.testng.annotations.Test;
import java.util.ArrayList;
import static org.mockito.Matchers.any;
import static org.mockito.Matchers.eq;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
public class DefaultMetadataServiceMockTest {
@Test
public void testShouldInvokeTypesRegistrarOnCreation() throws AtlasException {
IBootstrapTypesRegistrar typesRegistrar = mock(IBootstrapTypesRegistrar.class);
TypeSystem typeSystem = mock(TypeSystem.class);
when(typeSystem.isRegistered(any(String.class))).thenReturn(true);
DefaultMetadataService defaultMetadataService = new DefaultMetadataService(mock(MetadataRepository.class),
mock(ITypeStore.class),
typesRegistrar, new ArrayList<Provider<TypesChangeListener>>(), typeSystem);
verify(typesRegistrar).registerTypes(ReservedTypesRegistrar.getTypesDir(),
typeSystem, defaultMetadataService);
}
}
repository/src/test/java/org/apache/atlas/services/ReservedTypesRegistrarTest.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.services;
import org.apache.atlas.AtlasException;
import org.apache.atlas.TestUtils;
import org.apache.atlas.typesystem.TypesDef;
import org.apache.atlas.typesystem.json.TypesSerialization;
import org.apache.atlas.typesystem.types.TypeSystem;
import org.mockito.InOrder;
import org.mockito.Mock;
import org.mockito.MockitoAnnotations;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
import static org.mockito.Mockito.inOrder;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.verifyZeroInteractions;
import static org.mockito.Mockito.when;
public class ReservedTypesRegistrarTest {
@Mock
private TypeSystem typeSystem;
@Mock
private MetadataService metadataService;
@BeforeMethod
public void setup() {
MockitoAnnotations.initMocks(this);
}
@Test
public void testRegistrationWithNoFiles() throws AtlasException {
IBootstrapTypesRegistrar bootstrapTypesRegistrar = new ReservedTypesRegistrar();
bootstrapTypesRegistrar.registerTypes("/some/dir/", typeSystem, metadataService);
verifyZeroInteractions(typeSystem);
}
@Test
public void testRegisterFirstChecksClassTypeIsRegistered() throws AtlasException {
ReservedTypesRegistrar reservedTypesRegistrar = new ReservedTypesRegistrar();
TypesDef typesDef = TestUtils.defineHiveTypes();
String typesJson = TypesSerialization.toJson(typesDef);
reservedTypesRegistrar.registerType(typeSystem, metadataService, "/some/file/model.json", typesJson);
InOrder inOrder = inOrder(typeSystem, metadataService);
inOrder.verify(typeSystem).isRegistered(typesDef.classTypesAsJavaList().get(0).typeName);
inOrder.verify(metadataService).createType(typesJson);
}
@Test
public void testRegisterCreatesTypesUsingMetadataService() throws AtlasException {
ReservedTypesRegistrar reservedTypesRegistrar = new ReservedTypesRegistrar();
TypesDef typesDef = TestUtils.defineHiveTypes();
String typesJson = TypesSerialization.toJson(typesDef);
reservedTypesRegistrar.registerType(typeSystem, metadataService, "/some/file/model.json", typesJson);
verify(metadataService).createType(typesJson);
}
@Test(expectedExceptions = ReservedTypesRegistrationException.class)
public void testRegisterFailsIfErrorInJson() throws AtlasException {
ReservedTypesRegistrar reservedTypesRegistrar = new ReservedTypesRegistrar();
reservedTypesRegistrar.registerType(typeSystem, metadataService, "/some/file/model.json", "invalid json");
}
@Test(expectedExceptions = AtlasException.class)
public void testRegisterFailsOnTypeCreationException() throws AtlasException {
ReservedTypesRegistrar reservedTypesRegistrar = new ReservedTypesRegistrar();
TypesDef typesDef = TestUtils.defineHiveTypes();
String typesJson = TypesSerialization.toJson(typesDef);
when(metadataService.createType(typesJson)).thenThrow(new AtlasException("some exception"));
reservedTypesRegistrar.registerType(typeSystem, metadataService, "/some/file/model.json", typesJson);
}
@Test
public void testShouldNotRegisterIfTypeIsAlreadyRegistered() throws AtlasException {
ReservedTypesRegistrar reservedTypesRegistrar = new ReservedTypesRegistrar();
TypesDef typesDef = TestUtils.defineHiveTypes();
String typesJson = TypesSerialization.toJson(typesDef);
when(typeSystem.isRegistered(typesDef.classTypesAsJavaList().get(0).typeName)).thenReturn(true);
reservedTypesRegistrar.registerType(typeSystem, metadataService, "/some/file/model.json", typesJson);
verifyZeroInteractions(metadataService);
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment