Skip to content

A
atlas
Commit def9e385 by Madhan Neethiraj
Options

ATLAS-1272: updated types bootstrap to load from new format typedef JSON files

parent 2ea3a455
Showing with 1918 additions and 2619 deletions
addons/falcon-bridge/pom.xml
...@@ -355,24 +355,6 @@ ...@@ -355,24 +355,6 @@
<version>1.2.1</version> <version>1.2.1</version>
<inherited>false</inherited> <inherited>false</inherited>
<executions> <executions>
<execution>
<configuration>
<mainClass>org.apache.atlas.falcon.model.FalconDataModelGenerator</mainClass>
<systemProperties>
<systemProperty>
<key>atlas.conf</key>
<value>${project.build.directory}/../../../typesystem/target/test-classes</value>
</systemProperty>
</systemProperties>
<arguments>
<argument>${project.build.directory}/models/falcon_model.json</argument>
</arguments>
</configuration>
<phase>package</phase>
<goals>
<goal>java</goal>
</goals>
</execution>
</executions> </executions>
</plugin> </plugin>
</plugins> </plugins>
......
addons/falcon-bridge/src/main/java/org/apache/atlas/falcon/bridge/FalconBridge.java
...@@ -21,11 +21,8 @@ package org.apache.atlas.falcon.bridge; ...@@ -21,11 +21,8 @@ package org.apache.atlas.falcon.bridge;
import org.apache.atlas.AtlasClient; import org.apache.atlas.AtlasClient;
import org.apache.atlas.AtlasConstants; import org.apache.atlas.AtlasConstants;
import org.apache.atlas.falcon.Util.EventUtil; import org.apache.atlas.falcon.Util.EventUtil;
import org.apache.atlas.falcon.model.FalconDataModelGenerator;
import org.apache.atlas.falcon.model.FalconDataTypes; import org.apache.atlas.falcon.model.FalconDataTypes;
import org.apache.atlas.fs.model.FSDataTypes;
import org.apache.atlas.hive.bridge.HiveMetaStoreBridge; import org.apache.atlas.hive.bridge.HiveMetaStoreBridge;
import org.apache.atlas.hive.model.HiveDataModelGenerator;
import org.apache.atlas.hive.model.HiveDataTypes; import org.apache.atlas.hive.model.HiveDataTypes;
import org.apache.atlas.typesystem.Referenceable; import org.apache.atlas.typesystem.Referenceable;
import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.CollectionUtils;
...@@ -61,6 +58,15 @@ import java.util.Map; ...@@ -61,6 +58,15 @@ import java.util.Map;
public class FalconBridge { public class FalconBridge {
private static final Logger LOG = LoggerFactory.getLogger(FalconBridge.class); private static final Logger LOG = LoggerFactory.getLogger(FalconBridge.class);
public static final String COLO = "colo";
public static final String TAGS = "tags";
public static final String GROUPS = "groups";
public static final String PIPELINES = "pipelines";
public static final String WFPROPERTIES = "workflow-properties";
public static final String RUNSON = "runs-on";
public static final String STOREDIN = "stored-in";
public static final String FREQUENCY = "frequency";
/** /**
* Creates cluster entity * Creates cluster entity
* *
...@@ -77,14 +83,14 @@ public class FalconBridge { ...@@ -77,14 +83,14 @@ public class FalconBridge {
clusterRef.set(AtlasClient.DESCRIPTION, cluster.getDescription()); clusterRef.set(AtlasClient.DESCRIPTION, cluster.getDescription());
clusterRef.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, cluster.getName()); clusterRef.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, cluster.getName());
clusterRef.set(FalconDataModelGenerator.COLO, cluster.getColo()); clusterRef.set(FalconBridge.COLO, cluster.getColo());
if (cluster.getACL() != null) { if (cluster.getACL() != null) {
clusterRef.set(AtlasClient.OWNER, cluster.getACL().getGroup()); clusterRef.set(AtlasClient.OWNER, cluster.getACL().getGroup());
} }
if (StringUtils.isNotEmpty(cluster.getTags())) { if (StringUtils.isNotEmpty(cluster.getTags())) {
clusterRef.set(FalconDataModelGenerator.TAGS, clusterRef.set(FalconBridge.TAGS,
EventUtil.convertKeyValueStringToMap(cluster.getTags())); EventUtil.convertKeyValueStringToMap(cluster.getTags()));
} }
...@@ -100,19 +106,19 @@ public class FalconBridge { ...@@ -100,19 +106,19 @@ public class FalconBridge {
String feedQualifiedName = String feedQualifiedName =
getFeedQualifiedName(feed.getName(), (String) clusterReferenceable.get(AtlasClient.NAME)); getFeedQualifiedName(feed.getName(), (String) clusterReferenceable.get(AtlasClient.NAME));
feedEntity.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, feedQualifiedName); feedEntity.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, feedQualifiedName);
feedEntity.set(FalconDataModelGenerator.FREQUENCY, feed.getFrequency().toString()); feedEntity.set(FalconBridge.FREQUENCY, feed.getFrequency().toString());
feedEntity.set(FalconDataModelGenerator.STOREDIN, clusterReferenceable); feedEntity.set(FalconBridge.STOREDIN, clusterReferenceable);
if (feed.getACL() != null) { if (feed.getACL() != null) {
feedEntity.set(AtlasClient.OWNER, feed.getACL().getOwner()); feedEntity.set(AtlasClient.OWNER, feed.getACL().getOwner());
} }
if (StringUtils.isNotEmpty(feed.getTags())) { if (StringUtils.isNotEmpty(feed.getTags())) {
feedEntity.set(FalconDataModelGenerator.TAGS, feedEntity.set(FalconBridge.TAGS,
EventUtil.convertKeyValueStringToMap(feed.getTags())); EventUtil.convertKeyValueStringToMap(feed.getTags()));
} }
if (feed.getGroups() != null) { if (feed.getGroups() != null) {
feedEntity.set(FalconDataModelGenerator.GROUPS, feed.getGroups()); feedEntity.set(FalconBridge.GROUPS, feed.getGroups());
} }
return feedEntity; return feedEntity;
...@@ -165,7 +171,7 @@ public class FalconBridge { ...@@ -165,7 +171,7 @@ public class FalconBridge {
feedCreateEntity.set(AtlasClient.PROCESS_ATTRIBUTE_OUTPUTS, outputs); feedCreateEntity.set(AtlasClient.PROCESS_ATTRIBUTE_OUTPUTS, outputs);
} }
feedCreateEntity.set(FalconDataModelGenerator.STOREDIN, clusterReferenceable); feedCreateEntity.set(FalconBridge.STOREDIN, clusterReferenceable);
entities.add(feedCreateEntity); entities.add(feedCreateEntity);
} }
...@@ -244,7 +250,7 @@ public class FalconBridge { ...@@ -244,7 +250,7 @@ public class FalconBridge {
processEntity.set(AtlasClient.NAME, process.getName()); processEntity.set(AtlasClient.NAME, process.getName());
processEntity.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, processEntity.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME,
getProcessQualifiedName(process.getName(), cluster.getName())); getProcessQualifiedName(process.getName(), cluster.getName()));
processEntity.set(FalconDataModelGenerator.FREQUENCY, process.getFrequency().toString()); processEntity.set(FalconBridge.FREQUENCY, process.getFrequency().toString());
if (!inputs.isEmpty()) { if (!inputs.isEmpty()) {
processEntity.set(AtlasClient.PROCESS_ATTRIBUTE_INPUTS, inputs); processEntity.set(AtlasClient.PROCESS_ATTRIBUTE_INPUTS, inputs);
...@@ -254,7 +260,7 @@ public class FalconBridge { ...@@ -254,7 +260,7 @@ public class FalconBridge {
} }
// set cluster // set cluster
processEntity.set(FalconDataModelGenerator.RUNSON, clusterReferenceable); processEntity.set(FalconBridge.RUNSON, clusterReferenceable);
// Set user // Set user
if (process.getACL() != null) { if (process.getACL() != null) {
...@@ -262,15 +268,15 @@ public class FalconBridge { ...@@ -262,15 +268,15 @@ public class FalconBridge {
} }
if (StringUtils.isNotEmpty(process.getTags())) { if (StringUtils.isNotEmpty(process.getTags())) {
processEntity.set(FalconDataModelGenerator.TAGS, processEntity.set(FalconBridge.TAGS,
EventUtil.convertKeyValueStringToMap(process.getTags())); EventUtil.convertKeyValueStringToMap(process.getTags()));
} }
if (process.getPipelines() != null) { if (process.getPipelines() != null) {
processEntity.set(FalconDataModelGenerator.PIPELINES, process.getPipelines()); processEntity.set(FalconBridge.PIPELINES, process.getPipelines());
} }
processEntity.set(FalconDataModelGenerator.WFPROPERTIES, processEntity.set(FalconBridge.WFPROPERTIES,
getProcessEntityWFProperties(process.getWorkflow(), getProcessEntityWFProperties(process.getWorkflow(),
process.getName())); process.getName()));
...@@ -319,7 +325,7 @@ public class FalconBridge { ...@@ -319,7 +325,7 @@ public class FalconBridge {
private static List<Referenceable> fillHDFSDataSet(final String pathUri, final String clusterName) { private static List<Referenceable> fillHDFSDataSet(final String pathUri, final String clusterName) {
List<Referenceable> entities = new ArrayList<>(); List<Referenceable> entities = new ArrayList<>();
Referenceable ref = new Referenceable(FSDataTypes.HDFS_PATH().toString()); Referenceable ref = new Referenceable(HiveMetaStoreBridge.HDFS_PATH);
ref.set("path", pathUri); ref.set("path", pathUri);
// Path path = new Path(pathUri); // Path path = new Path(pathUri);
// ref.set("name", path.getName()); // ref.set("name", path.getName());
...@@ -352,7 +358,7 @@ public class FalconBridge { ...@@ -352,7 +358,7 @@ public class FalconBridge {
tableRef.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, tableRef.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME,
HiveMetaStoreBridge.getTableQualifiedName(clusterName, dbName, tableName)); HiveMetaStoreBridge.getTableQualifiedName(clusterName, dbName, tableName));
tableRef.set(AtlasClient.NAME, tableName.toLowerCase()); tableRef.set(AtlasClient.NAME, tableName.toLowerCase());
tableRef.set(HiveDataModelGenerator.DB, dbRef); tableRef.set(HiveMetaStoreBridge.DB, dbRef);
entities.add(tableRef); entities.add(tableRef);
return entities; return entities;
...@@ -364,7 +370,7 @@ public class FalconBridge { ...@@ -364,7 +370,7 @@ public class FalconBridge {
Referenceable clusterRef = new Referenceable(FalconDataTypes.FALCON_CLUSTER.getName()); Referenceable clusterRef = new Referenceable(FalconDataTypes.FALCON_CLUSTER.getName());
clusterRef.set(AtlasClient.NAME, String.format("%s", clusterName)); clusterRef.set(AtlasClient.NAME, String.format("%s", clusterName));
clusterRef.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, clusterName); clusterRef.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, clusterName);
clusterRef.set(FalconDataModelGenerator.COLO, colo); clusterRef.set(FalconBridge.COLO, colo);
return clusterRef; return clusterRef;
} }
...@@ -375,8 +381,8 @@ public class FalconBridge { ...@@ -375,8 +381,8 @@ public class FalconBridge {
feedDatasetRef.set(AtlasClient.NAME, feed.getName()); feedDatasetRef.set(AtlasClient.NAME, feed.getName());
feedDatasetRef.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, getFeedQualifiedName(feed.getName(), feedDatasetRef.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, getFeedQualifiedName(feed.getName(),
(String) clusterReference.get(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME))); (String) clusterReference.get(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME)));
feedDatasetRef.set(FalconDataModelGenerator.STOREDIN, clusterReference); feedDatasetRef.set(FalconBridge.STOREDIN, clusterReference);
feedDatasetRef.set(FalconDataModelGenerator.FREQUENCY, feed.getFrequency()); feedDatasetRef.set(FalconBridge.FREQUENCY, feed.getFrequency());
return feedDatasetRef; return feedDatasetRef;
} }
......
addons/falcon-bridge/src/main/java/org/apache/atlas/falcon/model/FalconDataModelGenerator.java deleted 100644 → 0
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.falcon.model;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import org.apache.atlas.AtlasClient;
import org.apache.atlas.AtlasException;
import org.apache.atlas.addons.ModelDefinitionDump;
import org.apache.atlas.typesystem.TypesDef;
import org.apache.atlas.typesystem.json.TypesSerialization;
import org.apache.atlas.typesystem.types.AttributeDefinition;
import org.apache.atlas.typesystem.types.ClassType;
import org.apache.atlas.typesystem.types.DataTypes;
import org.apache.atlas.typesystem.types.EnumType;
import org.apache.atlas.typesystem.types.EnumTypeDefinition;
import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition;
import org.apache.atlas.typesystem.types.Multiplicity;
import org.apache.atlas.typesystem.types.StructType;
import org.apache.atlas.typesystem.types.StructTypeDefinition;
import org.apache.atlas.typesystem.types.TraitType;
import org.apache.atlas.typesystem.types.utils.TypesUtil;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
/**
* Utility that generates falcon data model.
*/
public class FalconDataModelGenerator {
private static final Logger LOG = LoggerFactory.getLogger(FalconDataModelGenerator.class);
public static final String FREQUENCY = "frequency";
private final Map<String, HierarchicalTypeDefinition<ClassType>> classTypeDefinitions;
public static final String COLO = "colo";
public static final String TAGS = "tags";
public static final String GROUPS = "groups";
public static final String PIPELINES = "pipelines";
public static final String WFPROPERTIES = "workflow-properties";
public static final String RUNSON = "runs-on";
public static final String STOREDIN = "stored-in";
public FalconDataModelGenerator() {
classTypeDefinitions = new HashMap<>();
}
public void createDataModel() throws AtlasException {
LOG.info("Generating the Falcon Data Model");
// classes
createClusterEntityClass();
createProcessEntityClass();
createFeedCreationEntityClass();
createFeedEntityClass();
createReplicationFeedEntityClass();
}
private TypesDef getTypesDef() {
return TypesUtil.getTypesDef(ImmutableList.<EnumTypeDefinition>of(), ImmutableList.<StructTypeDefinition>of(),
getTraitTypeDefinitions(), getClassTypeDefinitions());
}
public String getDataModelAsJSON() {
return TypesSerialization.toJson(getTypesDef());
}
private ImmutableList<HierarchicalTypeDefinition<ClassType>> getClassTypeDefinitions() {
return ImmutableList.copyOf(classTypeDefinitions.values());
}
private ImmutableList<HierarchicalTypeDefinition<TraitType>> getTraitTypeDefinitions() {
return ImmutableList.of();
}
private void createClusterEntityClass() throws AtlasException {
AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
new AttributeDefinition(COLO, DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false,
false, true, null),
// map of tags
new AttributeDefinition(TAGS,
DataTypes.mapTypeName(DataTypes.STRING_TYPE.getName(), DataTypes.STRING_TYPE.getName()),
Multiplicity.OPTIONAL, false, null),};
HierarchicalTypeDefinition<ClassType> definition =
new HierarchicalTypeDefinition<>(ClassType.class, FalconDataTypes.FALCON_CLUSTER.getName(), null,
ImmutableSet.of(AtlasClient.INFRASTRUCTURE_SUPER_TYPE), attributeDefinitions);
classTypeDefinitions.put(FalconDataTypes.FALCON_CLUSTER.getName(), definition);
LOG.debug("Created definition for {}", FalconDataTypes.FALCON_CLUSTER.getName());
}
private void createFeedCreationEntityClass() throws AtlasException {
AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
new AttributeDefinition(STOREDIN, FalconDataTypes.FALCON_CLUSTER.getName(), Multiplicity.REQUIRED,
false, false, true, null)
};
HierarchicalTypeDefinition<ClassType> definition =
new HierarchicalTypeDefinition<>(ClassType.class, FalconDataTypes.FALCON_FEED_CREATION.getName(), null,
ImmutableSet.of(AtlasClient.PROCESS_SUPER_TYPE), attributeDefinitions);
classTypeDefinitions.put(FalconDataTypes.FALCON_FEED_CREATION.getName(), definition);
LOG.debug("Created definition for {}", FalconDataTypes.FALCON_FEED_CREATION.getName());
}
private void createFeedEntityClass() throws AtlasException {
AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
TypesUtil.createRequiredAttrDef(FREQUENCY, DataTypes.STRING_TYPE),
new AttributeDefinition(STOREDIN, FalconDataTypes.FALCON_CLUSTER.getName(), Multiplicity.REQUIRED,
false, false, true, null),
new AttributeDefinition(GROUPS, DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, null),
// map of tags
new AttributeDefinition(TAGS,
DataTypes.mapTypeName(DataTypes.STRING_TYPE.getName(), DataTypes.STRING_TYPE.getName()),
Multiplicity.OPTIONAL, false, null)
};
HierarchicalTypeDefinition<ClassType> definition =
new HierarchicalTypeDefinition<>(ClassType.class, FalconDataTypes.FALCON_FEED.getName(), null,
ImmutableSet.of(AtlasClient.DATA_SET_SUPER_TYPE), attributeDefinitions);
classTypeDefinitions.put(FalconDataTypes.FALCON_FEED.getName(), definition);
LOG.debug("Created definition for {}", FalconDataTypes.FALCON_FEED.getName());
}
private void createReplicationFeedEntityClass() throws AtlasException {
HierarchicalTypeDefinition<ClassType> definition =
new HierarchicalTypeDefinition<>(ClassType.class,
FalconDataTypes.FALCON_FEED_REPLICATION.getName(), null,
ImmutableSet.of(AtlasClient.PROCESS_SUPER_TYPE), null);
classTypeDefinitions.put(FalconDataTypes.FALCON_FEED_REPLICATION.getName(), definition);
LOG.debug("Created definition for {}", FalconDataTypes.FALCON_FEED_REPLICATION.getName());
}
private void createProcessEntityClass() throws AtlasException {
AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
TypesUtil.createRequiredAttrDef(FREQUENCY, DataTypes.STRING_TYPE),
new AttributeDefinition(RUNSON, FalconDataTypes.FALCON_CLUSTER.getName(), Multiplicity.REQUIRED,
false, false, true, null),
// map of tags
new AttributeDefinition(TAGS,
DataTypes.mapTypeName(DataTypes.STRING_TYPE.getName(), DataTypes.STRING_TYPE.getName()),
Multiplicity.OPTIONAL, false, null),
new AttributeDefinition(PIPELINES, DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, false, true, null),
// wf properties
new AttributeDefinition(WFPROPERTIES,
DataTypes.mapTypeName(DataTypes.STRING_TYPE.getName(), DataTypes.STRING_TYPE.getName()),
Multiplicity.OPTIONAL, false, null),};
HierarchicalTypeDefinition<ClassType> definition =
new HierarchicalTypeDefinition<>(ClassType.class, FalconDataTypes.FALCON_PROCESS.getName(), null,
ImmutableSet.of(AtlasClient.PROCESS_SUPER_TYPE), attributeDefinitions);
classTypeDefinitions.put(FalconDataTypes.FALCON_PROCESS.getName(), definition);
LOG.debug("Created definition for {}", FalconDataTypes.FALCON_PROCESS.getName());
}
public String getModelAsJson() throws AtlasException {
createDataModel();
return getDataModelAsJSON();
}
public static void main(String[] args) throws Exception {
FalconDataModelGenerator falconDataModelGenerator = new FalconDataModelGenerator();
String modelAsJson = falconDataModelGenerator.getModelAsJson();
if (args.length == 1) {
ModelDefinitionDump.dumpModelToFile(args[0], modelAsJson);
return;
}
System.out.println("falconDataModelAsJSON = " + modelAsJson);
TypesDef typesDef = falconDataModelGenerator.getTypesDef();
for (EnumTypeDefinition enumType : typesDef.enumTypesAsJavaList()) {
System.out.println(String.format("%s(%s) - values %s", enumType.name, EnumType.class.getSimpleName(),
Arrays.toString(enumType.enumValues)));
}
for (StructTypeDefinition structType : typesDef.structTypesAsJavaList()) {
System.out.println(
String.format("%s(%s) - attributes %s", structType.typeName, StructType.class.getSimpleName(),
Arrays.toString(structType.attributeDefinitions)));
}
for (HierarchicalTypeDefinition<ClassType> classType : typesDef.classTypesAsJavaList()) {
System.out.println(String.format("%s(%s) - super types [%s] - attributes %s", classType.typeName,
ClassType.class.getSimpleName(),
StringUtils.join(classType.superTypes, ","), Arrays.toString(classType.attributeDefinitions)));
}
for (HierarchicalTypeDefinition<TraitType> traitType : typesDef.traitTypesAsJavaList()) {
System.out.println(String.format("%s(%s) - %s", traitType.typeName, TraitType.class.getSimpleName(),
Arrays.toString(traitType.attributeDefinitions)));
}
}
}
addons/falcon-bridge/src/test/java/org/apache/atlas/falcon/hook/FalconHookIT.java
...@@ -23,9 +23,7 @@ import org.apache.atlas.ApplicationProperties; ...@@ -23,9 +23,7 @@ import org.apache.atlas.ApplicationProperties;
import org.apache.atlas.AtlasClient; import org.apache.atlas.AtlasClient;
import org.apache.atlas.AtlasServiceException; import org.apache.atlas.AtlasServiceException;
import org.apache.atlas.falcon.bridge.FalconBridge; import org.apache.atlas.falcon.bridge.FalconBridge;
import org.apache.atlas.falcon.model.FalconDataModelGenerator;
import org.apache.atlas.falcon.model.FalconDataTypes; import org.apache.atlas.falcon.model.FalconDataTypes;
import org.apache.atlas.fs.model.FSDataTypes;
import org.apache.atlas.hive.bridge.HiveMetaStoreBridge; import org.apache.atlas.hive.bridge.HiveMetaStoreBridge;
import org.apache.atlas.hive.model.HiveDataTypes; import org.apache.atlas.hive.model.HiveDataTypes;
import org.apache.atlas.typesystem.Referenceable; import org.apache.atlas.typesystem.Referenceable;
...@@ -47,7 +45,6 @@ import org.apache.falcon.entity.v0.feed.Location; ...@@ -47,7 +45,6 @@ import org.apache.falcon.entity.v0.feed.Location;
import org.apache.falcon.entity.v0.feed.LocationType; import org.apache.falcon.entity.v0.feed.LocationType;
import org.apache.falcon.entity.v0.process.Process; import org.apache.falcon.entity.v0.process.Process;
import org.apache.falcon.security.CurrentUser; import org.apache.falcon.security.CurrentUser;
import org.apache.hadoop.hive.conf.HiveConf;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.testng.annotations.BeforeClass; import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test; import org.testng.annotations.Test;
...@@ -84,24 +81,9 @@ public class FalconHookIT { ...@@ -84,24 +81,9 @@ public class FalconHookIT {
AtlasService service = new AtlasService(); AtlasService service = new AtlasService();
service.init(); service.init();
STORE.registerListener(service); STORE.registerListener(service);
registerFalconDataModel();
CurrentUser.authenticate(System.getProperty("user.name")); CurrentUser.authenticate(System.getProperty("user.name"));
} }
private void registerFalconDataModel() throws Exception {
if (isDataModelAlreadyRegistered()) {
LOG.info("Falcon data model is already registered!");
return;
}
HiveMetaStoreBridge hiveMetaStoreBridge = new HiveMetaStoreBridge(ApplicationProperties.get(), new HiveConf(), atlasClient);
hiveMetaStoreBridge.registerHiveDataModel();
FalconDataModelGenerator dataModelGenerator = new FalconDataModelGenerator();
LOG.info("Registering Falcon data model");
atlasClient.createType(dataModelGenerator.getModelAsJson());
}
private boolean isDataModelAlreadyRegistered() throws Exception { private boolean isDataModelAlreadyRegistered() throws Exception {
try { try {
atlasClient.getType(FalconDataTypes.FALCON_PROCESS.getName()); atlasClient.getType(FalconDataTypes.FALCON_PROCESS.getName());
...@@ -196,7 +178,7 @@ public class FalconHookIT { ...@@ -196,7 +178,7 @@ public class FalconHookIT {
String inputId = ((List<Id>) processEntity.get("inputs")).get(0).getId()._getId(); String inputId = ((List<Id>) processEntity.get("inputs")).get(0).getId()._getId();
Referenceable pathEntity = atlasClient.getEntity(inputId); Referenceable pathEntity = atlasClient.getEntity(inputId);
assertEquals(pathEntity.getTypeName(), FSDataTypes.HDFS_PATH().toString()); assertEquals(pathEntity.getTypeName(), HiveMetaStoreBridge.HDFS_PATH.toString());
List<Location> locations = FeedHelper.getLocations(feedCluster, feed); List<Location> locations = FeedHelper.getLocations(feedCluster, feed);
Location dataLocation = FileSystemStorage.getLocation(locations, LocationType.DATA); Location dataLocation = FileSystemStorage.getLocation(locations, LocationType.DATA);
...@@ -243,7 +225,7 @@ public class FalconHookIT { ...@@ -243,7 +225,7 @@ public class FalconHookIT {
private void assertFeedAttributes(String feedId) throws Exception { private void assertFeedAttributes(String feedId) throws Exception {
Referenceable feedEntity = atlasClient.getEntity(feedId); Referenceable feedEntity = atlasClient.getEntity(feedId);
assertEquals(feedEntity.get(AtlasClient.OWNER), "testuser"); assertEquals(feedEntity.get(AtlasClient.OWNER), "testuser");
assertEquals(feedEntity.get(FalconDataModelGenerator.FREQUENCY), "hours(1)"); assertEquals(feedEntity.get(FalconBridge.FREQUENCY), "hours(1)");
assertEquals(feedEntity.get(AtlasClient.DESCRIPTION), "test input"); assertEquals(feedEntity.get(AtlasClient.DESCRIPTION), "test input");
} }
......
addons/hdfs-model/pom.xml
...@@ -180,24 +180,6 @@ ...@@ -180,24 +180,6 @@
<version>1.2.1</version> <version>1.2.1</version>
<inherited>false</inherited> <inherited>false</inherited>
<executions> <executions>
<execution>
<configuration>
<mainClass>org.apache.atlas.fs.model.FSDataModelGenerator</mainClass>
<systemProperties>
<systemProperty>
<key>atlas.conf</key>
<value>${project.build.directory}/../../../typesystem/src/test/resources/</value>
</systemProperty>
</systemProperties>
<arguments>
<argument>${project.build.directory}/models/fs_model.json</argument>
</arguments>
</configuration>
<phase>package</phase>
<goals>
<goal>java</goal>
</goals>
</execution>
</executions> </executions>
</plugin> </plugin>
......
addons/hdfs-model/src/main/java/org/apache/atlas/fs/model/FSDataModelGenerator.java deleted 100644 → 0
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.fs.model;
import org.apache.atlas.addons.ModelDefinitionDump;
import org.apache.atlas.typesystem.TypesDef;
import org.apache.atlas.typesystem.json.TypesSerialization;
import java.io.IOException;
public class FSDataModelGenerator {
public static void main(String[] args) throws IOException {
FSDataModel.main(args);
TypesDef typesDef = FSDataModel.typesDef();
String fsTypesAsJSON = TypesSerialization.toJson(typesDef);
if (args.length == 1) {
ModelDefinitionDump.dumpModelToFile(args[0], fsTypesAsJSON);
return;
}
System.out.println("FS Data Model as JSON = " + fsTypesAsJSON);
}
}
addons/hdfs-model/src/main/scala/org/apache/atlas/fs/model/FSDataModel.scala deleted 100644 → 0
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.fs.model
import org.apache.atlas.{AtlasConstants, AtlasClient}
import org.apache.atlas.typesystem.TypesDef
import org.apache.atlas.typesystem.builders.TypesBuilder
import org.apache.atlas.typesystem.json.TypesSerialization
import org.apache.atlas.typesystem.types.DataTypes.MapType
import org.apache.hadoop.fs.permission.FsAction
import scala.tools.scalap.scalax.rules.scalasig.ClassFileParser.EnumConstValue
/**
* This represents the data model for a HDFS Path
*/
object FSDataModel extends App {
val typesBuilder = new TypesBuilder
import typesBuilder._
val typesDef : TypesDef = types {
// FS DataSet
_class(FSDataTypes.FS_PATH.toString, List(AtlasClient.DATA_SET_SUPER_TYPE)) {
//fully qualified path/URI to the filesystem path is stored in 'qualifiedName' and 'path'.
"path" ~ (string, required, indexed)
"createTime" ~ (date, optional, indexed)
"modifiedTime" ~ (date, optional, indexed)
//Is a regular file or a directory. If true, it is a file else a directory
"isFile" ~ (boolean, optional, indexed)
//Is a symlink or not
"isSymlink" ~ (boolean, optional)
//Optional and may not be set for a directory
"fileSize" ~ (long, optional)
"group" ~ (string, optional, indexed)
"posixPermissions" ~ (FSDataTypes.FS_PERMISSIONS.toString, optional, indexed)
}
enum(FSDataTypes.FS_ACTION.toString, FsAction.values().map(x => x.name()) : _*)
struct(FSDataTypes.FS_PERMISSIONS.toString) {
PosixPermissions.PERM_USER.toString ~ (FSDataTypes.FS_ACTION.toString, required, indexed)
PosixPermissions.PERM_GROUP.toString ~ (FSDataTypes.FS_ACTION.toString, required, indexed)
PosixPermissions.PERM_OTHER.toString ~ (FSDataTypes.FS_ACTION.toString, required, indexed)
PosixPermissions.STICKY_BIT.toString ~ (boolean, required, indexed)
}
//HDFS DataSet
_class(FSDataTypes.HDFS_PATH.toString, List(FSDataTypes.FS_PATH.toString)) {
//Making cluster optional since path is already unique containing the namenode URI
AtlasConstants.CLUSTER_NAME_ATTRIBUTE ~ (string, optional, indexed)
"numberOfReplicas" ~ (int, optional, indexed)
"extendedAttributes" ~ (map(string, string), optional)
}
//TODO - ACLs - https://hadoop.apache.org/docs/r2.7.1/hadoop-project-dist/hadoop-hdfs/HdfsPermissionsGuide.html#ACLs_Access_Control_Lists
}
// add the types to atlas
val typesAsJSON = TypesSerialization.toJson(typesDef)
println("FS Data Model as JSON: ")
println(typesAsJSON)
}
object FSDataTypes extends Enumeration {
type FSDataTypes = Value
val FS_ACTION = Value("file_action")
val FS_PATH = Value("fs_path")
val HDFS_PATH = Value("hdfs_path")
val FS_PERMISSIONS = Value("fs_permissions")
}
object PosixPermissions extends Enumeration {
type PosixPermissions = Value
val PERM_USER = Value("user")
val PERM_GROUP = Value("group")
val PERM_OTHER = Value("others")
val STICKY_BIT = Value("sticky")
}
addons/hdfs-model/src/test/java/org/apache/atlas/fs/model/HDFSModelTest.java deleted 100644 → 0
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.fs.model;
import javax.inject.Inject;
import org.apache.atlas.RepositoryMetadataModule;
import org.apache.atlas.repository.graph.AtlasGraphProvider;
import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.services.MetadataService;
import org.apache.atlas.typesystem.TypesDef;
import org.apache.atlas.typesystem.json.TypesSerialization;
import org.apache.atlas.typesystem.types.TypeSystem;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.testng.Assert;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Guice;
import org.testng.annotations.Test;
import scala.Enumeration;
import scala.collection.Iterator;
@Test
@Guice(modules = RepositoryMetadataModule.class)
public class HDFSModelTest {
public static final Logger LOG = LoggerFactory.getLogger(HDFSModelTest.class);
private static final String ATLAS_URL = "http://localhost:21000/";
@Inject
private MetadataService metadataService;
@BeforeClass
public void setUp() throws Exception {
}
@AfterClass
public void tearDown() throws Exception {
TypeSystem.getInstance().reset();
AtlasGraphProvider.cleanup();
}
@Test
public void testCreateDataModel() throws Exception {
FSDataModel.main(new String[]{});
TypesDef fsTypesDef = FSDataModel.typesDef();
String fsTypesAsJSON = TypesSerialization.toJson(fsTypesDef);
LOG.info("fsTypesAsJSON = {}", fsTypesAsJSON);
metadataService.createType(fsTypesAsJSON);
// verify types are registered
final Iterator<Enumeration.Value> valueIterator = FSDataTypes.values().iterator();
while (valueIterator.hasNext() ) {
final Enumeration.Value typeEnum = valueIterator.next();
String typeDefStr = metadataService.getTypeDefinition(typeEnum.toString());
Assert.assertNotNull(typeDefStr);
TypesDef typesDef = TypesSerialization.fromJson(typeDefStr);
Assert.assertNotNull(typesDef);
}
}
}
\ No newline at end of file
addons/hive-bridge/pom.xml
...@@ -399,24 +399,6 @@ ...@@ -399,24 +399,6 @@
<version>1.2.1</version> <version>1.2.1</version>
<inherited>false</inherited> <inherited>false</inherited>
<executions> <executions>
<execution>
<configuration>
<mainClass>org.apache.atlas.hive.model.HiveDataModelGenerator</mainClass>
<systemProperties>
<systemProperty>
<key>atlas.conf</key>
<value>${project.build.directory}/../../../typesystem/target/test-classes</value>
</systemProperty>
</systemProperties>
<arguments>
<argument>${project.build.directory}/models/hive_model.json</argument>
</arguments>
</configuration>
<phase>package</phase>
<goals>
<goal>java</goal>
</goals>
</execution>
</executions> </executions>
</plugin> </plugin>
</plugins> </plugins>
......
addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/ColumnLineageUtils.java
...@@ -19,8 +19,6 @@ ...@@ -19,8 +19,6 @@
package org.apache.atlas.hive.bridge; package org.apache.atlas.hive.bridge;
import org.apache.atlas.AtlasClient; import org.apache.atlas.AtlasClient;
import org.apache.atlas.hive.hook.HiveHook;
import org.apache.atlas.hive.model.HiveDataModelGenerator;
import org.apache.atlas.hive.model.HiveDataTypes; import org.apache.atlas.hive.model.HiveDataTypes;
import org.apache.atlas.typesystem.Referenceable; import org.apache.atlas.typesystem.Referenceable;
import org.apache.hadoop.hive.ql.hooks.LineageInfo; import org.apache.hadoop.hive.ql.hooks.LineageInfo;
...@@ -31,7 +29,6 @@ import java.util.ArrayList; ...@@ -31,7 +29,6 @@ import java.util.ArrayList;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Objects;
public class ColumnLineageUtils { public class ColumnLineageUtils {
public static final Logger LOG = LoggerFactory.getLogger(ColumnLineageUtils.class); public static final Logger LOG = LoggerFactory.getLogger(ColumnLineageUtils.class);
...@@ -92,7 +89,7 @@ public class ColumnLineageUtils { ...@@ -92,7 +89,7 @@ public class ColumnLineageUtils {
if (r.getTypeName().equals(HiveDataTypes.HIVE_TABLE.getName())) { if (r.getTypeName().equals(HiveDataTypes.HIVE_TABLE.getName())) {
String qName = (String) r.get(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME); String qName = (String) r.get(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME);
String[] qNameComps = extractComponents(qName); String[] qNameComps = extractComponents(qName);
for (Referenceable col : (List<Referenceable>) r.get(HiveDataModelGenerator.COLUMNS)) { for (Referenceable col : (List<Referenceable>) r.get(HiveMetaStoreBridge.COLUMNS)) {
String cName = (String) col.get(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME); String cName = (String) col.get(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME);
String[] colQNameComps = extractComponents(cName); String[] colQNameComps = extractComponents(cName);
String colQName = colQNameComps[0] + "." + colQNameComps[1] + "." + colQNameComps[2]; String colQName = colQNameComps[0] + "." + colQNameComps[1] + "." + colQNameComps[2];
......
addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java
...@@ -26,15 +26,11 @@ import org.apache.atlas.ApplicationProperties; ...@@ -26,15 +26,11 @@ import org.apache.atlas.ApplicationProperties;
import org.apache.atlas.AtlasClient; import org.apache.atlas.AtlasClient;
import org.apache.atlas.AtlasConstants; import org.apache.atlas.AtlasConstants;
import org.apache.atlas.AtlasServiceException; import org.apache.atlas.AtlasServiceException;
import org.apache.atlas.fs.model.FSDataModel;
import org.apache.atlas.fs.model.FSDataTypes;
import org.apache.atlas.hive.hook.HiveHook; import org.apache.atlas.hive.hook.HiveHook;
import org.apache.atlas.hive.model.HiveDataModelGenerator;
import org.apache.atlas.hive.model.HiveDataTypes; import org.apache.atlas.hive.model.HiveDataTypes;
import org.apache.atlas.typesystem.Referenceable; import org.apache.atlas.typesystem.Referenceable;
import org.apache.atlas.typesystem.Struct; import org.apache.atlas.typesystem.Struct;
import org.apache.atlas.typesystem.json.InstanceSerialization; import org.apache.atlas.typesystem.json.InstanceSerialization;
import org.apache.atlas.typesystem.json.TypesSerialization;
import org.apache.atlas.typesystem.persistence.Id; import org.apache.atlas.typesystem.persistence.Id;
import org.apache.atlas.utils.AuthenticationUtil; import org.apache.atlas.utils.AuthenticationUtil;
import org.apache.commons.cli.BasicParser; import org.apache.commons.cli.BasicParser;
...@@ -79,6 +75,25 @@ public class HiveMetaStoreBridge { ...@@ -79,6 +75,25 @@ public class HiveMetaStoreBridge {
public static final String ATLAS_ENDPOINT = "atlas.rest.address"; public static final String ATLAS_ENDPOINT = "atlas.rest.address";
public static final String COMMENT = "comment";
public static final String PARAMETERS = "parameters";
public static final String COLUMNS = "columns";
public static final String POSITION = "position";
public static final String PART_COLS = "partitionKeys";
public static final String TABLE_ALIAS_LIST = "aliases";
public static final String STORAGE_NUM_BUCKETS = "numBuckets";
public static final String STORAGE_IS_STORED_AS_SUB_DIRS = "storedAsSubDirectories";
public static final String TABLE = "table";
public static final String DB = "db";
public static final String STORAGE_DESC = "sd";
public static final String STORAGE_DESC_INPUT_FMT = "inputFormat";
public static final String STORAGE_DESC_OUTPUT_FMT = "outputFormat";
public static final String LOCATION = "location";
public static final String TABLE_TYPE_ATTR = "tableType";
public static final String CREATE_TIME = "createTime";
public static final String LAST_ACCESS_TIME = "lastAccessTime";
public static final String HDFS_PATH = "hdfs_path";
private static final Logger LOG = LoggerFactory.getLogger(HiveMetaStoreBridge.class); private static final Logger LOG = LoggerFactory.getLogger(HiveMetaStoreBridge.class);
public final Hive hiveClient; public final Hive hiveClient;
...@@ -174,8 +189,8 @@ public class HiveMetaStoreBridge { ...@@ -174,8 +189,8 @@ public class HiveMetaStoreBridge {
dbRef.set(AtlasClient.NAME, dbName); dbRef.set(AtlasClient.NAME, dbName);
dbRef.set(AtlasConstants.CLUSTER_NAME_ATTRIBUTE, clusterName); dbRef.set(AtlasConstants.CLUSTER_NAME_ATTRIBUTE, clusterName);
dbRef.set(DESCRIPTION_ATTR, hiveDB.getDescription()); dbRef.set(DESCRIPTION_ATTR, hiveDB.getDescription());
dbRef.set(HiveDataModelGenerator.LOCATION, hiveDB.getLocationUri()); dbRef.set(LOCATION, hiveDB.getLocationUri());
dbRef.set(HiveDataModelGenerator.PARAMETERS, hiveDB.getParameters()); dbRef.set(PARAMETERS, hiveDB.getParameters());
dbRef.set(AtlasClient.OWNER, hiveDB.getOwnerName()); dbRef.set(AtlasClient.OWNER, hiveDB.getOwnerName());
if (hiveDB.getOwnerType() != null) { if (hiveDB.getOwnerType() != null) {
dbRef.set("ownerType", hiveDB.getOwnerType().getValue()); dbRef.set("ownerType", hiveDB.getOwnerType().getValue());
...@@ -431,7 +446,7 @@ public class HiveMetaStoreBridge { ...@@ -431,7 +446,7 @@ public class HiveMetaStoreBridge {
try { try {
createDate = getTableCreatedTime(hiveTable); createDate = getTableCreatedTime(hiveTable);
LOG.debug("Setting create time to {} ", createDate); LOG.debug("Setting create time to {} ", createDate);
tableReference.set(HiveDataModelGenerator.CREATE_TIME, createDate); tableReference.set(CREATE_TIME, createDate);
} catch(Exception ne) { } catch(Exception ne) {
LOG.error("Error while setting createTime for the table {} ", hiveTable.getCompleteName(), ne); LOG.error("Error while setting createTime for the table {} ", hiveTable.getCompleteName(), ne);
} }
...@@ -441,19 +456,19 @@ public class HiveMetaStoreBridge { ...@@ -441,19 +456,19 @@ public class HiveMetaStoreBridge {
if ( hiveTable.getLastAccessTime() > 0) { if ( hiveTable.getLastAccessTime() > 0) {
lastAccessTime = new Date(hiveTable.getLastAccessTime() * MILLIS_CONVERT_FACTOR); lastAccessTime = new Date(hiveTable.getLastAccessTime() * MILLIS_CONVERT_FACTOR);
} }
tableReference.set(HiveDataModelGenerator.LAST_ACCESS_TIME, lastAccessTime); tableReference.set(LAST_ACCESS_TIME, lastAccessTime);
tableReference.set("retention", hiveTable.getRetention()); tableReference.set("retention", hiveTable.getRetention());
tableReference.set(HiveDataModelGenerator.COMMENT, hiveTable.getParameters().get(HiveDataModelGenerator.COMMENT)); tableReference.set(COMMENT, hiveTable.getParameters().get(COMMENT));
// add reference to the database // add reference to the database
tableReference.set(HiveDataModelGenerator.DB, dbReference); tableReference.set(DB, dbReference);
// add reference to the StorageDescriptor // add reference to the StorageDescriptor
Referenceable sdReferenceable = fillStorageDesc(hiveTable.getSd(), tableQualifiedName, getStorageDescQFName(tableQualifiedName), tableReference.getId()); Referenceable sdReferenceable = fillStorageDesc(hiveTable.getSd(), tableQualifiedName, getStorageDescQFName(tableQualifiedName), tableReference.getId());
tableReference.set(HiveDataModelGenerator.STORAGE_DESC, sdReferenceable); tableReference.set(STORAGE_DESC, sdReferenceable);
tableReference.set(HiveDataModelGenerator.PARAMETERS, hiveTable.getParameters()); tableReference.set(PARAMETERS, hiveTable.getParameters());
if (hiveTable.getViewOriginalText() != null) { if (hiveTable.getViewOriginalText() != null) {
tableReference.set("viewOriginalText", hiveTable.getViewOriginalText()); tableReference.set("viewOriginalText", hiveTable.getViewOriginalText());
...@@ -463,14 +478,14 @@ public class HiveMetaStoreBridge { ...@@ -463,14 +478,14 @@ public class HiveMetaStoreBridge {
tableReference.set("viewExpandedText", hiveTable.getViewExpandedText()); tableReference.set("viewExpandedText", hiveTable.getViewExpandedText());
} }
tableReference.set(HiveDataModelGenerator.TABLE_TYPE_ATTR, hiveTable.getTableType().name()); tableReference.set(TABLE_TYPE_ATTR, hiveTable.getTableType().name());
tableReference.set("temporary", hiveTable.isTemporary()); tableReference.set("temporary", hiveTable.isTemporary());
// add reference to the Partition Keys // add reference to the Partition Keys
List<Referenceable> partKeys = getColumns(hiveTable.getPartitionKeys(), tableReference); List<Referenceable> partKeys = getColumns(hiveTable.getPartitionKeys(), tableReference);
tableReference.set("partitionKeys", partKeys); tableReference.set("partitionKeys", partKeys);
tableReference.set(HiveDataModelGenerator.COLUMNS, getColumns(hiveTable.getCols(), tableReference)); tableReference.set(COLUMNS, getColumns(hiveTable.getCols(), tableReference));
return tableReference; return tableReference;
} }
...@@ -523,12 +538,12 @@ public class HiveMetaStoreBridge { ...@@ -523,12 +538,12 @@ public class HiveMetaStoreBridge {
serdeInfoStruct.set(AtlasClient.NAME, serdeInfo.getName()); serdeInfoStruct.set(AtlasClient.NAME, serdeInfo.getName());
serdeInfoStruct.set("serializationLib", serdeInfo.getSerializationLib()); serdeInfoStruct.set("serializationLib", serdeInfo.getSerializationLib());
serdeInfoStruct.set(HiveDataModelGenerator.PARAMETERS, serdeInfo.getParameters()); serdeInfoStruct.set(PARAMETERS, serdeInfo.getParameters());
sdReferenceable.set("serdeInfo", serdeInfoStruct); sdReferenceable.set("serdeInfo", serdeInfoStruct);
sdReferenceable.set(HiveDataModelGenerator.STORAGE_NUM_BUCKETS, storageDesc.getNumBuckets()); sdReferenceable.set(STORAGE_NUM_BUCKETS, storageDesc.getNumBuckets());
sdReferenceable sdReferenceable
.set(HiveDataModelGenerator.STORAGE_IS_STORED_AS_SUB_DIRS, storageDesc.isStoredAsSubDirectories()); .set(STORAGE_IS_STORED_AS_SUB_DIRS, storageDesc.isStoredAsSubDirectories());
List<Struct> sortColsStruct = new ArrayList<>(); List<Struct> sortColsStruct = new ArrayList<>();
for (Order sortcol : storageDesc.getSortCols()) { for (Order sortcol : storageDesc.getSortCols()) {
...@@ -543,7 +558,7 @@ public class HiveMetaStoreBridge { ...@@ -543,7 +558,7 @@ public class HiveMetaStoreBridge {
sdReferenceable.set("sortCols", sortColsStruct); sdReferenceable.set("sortCols", sortColsStruct);
} }
sdReferenceable.set(HiveDataModelGenerator.LOCATION, storageDesc.getLocation()); sdReferenceable.set(LOCATION, storageDesc.getLocation());
sdReferenceable.set("inputFormat", storageDesc.getInputFormat()); sdReferenceable.set("inputFormat", storageDesc.getInputFormat());
sdReferenceable.set("outputFormat", storageDesc.getOutputFormat()); sdReferenceable.set("outputFormat", storageDesc.getOutputFormat());
sdReferenceable.set("compressed", storageDesc.isCompressed()); sdReferenceable.set("compressed", storageDesc.isCompressed());
...@@ -552,15 +567,15 @@ public class HiveMetaStoreBridge { ...@@ -552,15 +567,15 @@ public class HiveMetaStoreBridge {
sdReferenceable.set("bucketCols", storageDesc.getBucketCols()); sdReferenceable.set("bucketCols", storageDesc.getBucketCols());
} }
sdReferenceable.set(HiveDataModelGenerator.PARAMETERS, storageDesc.getParameters()); sdReferenceable.set(PARAMETERS, storageDesc.getParameters());
sdReferenceable.set("storedAsSubDirectories", storageDesc.isStoredAsSubDirectories()); sdReferenceable.set("storedAsSubDirectories", storageDesc.isStoredAsSubDirectories());
sdReferenceable.set(HiveDataModelGenerator.TABLE, tableId); sdReferenceable.set(TABLE, tableId);
return sdReferenceable; return sdReferenceable;
} }
public Referenceable fillHDFSDataSet(String pathUri) { public Referenceable fillHDFSDataSet(String pathUri) {
Referenceable ref = new Referenceable(FSDataTypes.HDFS_PATH().toString()); Referenceable ref = new Referenceable(HDFS_PATH.toString());
ref.set("path", pathUri); ref.set("path", pathUri);
Path path = new Path(pathUri); Path path = new Path(pathUri);
ref.set(AtlasClient.NAME, Path.getPathWithoutSchemeAndAuthority(path).toString().toLowerCase()); ref.set(AtlasClient.NAME, Path.getPathWithoutSchemeAndAuthority(path).toString().toLowerCase());
...@@ -586,9 +601,9 @@ public class HiveMetaStoreBridge { ...@@ -586,9 +601,9 @@ public class HiveMetaStoreBridge {
colReferenceable.set(AtlasClient.NAME, fs.getName()); colReferenceable.set(AtlasClient.NAME, fs.getName());
colReferenceable.set(AtlasClient.OWNER, tableReference.get(AtlasClient.OWNER)); colReferenceable.set(AtlasClient.OWNER, tableReference.get(AtlasClient.OWNER));
colReferenceable.set("type", fs.getType()); colReferenceable.set("type", fs.getType());
colReferenceable.set(HiveDataModelGenerator.POSITION, columnPosition++); colReferenceable.set(POSITION, columnPosition++);
colReferenceable.set(HiveDataModelGenerator.COMMENT, fs.getComment()); colReferenceable.set(COMMENT, fs.getComment());
colReferenceable.set(HiveDataModelGenerator.TABLE, tableReference.getId()); colReferenceable.set(TABLE, tableReference.getId());
colList.add(colReferenceable); colList.add(colReferenceable);
...@@ -596,43 +611,6 @@ public class HiveMetaStoreBridge { ...@@ -596,43 +611,6 @@ public class HiveMetaStoreBridge {
return colList; return colList;
} }
/**
* Register the Hive DataModel in Atlas, if not already defined.
*
* The method checks for the presence of the type {@link HiveDataTypes#HIVE_PROCESS} with the Atlas server.
* If this type is defined, then we assume the Hive DataModel is registered.
* @throws Exception
*/
public synchronized void registerHiveDataModel() throws Exception {
HiveDataModelGenerator dataModelGenerator = new HiveDataModelGenerator();
AtlasClient dgiClient = getAtlasClient();
try {
dgiClient.getType(FSDataTypes.HDFS_PATH().toString());
LOG.info("HDFS data model is already registered!");
} catch(AtlasServiceException ase) {
if (ase.getStatus() == ClientResponse.Status.NOT_FOUND) {
//Trigger val definition
FSDataModel.main(null);
final String hdfsModelJson = TypesSerialization.toJson(FSDataModel.typesDef());
//Expected in case types do not exist
LOG.info("Registering HDFS data model : " + hdfsModelJson);
dgiClient.createType(hdfsModelJson);
}
}
try {
dgiClient.getType(HiveDataTypes.HIVE_PROCESS.getName());
LOG.info("Hive data model is already registered!");
} catch(AtlasServiceException ase) {
if (ase.getStatus() == ClientResponse.Status.NOT_FOUND) {
//Expected in case types do not exist
LOG.info("Registering Hive data model");
dgiClient.createType(dataModelGenerator.getModelAsJson());
}
}
}
public static void main(String[] args) throws Exception { public static void main(String[] args) throws Exception {
...@@ -661,7 +639,6 @@ public class HiveMetaStoreBridge { ...@@ -661,7 +639,6 @@ public class HiveMetaStoreBridge {
} }
HiveMetaStoreBridge hiveMetaStoreBridge = new HiveMetaStoreBridge(atlasConf, new HiveConf(), atlasClient); HiveMetaStoreBridge hiveMetaStoreBridge = new HiveMetaStoreBridge(atlasConf, new HiveConf(), atlasClient);
hiveMetaStoreBridge.registerHiveDataModel();
hiveMetaStoreBridge.importHiveMetadata(failOnError); hiveMetaStoreBridge.importHiveMetadata(failOnError);
} }
} }
addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java
...@@ -25,7 +25,6 @@ import org.apache.atlas.AtlasClient; ...@@ -25,7 +25,6 @@ import org.apache.atlas.AtlasClient;
import org.apache.atlas.AtlasConstants; import org.apache.atlas.AtlasConstants;
import org.apache.atlas.hive.bridge.HiveMetaStoreBridge; import org.apache.atlas.hive.bridge.HiveMetaStoreBridge;
import org.apache.atlas.hive.bridge.ColumnLineageUtils; import org.apache.atlas.hive.bridge.ColumnLineageUtils;
import org.apache.atlas.hive.model.HiveDataModelGenerator;
import org.apache.atlas.hive.model.HiveDataTypes; import org.apache.atlas.hive.model.HiveDataTypes;
import org.apache.atlas.hook.AtlasHook; import org.apache.atlas.hook.AtlasHook;
import org.apache.atlas.notification.hook.HookNotification; import org.apache.atlas.notification.hook.HookNotification;
...@@ -411,10 +410,10 @@ public class HiveHook extends AtlasHook implements ExecuteWithHookContext { ...@@ -411,10 +410,10 @@ public class HiveHook extends AtlasHook implements ExecuteWithHookContext {
Referenceable tableEntity = tables.get(Type.TABLE); Referenceable tableEntity = tables.get(Type.TABLE);
//Reset regular column QF Name to old Name and create a new partial notification request to replace old column QFName to newName to retain any existing traits //Reset regular column QF Name to old Name and create a new partial notification request to replace old column QFName to newName to retain any existing traits
replaceColumnQFName(event, (List<Referenceable>) tableEntity.get(HiveDataModelGenerator.COLUMNS), oldQualifiedName, newQualifiedName); replaceColumnQFName(event, (List<Referenceable>) tableEntity.get(HiveMetaStoreBridge.COLUMNS), oldQualifiedName, newQualifiedName);
//Reset partition key column QF Name to old Name and create a new partial notification request to replace old column QFName to newName to retain any existing traits //Reset partition key column QF Name to old Name and create a new partial notification request to replace old column QFName to newName to retain any existing traits
replaceColumnQFName(event, (List<Referenceable>) tableEntity.get(HiveDataModelGenerator.PART_COLS), oldQualifiedName, newQualifiedName); replaceColumnQFName(event, (List<Referenceable>) tableEntity.get(HiveMetaStoreBridge.PART_COLS), oldQualifiedName, newQualifiedName);
//Reset SD QF Name to old Name and create a new partial notification request to replace old SD QFName to newName to retain any existing traits //Reset SD QF Name to old Name and create a new partial notification request to replace old SD QFName to newName to retain any existing traits
replaceSDQFName(event, tableEntity, oldQualifiedName, newQualifiedName); replaceSDQFName(event, tableEntity, oldQualifiedName, newQualifiedName);
...@@ -437,7 +436,7 @@ public class HiveHook extends AtlasHook implements ExecuteWithHookContext { ...@@ -437,7 +436,7 @@ public class HiveHook extends AtlasHook implements ExecuteWithHookContext {
ArrayList<String> alias_list = new ArrayList<>(); ArrayList<String> alias_list = new ArrayList<>();
alias_list.add(oldTable.getTableName().toLowerCase()); alias_list.add(oldTable.getTableName().toLowerCase());
newEntity.set(HiveDataModelGenerator.TABLE_ALIAS_LIST, alias_list); newEntity.set(HiveMetaStoreBridge.TABLE_ALIAS_LIST, alias_list);
event.addMessage(new HookNotification.EntityPartialUpdateRequest(event.getUser(), event.addMessage(new HookNotification.EntityPartialUpdateRequest(event.getUser(),
HiveDataTypes.HIVE_TABLE.getName(), AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, HiveDataTypes.HIVE_TABLE.getName(), AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME,
oldTableQFName, newEntity)); oldTableQFName, newEntity));
...@@ -466,7 +465,7 @@ public class HiveHook extends AtlasHook implements ExecuteWithHookContext { ...@@ -466,7 +465,7 @@ public class HiveHook extends AtlasHook implements ExecuteWithHookContext {
private Referenceable replaceSDQFName(final HiveEventContext event, Referenceable tableEntity, final String oldTblQFName, final String newTblQFName) { private Referenceable replaceSDQFName(final HiveEventContext event, Referenceable tableEntity, final String oldTblQFName, final String newTblQFName) {
//Reset storage desc QF Name to old Name //Reset storage desc QF Name to old Name
final Referenceable sdRef = ((Referenceable) tableEntity.get(HiveDataModelGenerator.STORAGE_DESC)); final Referenceable sdRef = ((Referenceable) tableEntity.get(HiveMetaStoreBridge.STORAGE_DESC));
sdRef.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, HiveMetaStoreBridge.getStorageDescQFName(oldTblQFName)); sdRef.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, HiveMetaStoreBridge.getStorageDescQFName(oldTblQFName));
//Replace SD QF name first to retain tags //Replace SD QF name first to retain tags
......
addons/hive-bridge/src/main/java/org/apache/atlas/hive/model/HiveDataModelGenerator.java deleted 100755 → 0
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.hive.model;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import org.apache.atlas.AtlasClient;
import org.apache.atlas.AtlasConstants;
import org.apache.atlas.AtlasException;
import org.apache.atlas.addons.ModelDefinitionDump;
import org.apache.atlas.typesystem.TypesDef;
import org.apache.atlas.typesystem.json.TypesSerialization;
import org.apache.atlas.typesystem.types.AttributeDefinition;
import org.apache.atlas.typesystem.types.ClassType;
import org.apache.atlas.typesystem.types.DataTypes;
import org.apache.atlas.typesystem.types.EnumType;
import org.apache.atlas.typesystem.types.EnumTypeDefinition;
import org.apache.atlas.typesystem.types.EnumValue;
import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition;
import org.apache.atlas.typesystem.types.Multiplicity;
import org.apache.atlas.typesystem.types.StructType;
import org.apache.atlas.typesystem.types.StructTypeDefinition;
import org.apache.atlas.typesystem.types.TraitType;
import org.apache.atlas.typesystem.types.utils.TypesUtil;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
/**
* Utility that generates hive data model for both metastore entities and DDL/DML queries.
*/
public class HiveDataModelGenerator {
private static final Logger LOG = LoggerFactory.getLogger(HiveDataModelGenerator.class);
private static final DataTypes.MapType STRING_MAP_TYPE =
new DataTypes.MapType(DataTypes.STRING_TYPE, DataTypes.STRING_TYPE);
private final Map<String, HierarchicalTypeDefinition<ClassType>> classTypeDefinitions;
private final Map<String, EnumTypeDefinition> enumTypeDefinitionMap;
private final Map<String, StructTypeDefinition> structTypeDefinitionMap;
public static final String COMMENT = "comment";
public static final String PARAMETERS = "parameters";
public static final String COLUMNS = "columns";
public static final String POSITION = "position";
public static final String PART_COLS = "partitionKeys";
public static final String TABLE_ALIAS_LIST = "aliases";
public static final String STORAGE_NUM_BUCKETS = "numBuckets";
public static final String STORAGE_IS_STORED_AS_SUB_DIRS = "storedAsSubDirectories";
public static final String TABLE = "table";
public static final String DB = "db";
public static final String STORAGE_DESC = "sd";
public static final String STORAGE_DESC_INPUT_FMT = "inputFormat";
public static final String STORAGE_DESC_OUTPUT_FMT = "outputFormat";
public static final String LOCATION = "location";
public static final String TABLE_TYPE_ATTR = "tableType";
public static final String CREATE_TIME = "createTime";
public static final String LAST_ACCESS_TIME = "lastAccessTime";
public HiveDataModelGenerator() {
classTypeDefinitions = new HashMap<>();
enumTypeDefinitionMap = new HashMap<>();
structTypeDefinitionMap = new HashMap<>();
}
public void createDataModel() throws AtlasException {
LOG.info("Generating the Hive Data Model....");
// enums
createHivePrincipalTypeEnum();
// structs
createSerDeStruct();
createOrderStruct();
createStorageDescClass();
// classes
createDBClass();
createColumnClass();
createTableClass();
// DDL/DML Process
createProcessClass();
createColumnLineageClass();
}
public TypesDef getTypesDef() {
return TypesUtil.getTypesDef(getEnumTypeDefinitions(), getStructTypeDefinitions(), getTraitTypeDefinitions(),
getClassTypeDefinitions());
}
public String getDataModelAsJSON() {
return TypesSerialization.toJson(getTypesDef());
}
public ImmutableList<EnumTypeDefinition> getEnumTypeDefinitions() {
return ImmutableList.copyOf(enumTypeDefinitionMap.values());
}
public ImmutableList<StructTypeDefinition> getStructTypeDefinitions() {
return ImmutableList.copyOf(structTypeDefinitionMap.values());
}
public ImmutableList<HierarchicalTypeDefinition<ClassType>> getClassTypeDefinitions() {
return ImmutableList.copyOf(classTypeDefinitions.values());
}
public ImmutableList<HierarchicalTypeDefinition<TraitType>> getTraitTypeDefinitions() {
return ImmutableList.of();
}
private void createHivePrincipalTypeEnum() throws AtlasException {
EnumValue values[] = {new EnumValue("USER", 1), new EnumValue("ROLE", 2), new EnumValue("GROUP", 3),};
EnumTypeDefinition definition = new EnumTypeDefinition(HiveDataTypes.HIVE_PRINCIPAL_TYPE.getName(), values);
enumTypeDefinitionMap.put(HiveDataTypes.HIVE_PRINCIPAL_TYPE.getName(), definition);
LOG.debug("Created definition for " + HiveDataTypes.HIVE_PRINCIPAL_TYPE.getName());
}
private void createSerDeStruct() throws AtlasException {
AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
new AttributeDefinition(AtlasClient.NAME, DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, null),
new AttributeDefinition("serializationLib", DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL,
false, null),
new AttributeDefinition(HiveDataModelGenerator.PARAMETERS, STRING_MAP_TYPE.getName(), Multiplicity.OPTIONAL, false, null),};
StructTypeDefinition definition =
new StructTypeDefinition(HiveDataTypes.HIVE_SERDE.getName(), attributeDefinitions);
structTypeDefinitionMap.put(HiveDataTypes.HIVE_SERDE.getName(), definition);
LOG.debug("Created definition for " + HiveDataTypes.HIVE_SERDE.getName());
}
private void createOrderStruct() throws AtlasException {
AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
new AttributeDefinition("col", DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, null),
new AttributeDefinition("order", DataTypes.INT_TYPE.getName(), Multiplicity.REQUIRED, false, null),};
StructTypeDefinition definition =
new StructTypeDefinition(HiveDataTypes.HIVE_ORDER.getName(), attributeDefinitions);
structTypeDefinitionMap.put(HiveDataTypes.HIVE_ORDER.getName(), definition);
LOG.debug("Created definition for " + HiveDataTypes.HIVE_ORDER.getName());
}
private void createStorageDescClass() throws AtlasException {
AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
//Optional to keep it backward-compatible
new AttributeDefinition(TABLE, HiveDataTypes.HIVE_TABLE.getName(), Multiplicity.OPTIONAL, false,
STORAGE_DESC),
new AttributeDefinition(LOCATION, DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false,
null),
new AttributeDefinition("inputFormat", DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false,
null),
new AttributeDefinition("outputFormat", DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false,
null),
new AttributeDefinition("compressed", DataTypes.BOOLEAN_TYPE.getName(), Multiplicity.REQUIRED, false,
null),
new AttributeDefinition(STORAGE_NUM_BUCKETS, DataTypes.INT_TYPE.getName(), Multiplicity.OPTIONAL, false,
null),
new AttributeDefinition("serdeInfo", HiveDataTypes.HIVE_SERDE.getName(), Multiplicity.OPTIONAL, true,
null),
new AttributeDefinition("bucketCols", String.format("array<%s>", DataTypes.STRING_TYPE.getName()),
Multiplicity.OPTIONAL, false, null),
new AttributeDefinition("sortCols", String.format("array<%s>", HiveDataTypes.HIVE_ORDER.getName()),
Multiplicity.OPTIONAL, false, null),
new AttributeDefinition("parameters", STRING_MAP_TYPE.getName(), Multiplicity.OPTIONAL, false, null),
//new AttributeDefinition("skewedInfo", DefinedTypes.HIVE_SKEWEDINFO.getName(),
// Multiplicity.OPTIONAL, false, null),
new AttributeDefinition(STORAGE_IS_STORED_AS_SUB_DIRS, DataTypes.BOOLEAN_TYPE.getName(),
Multiplicity.OPTIONAL, false, null),};
HierarchicalTypeDefinition<ClassType> definition =
new HierarchicalTypeDefinition<>(ClassType.class, HiveDataTypes.HIVE_STORAGEDESC.getName(), null,
ImmutableSet.of(AtlasClient.REFERENCEABLE_SUPER_TYPE), attributeDefinitions);
classTypeDefinitions.put(HiveDataTypes.HIVE_STORAGEDESC.getName(), definition);
LOG.debug("Created definition for " + HiveDataTypes.HIVE_STORAGEDESC.getName());
}
/** Revisit later after nested array types are handled by the typesystem **/
private void createDBClass() throws AtlasException {
AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
new AttributeDefinition(AtlasConstants.CLUSTER_NAME_ATTRIBUTE, DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false,
false, true, null),
new AttributeDefinition(LOCATION, DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false,
null),
new AttributeDefinition(HiveDataModelGenerator.PARAMETERS, STRING_MAP_TYPE.getName(), Multiplicity.OPTIONAL, false, null),
new AttributeDefinition("ownerType", HiveDataTypes.HIVE_PRINCIPAL_TYPE.getName(), Multiplicity.OPTIONAL,
false, null),};
HierarchicalTypeDefinition<ClassType> definition =
new HierarchicalTypeDefinition<>(ClassType.class, HiveDataTypes.HIVE_DB.getName(), null,
ImmutableSet.of(AtlasClient.REFERENCEABLE_SUPER_TYPE, AtlasClient.ASSET_TYPE), attributeDefinitions);
classTypeDefinitions.put(HiveDataTypes.HIVE_DB.getName(), definition);
LOG.debug("Created definition for " + HiveDataTypes.HIVE_DB.getName());
}
private void createColumnClass() throws AtlasException {
AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
new AttributeDefinition("type", DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, false, true, null),
new AttributeDefinition(COMMENT, DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, null),
//Making this optional since this is an incompatible change
//Reverse attribute to 'columns' in Table
new AttributeDefinition(TABLE, HiveDataTypes.HIVE_TABLE.getName(), Multiplicity.OPTIONAL, false, COLUMNS),
new AttributeDefinition(POSITION, DataTypes.INT_TYPE.getName(), Multiplicity.OPTIONAL, false, false, false, null)};
HierarchicalTypeDefinition<ClassType> definition =
new HierarchicalTypeDefinition<>(ClassType.class, HiveDataTypes.HIVE_COLUMN.getName(), null, "1.1",
ImmutableSet.of(AtlasClient.DATA_SET_SUPER_TYPE), attributeDefinitions);
classTypeDefinitions.put(HiveDataTypes.HIVE_COLUMN.getName(), definition);
LOG.debug("Created definition for " + HiveDataTypes.HIVE_COLUMN.getName());
}
private void createTableClass() throws AtlasException {
AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
new AttributeDefinition(DB, HiveDataTypes.HIVE_DB.getName(), Multiplicity.REQUIRED, false, null),
new AttributeDefinition(CREATE_TIME, DataTypes.DATE_TYPE.getName(), Multiplicity.OPTIONAL, false,
null),
new AttributeDefinition(LAST_ACCESS_TIME, DataTypes.DATE_TYPE.getName(), Multiplicity.OPTIONAL, false,
null),
new AttributeDefinition(COMMENT, DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, null),
new AttributeDefinition("retention", DataTypes.INT_TYPE.getName(), Multiplicity.OPTIONAL, false, null),
new AttributeDefinition(STORAGE_DESC, HiveDataTypes.HIVE_STORAGEDESC.getName(), Multiplicity.OPTIONAL, true,
null),
new AttributeDefinition(PART_COLS, DataTypes.arrayTypeName(HiveDataTypes.HIVE_COLUMN.getName()),
Multiplicity.OPTIONAL, true, null),
new AttributeDefinition(TABLE_ALIAS_LIST, DataTypes.arrayTypeName(DataTypes.STRING_TYPE.getName()),
Multiplicity.OPTIONAL, true, null),
new AttributeDefinition("columns", DataTypes.arrayTypeName(HiveDataTypes.HIVE_COLUMN.getName()),
Multiplicity.OPTIONAL, true, null),
new AttributeDefinition(HiveDataModelGenerator.PARAMETERS, STRING_MAP_TYPE.getName(), Multiplicity.OPTIONAL, false, null),
new AttributeDefinition("viewOriginalText", DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL,
false, null),
new AttributeDefinition("viewExpandedText", DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL,
false, null),
new AttributeDefinition(HiveDataModelGenerator.TABLE_TYPE_ATTR, DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false,
null),
new AttributeDefinition("temporary", DataTypes.BOOLEAN_TYPE.getName(), Multiplicity.OPTIONAL, false,
false, true, null),};
HierarchicalTypeDefinition<ClassType> definition =
new HierarchicalTypeDefinition<>(ClassType.class, HiveDataTypes.HIVE_TABLE.getName(), null,
ImmutableSet.of(AtlasClient.DATA_SET_SUPER_TYPE), attributeDefinitions);
classTypeDefinitions.put(HiveDataTypes.HIVE_TABLE.getName(), definition);
LOG.debug("Created definition for " + HiveDataTypes.HIVE_TABLE.getName());
}
private void createProcessClass() throws AtlasException {
AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
new AttributeDefinition("startTime", DataTypes.DATE_TYPE.getName(), Multiplicity.REQUIRED, false, null),
new AttributeDefinition("endTime", DataTypes.DATE_TYPE.getName(), Multiplicity.REQUIRED, false, null),
new AttributeDefinition("userName", DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false,
false, true, null),
new AttributeDefinition("operationType", DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false,
false, true, null),
new AttributeDefinition("queryText", DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false,
null),
new AttributeDefinition("queryPlan", DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false,
null),
new AttributeDefinition("queryId", DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, null),
new AttributeDefinition("recentQueries", String.format("array<%s>", DataTypes.STRING_TYPE.getName()), Multiplicity.OPTIONAL, false, null),
new AttributeDefinition(AtlasConstants.CLUSTER_NAME_ATTRIBUTE, DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, null),
new AttributeDefinition("queryGraph", DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false,
null),};
HierarchicalTypeDefinition<ClassType> definition =
new HierarchicalTypeDefinition<>(ClassType.class, HiveDataTypes.HIVE_PROCESS.getName(), null,
ImmutableSet.of(AtlasClient.PROCESS_SUPER_TYPE), attributeDefinitions);
classTypeDefinitions.put(HiveDataTypes.HIVE_PROCESS.getName(), definition);
LOG.debug("Created definition for " + HiveDataTypes.HIVE_PROCESS.getName());
}
public String getModelAsJson() throws AtlasException {
createDataModel();
return getDataModelAsJSON();
}
public static void main(String[] args) throws Exception {
HiveDataModelGenerator hiveDataModelGenerator = new HiveDataModelGenerator();
String modelAsJson = hiveDataModelGenerator.getModelAsJson();
if (args.length==1) {
ModelDefinitionDump.dumpModelToFile(args[0], modelAsJson);
return;
}
System.out.println("hiveDataModelAsJSON = " + modelAsJson);
TypesDef typesDef = hiveDataModelGenerator.getTypesDef();
for (EnumTypeDefinition enumType : typesDef.enumTypesAsJavaList()) {
System.out.println(String.format("%s(%s) - values %s", enumType.name, EnumType.class.getSimpleName(),
Arrays.toString(enumType.enumValues)));
}
for (StructTypeDefinition structType : typesDef.structTypesAsJavaList()) {
System.out.println(String.format("%s(%s) - attributes %s", structType.typeName, StructType.class.getSimpleName(),
Arrays.toString(structType.attributeDefinitions)));
}
for (HierarchicalTypeDefinition<ClassType> classType : typesDef.classTypesAsJavaList()) {
System.out.println(String.format("%s(%s) - super types [%s] - attributes %s", classType.typeName, ClassType.class.getSimpleName(),
StringUtils.join(classType.superTypes, ","), Arrays.toString(classType.attributeDefinitions)));
}
for (HierarchicalTypeDefinition<TraitType> traitType : typesDef.traitTypesAsJavaList()) {
System.out.println(String.format("%s(%s) - %s", traitType.typeName, TraitType.class.getSimpleName(),
Arrays.toString(traitType.attributeDefinitions)));
}
}
private void createColumnLineageClass() throws AtlasException {
AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
new AttributeDefinition("query", HiveDataTypes.HIVE_PROCESS.getName(),
Multiplicity.REQUIRED, false, null),
new AttributeDefinition("depenendencyType",DataTypes.STRING_TYPE.getName(),
Multiplicity.REQUIRED, false, null),
new AttributeDefinition("expression",DataTypes.STRING_TYPE.getName(),
Multiplicity.OPTIONAL, false, null)
};
HierarchicalTypeDefinition<ClassType> definition =
new HierarchicalTypeDefinition<>(ClassType.class, HiveDataTypes.HIVE_COLUMN_LINEAGE.getName(), null,
ImmutableSet.of(AtlasClient.PROCESS_SUPER_TYPE), attributeDefinitions);
classTypeDefinitions.put(HiveDataTypes.HIVE_COLUMN_LINEAGE.getName(), definition);
LOG.debug("Created definition for " + HiveDataTypes.HIVE_COLUMN_LINEAGE.getName());
}
}
addons/hive-bridge/src/test/java/org/apache/atlas/hive/HiveITBase.java
...@@ -20,7 +20,6 @@ package org.apache.atlas.hive; ...@@ -20,7 +20,6 @@ package org.apache.atlas.hive;
import org.apache.atlas.ApplicationProperties; import org.apache.atlas.ApplicationProperties;
import org.apache.atlas.AtlasClient; import org.apache.atlas.AtlasClient;
import org.apache.atlas.fs.model.FSDataTypes;
import org.apache.atlas.hive.bridge.HiveMetaStoreBridge; import org.apache.atlas.hive.bridge.HiveMetaStoreBridge;
import org.apache.atlas.hive.hook.HiveHookIT; import org.apache.atlas.hive.hook.HiveHookIT;
import org.apache.atlas.hive.model.HiveDataTypes; import org.apache.atlas.hive.model.HiveDataTypes;
...@@ -93,7 +92,6 @@ public class HiveITBase { ...@@ -93,7 +92,6 @@ public class HiveITBase {
} }
hiveMetaStoreBridge = new HiveMetaStoreBridge(configuration, conf, atlasClient); hiveMetaStoreBridge = new HiveMetaStoreBridge(configuration, conf, atlasClient);
hiveMetaStoreBridge.registerHiveDataModel();
HiveConf conf = new HiveConf(); HiveConf conf = new HiveConf();
conf.set("hive.exec.post.hooks", ""); conf.set("hive.exec.post.hooks", "");
...@@ -232,7 +230,7 @@ public class HiveITBase { ...@@ -232,7 +230,7 @@ public class HiveITBase {
private String assertHDFSPathIsRegistered(String path) throws Exception { private String assertHDFSPathIsRegistered(String path) throws Exception {
LOG.debug("Searching for hdfs path {}", path); LOG.debug("Searching for hdfs path {}", path);
return assertEntityIsRegistered(FSDataTypes.HDFS_PATH().toString(), AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, path, null); return assertEntityIsRegistered(HiveMetaStoreBridge.HDFS_PATH, AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, path, null);
} }
protected String assertDatabaseIsRegistered(String dbName) throws Exception { protected String assertDatabaseIsRegistered(String dbName) throws Exception {
......
addons/hive-bridge/src/test/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridgeTest.java
...@@ -20,7 +20,6 @@ package org.apache.atlas.hive.bridge; ...@@ -20,7 +20,6 @@ package org.apache.atlas.hive.bridge;
import org.apache.atlas.AtlasClient; import org.apache.atlas.AtlasClient;
import org.apache.atlas.AtlasServiceException; import org.apache.atlas.AtlasServiceException;
import org.apache.atlas.hive.model.HiveDataModelGenerator;
import org.apache.atlas.hive.model.HiveDataTypes; import org.apache.atlas.hive.model.HiveDataTypes;
import org.apache.atlas.typesystem.Referenceable; import org.apache.atlas.typesystem.Referenceable;
import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.Path;
...@@ -32,7 +31,6 @@ import org.apache.hadoop.hive.ql.metadata.HiveException; ...@@ -32,7 +31,6 @@ import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.metadata.Partition; import org.apache.hadoop.hive.ql.metadata.Partition;
import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.hive.ql.metadata.Table;
import org.apache.hadoop.mapred.TextInputFormat; import org.apache.hadoop.mapred.TextInputFormat;
import org.codehaus.jettison.json.JSONArray;
import org.codehaus.jettison.json.JSONException; import org.codehaus.jettison.json.JSONException;
import org.mockito.ArgumentMatcher; import org.mockito.ArgumentMatcher;
import org.mockito.Mock; import org.mockito.Mock;
...@@ -110,7 +108,7 @@ public class HiveMetaStoreBridgeTest { ...@@ -110,7 +108,7 @@ public class HiveMetaStoreBridgeTest {
// verify update is called on table // verify update is called on table
verify(atlasClient).updateEntity(eq("82e06b34-9151-4023-aa9d-b82103a50e77"), verify(atlasClient).updateEntity(eq("82e06b34-9151-4023-aa9d-b82103a50e77"),
(Referenceable) argThat(new MatchesReferenceableProperty(HiveDataModelGenerator.TABLE_TYPE_ATTR, (Referenceable) argThat(new MatchesReferenceableProperty(HiveMetaStoreBridge.TABLE_TYPE_ATTR,
TableType.EXTERNAL_TABLE.name()))); TableType.EXTERNAL_TABLE.name())));
} }
...@@ -228,7 +226,7 @@ public class HiveMetaStoreBridgeTest { ...@@ -228,7 +226,7 @@ public class HiveMetaStoreBridgeTest {
private Referenceable createTableReference() { private Referenceable createTableReference() {
Referenceable tableReference = new Referenceable(HiveDataTypes.HIVE_TABLE.getName()); Referenceable tableReference = new Referenceable(HiveDataTypes.HIVE_TABLE.getName());
Referenceable sdReference = new Referenceable(HiveDataTypes.HIVE_STORAGEDESC.getName()); Referenceable sdReference = new Referenceable(HiveDataTypes.HIVE_STORAGEDESC.getName());
tableReference.set(HiveDataModelGenerator.STORAGE_DESC, sdReference); tableReference.set(HiveMetaStoreBridge.STORAGE_DESC, sdReference);
return tableReference; return tableReference;
} }
......
addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java
...@@ -23,10 +23,8 @@ import com.google.common.collect.ImmutableList; ...@@ -23,10 +23,8 @@ import com.google.common.collect.ImmutableList;
import com.sun.jersey.api.client.ClientResponse; import com.sun.jersey.api.client.ClientResponse;
import org.apache.atlas.AtlasClient; import org.apache.atlas.AtlasClient;
import org.apache.atlas.AtlasServiceException; import org.apache.atlas.AtlasServiceException;
import org.apache.atlas.fs.model.FSDataTypes;
import org.apache.atlas.hive.HiveITBase; import org.apache.atlas.hive.HiveITBase;
import org.apache.atlas.hive.bridge.HiveMetaStoreBridge; import org.apache.atlas.hive.bridge.HiveMetaStoreBridge;
import org.apache.atlas.hive.model.HiveDataModelGenerator;
import org.apache.atlas.hive.model.HiveDataTypes; import org.apache.atlas.hive.model.HiveDataTypes;
import org.apache.atlas.typesystem.Referenceable; import org.apache.atlas.typesystem.Referenceable;
import org.apache.atlas.typesystem.Struct; import org.apache.atlas.typesystem.Struct;
...@@ -45,7 +43,6 @@ import org.apache.hadoop.hive.ql.metadata.Table; ...@@ -45,7 +43,6 @@ import org.apache.hadoop.hive.ql.metadata.Table;
import org.apache.hadoop.hive.ql.plan.HiveOperation; import org.apache.hadoop.hive.ql.plan.HiveOperation;
import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.UserGroupInformation;
import org.codehaus.jettison.json.JSONArray;
import org.codehaus.jettison.json.JSONException; import org.codehaus.jettison.json.JSONException;
import org.codehaus.jettison.json.JSONObject; import org.codehaus.jettison.json.JSONObject;
import org.slf4j.Logger; import org.slf4j.Logger;
...@@ -80,7 +77,7 @@ public class HiveHookIT extends HiveITBase { ...@@ -80,7 +77,7 @@ public class HiveHookIT extends HiveITBase {
String dbId = assertDatabaseIsRegistered(dbName); String dbId = assertDatabaseIsRegistered(dbName);
Referenceable definition = atlasClient.getEntity(dbId); Referenceable definition = atlasClient.getEntity(dbId);
Map params = (Map) definition.get(HiveDataModelGenerator.PARAMETERS); Map params = (Map) definition.get(HiveMetaStoreBridge.PARAMETERS);
Assert.assertNotNull(params); Assert.assertNotNull(params);
Assert.assertEquals(params.size(), 2); Assert.assertEquals(params.size(), 2);
Assert.assertEquals(params.get("p1"), "v1"); Assert.assertEquals(params.get("p1"), "v1");
...@@ -150,8 +147,8 @@ public class HiveHookIT extends HiveITBase { ...@@ -150,8 +147,8 @@ public class HiveHookIT extends HiveITBase {
Referenceable colEntity = atlasClient.getEntity(colId); Referenceable colEntity = atlasClient.getEntity(colId);
Assert.assertEquals(colEntity.get(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME), String.format("%s.%s.%s@%s", dbName.toLowerCase(), Assert.assertEquals(colEntity.get(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME), String.format("%s.%s.%s@%s", dbName.toLowerCase(),
tableName.toLowerCase(), colName.toLowerCase(), CLUSTER_NAME)); tableName.toLowerCase(), colName.toLowerCase(), CLUSTER_NAME));
Assert.assertNotNull(colEntity.get(HiveDataModelGenerator.TABLE)); Assert.assertNotNull(colEntity.get(HiveMetaStoreBridge.TABLE));
Assert.assertEquals(((Id) colEntity.get(HiveDataModelGenerator.TABLE))._getId(), tableId); Assert.assertEquals(((Id) colEntity.get(HiveMetaStoreBridge.TABLE))._getId(), tableId);
//assert that column.owner = table.owner //assert that column.owner = table.owner
Referenceable tableRef = atlasClient.getEntity(tableId); Referenceable tableRef = atlasClient.getEntity(tableId);
...@@ -161,8 +158,8 @@ public class HiveHookIT extends HiveITBase { ...@@ -161,8 +158,8 @@ public class HiveHookIT extends HiveITBase {
tableName = createTable(); tableName = createTable();
tableId = assertTableIsRegistered(DEFAULT_DB, tableName); tableId = assertTableIsRegistered(DEFAULT_DB, tableName);
tableRef = atlasClient.getEntity(tableId); tableRef = atlasClient.getEntity(tableId);
Assert.assertEquals(tableRef.get(HiveDataModelGenerator.TABLE_TYPE_ATTR), TableType.MANAGED_TABLE.name()); Assert.assertEquals(tableRef.get(HiveMetaStoreBridge.TABLE_TYPE_ATTR), TableType.MANAGED_TABLE.name());
Assert.assertEquals(tableRef.get(HiveDataModelGenerator.COMMENT), "table comment"); Assert.assertEquals(tableRef.get(HiveMetaStoreBridge.COMMENT), "table comment");
String entityName = HiveMetaStoreBridge.getTableQualifiedName(CLUSTER_NAME, DEFAULT_DB, tableName); String entityName = HiveMetaStoreBridge.getTableQualifiedName(CLUSTER_NAME, DEFAULT_DB, tableName);
Assert.assertEquals(tableRef.get(AtlasClient.NAME), tableName.toLowerCase()); Assert.assertEquals(tableRef.get(AtlasClient.NAME), tableName.toLowerCase());
Assert.assertEquals(tableRef.get(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME), entityName); Assert.assertEquals(tableRef.get(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME), entityName);
...@@ -170,13 +167,13 @@ public class HiveHookIT extends HiveITBase { ...@@ -170,13 +167,13 @@ public class HiveHookIT extends HiveITBase {
Table t = hiveMetaStoreBridge.hiveClient.getTable(DEFAULT_DB, tableName); Table t = hiveMetaStoreBridge.hiveClient.getTable(DEFAULT_DB, tableName);
long createTime = Long.parseLong(t.getMetadata().getProperty(hive_metastoreConstants.DDL_TIME)) * HiveMetaStoreBridge.MILLIS_CONVERT_FACTOR; long createTime = Long.parseLong(t.getMetadata().getProperty(hive_metastoreConstants.DDL_TIME)) * HiveMetaStoreBridge.MILLIS_CONVERT_FACTOR;
verifyTimestamps(tableRef, HiveDataModelGenerator.CREATE_TIME, createTime); verifyTimestamps(tableRef, HiveMetaStoreBridge.CREATE_TIME, createTime);
verifyTimestamps(tableRef, HiveDataModelGenerator.LAST_ACCESS_TIME, createTime); verifyTimestamps(tableRef, HiveMetaStoreBridge.LAST_ACCESS_TIME, createTime);
final Referenceable sdRef = (Referenceable) tableRef.get(HiveDataModelGenerator.STORAGE_DESC); final Referenceable sdRef = (Referenceable) tableRef.get(HiveMetaStoreBridge.STORAGE_DESC);
Assert.assertEquals(sdRef.get(HiveDataModelGenerator.STORAGE_IS_STORED_AS_SUB_DIRS), false); Assert.assertEquals(sdRef.get(HiveMetaStoreBridge.STORAGE_IS_STORED_AS_SUB_DIRS), false);
Assert.assertNotNull(sdRef.get(HiveDataModelGenerator.TABLE)); Assert.assertNotNull(sdRef.get(HiveMetaStoreBridge.TABLE));
Assert.assertEquals(((Id) sdRef.get(HiveDataModelGenerator.TABLE))._getId(), tableId); Assert.assertEquals(((Id) sdRef.get(HiveMetaStoreBridge.TABLE))._getId(), tableId);
//Create table where database doesn't exist, will create database instance as well //Create table where database doesn't exist, will create database instance as well
assertDatabaseIsRegistered(DEFAULT_DB); assertDatabaseIsRegistered(DEFAULT_DB);
...@@ -878,7 +875,7 @@ public class HiveHookIT extends HiveITBase { ...@@ -878,7 +875,7 @@ public class HiveHookIT extends HiveITBase {
String tableId = assertTableIsRegistered(DEFAULT_DB, tableName); String tableId = assertTableIsRegistered(DEFAULT_DB, tableName);
Referenceable tableEntity = atlasClient.getEntity(tableId); Referenceable tableEntity = atlasClient.getEntity(tableId);
final String createTime = (String)tableEntity.get(HiveDataModelGenerator.CREATE_TIME); final String createTime = (String)tableEntity.get(HiveMetaStoreBridge.CREATE_TIME);
Assert.assertNotNull(createTime); Assert.assertNotNull(createTime);
String columnGuid = assertColumnIsRegistered(HiveMetaStoreBridge.getColumnQualifiedName(HiveMetaStoreBridge.getTableQualifiedName(CLUSTER_NAME, DEFAULT_DB, tableName), NAME)); String columnGuid = assertColumnIsRegistered(HiveMetaStoreBridge.getColumnQualifiedName(HiveMetaStoreBridge.getTableQualifiedName(CLUSTER_NAME, DEFAULT_DB, tableName), NAME));
...@@ -916,10 +913,10 @@ public class HiveHookIT extends HiveITBase { ...@@ -916,10 +913,10 @@ public class HiveHookIT extends HiveITBase {
assertTableIsRegistered(newDBName, newTableName, new AssertPredicate() { assertTableIsRegistered(newDBName, newTableName, new AssertPredicate() {
@Override @Override
public void assertOnEntity(final Referenceable entity) throws Exception { public void assertOnEntity(final Referenceable entity) throws Exception {
Referenceable sd = ((Referenceable) entity.get(HiveDataModelGenerator.STORAGE_DESC)); Referenceable sd = ((Referenceable) entity.get(HiveMetaStoreBridge.STORAGE_DESC));
String location = (String) sd.get(HiveDataModelGenerator.LOCATION); String location = (String) sd.get(HiveMetaStoreBridge.LOCATION);
assertTrue(location.contains(newTableName)); assertTrue(location.contains(newTableName));
Assert.assertEquals(entity.get(HiveDataModelGenerator.CREATE_TIME), createTime); Assert.assertEquals(entity.get(HiveMetaStoreBridge.CREATE_TIME), createTime);
} }
}); });
} }
...@@ -929,7 +926,7 @@ public class HiveHookIT extends HiveITBase { ...@@ -929,7 +926,7 @@ public class HiveHookIT extends HiveITBase {
Referenceable tableRef = atlasClient.getEntity(tableId); Referenceable tableRef = atlasClient.getEntity(tableId);
//with soft delete, the deleted columns are returned as well. So, filter the deleted ones //with soft delete, the deleted columns are returned as well. So, filter the deleted ones
List<Referenceable> columns = ((List<Referenceable>) tableRef.get(HiveDataModelGenerator.COLUMNS)); List<Referenceable> columns = ((List<Referenceable>) tableRef.get(HiveMetaStoreBridge.COLUMNS));
List<Referenceable> activeColumns = new ArrayList<>(); List<Referenceable> activeColumns = new ArrayList<>();
for (Referenceable col : columns) { for (Referenceable col : columns) {
if (col.getId().getState() == Id.EntityState.ACTIVE) { if (col.getId().getState() == Id.EntityState.ACTIVE) {
...@@ -1046,7 +1043,7 @@ public class HiveHookIT extends HiveITBase { ...@@ -1046,7 +1043,7 @@ public class HiveHookIT extends HiveITBase {
assertColumnIsRegistered(newColQualifiedName, new AssertPredicate() { assertColumnIsRegistered(newColQualifiedName, new AssertPredicate() {
@Override @Override
public void assertOnEntity(Referenceable entity) throws Exception { public void assertOnEntity(Referenceable entity) throws Exception {
assertEquals(entity.get(HiveDataModelGenerator.COMMENT), comment); assertEquals(entity.get(HiveMetaStoreBridge.COMMENT), comment);
} }
}); });
...@@ -1071,7 +1068,7 @@ public class HiveHookIT extends HiveITBase { ...@@ -1071,7 +1068,7 @@ public class HiveHookIT extends HiveITBase {
assertTableIsRegistered(DEFAULT_DB, tableName, new AssertPredicate() { assertTableIsRegistered(DEFAULT_DB, tableName, new AssertPredicate() {
@Override @Override
public void assertOnEntity(Referenceable entity) throws Exception { public void assertOnEntity(Referenceable entity) throws Exception {
List<Referenceable> columns = (List<Referenceable>) entity.get(HiveDataModelGenerator.COLUMNS); List<Referenceable> columns = (List<Referenceable>) entity.get(HiveMetaStoreBridge.COLUMNS);
assertEquals(columns.get(0).get(NAME), finalNewColName); assertEquals(columns.get(0).get(NAME), finalNewColName);
assertEquals(columns.get(1).get(NAME), "id"); assertEquals(columns.get(1).get(NAME), "id");
} }
...@@ -1099,7 +1096,7 @@ public class HiveHookIT extends HiveITBase { ...@@ -1099,7 +1096,7 @@ public class HiveHookIT extends HiveITBase {
assertTableIsRegistered(DEFAULT_DB, tableName, new AssertPredicate() { assertTableIsRegistered(DEFAULT_DB, tableName, new AssertPredicate() {
@Override @Override
public void assertOnEntity(Referenceable entity) throws Exception { public void assertOnEntity(Referenceable entity) throws Exception {
List<Referenceable> columns = (List<Referenceable>) entity.get(HiveDataModelGenerator.COLUMNS); List<Referenceable> columns = (List<Referenceable>) entity.get(HiveMetaStoreBridge.COLUMNS);
assertEquals(columns.get(1).get(NAME), finalNewColName2); assertEquals(columns.get(1).get(NAME), finalNewColName2);
assertEquals(columns.get(0).get(NAME), "id"); assertEquals(columns.get(0).get(NAME), "id");
} }
...@@ -1289,8 +1286,8 @@ public class HiveHookIT extends HiveITBase { ...@@ -1289,8 +1286,8 @@ public class HiveHookIT extends HiveITBase {
assertTableIsRegistered(DEFAULT_DB, tableName, new AssertPredicate() { assertTableIsRegistered(DEFAULT_DB, tableName, new AssertPredicate() {
@Override @Override
public void assertOnEntity(Referenceable tableRef) throws Exception { public void assertOnEntity(Referenceable tableRef) throws Exception {
Referenceable sdRef = (Referenceable) tableRef.get(HiveDataModelGenerator.STORAGE_DESC); Referenceable sdRef = (Referenceable) tableRef.get(HiveMetaStoreBridge.STORAGE_DESC);
Assert.assertEquals(new Path((String)sdRef.get(HiveDataModelGenerator.LOCATION)).toString(), new Path(testPath).toString()); Assert.assertEquals(new Path((String)sdRef.get(HiveMetaStoreBridge.LOCATION)).toString(), new Path(testPath).toString());
} }
}); });
...@@ -1313,18 +1310,18 @@ public class HiveHookIT extends HiveITBase { ...@@ -1313,18 +1310,18 @@ public class HiveHookIT extends HiveITBase {
assertTableIsRegistered(DEFAULT_DB, tableName, new AssertPredicate() { assertTableIsRegistered(DEFAULT_DB, tableName, new AssertPredicate() {
@Override @Override
public void assertOnEntity(Referenceable tableRef) throws Exception { public void assertOnEntity(Referenceable tableRef) throws Exception {
Referenceable sdRef = (Referenceable) tableRef.get(HiveDataModelGenerator.STORAGE_DESC); Referenceable sdRef = (Referenceable) tableRef.get(HiveMetaStoreBridge.STORAGE_DESC);
Assert.assertEquals(sdRef.get(HiveDataModelGenerator.STORAGE_DESC_INPUT_FMT), Assert.assertEquals(sdRef.get(HiveMetaStoreBridge.STORAGE_DESC_INPUT_FMT),
"org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"); "org.apache.hadoop.hive.ql.io.orc.OrcInputFormat");
Assert.assertEquals(sdRef.get(HiveDataModelGenerator.STORAGE_DESC_OUTPUT_FMT), Assert.assertEquals(sdRef.get(HiveMetaStoreBridge.STORAGE_DESC_OUTPUT_FMT),
"org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat"); "org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat");
Assert.assertNotNull(sdRef.get("serdeInfo")); Assert.assertNotNull(sdRef.get("serdeInfo"));
Struct serdeInfo = (Struct) sdRef.get("serdeInfo"); Struct serdeInfo = (Struct) sdRef.get("serdeInfo");
Assert.assertEquals(serdeInfo.get("serializationLib"), "org.apache.hadoop.hive.ql.io.orc.OrcSerde"); Assert.assertEquals(serdeInfo.get("serializationLib"), "org.apache.hadoop.hive.ql.io.orc.OrcSerde");
Assert.assertNotNull(serdeInfo.get(HiveDataModelGenerator.PARAMETERS)); Assert.assertNotNull(serdeInfo.get(HiveMetaStoreBridge.PARAMETERS));
Assert.assertEquals( Assert.assertEquals(
((Map<String, String>) serdeInfo.get(HiveDataModelGenerator.PARAMETERS)) ((Map<String, String>) serdeInfo.get(HiveMetaStoreBridge.PARAMETERS))
.get("serialization.format"), .get("serialization.format"),
"1"); "1");
} }
...@@ -1337,10 +1334,10 @@ public class HiveHookIT extends HiveITBase { ...@@ -1337,10 +1334,10 @@ public class HiveHookIT extends HiveITBase {
* runCommand(query); * runCommand(query);
* tableRef = atlasClient.getEntity(tableId); * tableRef = atlasClient.getEntity(tableId);
* sdRef = (Referenceable)tableRef.get(HiveDataModelGenerator.STORAGE_DESC); * sdRef = (Referenceable)tableRef.get(HiveMetaStoreBridge.STORAGE_DESC);
* Assert.assertEquals(sdRef.get(HiveDataModelGenerator.STORAGE_DESC_INPUT_FMT), "org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"); * Assert.assertEquals(sdRef.get(HiveMetaStoreBridge.STORAGE_DESC_INPUT_FMT), "org.apache.hadoop.hive.ql.io.orc.OrcInputFormat");
* Assert.assertEquals(sdRef.get(HiveDataModelGenerator.STORAGE_DESC_OUTPUT_FMT), "org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat"); * Assert.assertEquals(sdRef.get(HiveMetaStoreBridge.STORAGE_DESC_OUTPUT_FMT), "org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat");
* Assert.assertEquals(((Map) sdRef.get(HiveDataModelGenerator.PARAMETERS)).get("orc.compress"), "ZLIB"); * Assert.assertEquals(((Map) sdRef.get(HiveMetaStoreBridge.PARAMETERS)).get("orc.compress"), "ZLIB");
*/ */
} }
...@@ -1375,8 +1372,8 @@ public class HiveHookIT extends HiveITBase { ...@@ -1375,8 +1372,8 @@ public class HiveHookIT extends HiveITBase {
private void verifyBucketSortingProperties(Referenceable tableRef, int numBuckets, private void verifyBucketSortingProperties(Referenceable tableRef, int numBuckets,
ImmutableList<String> bucketColNames, ImmutableList<String> bucketColNames,
ImmutableList<String> sortcolNames) throws Exception { ImmutableList<String> sortcolNames) throws Exception {
Referenceable sdRef = (Referenceable) tableRef.get(HiveDataModelGenerator.STORAGE_DESC); Referenceable sdRef = (Referenceable) tableRef.get(HiveMetaStoreBridge.STORAGE_DESC);
Assert.assertEquals(((scala.math.BigInt) sdRef.get(HiveDataModelGenerator.STORAGE_NUM_BUCKETS)).intValue(), Assert.assertEquals(((scala.math.BigInt) sdRef.get(HiveMetaStoreBridge.STORAGE_NUM_BUCKETS)).intValue(),
numBuckets); numBuckets);
Assert.assertEquals(sdRef.get("bucketCols"), bucketColNames); Assert.assertEquals(sdRef.get("bucketCols"), bucketColNames);
...@@ -1661,7 +1658,7 @@ public class HiveHookIT extends HiveITBase { ...@@ -1661,7 +1658,7 @@ public class HiveHookIT extends HiveITBase {
assertTableIsRegistered(DEFAULT_DB, tableName, new AssertPredicate() { assertTableIsRegistered(DEFAULT_DB, tableName, new AssertPredicate() {
@Override @Override
public void assertOnEntity(Referenceable tableRef) throws Exception { public void assertOnEntity(Referenceable tableRef) throws Exception {
Referenceable sdRef = (Referenceable) tableRef.get(HiveDataModelGenerator.STORAGE_DESC); Referenceable sdRef = (Referenceable) tableRef.get(HiveMetaStoreBridge.STORAGE_DESC);
Struct serdeInfo = (Struct) sdRef.get("serdeInfo"); Struct serdeInfo = (Struct) sdRef.get("serdeInfo");
Assert.assertEquals(serdeInfo.get("serializationLib"), serdeLib); Assert.assertEquals(serdeInfo.get("serializationLib"), serdeLib);
verifyProperties(serdeInfo, expectedProps, false); verifyProperties(serdeInfo, expectedProps, false);
...@@ -1670,7 +1667,7 @@ public class HiveHookIT extends HiveITBase { ...@@ -1670,7 +1667,7 @@ public class HiveHookIT extends HiveITBase {
} }
private void verifyProperties(Struct referenceable, Map<String, String> expectedProps, boolean checkIfNotExists) { private void verifyProperties(Struct referenceable, Map<String, String> expectedProps, boolean checkIfNotExists) {
Map<String, String> parameters = (Map<String, String>) referenceable.get(HiveDataModelGenerator.PARAMETERS); Map<String, String> parameters = (Map<String, String>) referenceable.get(HiveMetaStoreBridge.PARAMETERS);
if (checkIfNotExists == false) { if (checkIfNotExists == false) {
//Check if properties exist //Check if properties exist
...@@ -1744,7 +1741,7 @@ public class HiveHookIT extends HiveITBase { ...@@ -1744,7 +1741,7 @@ public class HiveHookIT extends HiveITBase {
} }
private String getDSTypeName(Entity entity) { private String getDSTypeName(Entity entity) {
return Entity.Type.TABLE.equals(entity.getType()) ? HiveDataTypes.HIVE_TABLE.name() : FSDataTypes.HDFS_PATH().toString(); return Entity.Type.TABLE.equals(entity.getType()) ? HiveDataTypes.HIVE_TABLE.name() : HiveMetaStoreBridge.HDFS_PATH.toString();
} }
private <T extends Entity> SortedMap<T, Referenceable> getSortedProcessDataSets(Set<T> inputTbls) { private <T extends Entity> SortedMap<T, Referenceable> getSortedProcessDataSets(Set<T> inputTbls) {
......
addons/models/0010-base_model.json 0 → 100644
{
"enumDefs": [],
"structDefs": [],
"classificationDefs": [],
"entityDefs": [
{
"name": "Referenceable",
"superTypes": [],
"typeVersion": "1.0",
"attributeDefs": [
{
"name": "qualifiedName",
"typeName": "string",
"cardinality": "SINGLE",
"isIndexable": true,
"isOptional": false,
"isUnique": true
}
]
},
{
"name": "Asset",
"superTypes": [],
"typeVersion": "1.0",
"attributeDefs": [
{
"name": "name",
"typeName": "string",
"cardinality": "SINGLE",
"isIndexable": true,
"isOptional": false,
"isUnique": false
},
{
"name": "description",
"typeName": "string",
"cardinality": "SINGLE",
"isIndexable": false,
"isOptional": true,
"isUnique": false
},
{
"name": "owner",
"typeName": "string",
"cardinality": "SINGLE",
"isIndexable": true,
"isOptional": true,
"isUnique": false
}
]
},
{
"name": "DataSet",
"superTypes": [
"Referenceable",
"Asset"
],
"typeVersion": "1.0",
"attributeDefs": []
},
{
"name": "Infrastructure",
"superTypes": [
"Referenceable",
"Asset"
],
"typeVersion": "1.0",
"attributeDefs": []
},
{
"name": "Process",
"superTypes": [
"Referenceable",
"Asset"
],
"typeVersion": "1.0",
"attributeDefs": [
{
"name": "inputs",
"typeName": "array<DataSet>",
"cardinality": "SINGLE",
"isIndexable": false,
"isOptional": true,
"isUnique": false
},
{
"name": "outputs",
"typeName": "array<DataSet>",
"cardinality": "SINGLE",
"isIndexable": false,
"isOptional": true,
"isUnique": false
}
]
}
]
}
addons/models/0020-fs_model.json 0 → 100644
{
"enumDefs": [
{
"name": "file_action",
"typeVersion": "1.0",
"elementDefs": [
{
"ordinal": 0,
"value": "NONE"
},
{
"ordinal": 1,
"value": "EXECUTE"
},
{
"ordinal": 2,
"value": "WRITE"
},
{
"ordinal": 3,
"value": "WRITE_EXECUTE"
},
{
"ordinal": 4,
"value": "READ"
},
{
"ordinal": 5,
"value": "READ_EXECUTE"
},
{
"ordinal": 6,
"value": "READ_WRITE"
},
{
"ordinal": 7,
"value": "ALL"
}
]
}
],
"structDefs": [
{
"name": "fs_permissions",
"typeVersion": "1.0",
"attributeDefs": [
{
"name": "group",
"typeName": "file_action",
"cardinality": "SINGLE",
"isIndexable": true,
"isOptional": false,
"isUnique": false
},
{
"name": "user",
"typeName": "file_action",
"cardinality": "SINGLE",
"isIndexable": true,
"isOptional": false,
"isUnique": false
},
{
"name": "sticky",
"typeName": "boolean",
"cardinality": "SINGLE",
"isIndexable": true,
"isOptional": false,
"isUnique": false
},
{
"name": "others",
"typeName": "file_action",
"cardinality": "SINGLE",
"isIndexable": true,
"isOptional": false,
"isUnique": false
}
]
}
],
"classificationDefs": [],
"entityDefs": [
{
"name": "fs_path",
"superTypes": [
"DataSet"
],
"typeVersion": "1.0",
"attributeDefs": [
{
"name": "path",
"typeName": "string",
"cardinality": "SINGLE",
"isIndexable": true,
"isOptional": false,
"isUnique": false
},
{
"name": "createTime",
"typeName": "date",
"cardinality": "SINGLE",
"isIndexable": true,
"isOptional": true,
"isUnique": false
},
{
"name": "modifiedTime",
"typeName": "date",
"cardinality": "SINGLE",
"isIndexable": true,
"isOptional": true,
"isUnique": false
},
{
"name": "isFile",
"typeName": "boolean",
"cardinality": "SINGLE",
"isIndexable": true,
"isOptional": true,
"isUnique": false
},
{
"name": "isSymlink",
"typeName": "boolean",
"cardinality": "SINGLE",
"isIndexable": false,
"isOptional": true,
"isUnique": false
},
{
"name": "fileSize",
"typeName": "long",
"cardinality": "SINGLE",
"isIndexable": false,
"isOptional": true,
"isUnique": false
},
{
"name": "group",
"typeName": "string",
"cardinality": "SINGLE",
"isIndexable": true,
"isOptional": true,
"isUnique": false
},
{
"name": "posixPermissions",
"typeName": "fs_permissions",
"cardinality": "SINGLE",
"isIndexable": true,
"isOptional": true,
"isUnique": false
}
]
},
{
"name": "hdfs_path",
"superTypes": [
"fs_path"
],
"typeVersion": "1.0",
"attributeDefs": [
{
"name": "clusterName",
"typeName": "string",
"cardinality": "SINGLE",
"isIndexable": true,
"isOptional": true,
"isUnique": false
},
{
"name": "numberOfReplicas",
"typeName": "int",
"cardinality": "SINGLE",
"isIndexable": true,
"isOptional": true,
"isUnique": false
},
{
"name": "extendedAttributes",
"typeName": "map<string,string>",
"cardinality": "SINGLE",
"isIndexable": false,
"isOptional": true,
"isUnique": false
}
]
}
]
}
addons/models/0030-hive_model.json 0 → 100644
{
"enumDefs": [
{
"name": "hive_principal_type",
"typeVersion": "1.0",
"elementDefs": [
{
"ordinal": 1,
"value": "USER"
},
{
"ordinal": 2,
"value": "ROLE"
},
{
"ordinal": 3,
"value": "GROUP"
}
]
}
],
"structDefs": [
{
"name": "hive_order",
"typeVersion": "1.0",
"attributeDefs": [
{
"name": "order",
"typeName": "int",
"cardinality": "SINGLE",
"isIndexable": false,
"isOptional": false,
"isUnique": false
},
{
"name": "col",
"typeName": "string",
"cardinality": "SINGLE",
"isIndexable": false,
"isOptional": false,
"isUnique": false
}
]
},
{
"name": "hive_serde",
"typeVersion": "1.0",
"attributeDefs": [
{
"name": "name",
"typeName": "string",
"cardinality": "SINGLE",
"isIndexable": false,
"isOptional": true,
"isUnique": false
},
{
"name": "serializationLib",
"typeName": "string",
"cardinality": "SINGLE",
"isIndexable": false,
"isOptional": true,
"isUnique": false
},
{
"name": "parameters",
"typeName": "map<string,string>",
"cardinality": "SINGLE",
"isIndexable": false,
"isOptional": true,
"isUnique": false
}
]
}
],
"classificationDefs": [],
"entityDefs": [
{
"name": "hive_process",
"superTypes": [
"Process"
],
"typeVersion": "1.0",
"attributeDefs": [
{
"name": "startTime",
"typeName": "date",
"cardinality": "SINGLE",
"isIndexable": false,
"isOptional": false,
"isUnique": false
},
{
"name": "endTime",
"typeName": "date",
"cardinality": "SINGLE",
"isIndexable": false,
"isOptional": false,
"isUnique": false
},
{
"name": "userName",
"typeName": "string",
"cardinality": "SINGLE",
"isIndexable": true,
"isOptional": false,
"isUnique": false
},
{
"name": "operationType",
"typeName": "string",
"cardinality": "SINGLE",
"isIndexable": true,
"isOptional": false,
"isUnique": false
},
{
"name": "queryText",
"typeName": "string",
"cardinality": "SINGLE",
"isIndexable": false,
"isOptional": false,
"isUnique": false
},
{
"name": "queryPlan",
"typeName": "string",
"cardinality": "SINGLE",
"isIndexable": false,
"isOptional": false,
"isUnique": false
},
{
"name": "queryId",
"typeName": "string",
"cardinality": "SINGLE",
"isIndexable": false,
"isOptional": false,
"isUnique": false
},
{
"name": "recentQueries",
"typeName": "array<string>",
"cardinality": "SINGLE",
"isIndexable": false,
"isOptional": true,
"isUnique": false
},
{
"name": "clusterName",
"typeName": "string",
"cardinality": "SINGLE",
"isIndexable": false,
"isOptional": true,
"isUnique": false
},
{
"name": "queryGraph",
"typeName": "string",
"cardinality": "SINGLE",
"isIndexable": false,
"isOptional": true,
"isUnique": false
}
]
},
{
"name": "hive_table",
"superTypes": [
"DataSet"
],
"typeVersion": "1.0",
"attributeDefs": [
{
"name": "db",
"typeName": "hive_db",
"cardinality": "SINGLE",
"isIndexable": false,
"isOptional": false,
"isUnique": false
},
{
"name": "createTime",
"typeName": "date",
"cardinality": "SINGLE",
"isIndexable": false,
"isOptional": true,
"isUnique": false
},
{
"name": "lastAccessTime",
"typeName": "date",
"cardinality": "SINGLE",
"isIndexable": false,
"isOptional": true,
"isUnique": false
},
{
"name": "comment",
"typeName": "string",
"cardinality": "SINGLE",
"isIndexable": false,
"isOptional": true,
"isUnique": false
},
{
"name": "retention",
"typeName": "int",
"cardinality": "SINGLE",
"isIndexable": false,
"isOptional": true,
"isUnique": false
},
{
"name": "sd",
"typeName": "hive_storagedesc",
"cardinality": "SINGLE",
"constraintDefs": [
{
"type": "mappedFromRef",
"params": {
"refAttribute": "table"
}
}
],
"isIndexable": false,
"isOptional": true,
"isUnique": false
},
{
"name": "partitionKeys",
"typeName": "array<hive_column>",
"cardinality": "SINGLE",
"constraintDefs": [
{
"type": "foreignKey"
}
],
"isIndexable": false,
"isOptional": true,
"isUnique": false
},
{
"name": "aliases",
"typeName": "array<string>",
"cardinality": "SINGLE",
"isIndexable": false,
"isOptional": true,
"isUnique": false
},
{
"name": "columns",
"typeName": "array<hive_column>",
"cardinality": "SINGLE",
"constraintDefs": [
{
"type": "mappedFromRef",
"params": {
"refAttribute": "table"
}
}
],
"isIndexable": false,
"isOptional": true,
"isUnique": false
},
{
"name": "parameters",
"typeName": "map<string,string>",
"cardinality": "SINGLE",
"isIndexable": false,
"isOptional": true,
"isUnique": false
},
{
"name": "viewOriginalText",
"typeName": "string",
"cardinality": "SINGLE",
"isIndexable": false,
"isOptional": true,
"isUnique": false
},
{
"name": "viewExpandedText",
"typeName": "string",
"cardinality": "SINGLE",
"isIndexable": false,
"isOptional": true,
"isUnique": false
},
{
"name": "tableType",
"typeName": "string",
"cardinality": "SINGLE",
"isIndexable": false,
"isOptional": true,
"isUnique": false
},
{
"name": "temporary",
"typeName": "boolean",
"cardinality": "SINGLE",
"isIndexable": true,
"isOptional": true,
"isUnique": false
}
]
},
{
"name": "hive_storagedesc",
"superTypes": [
"Referenceable"
],
"typeVersion": "1.0",
"attributeDefs": [
{
"name": "table",
"typeName": "hive_table",
"cardinality": "SINGLE",
"constraintDefs": [
{
"type": "foreignKey",
"params": {
"onDelete": "cascade"
}
}
],
"isIndexable": false,
"isOptional": true,
"isUnique": false
},
{
"name": "location",
"typeName": "string",
"cardinality": "SINGLE",
"isIndexable": false,
"isOptional": true,
"isUnique": false
},
{
"name": "inputFormat",
"typeName": "string",
"cardinality": "SINGLE",
"isIndexable": false,
"isOptional": true,
"isUnique": false
},
{
"name": "outputFormat",
"typeName": "string",
"cardinality": "SINGLE",
"isIndexable": false,
"isOptional": true,
"isUnique": false
},
{
"name": "compressed",
"typeName": "boolean",
"cardinality": "SINGLE",
"isIndexable": false,
"isOptional": false,
"isUnique": false
},
{
"name": "numBuckets",
"typeName": "int",
"cardinality": "SINGLE",
"isIndexable": false,
"isOptional": true,
"isUnique": false
},
{
"name": "serdeInfo",
"typeName": "hive_serde",
"cardinality": "SINGLE",
"isIndexable": false,
"isOptional": true,
"isUnique": false
},
{
"name": "bucketCols",
"typeName": "array<string>",
"cardinality": "SINGLE",
"isIndexable": false,
"isOptional": true,
"isUnique": false
},
{
"name": "sortCols",
"typeName": "array<hive_order>",
"cardinality": "SINGLE",
"isIndexable": false,
"isOptional": true,
"isUnique": false
},
{
"name": "parameters",
"typeName": "map<string,string>",
"cardinality": "SINGLE",
"isIndexable": false,
"isOptional": true,
"isUnique": false
},
{
"name": "storedAsSubDirectories",
"typeName": "boolean",
"cardinality": "SINGLE",
"isIndexable": false,
"isOptional": true,
"isUnique": false
}
]
},
{
"name": "hive_db",
"superTypes": [
"Referenceable",
"Asset"
],
"typeVersion": "1.0",
"attributeDefs": [
{
"name": "clusterName",
"typeName": "string",
"cardinality": "SINGLE",
"isIndexable": true,
"isOptional": false,
"isUnique": false
},
{
"name": "location",
"typeName": "string",
"cardinality": "SINGLE",
"isIndexable": false,
"isOptional": true,
"isUnique": false
},
{
"name": "parameters",
"typeName": "map<string,string>",
"cardinality": "SINGLE",
"isIndexable": false,
"isOptional": true,
"isUnique": false
},
{
"name": "ownerType",
"typeName": "hive_principal_type",
"cardinality": "SINGLE",
"isIndexable": false,
"isOptional": true,
"isUnique": false
}
]
},
{
"name": "hive_column",
"superTypes": [
"DataSet"
],
"typeVersion": "1.0",
"attributeDefs": [
{
"name": "type",
"typeName": "string",
"cardinality": "SINGLE",
"isIndexable": true,
"isOptional": false,
"isUnique": false
},
{
"name": "comment",
"typeName": "string",
"cardinality": "SINGLE",
"isIndexable": false,
"isOptional": true,
"isUnique": false
},
{
"name": "table",
"typeName": "hive_table",
"cardinality": "SINGLE",
"constraintDefs": [
{
"type": "foreignKey",
"params": {
"onDelete": "cascade"
}
}
],
"isIndexable": false,
"isOptional": true,
"isUnique": false
}
]
}
]
}
addons/models/0040-sqoop_model.json 0 → 100644
{
"enumDefs": [],
"structDefs": [],
"classificationDefs": [],
"entityDefs": [
{
"name": "sqoop_process",
"superTypes": [
"Process"
],
"typeVersion": "1.0",
"attributeDefs": [
{
"name": "operation",
"typeName": "string",
"cardinality": "SINGLE",
"isIndexable": true,
"isOptional": false,
"isUnique": false
},
{
"name": "commandlineOpts",
"typeName": "map<string,string>",
"cardinality": "SINGLE",
"isIndexable": false,
"isOptional": false,
"isUnique": false
},
{
"name": "startTime",
"typeName": "date",
"cardinality": "SINGLE",
"isIndexable": false,
"isOptional": false,
"isUnique": false
},
{
"name": "endTime",
"typeName": "date",
"cardinality": "SINGLE",
"isIndexable": false,
"isOptional": false,
"isUnique": false
},
{
"name": "userName",
"typeName": "string",
"cardinality": "SINGLE",
"isIndexable": true,
"isOptional": true,
"isUnique": false
}
]
},
{
"name": "sqoop_dbdatastore",
"superTypes": [
"DataSet"
],
"typeVersion": "1.0",
"attributeDefs": [
{
"name": "dbStoreType",
"typeName": "string",
"cardinality": "SINGLE",
"isIndexable": true,
"isOptional": false,
"isUnique": false
},
{
"name": "storeUse",
"typeName": "string",
"cardinality": "SINGLE",
"isIndexable": false,
"isOptional": false,
"isUnique": false
},
{
"name": "storeUri",
"typeName": "string",
"cardinality": "SINGLE",
"isIndexable": false,
"isOptional": false,
"isUnique": false
},
{
"name": "source",
"typeName": "string",
"cardinality": "SINGLE",
"isIndexable": true,
"isOptional": true,
"isUnique": false
}
]
}
]
}
addons/models/0050-falcon_model.json 0 → 100644
{
"enumDefs": [],
"structDefs": [],
"classificationDefs": [],
"entityDefs": [
{
"name": "falcon_feed_replication",
"superTypes": [
"Process"
],
"typeVersion": "1.0",
"attributeDefs": []
},
{
"name": "falcon_cluster",
"superTypes": [
"Infrastructure"
],
"typeVersion": "1.0",
"attributeDefs": [
{
"name": "colo",
"typeName": "string",
"cardinality": "SINGLE",
"isIndexable": true,
"isOptional": false,
"isUnique": false
},
{
"name": "tags",
"typeName": "map<string,string>",
"cardinality": "SINGLE",
"isIndexable": false,
"isOptional": true,
"isUnique": false
}
]
},
{
"name": "falcon_feed",
"superTypes": [
"DataSet"
],
"typeVersion": "1.0",
"attributeDefs": [
{
"name": "frequency",
"typeName": "string",
"cardinality": "SINGLE",
"isIndexable": false,
"isOptional": false,
"isUnique": false
},
{
"name": "stored-in",
"typeName": "falcon_cluster",
"cardinality": "SINGLE",
"isIndexable": true,
"isOptional": false,
"isUnique": false
},
{
"name": "groups",
"typeName": "string",
"cardinality": "SINGLE",
"isIndexable": false,
"isOptional": true,
"isUnique": false
},
{
"name": "tags",
"typeName": "map<string,string>",
"cardinality": "SINGLE",
"isIndexable": false,
"isOptional": true,
"isUnique": false
}
]
},
{
"name": "falcon_process",
"superTypes": [
"Process"
],
"typeVersion": "1.0",
"attributeDefs": [
{
"name": "frequency",
"typeName": "string",
"cardinality": "SINGLE",
"isIndexable": false,
"isOptional": false,
"isUnique": false
},
{
"name": "runs-on",
"typeName": "falcon_cluster",
"cardinality": "SINGLE",
"isIndexable": true,
"isOptional": false,
"isUnique": false
},
{
"name": "tags",
"typeName": "map<string,string>",
"cardinality": "SINGLE",
"isIndexable": false,
"isOptional": true,
"isUnique": false
},
{
"name": "pipelines",
"typeName": "string",
"cardinality": "SINGLE",
"isIndexable": true,
"isOptional": true,
"isUnique": false
},
{
"name": "workflow-properties",
"typeName": "map<string,string>",
"cardinality": "SINGLE",
"isIndexable": false,
"isOptional": true,
"isUnique": false
}
]
},
{
"name": "falcon_feed_creation",
"superTypes": [
"Process"
],
"typeVersion": "1.0",
"attributeDefs": [
{
"name": "stored-in",
"typeName": "falcon_cluster",
"cardinality": "SINGLE",
"isIndexable": true,
"isOptional": false,
"isUnique": false
}
]
}
]
}
addons/models/0060-hbase_model.json 0 → 100644
{
"enumDefs": [],
"structDefs": [],
"classificationDefs": [],
"entityDefs": [
{
"name": "hbase_table",
"superTypes": [
"DataSet"
],
"typeVersion": "1.0",
"attributeDefs": [
{
"name": "uri",
"typeName": "string",
"cardinality": "SINGLE",
"isIndexable": false,
"isOptional": false,
"isUnique": false
}
]
}
]
}
addons/models/0070-kafka_model.json 0 → 100644
{
"enumDefs": [],
"structDefs": [],
"classificationDefs": [],
"entityDefs": [
{
"name": "kafka_topic",
"superTypes": [
"DataSet"
],
"typeVersion": "1.0",
"attributeDefs": [
{
"name": "topic",
"typeName": "string",
"cardinality": "SINGLE",
"isIndexable": true,
"isOptional": false,
"isUnique": true
},
{
"name": "uri",
"typeName": "string",
"cardinality": "SINGLE",
"isIndexable": false,
"isOptional": false,
"isUnique": false
}
]
},
{
"name": "jms_topic",
"superTypes": [
"DataSet"
],
"typeVersion": "1.0",
"attributeDefs": [
{
"name": "topic",
"typeName": "string",
"cardinality": "SINGLE",
"isIndexable": true,
"isOptional": false,
"isUnique": true
},
{
"name": "uri",
"typeName": "string",
"cardinality": "SINGLE",
"isIndexable": false,
"isOptional": false,
"isUnique": false
}
]
}
]
}
addons/models/0080-storm_model.json 0 → 100644
{
"enumDefs": [],
"structDefs": [],
"classificationDefs": [],
"entityDefs": [
{
"name": "storm_topology",
"superTypes": [
"Process"
],
"typeVersion": "1.0",
"attributeDefs": [
{
"name": "id",
"typeName": "string",
"cardinality": "SINGLE",
"isIndexable": true,
"isOptional": false,
"isUnique": true
},
{
"name": "startTime",
"typeName": "date",
"cardinality": "SINGLE",
"isIndexable": false,
"isOptional": true,
"isUnique": false
},
{
"name": "endTime",
"typeName": "date",
"cardinality": "SINGLE",
"isIndexable": false,
"isOptional": true,
"isUnique": false
},
{
"name": "conf",
"typeName": "map<string,string>",
"cardinality": "SINGLE",
"isIndexable": false,
"isOptional": true,
"isUnique": false
},
{
"name": "clusterName",
"typeName": "string",
"cardinality": "SINGLE",
"isIndexable": true,
"isOptional": true,
"isUnique": false
},
{
"name": "nodes",
"typeName": "array<storm_node>",
"cardinality": "LIST",
"constraintDefs": [
{
"type": "foreignKey"
}
],
"isIndexable": false,
"isOptional": false,
"isUnique": false
}
]
},
{
"name": "storm_node",
"superTypes": [],
"typeVersion": "1.0",
"attributeDefs": [
{
"name": "name",
"typeName": "string",
"cardinality": "SINGLE",
"isIndexable": true,
"isOptional": false,
"isUnique": false
},
{
"name": "description",
"typeName": "string",
"cardinality": "SINGLE",
"isIndexable": true,
"isOptional": true,
"isUnique": false
},
{
"name": "driverClass",
"typeName": "string",
"cardinality": "SINGLE",
"isIndexable": true,
"isOptional": false,
"isUnique": false
},
{
"name": "conf",
"typeName": "map<string,string>",
"cardinality": "SINGLE",
"isIndexable": false,
"isOptional": true,
"isUnique": false
}
]
},
{
"name": "storm_spout",
"superTypes": [
"storm_node"
],
"typeVersion": "1.0",
"attributeDefs": [
{
"name": "outputs",
"typeName": "array<string>",
"cardinality": "LIST",
"isIndexable": false,
"isOptional": false,
"isUnique": false
}
]
},
{
"name": "storm_bolt",
"superTypes": [
"storm_node"
],
"typeVersion": "1.0",
"attributeDefs": [
{
"name": "inputs",
"typeName": "array<string>",
"cardinality": "LIST",
"isIndexable": false,
"isOptional": false,
"isUnique": false
},
{
"name": "outputs",
"typeName": "array<string>",
"cardinality": "SINGLE",
"isIndexable": false,
"isOptional": true,
"isUnique": false
}
]
}
]
}
addons/models/patches/001-hive_column_add_position.json 0 → 100644
{
"patches": [
{
"action": "ADD_ATTRIBUTE",
"typeName": "hive_column",
"applyToVersion": "1.0",
"updateToVersion": "1.1",
"params": null,
"attributeDefs": [
{
"name": "position",
"typeName": "int",
"cardinality": "SINGLE",
"isIndexable": false,
"isOptional": true,
"isUnique": false
}
]
}
]
}
addons/sqoop-bridge/pom.xml
...@@ -411,24 +411,6 @@ ...@@ -411,24 +411,6 @@
<version>1.2.1</version> <version>1.2.1</version>
<inherited>false</inherited> <inherited>false</inherited>
<executions> <executions>
<execution>
<configuration>
<mainClass>org.apache.atlas.sqoop.model.SqoopDataModelGenerator</mainClass>
<systemProperties>
<systemProperty>
<key>atlas.conf</key>
<value>${project.build.directory}/../../../typesystem/target/test-classes</value>
</systemProperty>
</systemProperties>
<arguments>
<argument>${project.build.directory}/models/sqoop_model.json</argument>
</arguments>
</configuration>
<phase>package</phase>
<goals>
<goal>java</goal>
</goals>
</execution>
</executions> </executions>
</plugin> </plugin>
</plugins> </plugins>
......
addons/sqoop-bridge/src/main/java/org/apache/atlas/sqoop/hook/SqoopHook.java
...@@ -23,11 +23,9 @@ import org.apache.atlas.ApplicationProperties; ...@@ -23,11 +23,9 @@ import org.apache.atlas.ApplicationProperties;
import org.apache.atlas.AtlasClient; import org.apache.atlas.AtlasClient;
import org.apache.atlas.AtlasConstants; import org.apache.atlas.AtlasConstants;
import org.apache.atlas.hive.bridge.HiveMetaStoreBridge; import org.apache.atlas.hive.bridge.HiveMetaStoreBridge;
import org.apache.atlas.hive.model.HiveDataModelGenerator;
import org.apache.atlas.hive.model.HiveDataTypes; import org.apache.atlas.hive.model.HiveDataTypes;
import org.apache.atlas.hook.AtlasHook; import org.apache.atlas.hook.AtlasHook;
import org.apache.atlas.notification.hook.HookNotification; import org.apache.atlas.notification.hook.HookNotification;
import org.apache.atlas.sqoop.model.SqoopDataModelGenerator;
import org.apache.atlas.sqoop.model.SqoopDataTypes; import org.apache.atlas.sqoop.model.SqoopDataTypes;
import org.apache.atlas.typesystem.Referenceable; import org.apache.atlas.typesystem.Referenceable;
import org.apache.commons.configuration.Configuration; import org.apache.commons.configuration.Configuration;
...@@ -55,6 +53,20 @@ public class SqoopHook extends SqoopJobDataPublisher { ...@@ -55,6 +53,20 @@ public class SqoopHook extends SqoopJobDataPublisher {
public static final String ATLAS_CLUSTER_NAME = "atlas.cluster.name"; public static final String ATLAS_CLUSTER_NAME = "atlas.cluster.name";
public static final String DEFAULT_CLUSTER_NAME = "primary"; public static final String DEFAULT_CLUSTER_NAME = "primary";
public static final String USER = "userName";
public static final String DB_STORE_TYPE = "dbStoreType";
public static final String DB_STORE_USAGE = "storeUse";
public static final String SOURCE = "source";
public static final String DESCRIPTION = "description";
public static final String STORE_URI = "storeUri";
public static final String OPERATION = "operation";
public static final String START_TIME = "startTime";
public static final String END_TIME = "endTime";
public static final String CMD_LINE_OPTS = "commandlineOpts";
// multiple inputs and outputs for process
public static final String INPUTS = "inputs";
public static final String OUTPUTS = "outputs";
static { static {
org.apache.hadoop.conf.Configuration.addDefaultResource("sqoop-site.xml"); org.apache.hadoop.conf.Configuration.addDefaultResource("sqoop-site.xml");
} }
...@@ -75,7 +87,7 @@ public class SqoopHook extends SqoopJobDataPublisher { ...@@ -75,7 +87,7 @@ public class SqoopHook extends SqoopJobDataPublisher {
tableRef.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, tableRef.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME,
HiveMetaStoreBridge.getTableQualifiedName(clusterName, dbName, tableName)); HiveMetaStoreBridge.getTableQualifiedName(clusterName, dbName, tableName));
tableRef.set(AtlasClient.NAME, tableName.toLowerCase()); tableRef.set(AtlasClient.NAME, tableName.toLowerCase());
tableRef.set(HiveDataModelGenerator.DB, dbRef); tableRef.set(HiveMetaStoreBridge.DB, dbRef);
return tableRef; return tableRef;
} }
...@@ -94,11 +106,11 @@ public class SqoopHook extends SqoopJobDataPublisher { ...@@ -94,11 +106,11 @@ public class SqoopHook extends SqoopJobDataPublisher {
String name = getSqoopDBStoreName(data); String name = getSqoopDBStoreName(data);
storeRef.set(AtlasClient.NAME, name); storeRef.set(AtlasClient.NAME, name);
storeRef.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, name); storeRef.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, name);
storeRef.set(SqoopDataModelGenerator.DB_STORE_TYPE, data.getStoreType()); storeRef.set(SqoopHook.DB_STORE_TYPE, data.getStoreType());
storeRef.set(SqoopDataModelGenerator.DB_STORE_USAGE, usage); storeRef.set(SqoopHook.DB_STORE_USAGE, usage);
storeRef.set(SqoopDataModelGenerator.STORE_URI, data.getUrl()); storeRef.set(SqoopHook.STORE_URI, data.getUrl());
storeRef.set(SqoopDataModelGenerator.SOURCE, source); storeRef.set(SqoopHook.SOURCE, source);
storeRef.set(SqoopDataModelGenerator.DESCRIPTION, ""); storeRef.set(SqoopHook.DESCRIPTION, "");
storeRef.set(AtlasClient.OWNER, data.getUser()); storeRef.set(AtlasClient.OWNER, data.getUser());
return storeRef; return storeRef;
} }
...@@ -109,24 +121,24 @@ public class SqoopHook extends SqoopJobDataPublisher { ...@@ -109,24 +121,24 @@ public class SqoopHook extends SqoopJobDataPublisher {
final String sqoopProcessName = getSqoopProcessName(data, clusterName); final String sqoopProcessName = getSqoopProcessName(data, clusterName);
procRef.set(AtlasClient.NAME, sqoopProcessName); procRef.set(AtlasClient.NAME, sqoopProcessName);
procRef.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, sqoopProcessName); procRef.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, sqoopProcessName);
procRef.set(SqoopDataModelGenerator.OPERATION, data.getOperation()); procRef.set(SqoopHook.OPERATION, data.getOperation());
if (isImportOperation(data)) { if (isImportOperation(data)) {
procRef.set(SqoopDataModelGenerator.INPUTS, dbStoreRef); procRef.set(SqoopHook.INPUTS, dbStoreRef);
procRef.set(SqoopDataModelGenerator.OUTPUTS, hiveTableRef); procRef.set(SqoopHook.OUTPUTS, hiveTableRef);
} else { } else {
procRef.set(SqoopDataModelGenerator.INPUTS, hiveTableRef); procRef.set(SqoopHook.INPUTS, hiveTableRef);
procRef.set(SqoopDataModelGenerator.OUTPUTS, dbStoreRef); procRef.set(SqoopHook.OUTPUTS, dbStoreRef);
} }
procRef.set(SqoopDataModelGenerator.USER, data.getUser()); procRef.set(SqoopHook.USER, data.getUser());
procRef.set(SqoopDataModelGenerator.START_TIME, new Date(data.getStartTime())); procRef.set(SqoopHook.START_TIME, new Date(data.getStartTime()));
procRef.set(SqoopDataModelGenerator.END_TIME, new Date(data.getEndTime())); procRef.set(SqoopHook.END_TIME, new Date(data.getEndTime()));
Map<String, String> sqoopOptionsMap = new HashMap<>(); Map<String, String> sqoopOptionsMap = new HashMap<>();
Properties options = data.getOptions(); Properties options = data.getOptions();
for (Object k : options.keySet()) { for (Object k : options.keySet()) {
sqoopOptionsMap.put((String)k, (String) options.get(k)); sqoopOptionsMap.put((String)k, (String) options.get(k));
} }
procRef.set(SqoopDataModelGenerator.CMD_LINE_OPTS, sqoopOptionsMap); procRef.set(SqoopHook.CMD_LINE_OPTS, sqoopOptionsMap);
return procRef; return procRef;
} }
......
addons/sqoop-bridge/src/main/java/org/apache/atlas/sqoop/model/SqoopDataModelGenerator.java deleted 100644 → 0
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.sqoop.model;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import org.apache.atlas.AtlasClient;
import org.apache.atlas.AtlasException;
import org.apache.atlas.addons.ModelDefinitionDump;
import org.apache.atlas.typesystem.TypesDef;
import org.apache.atlas.typesystem.json.TypesSerialization;
import org.apache.atlas.typesystem.types.AttributeDefinition;
import org.apache.atlas.typesystem.types.ClassType;
import org.apache.atlas.typesystem.types.DataTypes;
import org.apache.atlas.typesystem.types.EnumType;
import org.apache.atlas.typesystem.types.EnumTypeDefinition;
import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition;
import org.apache.atlas.typesystem.types.Multiplicity;
import org.apache.atlas.typesystem.types.StructTypeDefinition;
import org.apache.atlas.typesystem.types.TraitType;
import org.apache.atlas.typesystem.types.utils.TypesUtil;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
/**
* Utility that generates Sqoop data model for both metastore entities and DDL/DML queries.
*/
public class SqoopDataModelGenerator {
private static final Logger LOG = LoggerFactory.getLogger(SqoopDataModelGenerator.class);
private final Map<String, HierarchicalTypeDefinition<ClassType>> classTypeDefinitions;
private final Map<String, EnumTypeDefinition> enumTypeDefinitionMap;
private final Map<String, StructTypeDefinition> structTypeDefinitionMap;
private static final DataTypes.MapType STRING_MAP_TYPE =
new DataTypes.MapType(DataTypes.STRING_TYPE, DataTypes.STRING_TYPE);
public static final String USER = "userName";
public static final String DB_STORE_TYPE = "dbStoreType";
public static final String DB_STORE_USAGE = "storeUse";
public static final String SOURCE = "source";
public static final String DESCRIPTION = "description";
public static final String STORE_URI = "storeUri";
public static final String OPERATION = "operation";
public static final String START_TIME = "startTime";
public static final String END_TIME = "endTime";
public static final String CMD_LINE_OPTS = "commandlineOpts";
// multiple inputs and outputs for process
public static final String INPUTS = "inputs";
public static final String OUTPUTS = "outputs";
public SqoopDataModelGenerator() {
classTypeDefinitions = new HashMap<>();
enumTypeDefinitionMap = new HashMap<>();
structTypeDefinitionMap = new HashMap<>();
}
public void createDataModel() throws AtlasException {
LOG.info("Generating the Sqoop Data Model....");
// enums
// structs
// classes
createSqoopDbStoreClass();
// DDL/DML Process
createSqoopProcessClass();
}
public TypesDef getTypesDef() {
return TypesUtil.getTypesDef(getEnumTypeDefinitions(), getStructTypeDefinitions(), getTraitTypeDefinitions(),
getClassTypeDefinitions());
}
public String getDataModelAsJSON() {
return TypesSerialization.toJson(getTypesDef());
}
public ImmutableList<EnumTypeDefinition> getEnumTypeDefinitions() {
return ImmutableList.copyOf(enumTypeDefinitionMap.values());
}
public ImmutableList<StructTypeDefinition> getStructTypeDefinitions() {
return ImmutableList.copyOf(structTypeDefinitionMap.values());
}
public ImmutableList<HierarchicalTypeDefinition<ClassType>> getClassTypeDefinitions() {
return ImmutableList.copyOf(classTypeDefinitions.values());
}
public ImmutableList<HierarchicalTypeDefinition<TraitType>> getTraitTypeDefinitions() {
return ImmutableList.of();
}
private void createSqoopDbStoreClass() throws AtlasException {
AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
new AttributeDefinition(DB_STORE_TYPE,
DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, false, true, null),
new AttributeDefinition(DB_STORE_USAGE,
DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, null),
new AttributeDefinition(STORE_URI,
DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, null),
new AttributeDefinition(SOURCE,
DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, false, true, null)
};
HierarchicalTypeDefinition<ClassType> definition =
new HierarchicalTypeDefinition<>(ClassType.class, SqoopDataTypes.SQOOP_DBDATASTORE.getName(), null,
ImmutableSet.of(AtlasClient.DATA_SET_SUPER_TYPE), attributeDefinitions);
classTypeDefinitions.put(SqoopDataTypes.SQOOP_DBDATASTORE.getName(), definition);
LOG.debug("Created definition for " + SqoopDataTypes.SQOOP_DBDATASTORE.getName());
}
private void createSqoopProcessClass() throws AtlasException {
AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
new AttributeDefinition(OPERATION,
DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, false, true, null),
new AttributeDefinition(CMD_LINE_OPTS, STRING_MAP_TYPE.getName(), Multiplicity.REQUIRED, false, null),
new AttributeDefinition(START_TIME, DataTypes.DATE_TYPE.getName(), Multiplicity.REQUIRED, false, null),
new AttributeDefinition(END_TIME, DataTypes.DATE_TYPE.getName(), Multiplicity.REQUIRED, false, null),
new AttributeDefinition(USER,
DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, false, true, null),
};
HierarchicalTypeDefinition<ClassType> definition =
new HierarchicalTypeDefinition<>(ClassType.class, SqoopDataTypes.SQOOP_PROCESS.getName(), null,
ImmutableSet.of(AtlasClient.PROCESS_SUPER_TYPE), attributeDefinitions);
classTypeDefinitions.put(SqoopDataTypes.SQOOP_PROCESS.getName(), definition);
LOG.debug("Created definition for " + SqoopDataTypes.SQOOP_PROCESS.getName());
}
public String getModelAsJson() throws AtlasException {
createDataModel();
return getDataModelAsJSON();
}
public static void main(String[] args) throws Exception {
SqoopDataModelGenerator dataModelGenerator = new SqoopDataModelGenerator();
String modelAsJson = dataModelGenerator.getModelAsJson();
if (args.length == 1) {
ModelDefinitionDump.dumpModelToFile(args[0], modelAsJson);
return;
}
System.out.println("sqoopDataModelAsJSON = " + modelAsJson);
TypesDef typesDef = dataModelGenerator.getTypesDef();
for (EnumTypeDefinition enumType : typesDef.enumTypesAsJavaList()) {
System.out.println(String.format("%s(%s) - values %s", enumType.name, EnumType.class.getSimpleName(),
Arrays.toString(enumType.enumValues)));
}
for (HierarchicalTypeDefinition<ClassType> classType : typesDef.classTypesAsJavaList()) {
System.out.println(
String.format("%s(%s) - super types [%s] - attributes %s", classType.typeName,
ClassType.class.getSimpleName(), StringUtils.join(classType.superTypes, ","),
Arrays.toString(classType.attributeDefinitions)));
}
}
}
addons/sqoop-bridge/src/test/java/org/apache/atlas/sqoop/hook/SqoopHookIT.java
...@@ -18,17 +18,13 @@ ...@@ -18,17 +18,13 @@
package org.apache.atlas.sqoop.hook; package org.apache.atlas.sqoop.hook;
import com.sun.jersey.api.client.ClientResponse;
import org.apache.atlas.ApplicationProperties; import org.apache.atlas.ApplicationProperties;
import org.apache.atlas.AtlasClient; import org.apache.atlas.AtlasClient;
import org.apache.atlas.AtlasServiceException;
import org.apache.atlas.hive.bridge.HiveMetaStoreBridge; import org.apache.atlas.hive.bridge.HiveMetaStoreBridge;
import org.apache.atlas.hive.model.HiveDataTypes; import org.apache.atlas.hive.model.HiveDataTypes;
import org.apache.atlas.sqoop.model.SqoopDataModelGenerator;
import org.apache.atlas.sqoop.model.SqoopDataTypes; import org.apache.atlas.sqoop.model.SqoopDataTypes;
import org.apache.atlas.utils.AuthenticationUtil; import org.apache.atlas.utils.AuthenticationUtil;
import org.apache.commons.configuration.Configuration; import org.apache.commons.configuration.Configuration;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.sqoop.SqoopJobDataPublisher; import org.apache.sqoop.SqoopJobDataPublisher;
import org.codehaus.jettison.json.JSONArray; import org.codehaus.jettison.json.JSONArray;
import org.codehaus.jettison.json.JSONObject; import org.codehaus.jettison.json.JSONObject;
...@@ -54,28 +50,6 @@ public class SqoopHookIT { ...@@ -54,28 +50,6 @@ public class SqoopHookIT {
} else { } else {
atlasClient = new AtlasClient(configuration.getStringArray(HiveMetaStoreBridge.ATLAS_ENDPOINT)); atlasClient = new AtlasClient(configuration.getStringArray(HiveMetaStoreBridge.ATLAS_ENDPOINT));
} }
registerDataModels(atlasClient);
}
private void registerDataModels(AtlasClient client) throws Exception {
// Make sure hive model exists
HiveMetaStoreBridge hiveMetaStoreBridge = new HiveMetaStoreBridge(ApplicationProperties.get(), new HiveConf(), atlasClient);
hiveMetaStoreBridge.registerHiveDataModel();
SqoopDataModelGenerator dataModelGenerator = new SqoopDataModelGenerator();
//Register sqoop data model if its not already registered
try {
client.getType(SqoopDataTypes.SQOOP_PROCESS.getName());
LOG.info("Sqoop data model is already registered!");
} catch(AtlasServiceException ase) {
if (ase.getStatus() == ClientResponse.Status.NOT_FOUND) {
//Expected in case types do not exist
LOG.info("Registering Sqoop data model");
client.createType(dataModelGenerator.getModelAsJson());
} else {
throw ase;
}
}
} }
@Test @Test
......
addons/storm-bridge/pom.xml
...@@ -529,24 +529,6 @@ ...@@ -529,24 +529,6 @@
<version>1.2.1</version> <version>1.2.1</version>
<inherited>false</inherited> <inherited>false</inherited>
<executions> <executions>
<execution>
<configuration>
<mainClass>org.apache.atlas.storm.model.StormDataModelGenerator</mainClass>
<systemProperties>
<systemProperty>
<key>atlas.conf</key>
<value>${project.build.directory}/../../../typesystem/target/test-classes</value>
</systemProperty>
</systemProperties>
<arguments>
<argument>${project.build.directory}/models/storm_model.json</argument>
</arguments>
</configuration>
<phase>package</phase>
<goals>
<goal>java</goal>
</goals>
</execution>
</executions> </executions>
</plugin> </plugin>
......
addons/storm-bridge/src/main/java/org/apache/atlas/storm/hook/StormAtlasHook.java
...@@ -26,9 +26,7 @@ import org.apache.storm.generated.TopologyInfo; ...@@ -26,9 +26,7 @@ import org.apache.storm.generated.TopologyInfo;
import org.apache.storm.utils.Utils; import org.apache.storm.utils.Utils;
import org.apache.atlas.AtlasClient; import org.apache.atlas.AtlasClient;
import org.apache.atlas.AtlasConstants; import org.apache.atlas.AtlasConstants;
import org.apache.atlas.fs.model.FSDataTypes;
import org.apache.atlas.hive.bridge.HiveMetaStoreBridge; import org.apache.atlas.hive.bridge.HiveMetaStoreBridge;
import org.apache.atlas.hive.model.HiveDataModelGenerator;
import org.apache.atlas.hook.AtlasHook; import org.apache.atlas.hook.AtlasHook;
import org.apache.atlas.storm.model.StormDataTypes; import org.apache.atlas.storm.model.StormDataTypes;
import org.apache.atlas.typesystem.Referenceable; import org.apache.atlas.typesystem.Referenceable;
...@@ -213,7 +211,7 @@ public class StormAtlasHook extends AtlasHook implements ISubmitterHook { ...@@ -213,7 +211,7 @@ public class StormAtlasHook extends AtlasHook implements ISubmitterHook {
break; break;
case "HdfsBolt": case "HdfsBolt":
dataSetReferenceable = new Referenceable(FSDataTypes.HDFS_PATH().toString()); dataSetReferenceable = new Referenceable(HiveMetaStoreBridge.HDFS_PATH);
String hdfsUri = config.get("HdfsBolt.rotationActions") == null String hdfsUri = config.get("HdfsBolt.rotationActions") == null
? config.get("HdfsBolt.fileNameFormat.path") ? config.get("HdfsBolt.fileNameFormat.path")
: config.get("HdfsBolt.rotationActions"); : config.get("HdfsBolt.rotationActions");
...@@ -241,7 +239,7 @@ public class StormAtlasHook extends AtlasHook implements ISubmitterHook { ...@@ -241,7 +239,7 @@ public class StormAtlasHook extends AtlasHook implements ISubmitterHook {
final String tableQualifiedName = HiveMetaStoreBridge.getTableQualifiedName(clusterName, final String tableQualifiedName = HiveMetaStoreBridge.getTableQualifiedName(clusterName,
databaseName, hiveTableName); databaseName, hiveTableName);
dataSetReferenceable.set(AtlasClient.NAME, hiveTableName); dataSetReferenceable.set(AtlasClient.NAME, hiveTableName);
dataSetReferenceable.set(HiveDataModelGenerator.DB, dbReferenceable); dataSetReferenceable.set(HiveMetaStoreBridge.DB, dbReferenceable);
dataSetReferenceable.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, tableQualifiedName); dataSetReferenceable.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, tableQualifiedName);
break; break;
......
addons/storm-bridge/src/main/java/org/apache/atlas/storm/model/StormDataModelGenerator.java deleted 100644 → 0
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.storm.model;
import org.apache.atlas.addons.ModelDefinitionDump;
import org.apache.atlas.typesystem.TypesDef;
import org.apache.atlas.typesystem.json.TypesSerialization;
import java.io.IOException;
public class StormDataModelGenerator {
public static void main(String[] args) throws IOException {
StormDataModel.main(new String[]{});
TypesDef typesDef = StormDataModel.typesDef();
String stormTypesAsJSON = TypesSerialization.toJson(typesDef);
if (args.length == 1) {
ModelDefinitionDump.dumpModelToFile(args[0], stormTypesAsJSON);
return;
}
System.out.println("stormTypesAsJSON = " + stormTypesAsJSON);
}
}
addons/storm-bridge/src/main/scala/org/apache/atlas/storm/model/StormDataModel.scala deleted 100644 → 0
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.storm.model
import org.apache.atlas.AtlasClient
import org.apache.atlas.typesystem.TypesDef
import org.apache.atlas.typesystem.builders.TypesBuilder
import org.apache.atlas.typesystem.json.TypesSerialization
/**
* This represents the data model for a storm topology.
*/
object StormDataModel extends App {
var typesDef : TypesDef = null
val typesBuilder = new TypesBuilder
import typesBuilder._
typesDef = types {
/**
* Model is represented as:
* Topology is a Process Super Type inheriting inputs/outputs
* Input DataSet(s) => Topology => Output DataSet(s)
* Also, Topology contains the Graph of Nodes
* Topology => Node(s) -> Spouts/Bolts
*/
_class(StormDataTypes.STORM_TOPOLOGY.getName, List(AtlasClient.PROCESS_SUPER_TYPE)) {
"id" ~ (string, required, indexed, unique)
"startTime" ~ date
"endTime" ~ date
"conf" ~ (map(string, string), optional)
"clusterName" ~ (string, optional, indexed)
// Nodes in the Graph
"nodes" ~ (array(StormDataTypes.STORM_NODE.getName), collection, composite)
}
// Base class for DataProducer aka Spouts and
// DataProcessor aka Bolts, also links from Topology
_class(StormDataTypes.STORM_NODE.getName) {
"name" ~ (string, required, indexed)
"description" ~ (string, optional, indexed)
// fully qualified driver java class name
"driverClass" ~ (string, required, indexed)
// spout or bolt configuration NVPs
"conf" ~ (map(string, string), optional)
}
// Data Producer and hence only outputs
_class(StormDataTypes.STORM_SPOUT.getName, List(StormDataTypes.STORM_NODE.getName)) {
// "outputs" ~ (array(StormDataTypes.STORM_NODE.getName), collection, composite)
"outputs" ~ (array(string), collection)
}
// Data Processor and hence both inputs and outputs (inherited from Spout)
_class(StormDataTypes.STORM_BOLT.getName, List(StormDataTypes.STORM_NODE.getName)) {
// "inputs" ~ (array(StormDataTypes.STORM_NODE.getName), collection, composite)
"inputs" ~ (array(string), collection)
"outputs" ~ (array(string), collection, optional)
}
// Kafka Data Set
_class(StormDataTypes.KAFKA_TOPIC.getName, List(AtlasClient.DATA_SET_SUPER_TYPE)) {
"topic" ~ (string, required, unique, indexed)
"uri" ~ (string, required)
}
// JMS Data Set
_class(StormDataTypes.JMS_TOPIC.getName, List(AtlasClient.DATA_SET_SUPER_TYPE)) {
"topic" ~ (string, required, unique, indexed)
"uri" ~ (string, required)
}
// HBase Data Set
_class(StormDataTypes.HBASE_TABLE.getName, List(AtlasClient.DATA_SET_SUPER_TYPE)) {
"uri" ~ (string, required)
}
// Hive table data set already exists in atlas.
}
// add the types to atlas
val typesAsJSON = TypesSerialization.toJson(typesDef)
println("Storm Data Model as JSON: ")
println(typesAsJSON)
}
addons/storm-bridge/src/test/java/org/apache/atlas/storm/hook/StormAtlasHookIT.java
...@@ -18,19 +18,11 @@ ...@@ -18,19 +18,11 @@
package org.apache.atlas.storm.hook; package org.apache.atlas.storm.hook;
import com.sun.jersey.api.client.ClientResponse;
import org.apache.atlas.ApplicationProperties; import org.apache.atlas.ApplicationProperties;
import org.apache.atlas.AtlasClient; import org.apache.atlas.AtlasClient;
import org.apache.atlas.AtlasException;
import org.apache.atlas.AtlasServiceException;
import org.apache.atlas.hive.bridge.HiveMetaStoreBridge; import org.apache.atlas.hive.bridge.HiveMetaStoreBridge;
import org.apache.atlas.hive.model.HiveDataModelGenerator;
import org.apache.atlas.hive.model.HiveDataTypes;
import org.apache.atlas.storm.model.StormDataModel;
import org.apache.atlas.storm.model.StormDataTypes; import org.apache.atlas.storm.model.StormDataTypes;
import org.apache.atlas.typesystem.Referenceable; import org.apache.atlas.typesystem.Referenceable;
import org.apache.atlas.typesystem.TypesDef;
import org.apache.atlas.typesystem.json.TypesSerialization;
import org.apache.atlas.utils.AuthenticationUtil; import org.apache.atlas.utils.AuthenticationUtil;
import org.apache.commons.configuration.Configuration; import org.apache.commons.configuration.Configuration;
import org.apache.storm.ILocalCluster; import org.apache.storm.ILocalCluster;
...@@ -67,37 +59,6 @@ public class StormAtlasHookIT { ...@@ -67,37 +59,6 @@ public class StormAtlasHookIT {
} else { } else {
atlasClient = new AtlasClient(configuration.getStringArray(HiveMetaStoreBridge.ATLAS_ENDPOINT)); atlasClient = new AtlasClient(configuration.getStringArray(HiveMetaStoreBridge.ATLAS_ENDPOINT));
} }
registerDataModel(new HiveDataModelGenerator());
}
private void registerDataModel(HiveDataModelGenerator dataModelGenerator) throws AtlasException,
AtlasServiceException {
try {
atlasClient.getType(HiveDataTypes.HIVE_PROCESS.getName());
LOG.info("Hive data model is already registered! Going ahead with registration of Storm Data model");
} catch(AtlasServiceException ase) {
if (ase.getStatus() == ClientResponse.Status.NOT_FOUND) {
//Expected in case types do not exist
LOG.info("Registering Hive data model");
atlasClient.createType(dataModelGenerator.getModelAsJson());
} else {
throw ase;
}
}
try {
atlasClient.getType(StormDataTypes.STORM_TOPOLOGY.getName());
} catch(AtlasServiceException ase) {
if (ase.getStatus() == ClientResponse.Status.NOT_FOUND) {
LOG.info("Registering Storm/Kafka data model");
StormDataModel.main(new String[]{});
TypesDef typesDef = StormDataModel.typesDef();
String stormTypesAsJSON = TypesSerialization.toJson(typesDef);
LOG.info("stormTypesAsJSON = {}", stormTypesAsJSON);
atlasClient.createType(stormTypesAsJSON);
}
}
} }
...@@ -109,23 +70,6 @@ public class StormAtlasHookIT { ...@@ -109,23 +70,6 @@ public class StormAtlasHookIT {
atlasClient = null; atlasClient = null;
} }
@Test
public void testCreateDataModel() throws Exception {
StormDataModel.main(new String[]{});
TypesDef stormTypesDef = StormDataModel.typesDef();
String stormTypesAsJSON = TypesSerialization.toJson(stormTypesDef);
LOG.info("stormTypesAsJSON = {}", stormTypesAsJSON);
registerDataModel(new HiveDataModelGenerator());
// verify types are registered
for (StormDataTypes stormDataType : StormDataTypes.values()) {
Assert.assertNotNull(atlasClient.getType(stormDataType.getName()));
}
}
@Test (dependsOnMethods = "testCreateDataModel")
public void testAddEntities() throws Exception { public void testAddEntities() throws Exception {
StormTopology stormTopology = StormTestUtil.createTestTopology(); StormTopology stormTopology = StormTestUtil.createTestTopology();
StormTestUtil.submitTopology(stormCluster, TOPOLOGY_NAME, stormTopology); StormTestUtil.submitTopology(stormCluster, TOPOLOGY_NAME, stormTopology);
......
distro/src/main/assemblies/standalone-package.xml
...@@ -106,9 +106,9 @@ ...@@ -106,9 +106,9 @@
<outputDirectory>examples</outputDirectory> <outputDirectory>examples</outputDirectory>
</fileSet> </fileSet>
<!-- addons/hdfs-model --> <!-- out-of-box-models -->
<fileSet> <fileSet>
<directory>../addons/hdfs-model/target/models</directory> <directory>../addons/models</directory>
<outputDirectory>models</outputDirectory> <outputDirectory>models</outputDirectory>
</fileSet> </fileSet>
...@@ -130,55 +130,29 @@ ...@@ -130,55 +130,29 @@
<outputDirectory>hook</outputDirectory> <outputDirectory>hook</outputDirectory>
</fileSet> </fileSet>
<fileSet>
<directory>../addons/hive-bridge/target/models</directory>
<outputDirectory>models</outputDirectory>
</fileSet>
<!-- addons/falcon --> <!-- addons/falcon -->
<fileSet> <fileSet>
<directory>../addons/falcon-bridge/target/dependency/hook</directory> <directory>../addons/falcon-bridge/target/dependency/hook</directory>
<outputDirectory>hook</outputDirectory> <outputDirectory>hook</outputDirectory>
</fileSet> </fileSet>
<fileSet>
<directory>../addons/falcon-bridge/target/models</directory>
<outputDirectory>models</outputDirectory>
</fileSet>
<!-- addons/sqoop --> <!-- addons/sqoop -->
<fileSet> <fileSet>
<directory>../addons/sqoop-bridge/target/dependency/hook</directory> <directory>../addons/sqoop-bridge/target/dependency/hook</directory>
<outputDirectory>hook</outputDirectory> <outputDirectory>hook</outputDirectory>
</fileSet> </fileSet>
<fileSet>
<directory>../addons/sqoop-bridge/target/models</directory>
<outputDirectory>models</outputDirectory>
</fileSet>
<!-- addons/storm --> <!-- addons/storm -->
<fileSet> <fileSet>
<directory>../addons/storm-bridge/target/dependency/hook</directory> <directory>../addons/storm-bridge/target/dependency/hook</directory>
<outputDirectory>hook</outputDirectory> <outputDirectory>hook</outputDirectory>
</fileSet> </fileSet>
<fileSet>
<directory>../addons/storm-bridge/target/models</directory>
<outputDirectory>models</outputDirectory>
</fileSet>
<!-- for kafka topic setup --> <!-- for kafka topic setup -->
<fileSet> <fileSet>
<directory>../notification/target/dependency/hook</directory> <directory>../notification/target/dependency/hook</directory>
<outputDirectory>hook</outputDirectory> <outputDirectory>hook</outputDirectory>
</fileSet> </fileSet>
<!-- for patches -->
<fileSet>
<directory>../addons/hive-bridge/src/patches</directory>
<outputDirectory>models/patches</outputDirectory>
</fileSet>
</fileSets> </fileSets>
<files> <files>
......
intg/src/main/java/org/apache/atlas/AtlasErrorCode.java
...@@ -53,7 +53,9 @@ public enum AtlasErrorCode { ...@@ -53,7 +53,9 @@ public enum AtlasErrorCode {
INTERNAL_ERROR(500, "ATLAS5001E", "Internal server error {0}"), INTERNAL_ERROR(500, "ATLAS5001E", "Internal server error {0}"),
INDEX_CREATION_FAILED(500, "ATLAS5002E", "Index creation failed for {0}"), INDEX_CREATION_FAILED(500, "ATLAS5002E", "Index creation failed for {0}"),
INDEX_ROLLBACK_FAILED(500, "ATLAS5003E", "Index rollback failed for {0}") INDEX_ROLLBACK_FAILED(500, "ATLAS5003E", "Index rollback failed for {0}"),
PATCH_NOT_APPLICABLE_FOR_TYPE(500, "ATLAS5004E", "{0} - invalid patch for type {1}"),
PATCH_FOR_UNKNOWN_TYPE(500, "ATLAS5005E", "{0} - patch references unknown type {1}")
; ;
private String errorCode; private String errorCode;
......
intg/src/main/java/org/apache/atlas/model/typedef/AtlasStructDef.java
...@@ -279,12 +279,12 @@ public class AtlasStructDef extends AtlasBaseTypeDef implements Serializable { ...@@ -279,12 +279,12 @@ public class AtlasStructDef extends AtlasBaseTypeDef implements Serializable {
List<AtlasConstraintDef> constraintDefs) { List<AtlasConstraintDef> constraintDefs) {
setName(name); setName(name);
setTypeName(typeName); setTypeName(typeName);
setOptional(isOptional); setIsOptional(isOptional);
setCardinality(cardinality); setCardinality(cardinality);
setValuesMinCount(valuesMinCount); setValuesMinCount(valuesMinCount);
setValuesMaxCount(valuesMaxCount); setValuesMaxCount(valuesMaxCount);
setUnique(isUnique); setIsUnique(isUnique);
setIndexable(isIndexable); setIsIndexable(isIndexable);
setConstraintDefs(constraintDefs); setConstraintDefs(constraintDefs);
} }
...@@ -292,12 +292,12 @@ public class AtlasStructDef extends AtlasBaseTypeDef implements Serializable { ...@@ -292,12 +292,12 @@ public class AtlasStructDef extends AtlasBaseTypeDef implements Serializable {
if (other != null) { if (other != null) {
setName(other.getName()); setName(other.getName());
setTypeName(other.getTypeName()); setTypeName(other.getTypeName());
setOptional(other.isOptional()); setIsOptional(other.getIsOptional());
setCardinality(other.getCardinality()); setCardinality(other.getCardinality());
setValuesMinCount(other.getValuesMinCount()); setValuesMinCount(other.getValuesMinCount());
setValuesMaxCount(other.getValuesMaxCount()); setValuesMaxCount(other.getValuesMaxCount());
setUnique(other.isUnique()); setIsUnique(other.getIsUnique());
setIndexable(other.isIndexable()); setIsIndexable(other.getIsIndexable());
setConstraintDefs(other.getConstraintDefs()); setConstraintDefs(other.getConstraintDefs());
} }
} }
...@@ -318,11 +318,11 @@ public class AtlasStructDef extends AtlasBaseTypeDef implements Serializable { ...@@ -318,11 +318,11 @@ public class AtlasStructDef extends AtlasBaseTypeDef implements Serializable {
this.typeName = typeName; this.typeName = typeName;
} }
public boolean isOptional() { public boolean getIsOptional() {
return isOptional; return isOptional;
} }
public void setOptional(boolean optional) { isOptional = optional; } public void setIsOptional(boolean optional) { isOptional = optional; }
public void setCardinality(Cardinality cardinality) { public void setCardinality(Cardinality cardinality) {
this.cardinality = cardinality; this.cardinality = cardinality;
...@@ -348,19 +348,19 @@ public class AtlasStructDef extends AtlasBaseTypeDef implements Serializable { ...@@ -348,19 +348,19 @@ public class AtlasStructDef extends AtlasBaseTypeDef implements Serializable {
this.valuesMaxCount = valuesMaxCount; this.valuesMaxCount = valuesMaxCount;
} }
public boolean isUnique() { public boolean getIsUnique() {
return isUnique; return isUnique;
} }
public void setUnique(boolean unique) { public void setIsUnique(boolean unique) {
isUnique = unique; isUnique = unique;
} }
public boolean isIndexable() { public boolean getIsIndexable() {
return isIndexable; return isIndexable;
} }
public void setIndexable(boolean idexable) { public void setIsIndexable(boolean idexable) {
isIndexable = idexable; isIndexable = idexable;
} }
...@@ -399,7 +399,7 @@ public class AtlasStructDef extends AtlasBaseTypeDef implements Serializable { ...@@ -399,7 +399,7 @@ public class AtlasStructDef extends AtlasBaseTypeDef implements Serializable {
sb.append("AtlasAttributeDef{"); sb.append("AtlasAttributeDef{");
sb.append("name='").append(name).append('\''); sb.append("name='").append(name).append('\'');
sb.append(", typeName='").append(typeName).append('\''); sb.append(", typeName='").append(typeName).append('\'');
sb.append(", isOptional=").append(isOptional); sb.append(", getIsOptional=").append(isOptional);
sb.append(", cardinality=").append(cardinality); sb.append(", cardinality=").append(cardinality);
sb.append(", valuesMinCount=").append(valuesMinCount); sb.append(", valuesMinCount=").append(valuesMinCount);
sb.append(", valuesMaxCount=").append(valuesMaxCount); sb.append(", valuesMaxCount=").append(valuesMaxCount);
......
intg/src/main/java/org/apache/atlas/model/typedef/AtlasTypesDef.java
...@@ -19,6 +19,7 @@ package org.apache.atlas.model.typedef; ...@@ -19,6 +19,7 @@ package org.apache.atlas.model.typedef;
import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.CollectionUtils;
import org.codehaus.jackson.annotate.JsonAutoDetect; import org.codehaus.jackson.annotate.JsonAutoDetect;
import org.codehaus.jackson.annotate.JsonIgnore;
import org.codehaus.jackson.annotate.JsonIgnoreProperties; import org.codehaus.jackson.annotate.JsonIgnoreProperties;
import org.codehaus.jackson.map.annotate.JsonSerialize; import org.codehaus.jackson.map.annotate.JsonSerialize;
...@@ -91,6 +92,7 @@ public class AtlasTypesDef { ...@@ -91,6 +92,7 @@ public class AtlasTypesDef {
this.entityDefs = entityDefs; this.entityDefs = entityDefs;
} }
@JsonIgnore
public boolean isEmpty() { public boolean isEmpty() {
return CollectionUtils.isEmpty(enumDefs) && return CollectionUtils.isEmpty(enumDefs) &&
CollectionUtils.isEmpty(structDefs) && CollectionUtils.isEmpty(structDefs) &&
......
intg/src/main/java/org/apache/atlas/type/AtlasStructType.java
...@@ -198,7 +198,7 @@ public class AtlasStructType extends AtlasType { ...@@ -198,7 +198,7 @@ public class AtlasStructType extends AtlasType {
if (value != null) { if (value != null) {
ret = dataType.validateValue(value, fieldName, messages) && ret; ret = dataType.validateValue(value, fieldName, messages) && ret;
} else if (!attributeDef.isOptional()) { } else if (!attributeDef.getIsOptional()) {
ret = false; ret = false;
messages.add(fieldName + ": mandatory attribute value missing in type " + getTypeName()); messages.add(fieldName + ": mandatory attribute value missing in type " + getTypeName());
...@@ -218,7 +218,7 @@ public class AtlasStructType extends AtlasType { ...@@ -218,7 +218,7 @@ public class AtlasStructType extends AtlasType {
if (value != null) { if (value != null) {
ret = dataType.validateValue(value, fieldName, messages) && ret; ret = dataType.validateValue(value, fieldName, messages) && ret;
} else if (!attributeDef.isOptional()) { } else if (!attributeDef.getIsOptional()) {
ret = false; ret = false;
messages.add(fieldName + ": mandatory attribute value missing in type " + getTypeName()); messages.add(fieldName + ": mandatory attribute value missing in type " + getTypeName());
...@@ -244,7 +244,7 @@ public class AtlasStructType extends AtlasType { ...@@ -244,7 +244,7 @@ public class AtlasStructType extends AtlasType {
Object attributeValue = getNormalizedValue(obj.getAttribute(attributeName), attributeDef); Object attributeValue = getNormalizedValue(obj.getAttribute(attributeName), attributeDef);
obj.setAttribute(attributeName, attributeValue); obj.setAttribute(attributeName, attributeValue);
} else if (!attributeDef.isOptional()) { } else if (!attributeDef.getIsOptional()) {
obj.setAttribute(attributeName, createDefaultValue(attributeDef)); obj.setAttribute(attributeName, createDefaultValue(attributeDef));
} }
} }
...@@ -260,7 +260,7 @@ public class AtlasStructType extends AtlasType { ...@@ -260,7 +260,7 @@ public class AtlasStructType extends AtlasType {
Object attributeValue = getNormalizedValue(obj.get(attributeName), attributeDef); Object attributeValue = getNormalizedValue(obj.get(attributeName), attributeDef);
obj.put(attributeName, attributeValue); obj.put(attributeName, attributeValue);
} else if (!attributeDef.isOptional()) { } else if (!attributeDef.getIsOptional()) {
obj.put(attributeName, createDefaultValue(attributeDef)); obj.put(attributeName, createDefaultValue(attributeDef));
} }
} }
...@@ -276,7 +276,7 @@ public class AtlasStructType extends AtlasType { ...@@ -276,7 +276,7 @@ public class AtlasStructType extends AtlasType {
} }
for (AtlasAttributeDef attributeDef : structDef.getAttributeDefs()) { for (AtlasAttributeDef attributeDef : structDef.getAttributeDefs()) {
if (!attributeDef.isOptional()) { if (!attributeDef.getIsOptional()) {
attributes.put(attributeDef.getName(), createDefaultValue(attributeDef)); attributes.put(attributeDef.getName(), createDefaultValue(attributeDef));
} }
} }
...@@ -310,7 +310,7 @@ public class AtlasStructType extends AtlasType { ...@@ -310,7 +310,7 @@ public class AtlasStructType extends AtlasType {
ret = false; // invalid value ret = false; // invalid value
} }
} }
} else if (!attributeDef.isOptional()) { } else if (!attributeDef.getIsOptional()) {
ret = false; // mandatory attribute not present ret = false; // mandatory attribute not present
} }
...@@ -322,7 +322,7 @@ public class AtlasStructType extends AtlasType { ...@@ -322,7 +322,7 @@ public class AtlasStructType extends AtlasType {
if (attrType != null) { if (attrType != null) {
if (value == null) { if (value == null) {
if (!attributeDef.isOptional()) { if (!attributeDef.getIsOptional()) {
return attrType.createDefaultValue(); return attrType.createDefaultValue();
} }
} else { } else {
......
intg/src/main/java/org/apache/atlas/type/AtlasTypeRegistry.java
...@@ -64,6 +64,10 @@ public class AtlasTypeRegistry { ...@@ -64,6 +64,10 @@ public class AtlasTypeRegistry {
public Collection<String> getAllTypeNames() { return registryData.allTypes.getAllTypeNames(); } public Collection<String> getAllTypeNames() { return registryData.allTypes.getAllTypeNames(); }
public boolean isRegisteredType(String typeName) {
return registryData.allTypes.isKnownType(typeName);
}
public AtlasType getType(String typeName) throws AtlasBaseException { public AtlasType getType(String typeName) throws AtlasBaseException {
if (LOG.isDebugEnabled()) { if (LOG.isDebugEnabled()) {
LOG.debug("==> AtlasTypeRegistry.getType({})", typeName); LOG.debug("==> AtlasTypeRegistry.getType({})", typeName);
...@@ -677,6 +681,10 @@ class TypeCache { ...@@ -677,6 +681,10 @@ class TypeCache {
} }
} }
public boolean isKnownType(String typeName) {
return typeNameMap.containsKey(typeName);
}
public Collection<String> getAllTypeNames() { public Collection<String> getAllTypeNames() {
return Collections.unmodifiableCollection(typeNameMap.keySet()); return Collections.unmodifiableCollection(typeNameMap.keySet());
} }
......
repository/src/main/java/org/apache/atlas/RepositoryMetadataModule.java
...@@ -44,9 +44,7 @@ import org.apache.atlas.repository.typestore.GraphBackedTypeStore; ...@@ -44,9 +44,7 @@ import org.apache.atlas.repository.typestore.GraphBackedTypeStore;
import org.apache.atlas.repository.typestore.ITypeStore; import org.apache.atlas.repository.typestore.ITypeStore;
import org.apache.atlas.service.Service; import org.apache.atlas.service.Service;
import org.apache.atlas.services.DefaultMetadataService; import org.apache.atlas.services.DefaultMetadataService;
import org.apache.atlas.services.IBootstrapTypesRegistrar;
import org.apache.atlas.services.MetadataService; import org.apache.atlas.services.MetadataService;
import org.apache.atlas.services.ReservedTypesRegistrar;
import org.apache.atlas.store.AtlasTypeDefStore; import org.apache.atlas.store.AtlasTypeDefStore;
import org.apache.atlas.type.AtlasTypeRegistry; import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.atlas.typesystem.types.TypeSystem; import org.apache.atlas.typesystem.types.TypeSystem;
...@@ -92,8 +90,6 @@ public class RepositoryMetadataModule extends com.google.inject.AbstractModule { ...@@ -92,8 +90,6 @@ public class RepositoryMetadataModule extends com.google.inject.AbstractModule {
// bind the MetadataService interface to an implementation // bind the MetadataService interface to an implementation
bind(MetadataService.class).to(DefaultMetadataService.class).asEagerSingleton(); bind(MetadataService.class).to(DefaultMetadataService.class).asEagerSingleton();
bind(IBootstrapTypesRegistrar.class).to(ReservedTypesRegistrar.class);
// bind the DiscoveryService interface to an implementation // bind the DiscoveryService interface to an implementation
bind(DiscoveryService.class).to(GraphBackedDiscoveryService.class).asEagerSingleton(); bind(DiscoveryService.class).to(GraphBackedDiscoveryService.class).asEagerSingleton();
......
repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java
...@@ -267,8 +267,8 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang ...@@ -267,8 +267,8 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
AtlasAttributeDef attributeDef) { AtlasAttributeDef attributeDef) {
final String propertyName = GraphHelper.encodePropertyKey(typeName + "." + attributeDef.getName()); final String propertyName = GraphHelper.encodePropertyKey(typeName + "." + attributeDef.getName());
AtlasCardinality cardinality = toAtlasCardinality(attributeDef.getCardinality()); AtlasCardinality cardinality = toAtlasCardinality(attributeDef.getCardinality());
boolean isUnique = attributeDef.isUnique(); boolean isUnique = attributeDef.getIsUnique();
boolean isIndexable = attributeDef.isIndexable(); boolean isIndexable = attributeDef.getIsIndexable();
String attribTypeName = attributeDef.getTypeName(); String attribTypeName = attributeDef.getTypeName();
boolean isBuiltInType = AtlasTypeUtil.isBuiltInType(attribTypeName); boolean isBuiltInType = AtlasTypeUtil.isBuiltInType(attribTypeName);
boolean isArrayType = AtlasTypeUtil.isArrayType(attribTypeName); boolean isArrayType = AtlasTypeUtil.isArrayType(attribTypeName);
......
repository/src/main/java/org/apache/atlas/repository/store/bootstrap/AtlasTypeDefStoreInitializer.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.repository.store.bootstrap;
import org.apache.atlas.AtlasErrorCode;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.typedef.AtlasBaseTypeDef;
import org.apache.atlas.model.typedef.AtlasClassificationDef;
import org.apache.atlas.model.typedef.AtlasEntityDef;
import org.apache.atlas.model.typedef.AtlasEnumDef;
import org.apache.atlas.model.typedef.AtlasStructDef;
import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef;
import org.apache.atlas.model.typedef.AtlasTypesDef;
import org.apache.atlas.store.AtlasTypeDefStore;
import org.apache.atlas.type.AtlasType;
import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.commons.collections.CollectionUtils;
import org.codehaus.jackson.annotate.JsonAutoDetect;
import org.codehaus.jackson.annotate.JsonIgnoreProperties;
import org.codehaus.jackson.map.annotate.JsonSerialize;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlRootElement;
import java.io.File;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import static org.codehaus.jackson.annotate.JsonAutoDetect.Visibility.NONE;
import static org.codehaus.jackson.annotate.JsonAutoDetect.Visibility.PUBLIC_ONLY;
/**
* Class that handles initial loading of models and patches into typedef store
*/
public class AtlasTypeDefStoreInitializer {
private static final Logger LOG = LoggerFactory.getLogger(AtlasTypeDefStoreInitializer.class);
public void initializeStore(AtlasTypeDefStore typeDefStore, AtlasTypeRegistry typeRegistry, String typesDirName) {
File typesDir = new File(typesDirName);
File[] typeDefFiles = typesDir.exists() ? typesDir.listFiles() : null;
if (typeDefFiles == null || typeDefFiles.length == 0) {
LOG.info("Types directory {} does not exist or not readable or has no typedef files", typesDirName);
return;
}
// sort the files by filename
Arrays.sort(typeDefFiles);
for (File typeDefFile : typeDefFiles) {
if (!typeDefFile.isFile()) {
continue;
}
try {
String jsonStr = new String(Files.readAllBytes(typeDefFile.toPath()), StandardCharsets.UTF_8);
AtlasTypesDef typesDef = AtlasType.fromJson(jsonStr, AtlasTypesDef.class);
if (typesDef == null || typesDef.isEmpty()) {
LOG.info("No type in file {}", typeDefFile.getAbsolutePath());
continue;
}
AtlasTypesDef typesToCreate = new AtlasTypesDef();
if (CollectionUtils.isNotEmpty(typesDef.getEnumDefs())) {
for (AtlasEnumDef enumDef : typesDef.getEnumDefs()) {
if (!typeRegistry.isRegisteredType(enumDef.getName())) {
typesToCreate.getEnumDefs().add(enumDef);
}
}
}
if (CollectionUtils.isNotEmpty(typesDef.getStructDefs())) {
for (AtlasStructDef structDef : typesDef.getStructDefs()) {
if (!typeRegistry.isRegisteredType(structDef.getName())) {
typesToCreate.getStructDefs().add(structDef);
}
}
}
if (CollectionUtils.isNotEmpty(typesDef.getClassificationDefs())) {
for (AtlasClassificationDef classificationDef : typesDef.getClassificationDefs()) {
if (!typeRegistry.isRegisteredType(classificationDef.getName())) {
typesToCreate.getClassificationDefs().add(classificationDef);
}
}
}
if (CollectionUtils.isNotEmpty(typesDef.getEntityDefs())) {
for (AtlasEntityDef entityDef : typesDef.getEntityDefs()) {
if (!typeRegistry.isRegisteredType(entityDef.getName())) {
typesToCreate.getEntityDefs().add(entityDef);
}
}
}
if (typesToCreate.isEmpty()) {
LOG.info("No new type in file {}", typeDefFile.getAbsolutePath());
continue;
}
LOG.info("Loading types defined in file {}", typeDefFile.getAbsolutePath());
typeDefStore.createTypesDef(typesDef);
} catch (Throwable t) {
LOG.error("error while registering types in file " + typeDefFile.getAbsolutePath(), t);
}
}
applyTypePatches(typeDefStore, typeRegistry, typesDirName);
}
private void applyTypePatches(AtlasTypeDefStore typeDefStore, AtlasTypeRegistry typeRegistry, String typesDirName) {
String typePatchesDirName = typesDirName + File.separator + "patches";
File typePatchesDir = new File(typePatchesDirName);
File[] typePatchFiles = typePatchesDir.exists() ? typePatchesDir.listFiles() : null;
if (typePatchFiles == null || typePatchFiles.length == 0) {
LOG.info("Type patches directory {} does not exist or not readable or has no patches", typePatchesDirName);
return;
}
// sort the files by filename
Arrays.sort(typePatchFiles);
PatchHandler[] patchHandlers = new PatchHandler[] { new AddAttributePatchHandler(typeDefStore, typeRegistry) };
Map<String, PatchHandler> patchHandlerRegistry = new HashMap<>();
for (PatchHandler patchHandler : patchHandlers) {
for (String supportedAction : patchHandler.getSupportedActions()) {
patchHandlerRegistry.put(supportedAction, patchHandler);
}
}
for (File typePatchFile : typePatchFiles) {
if (!typePatchFile.isFile()) {
continue;
}
try {
String jsonStr = new String(Files.readAllBytes(typePatchFile.toPath()), StandardCharsets.UTF_8);
TypeDefPatches patches = AtlasType.fromJson(jsonStr, TypeDefPatches.class);
if (patches == null || CollectionUtils.isEmpty(patches.getPatches())) {
LOG.info("No patches in file {}", typePatchFile.getAbsolutePath());
continue;
}
for (TypeDefPatch patch : patches.getPatches()) {
PatchHandler patchHandler = patchHandlerRegistry.get(patch.getAction());
if (patchHandler == null) {
LOG.error("Unknown patch action {} in file {}. Ignored",
patch.getAction(), typePatchFile.getAbsolutePath());
continue;
}
try {
patchHandler.applyPatch(patch);
} catch (AtlasBaseException excp) {
LOG.error("Failed to apply " + patch.getAction() + " patch in file " +
typePatchFile.getAbsolutePath() + ". Ignored", excp);
}
}
} catch (Throwable t) {
LOG.error("Failed to apply patches in file " + typePatchFile.getAbsolutePath() + ". Ignored", t);
}
}
}
/**
* typedef patch details
*/
@JsonAutoDetect(getterVisibility = PUBLIC_ONLY, setterVisibility = PUBLIC_ONLY, fieldVisibility = NONE)
@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
@JsonIgnoreProperties(ignoreUnknown = true)
@XmlRootElement
@XmlAccessorType(XmlAccessType.PROPERTY)
static class TypeDefPatch {
private String action;
private String typeName;
private String applyToVersion;
private String updateToVersion;
private Map<String, Object> params;
private List<AtlasAttributeDef> attributeDefs;
public String getAction() {
return action;
}
public void setAction(String action) {
this.action = action;
}
public String getTypeName() {
return typeName;
}
public void setTypeName(String typeName) {
this.typeName = typeName;
}
public String getApplyToVersion() {
return applyToVersion;
}
public void setApplyToVersion(String applyToVersion) {
this.applyToVersion = applyToVersion;
}
public String getUpdateToVersion() {
return updateToVersion;
}
public void setUpdateToVersion(String updateToVersion) {
this.updateToVersion = updateToVersion;
}
public Map<String, Object> getParams() {
return params;
}
public void setParams(Map<String, Object> params) {
this.params = params;
}
public List<AtlasAttributeDef> getAttributeDefs() {
return attributeDefs;
}
public void setAttributeDefs(List<AtlasAttributeDef> attributeDefs) {
this.attributeDefs = attributeDefs;
}
}
/**
* list of typedef patches
*/
@JsonAutoDetect(getterVisibility = PUBLIC_ONLY, setterVisibility = PUBLIC_ONLY, fieldVisibility = NONE)
@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
@JsonIgnoreProperties(ignoreUnknown = true)
@XmlRootElement
@XmlAccessorType(XmlAccessType.PROPERTY)
static class TypeDefPatches {
private List<TypeDefPatch> patches;
public List<TypeDefPatch> getPatches() {
return patches;
}
public void setPatches(List<TypeDefPatch> patches) {
this.patches = patches;
}
}
abstract class PatchHandler {
protected final AtlasTypeDefStore typeDefStore;
protected final AtlasTypeRegistry typeRegistry;
protected final String[] supportedActions;
protected PatchHandler(AtlasTypeDefStore typeDefStore, AtlasTypeRegistry typeRegistry, String[] supportedActions) {
this.typeDefStore = typeDefStore;
this.typeRegistry = typeRegistry;
this.supportedActions = supportedActions;
}
public String[] getSupportedActions() { return supportedActions; }
public abstract void applyPatch(TypeDefPatch patch) throws AtlasBaseException;
protected boolean isPatchApplicable(TypeDefPatch patch, AtlasBaseTypeDef currentTypeDef) {
String currentVersion = currentTypeDef.getTypeVersion();
String applyToVersion = patch.getApplyToVersion();
return currentVersion == null ||
currentVersion.equalsIgnoreCase(applyToVersion) ||
currentVersion.startsWith(applyToVersion + ".");
}
}
class AddAttributePatchHandler extends PatchHandler {
public AddAttributePatchHandler(AtlasTypeDefStore typeDefStore, AtlasTypeRegistry typeRegistry) {
super(typeDefStore, typeRegistry, new String[] { "ADD_ATTRIBUTE" });
}
@Override
public void applyPatch(TypeDefPatch patch) throws AtlasBaseException {
String typeName = patch.getTypeName();
if (!typeRegistry.isRegisteredType(typeName)) {
throw new AtlasBaseException(AtlasErrorCode.PATCH_FOR_UNKNOWN_TYPE, patch.getAction(), typeName);
}
AtlasBaseTypeDef typeDef = typeRegistry.getTypeDefByName(typeName);
if (isPatchApplicable(patch, typeDef)) {
if (typeDef.getClass().equals(AtlasEntityDef.class)) {
AtlasEntityDef updatedDef = new AtlasEntityDef((AtlasEntityDef)typeDef);
for (AtlasAttributeDef attributeDef : patch.getAttributeDefs()) {
updatedDef.addAttribute(attributeDef);
}
updatedDef.setTypeVersion(patch.getUpdateToVersion());
typeDefStore.updateEntityDefByName(typeName, updatedDef);
} else if (typeDef.getClass().equals(AtlasClassificationDef.class)) {
AtlasClassificationDef updatedDef = new AtlasClassificationDef((AtlasClassificationDef)typeDef);
for (AtlasAttributeDef attributeDef : patch.getAttributeDefs()) {
updatedDef.addAttribute(attributeDef);
}
updatedDef.setTypeVersion(patch.getUpdateToVersion());
typeDefStore.updateClassificationDefByName(typeName, updatedDef);
} else if (typeDef.getClass().equals(AtlasStructDef.class)) {
AtlasStructDef updatedDef = new AtlasStructDef((AtlasStructDef)typeDef);
for (AtlasAttributeDef attributeDef : patch.getAttributeDefs()) {
updatedDef.addAttribute(attributeDef);
}
updatedDef.setTypeVersion(patch.getUpdateToVersion());
typeDefStore.updateStructDefByName(typeName, updatedDef);
} else {
throw new AtlasBaseException(AtlasErrorCode.PATCH_NOT_APPLICABLE_FOR_TYPE,
patch.getAction(), typeDef.getClass().getSimpleName());
}
} else {
LOG.info("patch skipped: typeName={}; applyToVersion={}; updateToVersion={}",
patch.getTypeName(), patch.getApplyToVersion(), patch.getUpdateToVersion());
}
}
}
}
\ No newline at end of file
repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasTypeDefGraphStore.java
...@@ -35,6 +35,7 @@ import org.apache.atlas.model.typedef.AtlasEnumDef.AtlasEnumDefs; ...@@ -35,6 +35,7 @@ import org.apache.atlas.model.typedef.AtlasEnumDef.AtlasEnumDefs;
import org.apache.atlas.model.typedef.AtlasStructDef; import org.apache.atlas.model.typedef.AtlasStructDef;
import org.apache.atlas.model.typedef.AtlasStructDef.AtlasStructDefs; import org.apache.atlas.model.typedef.AtlasStructDef.AtlasStructDefs;
import org.apache.atlas.model.typedef.AtlasTypesDef; import org.apache.atlas.model.typedef.AtlasTypesDef;
import org.apache.atlas.repository.store.bootstrap.AtlasTypeDefStoreInitializer;
import org.apache.atlas.repository.util.FilterUtil; import org.apache.atlas.repository.util.FilterUtil;
import org.apache.atlas.store.AtlasTypeDefStore; import org.apache.atlas.store.AtlasTypeDefStore;
import org.apache.atlas.type.AtlasTypeRegistry; import org.apache.atlas.type.AtlasTypeRegistry;
...@@ -45,6 +46,7 @@ import org.apache.commons.lang3.StringUtils; ...@@ -45,6 +46,7 @@ import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import java.io.File;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collection; import java.util.Collection;
import java.util.Collections; import java.util.Collections;
...@@ -89,6 +91,8 @@ public abstract class AtlasTypeDefGraphStore implements AtlasTypeDefStore, Activ ...@@ -89,6 +91,8 @@ public abstract class AtlasTypeDefGraphStore implements AtlasTypeDefStore, Activ
ttr.addTypes(typesDef); ttr.addTypes(typesDef);
typeRegistry.commitTransientTypeRegistry(ttr); typeRegistry.commitTransientTypeRegistry(ttr);
bootstrapTypes();
} }
@Override @Override
...@@ -886,6 +890,15 @@ public abstract class AtlasTypeDefGraphStore implements AtlasTypeDefStore, Activ ...@@ -886,6 +890,15 @@ public abstract class AtlasTypeDefGraphStore implements AtlasTypeDefStore, Activ
LOG.info("Not reacting to a Passive state change"); LOG.info("Not reacting to a Passive state change");
} }
private void bootstrapTypes() {
AtlasTypeDefStoreInitializer storeInitializer = new AtlasTypeDefStoreInitializer();
String atlasHomeDir = System.getProperty("atlas.home");
String typesDirName = (StringUtils.isEmpty(atlasHomeDir) ? "." : atlasHomeDir) + File.separator + "models";
storeInitializer.initializeStore(this, typeRegistry, typesDirName);
}
private void updateTypeRegistryPostCommit(AtlasTransientTypeRegistry ttr) { private void updateTypeRegistryPostCommit(AtlasTransientTypeRegistry ttr) {
new TypeRegistryUpdateHook(ttr); new TypeRegistryUpdateHook(ttr);
} }
......
repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasStructDefStoreV1.java
...@@ -421,7 +421,7 @@ public class AtlasStructDefStoreV1 extends AtlasAbstractDefStoreV1 implements At ...@@ -421,7 +421,7 @@ public class AtlasStructDefStoreV1 extends AtlasAbstractDefStoreV1 implements At
for (AtlasAttributeDef attributeDef : structDef.getAttributeDefs()) { for (AtlasAttributeDef attributeDef : structDef.getAttributeDefs()) {
if (CollectionUtils.isEmpty(currAttrNames) || !currAttrNames.contains(attributeDef.getName())) { if (CollectionUtils.isEmpty(currAttrNames) || !currAttrNames.contains(attributeDef.getName())) {
// new attribute - only allow if optional // new attribute - only allow if optional
if (!attributeDef.isOptional()) { if (!attributeDef.getIsOptional()) {
throw new AtlasBaseException(AtlasErrorCode.CANNOT_ADD_MANDATORY_ATTRIBUTE, structDef.getName(), attributeDef.getName()); throw new AtlasBaseException(AtlasErrorCode.CANNOT_ADD_MANDATORY_ATTRIBUTE, structDef.getName(), attributeDef.getName());
} }
} }
...@@ -510,13 +510,30 @@ public class AtlasStructDefStoreV1 extends AtlasAbstractDefStoreV1 implements At ...@@ -510,13 +510,30 @@ public class AtlasStructDefStoreV1 extends AtlasAbstractDefStoreV1 implements At
attribInfo.put("name", attributeDef.getName()); attribInfo.put("name", attributeDef.getName());
attribInfo.put("dataType", attributeDef.getTypeName()); attribInfo.put("dataType", attributeDef.getTypeName());
attribInfo.put("isUnique", attributeDef.isUnique()); attribInfo.put("isUnique", attributeDef.getIsUnique());
attribInfo.put("isIndexable", attributeDef.isIndexable()); attribInfo.put("isIndexable", attributeDef.getIsIndexable());
attribInfo.put("isComposite", isComposite); attribInfo.put("isComposite", isComposite);
attribInfo.put("reverseAttributeName", reverseAttribName); attribInfo.put("reverseAttributeName", reverseAttribName);
final int lower;
final int upper;
if (attributeDef.getCardinality() == AtlasAttributeDef.Cardinality.SINGLE) {
lower = attributeDef.getIsOptional() ? 0 : 1;
upper = 1;
} else {
if(attributeDef.getIsOptional()) {
lower = 0;
} else {
lower = attributeDef.getValuesMinCount() < 1 ? 1 : attributeDef.getValuesMinCount();
}
upper = attributeDef.getValuesMaxCount() < 2 ? Integer.MAX_VALUE : attributeDef.getValuesMaxCount();
}
Map<String, Object> multiplicity = new HashMap<>(); Map<String, Object> multiplicity = new HashMap<>();
multiplicity.put("lower", attributeDef.getValuesMinCount()); multiplicity.put("lower", lower);
multiplicity.put("upper", attributeDef.getValuesMaxCount()); multiplicity.put("upper", upper);
multiplicity.put("isUnique", AtlasAttributeDef.Cardinality.SET.equals(attributeDef.getCardinality())); multiplicity.put("isUnique", AtlasAttributeDef.Cardinality.SET.equals(attributeDef.getCardinality()));
attribInfo.put("multiplicity", AtlasType.toJson(multiplicity)); attribInfo.put("multiplicity", AtlasType.toJson(multiplicity));
...@@ -532,8 +549,8 @@ public class AtlasStructDefStoreV1 extends AtlasAbstractDefStoreV1 implements At ...@@ -532,8 +549,8 @@ public class AtlasStructDefStoreV1 extends AtlasAbstractDefStoreV1 implements At
ret.setName((String) attribInfo.get("name")); ret.setName((String) attribInfo.get("name"));
ret.setTypeName((String) attribInfo.get("dataType")); ret.setTypeName((String) attribInfo.get("dataType"));
ret.setUnique((Boolean) attribInfo.get("isUnique")); ret.setIsUnique((Boolean) attribInfo.get("isUnique"));
ret.setIndexable((Boolean) attribInfo.get("isIndexable")); ret.setIsIndexable((Boolean) attribInfo.get("isIndexable"));
String attrTypeName = ret.getTypeName(); String attrTypeName = ret.getTypeName();
...@@ -608,10 +625,10 @@ public class AtlasStructDefStoreV1 extends AtlasAbstractDefStoreV1 implements At ...@@ -608,10 +625,10 @@ public class AtlasStructDefStoreV1 extends AtlasAbstractDefStoreV1 implements At
Boolean isUnique = (Boolean) multiplicity.get("isUnique"); Boolean isUnique = (Boolean) multiplicity.get("isUnique");
if (minCount == null || minCount.intValue() == 0) { if (minCount == null || minCount.intValue() == 0) {
ret.setOptional(true); ret.setIsOptional(true);
ret.setValuesMinCount(0); ret.setValuesMinCount(0);
} else { } else {
ret.setOptional(false); ret.setIsOptional(false);
ret.setValuesMinCount(minCount.intValue()); ret.setValuesMinCount(minCount.intValue());
} }
......
repository/src/main/java/org/apache/atlas/services/AtlasPatchHandler.java deleted 100644 → 0
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.services;
import com.google.gson.FieldNamingPolicy;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.google.gson.JsonDeserializationContext;
import com.google.gson.JsonDeserializer;
import com.google.gson.JsonElement;
import com.google.gson.JsonParseException;
import com.google.gson.JsonSyntaxException;
import org.apache.atlas.services.AtlasTypePatch.PatchContent;
import org.apache.atlas.services.AtlasTypePatch.PatchData;
import org.apache.atlas.services.AtlasTypePatch.PatchResult;
import org.apache.atlas.services.AtlasTypePatch.PatchStatus;
import org.apache.atlas.typesystem.types.Multiplicity;
import org.apache.atlas.typesystem.types.TypeSystem;
import org.apache.atlas.typesystem.types.TypeUpdateException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
import java.lang.reflect.Type;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
public class AtlasPatchHandler {
private static final Logger LOG = LoggerFactory.getLogger(AtlasPatchHandler.class);
public static void handlePatches(DefaultMetadataService metadataService, TypeSystem typeSystem) throws TypeUpdateException {
Map<String, AtlasTypePatch> patchHandlerMap = initializePatchHandlerMap(metadataService, typeSystem);
if (patchHandlerMap == null || patchHandlerMap.isEmpty())
return;
String patchDirName = ReservedTypesRegistrar.getTypesDir() + File.separator + "patches";
LOG.info("Checking for any patches to be applied to the type system in " + patchDirName);
File patchDir = new File(patchDirName);
if (!patchDir.exists()) {
LOG.info("Patch directory {} doesn't exist, not applying any patches", patchDirName);
return;
}
File[] patchFiles = patchDir.listFiles();
if (patchFiles == null || patchFiles.length == 0) {
LOG.info("No patch files found in {}, not applying any patches", patchDirName);
return;
}
// Sort the patch files based on file name.
Arrays.sort(patchFiles, new Comparator<File>() {
public int compare(File f1, File f2) {
return String.valueOf(f1.getName()).compareTo(f2.getName());
}
});
LOG.info("Found " + patchFiles.length + " patch files to process.");
int patchNumber = 0;
Gson gson = initializeGson();
AtlasTypePatch typePatch;
for (File patchFile : patchFiles) {
try {
LOG.info("Processing patch file " + (++patchNumber) + " - " + patchFile.getAbsolutePath());
String content = new String(Files.readAllBytes(patchFile.toPath()), StandardCharsets.UTF_8);
PatchContent patchContent = gson.fromJson(content, PatchContent.class);
PatchData[] patchDatas = patchContent.getPatches();
PatchResult result;
int patchCounter = 0;
for (PatchData patch : patchDatas) {
typePatch = patchHandlerMap.get(patch.getAction());
if (typePatch != null) {
result = typePatch.applyPatch(patch);
if (result != null) {
LOG.info(result.getMessage() + " Patch " + (++patchCounter) + " out of " + patchDatas.length + " processed in : " + patchFile.toPath());
if (result.getStatus().equals(PatchStatus.FAILED)) {
throw new TypeUpdateException(result.getMessage() + " patch " + patchNumber + " failed in :" + patchFile.getAbsolutePath());
}
}
}
}
} catch (IOException e) {
throw new TypeUpdateException("Unable to read patch file from " + patchFile.getAbsolutePath());
} catch (JsonSyntaxException e) {
throw new TypeUpdateException("Invalid non-parseable JSON patch file in " + patchFile.getAbsolutePath());
}
}
LOG.info("Processed " + patchFiles.length + " patch files.");
}
private static Map<String, AtlasTypePatch> initializePatchHandlerMap(DefaultMetadataService metadataService, TypeSystem typeSystem) {
Map<String, AtlasTypePatch> patchHandlerMap = new HashMap<String, AtlasTypePatch>();
List<AtlasTypePatch> patchers = new ArrayList<AtlasTypePatch>();
// Register new patch classes here
patchers.add(new AtlasTypeAttributePatch(metadataService, typeSystem));
for (AtlasTypePatch patcher : patchers) {
for (String action : patcher.getSupportedActions()) {
patchHandlerMap.put(action, patcher);
}
}
return patchHandlerMap;
}
public static Gson initializeGson() {
GsonBuilder gsonBuilder = new GsonBuilder();
gsonBuilder.registerTypeAdapter(Multiplicity.class, new MultiplicityDeserializer());
gsonBuilder.setFieldNamingPolicy(FieldNamingPolicy.IDENTITY);
Gson gson = gsonBuilder.create();
return gson;
}
static class MultiplicityDeserializer implements JsonDeserializer<Multiplicity> {
@Override
public Multiplicity deserialize(JsonElement json, Type typeOfT, JsonDeserializationContext context)
throws JsonParseException {
String multiplicityString = json.getAsString().toLowerCase();
Multiplicity m = null;
switch (multiplicityString) {
case "optional":
m = Multiplicity.OPTIONAL;
break;
case "required":
m = Multiplicity.REQUIRED;
break;
case "collection":
m = Multiplicity.COLLECTION;
break;
case "set":
m = Multiplicity.SET;
break;
default:
break;
}
return m;
}
}
}
\ No newline at end of file
repository/src/main/java/org/apache/atlas/services/AtlasTypeAttributePatch.java deleted 100644 → 0
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.services;
import com.google.common.collect.ImmutableList;
import org.apache.atlas.AtlasException;
import org.apache.atlas.typesystem.TypesDef;
import org.apache.atlas.typesystem.json.TypesSerialization;
import org.apache.atlas.typesystem.types.AttributeDefinition;
import org.apache.atlas.typesystem.types.ClassType;
import org.apache.atlas.typesystem.types.EnumTypeDefinition;
import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition;
import org.apache.atlas.typesystem.types.IDataType;
import org.apache.atlas.typesystem.types.Multiplicity;
import org.apache.atlas.typesystem.types.StructTypeDefinition;
import org.apache.atlas.typesystem.types.TraitType;
import org.apache.atlas.typesystem.types.TypeSystem;
import org.apache.atlas.typesystem.types.utils.TypesUtil;
import org.codehaus.jettison.json.JSONException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
public class AtlasTypeAttributePatch extends AtlasTypePatch {
private static final Logger LOG = LoggerFactory.getLogger(AtlasTypeAttributePatch.class);
private static final String STRUCT_TYPE = "STRUCT";
private static final String CLASS_TYPE = "CLASS";
private static final String TRAIT_TYPE = "TRAIT";
private static final String[] SUPPORTED_ACTIONS = new String[] { "ADD_ATTRIBUTE", "UPDATE_ATTRIBUTE", "DELETE_ATTRIBUTE" };
public AtlasTypeAttributePatch(DefaultMetadataService metadataService, TypeSystem typeSystem) {
super(metadataService, typeSystem, SUPPORTED_ACTIONS);
}
/********* SAMPLE PATCH DATA ***********
*
*
{
"patches": [
{
"action": "ADD_ATTRIBUTE",
"typeName": "hive_column",
"applyToVersion": "1.0",
"updateToVersion": "2.0",
"params": {
"optionalParam1" : "true",
"optionalParam2" : "false"
},
"attributeDefinitions": [
{
"name": "position",
"dataTypeName": "string",
"multiplicity": "optional",
"isComposite": false,
"isUnique": false,
"isIndexable": false,
"reverseAttributeName": null
}
]
}
]
}
*
*/
@Override
public PatchResult applyPatch(PatchData patchData) {
String typeName = patchData.getTypeName();
String applyVersion = patchData.getApplyToVersion();
TypesDef updatedTypeDef;
PatchResult result;
try {
// Check if type exists in type system
if (checkIfTypeExists(typeName, metadataService)) {
TypesDef typesDef = getTypeDef(typeName);
String currentVersion = getTypeVersion(typeName);
// Check version to apply patch
if (currentVersion == null || currentVersion.equalsIgnoreCase(applyVersion) || currentVersion.startsWith(applyVersion + ".")) {
updatedTypeDef = updateTypesDef(typesDef, patchData);
if (updatedTypeDef != null) {
metadataService.updateType(TypesSerialization.toJson(updatedTypeDef));
LOG.info("updated " + patchData.getTypeName() + " type from version " + patchData.getApplyToVersion() + " to " + patchData.getUpdateToVersion());
result = new PatchResult("patch applied successfully!", PatchStatus.SUCCESS);
} else {
LOG.error("failed to create updated typedef for type=" +typeName+ "; applyToVersion=" + applyVersion + "; updateToVersion=" + patchData.getUpdateToVersion() );
result = new PatchResult("patch failed!", PatchStatus.FAILED);
}
} else {
LOG.info("patch skipped for " + patchData.getTypeName());
result = new PatchResult("patch skipped!", PatchStatus.SKIPPED);
}
} else {
LOG.error("failed to apply patch (typeName=" + typeName + "; applyToVersion=" + applyVersion + "; updateToVersion=" + patchData.getUpdateToVersion() + "): type doesn't exist");
result = new PatchResult("patch failed!", PatchStatus.FAILED);
}
} catch (AtlasException e) {
LOG.error("error in updating type for " + patchData.getTypeName());
result = new PatchResult("unable to update type", PatchStatus.FAILED);
} catch (JSONException e) {
LOG.error("error in updating typedef for " + patchData.getTypeName());
result = new PatchResult("unable to update typedef", PatchStatus.FAILED);
}
return result;
}
public TypesDef updateTypesDef(TypesDef typesDef, PatchData patchData) throws AtlasException, JSONException {
AttributeDefinition[] typeAttributes = getAttributesFromTypesDef(typesDef, patchData.getTypeName());
AttributeDefinition[] patchAttributes = patchData.getAttributeDefinitions();
AttributeDefinition[] newAttributes = new AttributeDefinition[0];
String patchAction = patchData.getAction();
TypesDef newTypeDef = null;
if (patchAction != null && typeAttributes != null && patchAttributes != null) {
switch (patchAction) {
case "ADD_ATTRIBUTE":
LOG.info("adding new attribute to type {}", patchData.getTypeName());
newAttributes = addAttributes(typeAttributes, patchAttributes);
break;
case "DELETE_ATTRIBUTE":
LOG.info("deleting attribute from type {}", patchData.getTypeName());
newAttributes = deleteAttributes(typeAttributes, patchAttributes);
break;
case "UPDATE_ATTRIBUTE":
LOG.info("updating attribute in type {}", patchData.getTypeName());
newAttributes = updateAttributes(typeAttributes, patchAttributes);
break;
default:
LOG.info("invalid action " + patchAction + ", supported actions: " + Arrays.toString(SUPPORTED_ACTIONS));
break;
}
newTypeDef = createTypeDefWithNewAttributes(typesDef, patchData.getTypeName(), newAttributes, patchData.getUpdateToVersion());
}
return newTypeDef;
}
private AttributeDefinition[] addAttributes(AttributeDefinition[] typeAttributes, AttributeDefinition[] patchAttributes) {
ArrayList<AttributeDefinition> newAttrList = new ArrayList<AttributeDefinition>(Arrays.asList(typeAttributes));
Map<String, AttributeDefinition> typeAttrs = getAttributeNamesFromDefinition(typeAttributes);
for (AttributeDefinition attr : patchAttributes) {
if (!typeAttrs.containsKey(attr.name))
newAttrList.add(attr);
}
return newAttrList.toArray(new AttributeDefinition[newAttrList.size()]);
}
private AttributeDefinition[] deleteAttributes(AttributeDefinition[] typeAttributes, AttributeDefinition[] patchAttributes) {
ArrayList<AttributeDefinition> newAttrList = new ArrayList<AttributeDefinition>();
Map<String, AttributeDefinition> patchAttrs = getAttributeNamesFromDefinition(patchAttributes);
for (AttributeDefinition attr : typeAttributes) {
if (!patchAttrs.containsKey(attr.name))
newAttrList.add(attr);
}
return newAttrList.toArray(new AttributeDefinition[newAttrList.size()]);
}
private AttributeDefinition[] updateAttributes(AttributeDefinition[] typeAttributes, AttributeDefinition[] patchAttributes) {
ArrayList<AttributeDefinition> newAttrList = new ArrayList<AttributeDefinition>();
Map<String, AttributeDefinition> patchAttrs = getAttributeNamesFromDefinition(patchAttributes);
AttributeDefinition newAttr;
for (AttributeDefinition attr : typeAttributes) {
newAttr = patchAttrs.get(attr.name);
if (patchAttrs.containsKey(attr.name) && checkIfAttributeUpdatable(attr, newAttr)) {
newAttrList.add(newAttr);
} else {
newAttrList.add(attr);
}
}
return newAttrList.toArray(new AttributeDefinition[newAttrList.size()]);
}
private TypesDef createTypeDefWithNewAttributes(TypesDef typesDef, String typeName, AttributeDefinition[] newAttributes, String newVersion) throws AtlasException {
ImmutableList.Builder<EnumTypeDefinition> enums = ImmutableList.builder();
ImmutableList.Builder<StructTypeDefinition> structs = ImmutableList.builder();
ImmutableList.Builder<HierarchicalTypeDefinition<ClassType>> classTypes = ImmutableList.builder();
ImmutableList.Builder<HierarchicalTypeDefinition<TraitType>> traits = ImmutableList.builder();
String dataType = getDataType(typeName).toUpperCase();
switch (dataType) {
case STRUCT_TYPE:
StructTypeDefinition structType = typesDef.structTypesAsJavaList().get(0);
structs.add(new StructTypeDefinition(structType.typeName, structType.typeDescription, newVersion, newAttributes));
break;
case CLASS_TYPE:
HierarchicalTypeDefinition<ClassType> classType = typesDef.classTypesAsJavaList().get(0);
classTypes.add(new HierarchicalTypeDefinition(ClassType.class, classType.typeName, classType.typeDescription, newVersion, classType.superTypes, newAttributes));
break;
case TRAIT_TYPE:
HierarchicalTypeDefinition<TraitType> traitType = typesDef.traitTypesAsJavaList().get(0);
traits.add(new HierarchicalTypeDefinition(TraitType.class, traitType.typeName, traitType.typeDescription, newVersion, traitType.superTypes, newAttributes));
break;
default:
LOG.error("unhandled datatype {} when creating new typedef", dataType);
}
return TypesUtil.getTypesDef(enums.build(), structs.build(), traits.build(), classTypes.build());
}
private AttributeDefinition[] getAttributesFromTypesDef(TypesDef typesDef, String typeName) throws AtlasException {
AttributeDefinition[] typeAttributes = null;
String dataType = getDataType(typeName).toUpperCase();
switch (dataType) {
case STRUCT_TYPE:
typeAttributes = typesDef.structTypesAsJavaList().get(0).attributeDefinitions;
break;
case CLASS_TYPE:
typeAttributes = typesDef.classTypesAsJavaList().get(0).attributeDefinitions;
break;
case TRAIT_TYPE:
typeAttributes = typesDef.traitTypesAsJavaList().get(0).attributeDefinitions;
break;
default:
LOG.error("unhandled datatype {}", dataType);
}
return typeAttributes;
}
private Map<String, AttributeDefinition> getAttributeNamesFromDefinition(AttributeDefinition[] attribDef) {
Map<String, AttributeDefinition> attrsMap = new HashMap<String, AttributeDefinition>();
for (AttributeDefinition attr : attribDef) {
attrsMap.put(attr.name, attr);
}
return attrsMap;
}
private boolean checkIfAttributeUpdatable(AttributeDefinition attr, AttributeDefinition newAttr) {
boolean result = false;
if (!attr.equals(newAttr) && (attr.multiplicity == Multiplicity.REQUIRED
&& newAttr.multiplicity == Multiplicity.OPTIONAL)) {
result = true;
}
return result;
}
// Returns the datatype the typename belongs to - PRIMITIVE, ENUM, ARRAY, MAP, STRUCT, TRAIT, CLASS
private String getDataType(String typeName) throws AtlasException {
IDataType dataType = typeSystem.getDataType(IDataType.class, typeName);
return dataType.getTypeCategory().toString();
}
private String getTypeVersion(String typeName) throws AtlasException {
return typeSystem.getDataType(IDataType.class, typeName).getVersion();
}
private TypesDef getTypeDef(String typeName) throws AtlasException {
return TypesSerialization.fromJson(metadataService.getTypeDefinition(typeName));
}
private static boolean checkIfTypeExists(String typeName, DefaultMetadataService metadataService) {
boolean result = true;
try {
metadataService.getTypeDefinition(typeName);
} catch (AtlasException e) {
result = false;
}
return result;
}
}
\ No newline at end of file
repository/src/main/java/org/apache/atlas/services/AtlasTypePatch.java deleted 100644 → 0
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.services;
import org.apache.atlas.typesystem.types.AttributeDefinition;
import org.apache.atlas.typesystem.types.TypeSystem;
import java.util.Map;
public abstract class AtlasTypePatch {
protected final TypeSystem typeSystem;
protected final DefaultMetadataService metadataService;
protected final String[] supportedActions;
protected AtlasTypePatch(DefaultMetadataService metadataService, TypeSystem typeSystem, String[] supportedActions) {
this.metadataService = metadataService;
this.typeSystem = typeSystem;
this.supportedActions = supportedActions;
}
public final String[] getSupportedActions() { return supportedActions; }
public abstract PatchResult applyPatch(PatchData patch);
public enum PatchStatus { SUCCESS, FAILED, SKIPPED }
public class PatchResult {
private String message;
private PatchStatus status;
public PatchResult(String message, PatchStatus status) {
this.message = message;
this.status = status;
}
public String getMessage() { return message; }
public void setMessage(String message) { this.message = message; }
public PatchStatus getStatus() { return status; }
public void setStatus(PatchStatus status) { this.status = status; }
}
/**
* A class to capture patch content.
*/
public class PatchContent {
private PatchData[] patches;
public PatchData[] getPatches() {
return patches;
}
}
public static class PatchData {
private String action;
private String typeName;
private String applyToVersion;
private String updateToVersion;
private Map<String, String> params;
private AttributeDefinition[] attributeDefinitions;
public PatchData(String action, String typeName, String applyToVersion, String updateToVersion, Map<String, String> params, AttributeDefinition[] attributeDefinitions) {
this.action = action;
this.typeName = typeName;
this.applyToVersion = applyToVersion;
this.updateToVersion = updateToVersion;
this.params = params;
this.attributeDefinitions = attributeDefinitions;
}
public String getAction() { return action; }
public String getTypeName() { return typeName; }
public String getApplyToVersion() { return applyToVersion; }
public String getUpdateToVersion() { return updateToVersion; }
public Map<String, String> getParams() { return params; }
public AttributeDefinition[] getAttributeDefinitions() { return attributeDefinitions; }
}
}
repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java
...@@ -20,7 +20,6 @@ package org.apache.atlas.services; ...@@ -20,7 +20,6 @@ package org.apache.atlas.services;
import com.google.common.base.Preconditions; import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import com.google.inject.Provider; import com.google.inject.Provider;
import org.apache.atlas.ApplicationProperties; import org.apache.atlas.ApplicationProperties;
...@@ -29,7 +28,6 @@ import org.apache.atlas.AtlasErrorCode; ...@@ -29,7 +28,6 @@ import org.apache.atlas.AtlasErrorCode;
import org.apache.atlas.AtlasException; import org.apache.atlas.AtlasException;
import org.apache.atlas.EntityAuditEvent; import org.apache.atlas.EntityAuditEvent;
import org.apache.atlas.RequestContext; import org.apache.atlas.RequestContext;
import org.apache.atlas.classification.InterfaceAudience;
import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.ha.HAConfiguration; import org.apache.atlas.ha.HAConfiguration;
import org.apache.atlas.listener.ActiveStateChangeHandler; import org.apache.atlas.listener.ActiveStateChangeHandler;
...@@ -54,7 +52,6 @@ import org.apache.atlas.typesystem.json.InstanceSerialization; ...@@ -54,7 +52,6 @@ import org.apache.atlas.typesystem.json.InstanceSerialization;
import org.apache.atlas.typesystem.json.TypesSerialization; import org.apache.atlas.typesystem.json.TypesSerialization;
import org.apache.atlas.typesystem.persistence.Id; import org.apache.atlas.typesystem.persistence.Id;
import org.apache.atlas.typesystem.persistence.ReferenceableInstance; import org.apache.atlas.typesystem.persistence.ReferenceableInstance;
import org.apache.atlas.typesystem.types.AttributeDefinition;
import org.apache.atlas.typesystem.types.AttributeInfo; import org.apache.atlas.typesystem.types.AttributeInfo;
import org.apache.atlas.typesystem.types.ClassType; import org.apache.atlas.typesystem.types.ClassType;
import org.apache.atlas.typesystem.types.DataTypes; import org.apache.atlas.typesystem.types.DataTypes;
...@@ -83,9 +80,6 @@ import java.util.Map; ...@@ -83,9 +80,6 @@ import java.util.Map;
import javax.inject.Inject; import javax.inject.Inject;
import javax.inject.Singleton; import javax.inject.Singleton;
import static org.apache.atlas.AtlasClient.PROCESS_ATTRIBUTE_INPUTS;
import static org.apache.atlas.AtlasClient.PROCESS_ATTRIBUTE_OUTPUTS;
/** /**
...@@ -103,35 +97,29 @@ public class DefaultMetadataService implements MetadataService, ActiveStateChang ...@@ -103,35 +97,29 @@ public class DefaultMetadataService implements MetadataService, ActiveStateChang
private final TypeSystem typeSystem; private final TypeSystem typeSystem;
private final MetadataRepository repository; private final MetadataRepository repository;
private final ITypeStore typeStore; private final ITypeStore typeStore;
private IBootstrapTypesRegistrar typesRegistrar;
private final Collection<TypesChangeListener> typeChangeListeners = new LinkedHashSet<>(); private final Collection<TypesChangeListener> typeChangeListeners = new LinkedHashSet<>();
private final Collection<EntityChangeListener> entityChangeListeners = new LinkedHashSet<>(); private final Collection<EntityChangeListener> entityChangeListeners = new LinkedHashSet<>();
private boolean wasInitialized = false;
@Inject @Inject
private EntityAuditRepository auditRepository; private EntityAuditRepository auditRepository;
@Inject @Inject
DefaultMetadataService(final MetadataRepository repository, final ITypeStore typeStore, DefaultMetadataService(final MetadataRepository repository, final ITypeStore typeStore,
final IBootstrapTypesRegistrar typesRegistrar,
final Collection<Provider<TypesChangeListener>> typeListenerProviders, final Collection<Provider<TypesChangeListener>> typeListenerProviders,
final Collection<Provider<EntityChangeListener>> entityListenerProviders, TypeCache typeCache) final Collection<Provider<EntityChangeListener>> entityListenerProviders, TypeCache typeCache)
throws AtlasException { throws AtlasException {
this(repository, typeStore, typesRegistrar, typeListenerProviders, entityListenerProviders, this(repository, typeStore, typeListenerProviders, entityListenerProviders,
TypeSystem.getInstance(), ApplicationProperties.get(), typeCache); TypeSystem.getInstance(), ApplicationProperties.get(), typeCache);
} }
//for testing only //for testing only
public DefaultMetadataService(final MetadataRepository repository, final ITypeStore typeStore, public DefaultMetadataService(final MetadataRepository repository, final ITypeStore typeStore,
final IBootstrapTypesRegistrar typesRegistrar,
final Collection<Provider<TypesChangeListener>> typeListenerProviders, final Collection<Provider<TypesChangeListener>> typeListenerProviders,
final Collection<Provider<EntityChangeListener>> entityListenerProviders, final Collection<Provider<EntityChangeListener>> entityListenerProviders,
final TypeSystem typeSystem, final TypeSystem typeSystem,
final Configuration configuration, TypeCache typeCache) throws AtlasException { final Configuration configuration, TypeCache typeCache) throws AtlasException {
this.typeStore = typeStore; this.typeStore = typeStore;
this.typesRegistrar = typesRegistrar;
this.typeSystem = typeSystem; this.typeSystem = typeSystem;
/** /**
* Ideally a TypeCache implementation should have been injected in the TypeSystemProvider, * Ideally a TypeCache implementation should have been injected in the TypeSystemProvider,
...@@ -163,70 +151,20 @@ public class DefaultMetadataService implements MetadataService, ActiveStateChang ...@@ -163,70 +151,20 @@ public class DefaultMetadataService implements MetadataService, ActiveStateChang
private void restoreTypeSystem() throws AtlasException { private void restoreTypeSystem() throws AtlasException {
LOG.info("Restoring type system from the store"); LOG.info("Restoring type system from the store");
TypesDef typesDef = typeStore.restore(); TypesDef typesDef = typeStore.restore();
if (!wasInitialized) {
LOG.info("Initializing type system for the first time."); refreshCache(typesDef);
typeSystem.defineTypes(typesDef);
// restore types before creating super types
createSuperTypes();
typesRegistrar.registerTypes(ReservedTypesRegistrar.getTypesDir(), typeSystem, this);
wasInitialized = true;
} else {
LOG.info("Type system was already initialized, refreshing cache.");
refreshCache(typesDef);
}
LOG.info("Restored type system from the store"); LOG.info("Restored type system from the store");
} }
private void refreshCache(TypesDef typesDef) throws AtlasException { private void refreshCache(TypesDef typesDef) throws AtlasException {
TypeSystem.TransientTypeSystem transientTypeSystem if (typesDef != null && !typesDef.isEmpty()) {
= typeSystem.createTransientTypeSystem(typesDef, true); TypeSystem.TransientTypeSystem transientTypeSystem = typeSystem.createTransientTypeSystem(typesDef, true);
Map<String, IDataType> typesAdded = transientTypeSystem.getTypesAdded(); Map<String, IDataType> typesAdded = transientTypeSystem.getTypesAdded();
LOG.info("Number of types got from transient type system: " + typesAdded.size()); LOG.info("Number of types got from transient type system: " + typesAdded.size());
typeSystem.commitTypes(typesAdded); typeSystem.commitTypes(typesAdded);
}
@InterfaceAudience.Private
private void createSuperTypes() throws AtlasException {
HierarchicalTypeDefinition<ClassType> referenceableType = TypesUtil
.createClassTypeDef(AtlasClient.REFERENCEABLE_SUPER_TYPE, ImmutableSet.<String>of(),
new AttributeDefinition(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, true, true, null));
createType(referenceableType);
HierarchicalTypeDefinition<ClassType> assetType = TypesUtil
.createClassTypeDef(AtlasClient.ASSET_TYPE, ImmutableSet.<String>of(),
new AttributeDefinition(AtlasClient.NAME, DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, false, true, null),
TypesUtil.createOptionalAttrDef(AtlasClient.DESCRIPTION, DataTypes.STRING_TYPE),
new AttributeDefinition(AtlasClient.OWNER, DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, false, true, null));
createType(assetType);
HierarchicalTypeDefinition<ClassType> infraType = TypesUtil
.createClassTypeDef(AtlasClient.INFRASTRUCTURE_SUPER_TYPE,
ImmutableSet.of(AtlasClient.REFERENCEABLE_SUPER_TYPE, AtlasClient.ASSET_TYPE));
createType(infraType);
HierarchicalTypeDefinition<ClassType> datasetType = TypesUtil
.createClassTypeDef(AtlasClient.DATA_SET_SUPER_TYPE,
ImmutableSet.of(AtlasClient.REFERENCEABLE_SUPER_TYPE, AtlasClient.ASSET_TYPE));
createType(datasetType);
HierarchicalTypeDefinition<ClassType> processType = TypesUtil
.createClassTypeDef(AtlasClient.PROCESS_SUPER_TYPE,
ImmutableSet.of(AtlasClient.REFERENCEABLE_SUPER_TYPE, AtlasClient.ASSET_TYPE),
new AttributeDefinition(PROCESS_ATTRIBUTE_INPUTS, DataTypes.arrayTypeName(AtlasClient.DATA_SET_SUPER_TYPE),
Multiplicity.OPTIONAL, false, null),
new AttributeDefinition(PROCESS_ATTRIBUTE_OUTPUTS, DataTypes.arrayTypeName(AtlasClient.DATA_SET_SUPER_TYPE),
Multiplicity.OPTIONAL, false, null));
createType(processType);
}
private void createType(HierarchicalTypeDefinition<ClassType> type) throws AtlasException {
if (!typeSystem.isRegistered(type.typeName)) {
TypesDef typesDef = TypesUtil.getTypesDef(ImmutableList.<EnumTypeDefinition>of(), ImmutableList.<StructTypeDefinition>of(),
ImmutableList.<HierarchicalTypeDefinition<TraitType>>of(),
ImmutableList.of(type));
createType(TypesSerialization.toJson(typesDef));
} }
} }
......
repository/src/main/java/org/apache/atlas/services/IBootstrapTypesRegistrar.java deleted 100644 → 0
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.services;
import org.apache.atlas.AtlasException;
import org.apache.atlas.typesystem.types.TypeSystem;
public interface IBootstrapTypesRegistrar {
void registerTypes(String typesDirName, TypeSystem typeSystem, MetadataService metadataService)
throws AtlasException;
}
repository/src/main/java/org/apache/atlas/services/ReservedTypesRegistrar.java deleted 100644 → 0
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.services;
import com.google.common.collect.ImmutableList;
import org.apache.atlas.AtlasException;
import org.apache.atlas.typesystem.TypesDef;
import org.apache.atlas.typesystem.json.TypesSerialization;
import org.apache.atlas.typesystem.types.*;
import org.apache.atlas.typesystem.types.utils.TypesUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Comparator;
import java.util.List;
public class ReservedTypesRegistrar implements IBootstrapTypesRegistrar {
private static final Logger LOG = LoggerFactory.getLogger(ReservedTypesRegistrar.class);
static String getTypesDir() {
return System.getProperty("atlas.home")+ File.separator+"models";
}
@Override
public void registerTypes(String typesDirName, TypeSystem typeSystem, MetadataService metadataService)
throws AtlasException {
File typesDir = new File(typesDirName);
if (!typesDir.exists()) {
LOG.info("No types directory {} found - not registering any reserved types", typesDirName);
return;
}
File[] typeDefFiles = typesDir.listFiles();
//TODO - Enforce a dependency order among models registered by definition and not by modifiedTime as below
// Workaround - Sort by modifiedTime to get the dependency of models in the right order - first hdfs, followed by hive and hive is needed by storm, falcon models.
// Sorting them by time will ensure the right order since the modules are in the correct order in pom.
Arrays.sort(typeDefFiles, new Comparator<File>() {
public int compare(File f1, File f2) {
return Long.valueOf(f1.lastModified()).compareTo(f2.lastModified());
}
});
for (File typeDefFile : typeDefFiles) {
try {
if (typeDefFile.isFile()) {
String typeDefJSON = new String(Files.readAllBytes(typeDefFile.toPath()), StandardCharsets.UTF_8);
registerType(typeSystem, metadataService, typeDefFile.getAbsolutePath(), typeDefJSON);
}
} catch (IOException e) {
LOG.error("error while registering types in file " + typeDefFile.getAbsolutePath(), e);
} catch (AtlasException e) {
LOG.error("error while registering types in file " + typeDefFile.getAbsolutePath(), e);
throw e;
}
}
}
void registerType(TypeSystem typeSystem, MetadataService metadataService, String typeDefName, String typeDefJSON)
throws AtlasException {
TypesDef typesDef = null;
try {
typesDef = TypesSerialization.fromJson(typeDefJSON);
} catch (Exception e) {
LOG.error("Error while deserializing JSON in {}", typeDefName);
throw new ReservedTypesRegistrationException("Error while deserializing JSON in " + typeDefName, e);
}
List<HierarchicalTypeDefinition<ClassType>> createClassDefList = new ArrayList<>();
List<HierarchicalTypeDefinition<TraitType>> createTraitDefList = new ArrayList<>();
List<EnumTypeDefinition> createEnumDefList = new ArrayList<>();
List<StructTypeDefinition> createStructDefList = new ArrayList<>();
for(HierarchicalTypeDefinition<ClassType> classTypeDef:typesDef.classTypesAsJavaList()){
if(!typeSystem.isRegistered(classTypeDef.typeName)){
LOG.debug("ClassType {} is not registered. Adding to create type list", classTypeDef.typeName);
createClassDefList.add(classTypeDef);
}
}
for(HierarchicalTypeDefinition<TraitType> traitTypeDef:typesDef.traitTypesAsJavaList()){
if(!typeSystem.isRegistered(traitTypeDef.typeName)){
LOG.debug("TraitType {} is not registered. Adding to create type list", traitTypeDef.typeName);
createTraitDefList.add(traitTypeDef);
}
}
for(StructTypeDefinition structTypeDef:typesDef.structTypesAsJavaList()){
if(!typeSystem.isRegistered(structTypeDef.typeName)){
LOG.debug("StructType {} is not registered. Adding to create type list", structTypeDef.typeName);
createStructDefList.add(structTypeDef);
}
}
for(EnumTypeDefinition enumTypeDef:typesDef.enumTypesAsJavaList()){
if(!typeSystem.isRegistered(enumTypeDef.name)){
LOG.debug("EnumType {} is not registered. Adding to create type list", enumTypeDef.name);
createEnumDefList.add(enumTypeDef);
}
}
TypesDef createTypes = TypesUtil.getTypesDef(ImmutableList.copyOf(createEnumDefList), ImmutableList.copyOf(createStructDefList),
ImmutableList.copyOf(createTraitDefList), ImmutableList.copyOf(createClassDefList));
if (! createTypes.isEmpty()) {
String createTypeJSON = TypesSerialization.toJson(createTypes);
if (createTypeJSON != null) {
metadataService.createType(createTypeJSON);
LOG.info("Created types definition JSON {}", createTypeJSON);
}
}
}
}
repository/src/main/java/org/apache/atlas/services/ReservedTypesRegistrationException.java deleted 100644 → 0
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.services;
public class ReservedTypesRegistrationException extends RuntimeException {
public ReservedTypesRegistrationException(String message, Exception e) {
super(message, e);
}
}
repository/src/test/java/org/apache/atlas/BaseRepositoryTest.java
...@@ -44,6 +44,9 @@ import org.apache.atlas.typesystem.types.TypeSystem; ...@@ -44,6 +44,9 @@ import org.apache.atlas.typesystem.types.TypeSystem;
import org.apache.atlas.typesystem.types.utils.TypesUtil; import org.apache.atlas.typesystem.types.utils.TypesUtil;
import org.testng.annotations.Guice; import org.testng.annotations.Guice;
import static org.apache.atlas.AtlasClient.PROCESS_ATTRIBUTE_INPUTS;
import static org.apache.atlas.AtlasClient.PROCESS_ATTRIBUTE_OUTPUTS;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Date; import java.util.Date;
import java.util.List; import java.util.List;
...@@ -67,6 +70,7 @@ public class BaseRepositoryTest { ...@@ -67,6 +70,7 @@ public class BaseRepositoryTest {
protected void setUp() throws Exception { protected void setUp() throws Exception {
//force graph initialization / built in type registration //force graph initialization / built in type registration
TestUtils.getGraph(); TestUtils.getGraph();
setUpDefaultTypes();
setUpTypes(); setUpTypes();
new GraphBackedSearchIndexer(new AtlasTypeRegistry()); new GraphBackedSearchIndexer(new AtlasTypeRegistry());
TestUtils.resetRequestContext(); TestUtils.resetRequestContext();
...@@ -395,4 +399,42 @@ public class BaseRepositoryTest { ...@@ -395,4 +399,42 @@ public class BaseRepositoryTest {
// return the reference to created instance with guid // return the reference to created instance with guid
return new Id(guids.get(guids.size() - 1), 0, referenceable.getTypeName()); return new Id(guids.get(guids.size() - 1), 0, referenceable.getTypeName());
} }
private void setUpDefaultTypes() throws Exception {
TypesDef typesDef = createDefaultTypeDefinitions();
String typesAsJSON = TypesSerialization.toJson(typesDef);
metadataService.createType(typesAsJSON);
}
TypesDef createDefaultTypeDefinitions() {
HierarchicalTypeDefinition<ClassType> referenceableType = TypesUtil
.createClassTypeDef(AtlasClient.REFERENCEABLE_SUPER_TYPE, ImmutableSet.<String>of(),
new AttributeDefinition(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, true, true, null));
HierarchicalTypeDefinition<ClassType> assetType = TypesUtil
.createClassTypeDef(AtlasClient.ASSET_TYPE, ImmutableSet.<String>of(),
new AttributeDefinition(AtlasClient.NAME, DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, false, true, null),
TypesUtil.createOptionalAttrDef(AtlasClient.DESCRIPTION, DataTypes.STRING_TYPE),
new AttributeDefinition(AtlasClient.OWNER, DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, false, true, null));
HierarchicalTypeDefinition<ClassType> infraType = TypesUtil
.createClassTypeDef(AtlasClient.INFRASTRUCTURE_SUPER_TYPE,
ImmutableSet.of(AtlasClient.REFERENCEABLE_SUPER_TYPE, AtlasClient.ASSET_TYPE));
HierarchicalTypeDefinition<ClassType> datasetType = TypesUtil
.createClassTypeDef(AtlasClient.DATA_SET_SUPER_TYPE,
ImmutableSet.of(AtlasClient.REFERENCEABLE_SUPER_TYPE, AtlasClient.ASSET_TYPE));
HierarchicalTypeDefinition<ClassType> processType = TypesUtil
.createClassTypeDef(AtlasClient.PROCESS_SUPER_TYPE,
ImmutableSet.of(AtlasClient.REFERENCEABLE_SUPER_TYPE, AtlasClient.ASSET_TYPE),
new AttributeDefinition(PROCESS_ATTRIBUTE_INPUTS, DataTypes.arrayTypeName(AtlasClient.DATA_SET_SUPER_TYPE),
Multiplicity.OPTIONAL, false, null),
new AttributeDefinition(PROCESS_ATTRIBUTE_OUTPUTS, DataTypes.arrayTypeName(AtlasClient.DATA_SET_SUPER_TYPE),
Multiplicity.OPTIONAL, false, null));
return TypesUtil.getTypesDef(ImmutableList.<EnumTypeDefinition>of(), ImmutableList.<StructTypeDefinition>of(),
ImmutableList.<HierarchicalTypeDefinition<TraitType>>of(),
ImmutableList.of(referenceableType, assetType, infraType, datasetType, processType));
}
} }
repository/src/test/java/org/apache/atlas/TestUtils.java
...@@ -33,7 +33,6 @@ import org.apache.atlas.repository.typestore.GraphBackedTypeStore; ...@@ -33,7 +33,6 @@ import org.apache.atlas.repository.typestore.GraphBackedTypeStore;
import org.apache.atlas.repository.typestore.ITypeStore; import org.apache.atlas.repository.typestore.ITypeStore;
import org.apache.atlas.services.DefaultMetadataService; import org.apache.atlas.services.DefaultMetadataService;
import org.apache.atlas.services.MetadataService; import org.apache.atlas.services.MetadataService;
import org.apache.atlas.services.ReservedTypesRegistrar;
import org.apache.atlas.type.AtlasTypeRegistry; import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.atlas.typesystem.ITypedReferenceableInstance; import org.apache.atlas.typesystem.ITypedReferenceableInstance;
import org.apache.atlas.typesystem.Referenceable; import org.apache.atlas.typesystem.Referenceable;
...@@ -527,7 +526,6 @@ public final class TestUtils { ...@@ -527,7 +526,6 @@ public final class TestUtils {
ITypeStore typeStore = new GraphBackedTypeStore(); ITypeStore typeStore = new GraphBackedTypeStore();
DefaultMetadataService defaultMetadataService = new DefaultMetadataService(repo, DefaultMetadataService defaultMetadataService = new DefaultMetadataService(repo,
typeStore, typeStore,
new ReservedTypesRegistrar(),
Collections.singletonList(indexerProvider), Collections.singletonList(indexerProvider),
new ArrayList<Provider<EntityChangeListener>>(), TypeSystem.getInstance(), config, typeCache); new ArrayList<Provider<EntityChangeListener>>(), TypeSystem.getInstance(), config, typeCache);
......
repository/src/test/java/org/apache/atlas/repository/store/graph/AtlasTypeDefGraphStoreTest.java
...@@ -190,7 +190,7 @@ public class AtlasTypeDefGraphStoreTest { ...@@ -190,7 +190,7 @@ public class AtlasTypeDefGraphStoreTest {
try { try {
existingTypesDef = typeDefStore.searchTypesDef(new SearchFilter()); existingTypesDef = typeDefStore.searchTypesDef(new SearchFilter());
} catch (AtlasBaseException e) { } catch (AtlasBaseException e) {
fail("Shouldn't have failed during Search"); // ignore
} }
assertNotEquals(atlasTypesDef, existingTypesDef, "Types to be created already exist in the system"); assertNotEquals(atlasTypesDef, existingTypesDef, "Types to be created already exist in the system");
...@@ -204,7 +204,7 @@ public class AtlasTypeDefGraphStoreTest { ...@@ -204,7 +204,7 @@ public class AtlasTypeDefGraphStoreTest {
assertTrue(createdTypesDef.getEntityDefs().containsAll(atlasTypesDef.getEntityDefs()), "EntityDef creation failed"); assertTrue(createdTypesDef.getEntityDefs().containsAll(atlasTypesDef.getEntityDefs()), "EntityDef creation failed");
} catch (AtlasBaseException e) { } catch (AtlasBaseException e) {
fail("Creation of Types should've been a success"); fail("Creation of Types should've been a success", e);
} }
} }
......
repository/src/test/java/org/apache/atlas/services/DefaultMetadataServiceMockTest.java deleted 100644 → 0
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.services;
import com.google.inject.Provider;
import org.apache.atlas.AtlasException;
import org.apache.atlas.listener.EntityChangeListener;
import org.apache.atlas.listener.TypesChangeListener;
import org.apache.atlas.repository.MetadataRepository;
import org.apache.atlas.repository.typestore.ITypeStore;
import org.apache.atlas.typesystem.types.TypeSystem;
import org.apache.atlas.ha.HAConfiguration;
import org.apache.atlas.typesystem.TypesDef;
import org.apache.atlas.typesystem.types.IDataType;
import org.apache.commons.configuration.Configuration;
import org.mockito.Matchers;
import org.mockito.Mock;
import org.mockito.MockitoAnnotations;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
import java.util.ArrayList;
import java.util.HashMap;
import static org.mockito.Matchers.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.verifyZeroInteractions;
import static org.mockito.Mockito.when;
public class DefaultMetadataServiceMockTest {
@Mock
private IBootstrapTypesRegistrar typesRegistrar;
@Mock
private TypeSystem typeSystem;
@Mock
private MetadataRepository metadataRepository;
@Mock
private ITypeStore typeStore;
@Mock
private Configuration configuration;
@BeforeMethod
public void setup() {
MockitoAnnotations.initMocks(this);
}
@Test
public void testShouldInvokeTypesRegistrarOnCreation() throws AtlasException {
when(typeSystem.isRegistered(any(String.class))).thenReturn(true);
when(configuration.containsKey(HAConfiguration.ATLAS_SERVER_HA_ENABLED_KEY)).thenReturn(true);
when(configuration.getBoolean(HAConfiguration.ATLAS_SERVER_HA_ENABLED_KEY)).thenReturn(false);
DefaultMetadataService defaultMetadataService = new DefaultMetadataService(mock(MetadataRepository.class),
mock(ITypeStore.class),
typesRegistrar, new ArrayList<Provider<TypesChangeListener>>(),
new ArrayList<Provider<EntityChangeListener>>(), typeSystem, configuration, null);
verify(typesRegistrar).registerTypes(ReservedTypesRegistrar.getTypesDir(),
typeSystem, defaultMetadataService);
}
@Test
public void testShouldNotRestoreTypesIfHAIsEnabled() throws AtlasException {
when(configuration.containsKey(HAConfiguration.ATLAS_SERVER_HA_ENABLED_KEY)).thenReturn(true);
when(configuration.getBoolean(HAConfiguration.ATLAS_SERVER_HA_ENABLED_KEY)).thenReturn(true);
new DefaultMetadataService(metadataRepository, typeStore,
typesRegistrar, new ArrayList<Provider<TypesChangeListener>>(),
new ArrayList<Provider<EntityChangeListener>>(), typeSystem, configuration, null);
verifyZeroInteractions(typeStore);
verify(typeSystem, never()).defineTypes(Matchers.<TypesDef>any());
verifyZeroInteractions(typesRegistrar);
}
@Test
public void testShouldRestoreTypeSystemOnServerActive() throws AtlasException {
when(configuration.containsKey(HAConfiguration.ATLAS_SERVER_HA_ENABLED_KEY)).thenReturn(true);
when(configuration.getBoolean(HAConfiguration.ATLAS_SERVER_HA_ENABLED_KEY)).thenReturn(true);
TypesDef typesDef = mock(TypesDef.class);
when(typeStore.restore()).thenReturn(typesDef);
when(typeSystem.isRegistered(any(String.class))).thenReturn(true);
DefaultMetadataService defaultMetadataService = new DefaultMetadataService(metadataRepository,
typeStore,
typesRegistrar, new ArrayList<Provider<TypesChangeListener>>(),
new ArrayList<Provider<EntityChangeListener>>(), typeSystem, configuration, null);
defaultMetadataService.instanceIsActive();
verify(typeStore).restore();
verify(typeSystem).defineTypes(typesDef);
verify(typesRegistrar).registerTypes(ReservedTypesRegistrar.getTypesDir(),
typeSystem, defaultMetadataService);
}
@Test
public void testShouldOnlyRestoreCacheOnServerActiveIfAlreadyDoneOnce() throws AtlasException {
when(configuration.containsKey(HAConfiguration.ATLAS_SERVER_HA_ENABLED_KEY)).thenReturn(true);
when(configuration.getBoolean(HAConfiguration.ATLAS_SERVER_HA_ENABLED_KEY)).thenReturn(true);
TypesDef typesDef = mock(TypesDef.class);
when(typeStore.restore()).thenReturn(typesDef);
when(typeSystem.isRegistered(any(String.class))).thenReturn(true);
TypeSystem.TransientTypeSystem transientTypeSystem = mock(TypeSystem.TransientTypeSystem.class);
HashMap<String, IDataType> typesAdded = new HashMap<>();
when(transientTypeSystem.getTypesAdded()).thenReturn(typesAdded);
when(typeSystem.createTransientTypeSystem(typesDef, true)).
thenReturn(transientTypeSystem);
DefaultMetadataService defaultMetadataService = new DefaultMetadataService(metadataRepository,
typeStore,
typesRegistrar, new ArrayList<Provider<TypesChangeListener>>(),
new ArrayList<Provider<EntityChangeListener>>(), typeSystem, configuration, null);
defaultMetadataService.instanceIsActive();
defaultMetadataService.instanceIsPassive();
defaultMetadataService.instanceIsActive();
verify(typeStore, times(2)).restore();
verify(typeSystem, times(1)).defineTypes(typesDef);
verify(typesRegistrar, times(1)).
registerTypes(ReservedTypesRegistrar.getTypesDir(), typeSystem, defaultMetadataService);
verify(typeSystem, times(1)).createTransientTypeSystem(typesDef, true);
verify(typeSystem, times(1)).commitTypes(typesAdded);
}
}
repository/src/test/java/org/apache/atlas/services/ReservedTypesRegistrarTest.java deleted 100644 → 0
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.services;
import org.apache.atlas.AtlasException;
import org.apache.atlas.TestUtils;
import org.apache.atlas.typesystem.TypesDef;
import org.apache.atlas.typesystem.json.TypesSerialization;
import org.apache.atlas.typesystem.types.TypeSystem;
import org.apache.atlas.typesystem.types.TypeUtils;
import org.apache.atlas.typesystem.types.utils.TypesUtil;
import org.mockito.InOrder;
import org.mockito.Mock;
import org.mockito.MockitoAnnotations;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
import static org.mockito.Mockito.inOrder;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.verifyZeroInteractions;
import static org.mockito.Mockito.when;
public class ReservedTypesRegistrarTest {
@Mock
private TypeSystem typeSystem;
@Mock
private MetadataService metadataService;
@BeforeMethod
public void setup() {
MockitoAnnotations.initMocks(this);
}
@Test
public void testRegistrationWithNoFiles() throws AtlasException {
IBootstrapTypesRegistrar bootstrapTypesRegistrar = new ReservedTypesRegistrar();
bootstrapTypesRegistrar.registerTypes("/some/dir/", typeSystem, metadataService);
verifyZeroInteractions(typeSystem);
}
@Test
public void testRegisterCreatesTypesUsingMetadataService() throws AtlasException {
ReservedTypesRegistrar reservedTypesRegistrar = new ReservedTypesRegistrar();
TypesDef typesDef = TestUtils.defineHiveTypes();
String typesJson = TypesSerialization.toJson(typesDef);
reservedTypesRegistrar.registerType(typeSystem, metadataService, "/some/file/model.json", typesJson);
verify(metadataService).createType(typesJson);
}
@Test(expectedExceptions = ReservedTypesRegistrationException.class)
public void testRegisterFailsIfErrorInJson() throws AtlasException {
ReservedTypesRegistrar reservedTypesRegistrar = new ReservedTypesRegistrar();
reservedTypesRegistrar.registerType(typeSystem, metadataService, "/some/file/model.json", "invalid json");
}
@Test(expectedExceptions = AtlasException.class)
public void testRegisterFailsOnTypeCreationException() throws AtlasException {
ReservedTypesRegistrar reservedTypesRegistrar = new ReservedTypesRegistrar();
TypesDef typesDef = TestUtils.defineHiveTypes();
String typesJson = TypesSerialization.toJson(typesDef);
when(metadataService.createType(typesJson)).thenThrow(new AtlasException("some exception"));
reservedTypesRegistrar.registerType(typeSystem, metadataService, "/some/file/model.json", typesJson);
}
@Test
public void testCreateAndUpdateType() throws AtlasException{
ReservedTypesRegistrar reservedTypesRegistrar = new ReservedTypesRegistrar();
TypesDef typesDef = TestUtils.simpleType();
String typesJson = TypesSerialization.toJson(typesDef);
reservedTypesRegistrar.registerType(typeSystem, metadataService, "/some/file/model.json", typesJson);
verify(metadataService).createType(typesJson);
//test update simple type
TypesDef updatedTypesDef = TestUtils.simpleTypeUpdated();
String updatedTypesJson = TypesSerialization.toJson(updatedTypesDef);
TypesDef simpleTypeUpdatedDiff = TestUtils.simpleTypeUpdatedDiff();
String simpleTypeUpdatedDiffJson = TypesSerialization.toJson(simpleTypeUpdatedDiff);
when(typeSystem.isRegistered("h_type")).thenReturn(true);
when(typeSystem.isRegistered("t_type")).thenReturn(true);
when(typeSystem.isRegistered("s_type")).thenReturn(true);
when(typeSystem.isRegistered("e_type")).thenReturn(true);
reservedTypesRegistrar.registerType(typeSystem, metadataService, "/some/file/model.json", updatedTypesJson);
verify(metadataService).createType(simpleTypeUpdatedDiffJson);
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment