Skip to content

A
atlas
Commit 1dcc3073 by David Radley Committed by Madhan Neethiraj
Options

ATLAS-2112: move the Atlas models to subfolders

Signed-off-by: 's avatarMadhan Neethiraj <madhan@apache.org>
parent 9d38c6a9
Showing with 127 additions and 91 deletions
addons/falcon-bridge/pom.xml
......@@ -368,10 +368,10 @@
<goal>copy-resources</goal>
</goals>
<configuration>
<outputDirectory>${basedir}/target/models</outputDirectory>
<outputDirectory>${basedir}/target/models/1000-Hadoop</outputDirectory>
<resources>
<resource>
<directory>${basedir}/../models</directory>
<directory>${basedir}/../models/1000-Hadoop</directory>
<filtering>true</filtering>
</resource>
</resources>
......
addons/hive-bridge/pom.xml
......@@ -417,10 +417,10 @@
<goal>copy-resources</goal>
</goals>
<configuration>
<outputDirectory>${basedir}/target/models</outputDirectory>
<outputDirectory>${basedir}/target/models/1000-Hadoop</outputDirectory>
<resources>
<resource>
<directory>${basedir}/../models</directory>
<directory>${basedir}/../models/1000-Hadoop</directory>
<filtering>true</filtering>
</resource>
</resources>
......
addons/sqoop-bridge/pom.xml
......@@ -429,10 +429,10 @@
<goal>copy-resources</goal>
</goals>
<configuration>
<outputDirectory>${basedir}/target/models</outputDirectory>
<outputDirectory>${basedir}/target/models/1000-Hadoop</outputDirectory>
<resources>
<resource>
<directory>${basedir}/../models</directory>
<directory>${basedir}/../models/1000-Hadoop</directory>
<filtering>true</filtering>
</resource>
</resources>
......
addons/storm-bridge/pom.xml
......@@ -501,10 +501,10 @@
<goal>copy-resources</goal>
</goals>
<configuration>
<outputDirectory>${basedir}/target/models</outputDirectory>
<outputDirectory>${basedir}/target/models/1000-Hadoop</outputDirectory>
<resources>
<resource>
<directory>${basedir}/../models</directory>
<directory>${basedir}/../models/1000-Hadoop</directory>
<filtering>true</filtering>
</resource>
</resources>
......
repository/src/main/java/org/apache/atlas/repository/store/bootstrap/AtlasTypeDefStoreInitializer.java
......@@ -69,6 +69,7 @@ import static org.codehaus.jackson.annotate.JsonAutoDetect.Visibility.PUBLIC_ONL
@Service
public class AtlasTypeDefStoreInitializer implements ActiveStateChangeHandler {
private static final Logger LOG = LoggerFactory.getLogger(AtlasTypeDefStoreInitializer.class);
public static final String PATCHES_FOLDER_NAME = "patches";
private final AtlasTypeDefStore atlasTypeDefStore;
private final AtlasTypeRegistry atlasTypeRegistry;
......@@ -97,57 +98,93 @@ public class AtlasTypeDefStoreInitializer implements ActiveStateChangeHandler {
LOG.info("<== AtlasTypeDefStoreInitializer.init()");
}
/**
* This method is looking for folders in alphabetical order in the models directory. It loads each of these folders and their associated patches in order.
* It then loads any models in the top level folder and its patches.
*
* This allows models to be grouped into folders to help managability.
*
*/
private void loadBootstrapTypeDefs() {
LOG.info("==> AtlasTypeDefStoreInitializer.loadBootstrapTypeDefs()");
String atlasHomeDir = System.getProperty("atlas.home");
String typesDirName = (StringUtils.isEmpty(atlasHomeDir) ? "." : atlasHomeDir) + File.separator + "models";
File typesDir = new File(typesDirName);
File[] typeDefFiles = typesDir.exists() ? typesDir.listFiles() : null;
String atlasHomeDir = System.getProperty("atlas.home");
String modelsDirName = (StringUtils.isEmpty(atlasHomeDir) ? "." : atlasHomeDir) + File.separator + "models";
if (typeDefFiles == null || typeDefFiles.length == 0) {
LOG.info("Types directory {} does not exist or not readable or has no typedef files", typesDirName);
if (modelsDirName == null || modelsDirName.length() == 0) {
LOG.info("Types directory {} does not exist or not readable or has no typedef files", modelsDirName);
} else {
// look for folders we need to load models from
File topModeltypesDir = new File(modelsDirName);
File[] modelsDirContents = topModeltypesDir.exists() ? topModeltypesDir.listFiles() : null;
Arrays.sort(modelsDirContents);
return;
for (File folder : modelsDirContents) {
if (folder.isFile()) {
// ignore files
continue;
} else if (!folder.getName().equals(PATCHES_FOLDER_NAME)){
// load the models alphabetically in the subfolders apart from patches
loadModelsInFolder(folder);
}
}
// load any files in the top models folder and any associated patches.
loadModelsInFolder(topModeltypesDir);
}
LOG.info("<== AtlasTypeDefStoreInitializer.loadBootstrapTypeDefs()");
}
// sort the files by filename
Arrays.sort(typeDefFiles);
/**
* Load all the model files in the supplied folder followed by the contents of the patches folder.
* @param typesDir
*/
private void loadModelsInFolder(File typesDir) {
LOG.info("==> AtlasTypeDefStoreInitializer({})", typesDir);
for (File typeDefFile : typeDefFiles) {
if (!typeDefFile.isFile()) {
continue;
}
String typesDirName = typesDir.getName();
File[] typeDefFiles = typesDir.exists() ? typesDir.listFiles() : null;
try {
String jsonStr = new String(Files.readAllBytes(typeDefFile.toPath()), StandardCharsets.UTF_8);
AtlasTypesDef typesDef = AtlasType.fromJson(jsonStr, AtlasTypesDef.class);
if (typeDefFiles == null || typeDefFiles.length == 0) {
LOG.info("Types directory {} does not exist or not readable or has no typedef files", typesDirName );
} else {
if (typesDef == null || typesDef.isEmpty()) {
LOG.info("No type in file {}", typeDefFile.getAbsolutePath());
// sort the files by filename
Arrays.sort(typeDefFiles);
continue;
}
for (File typeDefFile : typeDefFiles) {
if (typeDefFile.isFile()) {
try {
String jsonStr = new String(Files.readAllBytes(typeDefFile.toPath()), StandardCharsets.UTF_8);
AtlasTypesDef typesDef = AtlasType.fromJson(jsonStr, AtlasTypesDef.class);
AtlasTypesDef typesToCreate = getTypesToCreate(typesDef, atlasTypeRegistry);
AtlasTypesDef typesToUpdate = getTypesToUpdate(typesDef, atlasTypeRegistry);
if (typesDef == null || typesDef.isEmpty()) {
LOG.info("No type in file {}", typeDefFile.getAbsolutePath());
if (!typesToCreate.isEmpty() || !typesToUpdate.isEmpty()) {
atlasTypeDefStore.createUpdateTypesDef(typesToCreate, typesToUpdate);
continue;
}
LOG.info("Created/Updated types defined in file {}", typeDefFile.getAbsolutePath());
} else {
LOG.info("No new type in file {}", typeDefFile.getAbsolutePath());
}
AtlasTypesDef typesToCreate = getTypesToCreate(typesDef, atlasTypeRegistry);
AtlasTypesDef typesToUpdate = getTypesToUpdate(typesDef, atlasTypeRegistry);
} catch (Throwable t) {
LOG.error("error while registering types in file {}", typeDefFile.getAbsolutePath(), t);
}
}
if (!typesToCreate.isEmpty() || !typesToUpdate.isEmpty()) {
atlasTypeDefStore.createUpdateTypesDef(typesToCreate, typesToUpdate);
applyTypePatches(typesDirName);
LOG.info("Created/Updated types defined in file {}", typeDefFile.getAbsolutePath());
} else {
LOG.info("No new type in file {}", typeDefFile.getAbsolutePath());
}
LOG.info("<== AtlasTypeDefStoreInitializer.loadBootstrapTypeDefs()");
} catch (Throwable t) {
LOG.error("error while registering types in file {}", typeDefFile.getAbsolutePath(), t);
}
}
}
applyTypePatches(typesDir.getPath());
}
LOG.info("<== AtlasTypeDefStoreInitializer({})", typesDir);
}
public static AtlasTypesDef getTypesToCreate(AtlasTypesDef typesDef, AtlasTypeRegistry typeRegistry) {
......@@ -327,68 +364,67 @@ public class AtlasTypeDefStoreInitializer implements ActiveStateChangeHandler {
}
private void applyTypePatches(String typesDirName) {
String typePatchesDirName = typesDirName + File.separator + "patches";
String typePatchesDirName = typesDirName + File.separator + PATCHES_FOLDER_NAME;
File typePatchesDir = new File(typePatchesDirName);
File[] typePatchFiles = typePatchesDir.exists() ? typePatchesDir.listFiles() : null;
if (typePatchFiles == null || typePatchFiles.length == 0) {
LOG.info("Type patches directory {} does not exist or not readable or has no patches", typePatchesDirName);
} else {
LOG.info("Type patches directory {} is being processed", typePatchesDirName);
return;
}
// sort the files by filename
Arrays.sort(typePatchFiles);
// sort the files by filename
Arrays.sort(typePatchFiles);
PatchHandler[] patchHandlers = new PatchHandler[] {
new AddAttributePatchHandler(atlasTypeDefStore, atlasTypeRegistry),
new UpdateTypeDefOptionsPatchHandler(atlasTypeDefStore, atlasTypeRegistry),
new UpdateAttributePatchHandler(atlasTypeDefStore, atlasTypeRegistry)
};
PatchHandler[] patchHandlers = new PatchHandler[] {
new AddAttributePatchHandler(atlasTypeDefStore, atlasTypeRegistry),
new UpdateTypeDefOptionsPatchHandler(atlasTypeDefStore, atlasTypeRegistry),
new UpdateAttributePatchHandler(atlasTypeDefStore, atlasTypeRegistry)
};
Map<String, PatchHandler> patchHandlerRegistry = new HashMap<>();
Map<String, PatchHandler> patchHandlerRegistry = new HashMap<>();
for (PatchHandler patchHandler : patchHandlers) {
for (String supportedAction : patchHandler.getSupportedActions()) {
patchHandlerRegistry.put(supportedAction, patchHandler);
for (PatchHandler patchHandler : patchHandlers) {
for (String supportedAction : patchHandler.getSupportedActions()) {
patchHandlerRegistry.put(supportedAction, patchHandler);
}
}
}
for (File typePatchFile : typePatchFiles) {
if (!typePatchFile.isFile()) {
continue;
}
for (File typePatchFile : typePatchFiles) {
if (typePatchFile.isFile()) {
LOG.info("Applying patches in file {}", typePatchFile.getAbsolutePath());
LOG.info("Applying patches in file {}", typePatchFile.getAbsolutePath());
try {
String jsonStr = new String(Files.readAllBytes(typePatchFile.toPath()), StandardCharsets.UTF_8);
TypeDefPatches patches = AtlasType.fromJson(jsonStr, TypeDefPatches.class);
try {
String jsonStr = new String(Files.readAllBytes(typePatchFile.toPath()), StandardCharsets.UTF_8);
TypeDefPatches patches = AtlasType.fromJson(jsonStr, TypeDefPatches.class);
if (patches == null || CollectionUtils.isEmpty(patches.getPatches())) {
LOG.info("No patches in file {}", typePatchFile.getAbsolutePath());
if (patches == null || CollectionUtils.isEmpty(patches.getPatches())) {
LOG.info("No patches in file {}", typePatchFile.getAbsolutePath());
continue;
}
continue;
}
for (TypeDefPatch patch : patches.getPatches()) {
PatchHandler patchHandler = patchHandlerRegistry.get(patch.getAction());
for (TypeDefPatch patch : patches.getPatches()) {
PatchHandler patchHandler = patchHandlerRegistry.get(patch.getAction());
if (patchHandler == null) {
LOG.error("Unknown patch action {} in file {}. Ignored",
patch.getAction(), typePatchFile.getAbsolutePath());
if (patchHandler == null) {
LOG.error("Unknown patch action {} in file {}. Ignored",
patch.getAction(), typePatchFile.getAbsolutePath());
continue;
}
continue;
}
try {
patchHandler.applyPatch(patch);
} catch (AtlasBaseException excp) {
LOG.error("Failed to apply {} patch in file {}. Ignored", patch.getAction(), typePatchFile.getAbsolutePath(), excp);
try {
patchHandler.applyPatch(patch);
} catch (AtlasBaseException excp) {
LOG.error("Failed to apply {} patch in file {}. Ignored", patch.getAction(), typePatchFile.getAbsolutePath(), excp);
}
}
} catch (Throwable t) {
LOG.error("Failed to apply patches in file {}. Ignored", typePatchFile.getAbsolutePath(), t);
}
}
} catch (Throwable t) {
LOG.error("Failed to apply patches in file {}. Ignored", typePatchFile.getAbsolutePath(), t);
}
}
}
......
repository/src/test/java/org/apache/atlas/repository/impexp/ImportServiceTest.java
......@@ -68,7 +68,7 @@ public class ImportServiceTest {
@Test(dataProvider = "sales")
public void importDB1(ZipSource zipSource) throws AtlasBaseException, IOException {
loadModelFromJson("0010-base_model.json", typeDefStore, typeRegistry);
loadModelFromJson("0000-Area0/0010-base_model.json", typeDefStore, typeRegistry);
runAndVerifyQuickStart_v1_Import(new ImportService(typeDefStore, entityStore, typeRegistry), zipSource);
}
......@@ -79,7 +79,7 @@ public class ImportServiceTest {
@Test(dataProvider = "reporting")
public void importDB2(ZipSource zipSource) throws AtlasBaseException, IOException {
loadModelFromJson("0010-base_model.json", typeDefStore, typeRegistry);
loadModelFromJson("0000-Area0/0010-base_model.json", typeDefStore, typeRegistry);
runAndVerifyQuickStart_v1_Import(new ImportService(typeDefStore, entityStore, typeRegistry), zipSource);
}
......@@ -90,7 +90,7 @@ public class ImportServiceTest {
@Test(dataProvider = "logging")
public void importDB3(ZipSource zipSource) throws AtlasBaseException, IOException {
loadModelFromJson("0010-base_model.json", typeDefStore, typeRegistry);
loadModelFromJson("0000-Area0/0010-base_model.json", typeDefStore, typeRegistry);
runAndVerifyQuickStart_v1_Import(new ImportService(typeDefStore, entityStore, typeRegistry), zipSource);
}
......@@ -101,7 +101,7 @@ public class ImportServiceTest {
@Test(dataProvider = "salesNewTypeAttrs", dependsOnMethods = "importDB1")
public void importDB4(ZipSource zipSource) throws AtlasBaseException, IOException {
loadModelFromJson("0010-base_model.json", typeDefStore, typeRegistry);
loadModelFromJson("0000-Area0/0010-base_model.json", typeDefStore, typeRegistry);
runImportWithParameters(new ImportService(typeDefStore, entityStore, typeRegistry), getDefaultImportRequest(), zipSource);
}
......@@ -150,8 +150,8 @@ public class ImportServiceTest {
@Test(dataProvider = "ctas")
public void importCTAS(ZipSource zipSource) throws IOException, AtlasBaseException {
loadModelFromJson("0010-base_model.json", typeDefStore, typeRegistry);
loadModelFromJson("1030-hive_model.json", typeDefStore, typeRegistry);
loadModelFromJson("0000-Area0/0010-base_model.json", typeDefStore, typeRegistry);
loadModelFromJson("1000-Hadoop/1030-hive_model.json", typeDefStore, typeRegistry);
runImportWithNoParameters(getImportService(), zipSource);
}
......
webapp/pom.xml
......@@ -688,10 +688,10 @@
<goal>copy-resources</goal>
</goals>
<configuration>
<outputDirectory>${basedir}/target/models</outputDirectory>
<outputDirectory>${basedir}/target/models/1000-Hadoop</outputDirectory>
<resources>
<resource>
<directory>${basedir}/../addons/models</directory>
<directory>${basedir}/../addons/models/1000-Hadoop</directory>
<filtering>true</filtering>
</resource>
</resources>
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment