Skip to content

A
atlas
Commit 34cdef44 by Dan Markwat
Options

Merge remote-tracking branch 'origin/guice-enabled'

Conflicts: repository/pom.xml repository/src/main/java/org/apache/hadoop/metadata/services/GraphBackedMetadataRepository.java repository/src/main/java/org/apache/hadoop/metadata/services/MetadataRepository.java repository/src/main/java/org/apache/hadoop/metadata/services/TitanGraphService.java repository/src/test/java/org/apache/hadoop/metadata/services/GraphBackedMetadataRepositoryTest.java webapp/src/main/java/org/apache/hadoop/metadata/web/resources/EntityResource.java webapp/src/main/resources/application.properties webapp/src/test/java/org/apache/hadoop/metadata/GraphRepositoryServiceIT.java
parents faf0b48b 96350bda
Showing with 2353 additions and 2725 deletions
bridge/pom.xml
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion> <modelVersion>4.0.0</modelVersion>
<parent> <parent>
<groupId>org.apache.hadoop.metadata</groupId> <groupId>org.apache.hadoop.metadata</groupId>
<artifactId>metadata-governance</artifactId> <artifactId>metadata-governance</artifactId>
<version>0.1-incubating-SNAPSHOT</version> <version>0.1-incubating-SNAPSHOT</version>
</parent> </parent>
<artifactId>metadata-bridge</artifactId> <artifactId>metadata-bridge</artifactId>
<dependencies> <dependencies>
<dependency> <dependency>
<groupId>org.apache.hive</groupId> <groupId>org.apache.hive</groupId>
<artifactId>hive-metastore</artifactId> <artifactId>hive-metastore</artifactId>
<version>0.14.0</version> <version>0.14.0</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>com.google.code.gson</groupId> <groupId>com.google.code.gson</groupId>
<artifactId>gson</artifactId> <artifactId>gson</artifactId>
<version>2.2.2</version> <version>2.2.2</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>junit</groupId> <groupId>junit</groupId>
<artifactId>junit</artifactId> <artifactId>junit</artifactId>
<version>4.10</version> <version>4.10</version>
<scope>test</scope> <scope>test</scope>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.apache.hadoop.metadata</groupId> <groupId>org.apache.hadoop.metadata</groupId>
<artifactId>metadata-typesystem</artifactId> <artifactId>metadata-typesystem</artifactId>
<version>0.1-incubating-SNAPSHOT</version> <version>0.1-incubating-SNAPSHOT</version>
</dependency> </dependency>
</dependencies> </dependencies>
</project> </project>
\ No newline at end of file
bridge/src/main/java/org/apache/hadoop/metadata/bridge/Bridge.java
package org.apache.hadoop.metadata.bridge; package org.apache.hadoop.metadata.bridge;
import org.apache.hadoop.metadata.types.AttributeDefinition; import org.apache.hadoop.metadata.types.AttributeDefinition;
import org.apache.hadoop.metadata.types.ClassType; import org.apache.hadoop.metadata.types.ClassType;
import org.apache.hadoop.metadata.types.HierarchicalTypeDefinition; import org.apache.hadoop.metadata.types.HierarchicalTypeDefinition;
import org.apache.hadoop.metadata.types.TypeSystem; import org.apache.hadoop.metadata.types.TypeSystem;
import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableList;
public interface Bridge { public interface Bridge {
boolean defineBridgeTypes(TypeSystem ts); boolean defineBridgeTypes(TypeSystem ts);
} }
bridge/src/main/java/org/apache/hadoop/metadata/bridge/BridgeAssistant.java
package org.apache.hadoop.metadata.bridge; package org.apache.hadoop.metadata.bridge;
import org.apache.hadoop.metadata.types.AttributeDefinition; import org.apache.hadoop.metadata.types.AttributeDefinition;
import org.apache.hadoop.metadata.types.ClassType; import org.apache.hadoop.metadata.types.ClassType;
import org.apache.hadoop.metadata.types.HierarchicalTypeDefinition; import org.apache.hadoop.metadata.types.HierarchicalTypeDefinition;
import org.apache.hadoop.metadata.types.TypeSystem; import org.apache.hadoop.metadata.types.TypeSystem;
import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableList;
public class BridgeAssistant { public class BridgeAssistant {
protected HierarchicalTypeDefinition<ClassType> createClassTypeDef(String name, ImmutableList<String> superTypes, AttributeDefinition... attrDefs) {return new HierarchicalTypeDefinition(ClassType.class, name, superTypes, attrDefs);} protected HierarchicalTypeDefinition<ClassType> createClassTypeDef(String name, ImmutableList<String> superTypes, AttributeDefinition... attrDefs) {return new HierarchicalTypeDefinition(ClassType.class, name, superTypes, attrDefs);}
} }
bridge/src/main/java/org/apache/hadoop/metadata/bridge/BridgeException.java
package org.apache.hadoop.metadata.bridge; package org.apache.hadoop.metadata.bridge;
import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.MetaException;
public class BridgeException extends MetaException { public class BridgeException extends MetaException {
/** /**
* *
*/ */
private static final long serialVersionUID = -384401342591560473L; private static final long serialVersionUID = -384401342591560473L;
} }
bridge/src/main/java/org/apache/hadoop/metadata/bridge/BridgeManager.java
package org.apache.hadoop.metadata.bridge; package org.apache.hadoop.metadata.bridge;
import org.apache.hadoop.metadata.types.TypeSystem; import org.apache.hadoop.metadata.types.TypeSystem;
public class BridgeManager { public class BridgeManager {
TypeSystem ts; TypeSystem ts;
BridgeManager(TypeSystem ts){ BridgeManager(TypeSystem ts){
this.ts = ts; this.ts = ts;
} }
} }
bridge/src/main/java/org/apache/hadoop/metadata/bridge/hivelineage/HiveLineageBridge.java
package org.apache.hadoop.metadata.bridge.hivelineage; package org.apache.hadoop.metadata.bridge.hivelineage;
import org.apache.hadoop.metadata.bridge.Bridge; import org.apache.hadoop.metadata.bridge.Bridge;
import org.apache.hadoop.metadata.types.TypeSystem; import org.apache.hadoop.metadata.types.TypeSystem;
public class HiveLineageBridge implements Bridge { public class HiveLineageBridge implements Bridge {
@Override @Override
public boolean defineBridgeTypes(TypeSystem ts) { public boolean defineBridgeTypes(TypeSystem ts) {
// TODO Auto-generated method stub // TODO Auto-generated method stub
return false; return false;
} }
} }
bridge/src/main/java/org/apache/hadoop/metadata/bridge/hivestructure/HiveMetaImporter.java
package org.apache.hadoop.metadata.bridge.hivestructure; package org.apache.hadoop.metadata.bridge.hivestructure;
import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
import org.apache.hadoop.hive.metastore.api.Database; import org.apache.hadoop.hive.metastore.api.Database;
import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.MetaException;
import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
import org.apache.hadoop.hive.metastore.api.Table; import org.apache.hadoop.hive.metastore.api.Table;
import org.apache.hadoop.hive.metastore.api.UnknownDBException; import org.apache.hadoop.hive.metastore.api.UnknownDBException;
import org.apache.hadoop.hive.metastore.api.UnknownTableException; import org.apache.hadoop.hive.metastore.api.UnknownTableException;
import org.apache.hadoop.metadata.MetadataException; import org.apache.hadoop.metadata.MetadataException;
import org.apache.hadoop.metadata.MetadataService; import org.apache.hadoop.metadata.MetadataService;
import org.apache.hadoop.metadata.Referenceable; import org.apache.hadoop.metadata.Referenceable;
import org.apache.hadoop.metadata.storage.RepositoryException; import org.apache.hadoop.metadata.storage.RepositoryException;
import org.apache.hadoop.metadata.types.ClassType; import org.apache.hadoop.metadata.types.ClassType;
import org.apache.thrift.TException; import org.apache.thrift.TException;
/* /*
* Initial pass at one time importer TODO - needs re-write * Initial pass at one time importer TODO - needs re-write
*/ */
public class HiveMetaImporter { public class HiveMetaImporter {
private static HiveMetaStoreClient msc; private static HiveMetaStoreClient msc;
private static MetadataService ms; private static MetadataService ms;
public HiveMetaImporter(MetadataService ms){ public HiveMetaImporter(MetadataService ms){
try { try {
this.ms = ms; this.ms = ms;
msc = new HiveMetaStoreClient(new HiveConf()); msc = new HiveMetaStoreClient(new HiveConf());
} catch (MetaException e) { } catch (MetaException e) {
// TODO Auto-generated catch block // TODO Auto-generated catch block
e.printStackTrace(); e.printStackTrace();
} }
} }
public static boolean fullImport(){ public static boolean fullImport(){
try{ try{
databasesImport(); databasesImport();
for (String dbName : msc.getAllDatabases()){ for (String dbName : msc.getAllDatabases()){
tablesImport(dbName); tablesImport(dbName);
for(String tbName : msc.getAllTables(dbName)){ for(String tbName : msc.getAllTables(dbName)){
fieldsImport(dbName,tbName); fieldsImport(dbName,tbName);
} }
return true; return true;
} }
}catch(MetaException me){ }catch(MetaException me){
me.printStackTrace(); me.printStackTrace();
}catch(RepositoryException re){ }catch(RepositoryException re){
re.printStackTrace(); re.printStackTrace();
} }
return false; return false;
} }
public static boolean databasesImport() throws MetaException, RepositoryException{ public static boolean databasesImport() throws MetaException, RepositoryException{
ClassType classType = null; ClassType classType = null;
try { try {
classType = ms.getTypeSystem().getDataType(ClassType.class, HiveStructureBridge.DB_CLASS_TYPE); classType = ms.getTypeSystem().getDataType(ClassType.class, HiveStructureBridge.DB_CLASS_TYPE);
} catch (MetadataException e1) { } catch (MetadataException e1) {
e1.printStackTrace(); e1.printStackTrace();
} }
for(String dbName : msc.getAllDatabases()){ for(String dbName : msc.getAllDatabases()){
databaseImport(dbName); databaseImport(dbName);
} }
return true; return true;
} }
public static boolean databaseImport(String dbName) throws MetaException, RepositoryException{ public static boolean databaseImport(String dbName) throws MetaException, RepositoryException{
try { try {
Database db = msc.getDatabase(dbName); Database db = msc.getDatabase(dbName);
Referenceable dbRef = new Referenceable(HiveStructureBridge.DB_CLASS_TYPE); Referenceable dbRef = new Referenceable(HiveStructureBridge.DB_CLASS_TYPE);
dbRef.set("DESC", db.getDescription()); dbRef.set("DESC", db.getDescription());
dbRef.set("DB_LOCATION_URI", db.getLocationUri()); dbRef.set("DB_LOCATION_URI", db.getLocationUri());
dbRef.set("NAME", db.getName()); dbRef.set("NAME", db.getName());
if(db.isSetOwnerType()){dbRef.set("OWNER_TYPE", db.getOwnerType());} if(db.isSetOwnerType()){dbRef.set("OWNER_TYPE", db.getOwnerType());}
if(db.isSetOwnerName()){dbRef.set("OWNER_NAME", db.getOwnerName());} if(db.isSetOwnerName()){dbRef.set("OWNER_NAME", db.getOwnerName());}
ms.getRepository().create(dbRef); ms.getRepository().create(dbRef);
} catch (NoSuchObjectException e) { } catch (NoSuchObjectException e) {
// TODO Auto-generated catch block // TODO Auto-generated catch block
e.printStackTrace(); e.printStackTrace();
} catch (TException e) { } catch (TException e) {
// TODO Auto-generated catch block // TODO Auto-generated catch block
e.printStackTrace(); e.printStackTrace();
} }
return true; return true;
} }
public static boolean tablesImport(String dbName) throws MetaException, RepositoryException{ public static boolean tablesImport(String dbName) throws MetaException, RepositoryException{
ClassType classType = null; ClassType classType = null;
try { try {
classType = ms.getTypeSystem().getDataType(ClassType.class, HiveStructureBridge.TB_CLASS_TYPE); classType = ms.getTypeSystem().getDataType(ClassType.class, HiveStructureBridge.TB_CLASS_TYPE);
} catch (MetadataException e1) { } catch (MetadataException e1) {
e1.printStackTrace(); e1.printStackTrace();
} }
for(String tbName : msc.getAllTables(dbName)){ for(String tbName : msc.getAllTables(dbName)){
tableImport(dbName, tbName); tableImport(dbName, tbName);
} }
return true; return true;
} }
public static boolean tableImport(String dbName, String tbName) throws MetaException, RepositoryException{ public static boolean tableImport(String dbName, String tbName) throws MetaException, RepositoryException{
try { try {
Table tb = msc.getTable(dbName, tbName); Table tb = msc.getTable(dbName, tbName);
Referenceable tbRef = new Referenceable(HiveStructureBridge.TB_CLASS_TYPE); Referenceable tbRef = new Referenceable(HiveStructureBridge.TB_CLASS_TYPE);
tbRef.set("CREATE_TIME", tb.getCreateTime()); tbRef.set("CREATE_TIME", tb.getCreateTime());
tbRef.set("LAST_ACCESS_TIME", tb.getLastAccessTime()); tbRef.set("LAST_ACCESS_TIME", tb.getLastAccessTime());
tbRef.set("OWNER", tb.getOwner()); tbRef.set("OWNER", tb.getOwner());
tbRef.set("TBL_NAME", tb.getTableName()); tbRef.set("TBL_NAME", tb.getTableName());
tbRef.set("TBL_TYPE", tb.getTableType()); tbRef.set("TBL_TYPE", tb.getTableType());
if(tb.isSetViewExpandedText()){tbRef.set("VIEW_EXPANDED_TEXT", tb.getViewExpandedText());} if(tb.isSetViewExpandedText()){tbRef.set("VIEW_EXPANDED_TEXT", tb.getViewExpandedText());}
if(tb.isSetViewOriginalText()){tbRef.set("VIEW_ORIGINAL_TEXT", tb.getViewOriginalText());} if(tb.isSetViewOriginalText()){tbRef.set("VIEW_ORIGINAL_TEXT", tb.getViewOriginalText());}
ms.getRepository().create(tbRef); ms.getRepository().create(tbRef);
} catch (NoSuchObjectException e) { } catch (NoSuchObjectException e) {
// TODO Auto-generated catch block // TODO Auto-generated catch block
e.printStackTrace(); e.printStackTrace();
} catch (TException e) { } catch (TException e) {
// TODO Auto-generated catch block // TODO Auto-generated catch block
e.printStackTrace(); e.printStackTrace();
} }
return true; return true;
} }
public static boolean fieldsImport (String dbName, String tbName) throws MetaException, RepositoryException{ public static boolean fieldsImport (String dbName, String tbName) throws MetaException, RepositoryException{
ClassType classType = null; ClassType classType = null;
try { try {
classType = ms.getTypeSystem().getDataType(ClassType.class, HiveStructureBridge.FD_CLASS_TYPE); classType = ms.getTypeSystem().getDataType(ClassType.class, HiveStructureBridge.FD_CLASS_TYPE);
} catch (MetadataException e1) { } catch (MetadataException e1) {
e1.printStackTrace(); e1.printStackTrace();
} }
try { try {
for(FieldSchema fs : msc.getFields(dbName, tbName)){ for(FieldSchema fs : msc.getFields(dbName, tbName)){
Referenceable fdRef = new Referenceable(HiveStructureBridge.FD_CLASS_TYPE); Referenceable fdRef = new Referenceable(HiveStructureBridge.FD_CLASS_TYPE);
if(fs.isSetComment()){fdRef.set("COMMENT", fs.getName());} if(fs.isSetComment()){fdRef.set("COMMENT", fs.getName());}
fdRef.set("COLUMN_NAME", fs.getName()); fdRef.set("COLUMN_NAME", fs.getName());
fdRef.set("TYPE_NAME", fs.getType()); fdRef.set("TYPE_NAME", fs.getType());
ms.getRepository().create(fdRef); ms.getRepository().create(fdRef);
} }
} catch (UnknownTableException e) { } catch (UnknownTableException e) {
// TODO Auto-generated catch block // TODO Auto-generated catch block
e.printStackTrace(); e.printStackTrace();
} catch (UnknownDBException e) { } catch (UnknownDBException e) {
// TODO Auto-generated catch block // TODO Auto-generated catch block
e.printStackTrace(); e.printStackTrace();
} catch (TException e) { } catch (TException e) {
// TODO Auto-generated catch block // TODO Auto-generated catch block
e.printStackTrace(); e.printStackTrace();
} }
return true; return true;
} }
public static boolean fieldImport(String dbName, String tbName, String fdName) throws MetaException{ public static boolean fieldImport(String dbName, String tbName, String fdName) throws MetaException{
try { try {
for(FieldSchema fs : msc.getFields(dbName, tbName)){ for(FieldSchema fs : msc.getFields(dbName, tbName)){
if (fs.getName().equals(fs)){ if (fs.getName().equals(fs)){
Referenceable fdRef = new Referenceable(HiveStructureBridge.TB_CLASS_TYPE); Referenceable fdRef = new Referenceable(HiveStructureBridge.TB_CLASS_TYPE);
if(fs.isSetComment()){fdRef.set("COMMENT", fs.getName());} if(fs.isSetComment()){fdRef.set("COMMENT", fs.getName());}
fdRef.set("COLUMN_NAME", fs.getName()); fdRef.set("COLUMN_NAME", fs.getName());
fdRef.set("TYPE_NAME", fs.getType()); fdRef.set("TYPE_NAME", fs.getType());
//SaveObject to MS Backend //SaveObject to MS Backend
return true; return true;
} }
} }
} catch (UnknownTableException e) { } catch (UnknownTableException e) {
// TODO Auto-generated catch block // TODO Auto-generated catch block
e.printStackTrace(); e.printStackTrace();
} catch (UnknownDBException e) { } catch (UnknownDBException e) {
// TODO Auto-generated catch block // TODO Auto-generated catch block
e.printStackTrace(); e.printStackTrace();
} catch (TException e) { } catch (TException e) {
// TODO Auto-generated catch block // TODO Auto-generated catch block
e.printStackTrace(); e.printStackTrace();
} }
return true; return true;
} }
} }
bridge/src/main/java/org/apache/hadoop/metadata/bridge/hivestructure/HiveStructureBridge.java
package org.apache.hadoop.metadata.bridge.hivestructure; package org.apache.hadoop.metadata.bridge.hivestructure;
import java.util.ArrayList; import java.util.ArrayList;
import org.apache.hadoop.metadata.MetadataException; import org.apache.hadoop.metadata.MetadataException;
import org.apache.hadoop.metadata.bridge.Bridge; import org.apache.hadoop.metadata.bridge.Bridge;
import org.apache.hadoop.metadata.bridge.BridgeAssistant; import org.apache.hadoop.metadata.bridge.BridgeAssistant;
import org.apache.hadoop.metadata.types.AttributeDefinition; import org.apache.hadoop.metadata.types.AttributeDefinition;
import org.apache.hadoop.metadata.types.ClassType; import org.apache.hadoop.metadata.types.ClassType;
import org.apache.hadoop.metadata.types.HierarchicalTypeDefinition; import org.apache.hadoop.metadata.types.HierarchicalTypeDefinition;
import org.apache.hadoop.metadata.types.Multiplicity; import org.apache.hadoop.metadata.types.Multiplicity;
import org.apache.hadoop.metadata.types.TypeSystem; import org.apache.hadoop.metadata.types.TypeSystem;
public class HiveStructureBridge extends BridgeAssistant implements Bridge{ public class HiveStructureBridge extends BridgeAssistant implements Bridge{
static final String DB_CLASS_TYPE = "HiveDatabase"; static final String DB_CLASS_TYPE = "HiveDatabase";
static final String TB_CLASS_TYPE = "HiveTable"; static final String TB_CLASS_TYPE = "HiveTable";
static final String FD_CLASS_TYPE = "HiveField"; static final String FD_CLASS_TYPE = "HiveField";
@Override @Override
public boolean defineBridgeTypes(TypeSystem ts) { public boolean defineBridgeTypes(TypeSystem ts) {
ArrayList<HierarchicalTypeDefinition<?>> al = new ArrayList<HierarchicalTypeDefinition<?>>(); ArrayList<HierarchicalTypeDefinition<?>> al = new ArrayList<HierarchicalTypeDefinition<?>>();
try{ try{
HierarchicalTypeDefinition<ClassType> databaseClassTypeDef = new HierarchicalTypeDefinition<ClassType>("ClassType",DB_CLASS_TYPE, null, HierarchicalTypeDefinition<ClassType> databaseClassTypeDef = new HierarchicalTypeDefinition<ClassType>("ClassType",DB_CLASS_TYPE, null,
new AttributeDefinition[]{ new AttributeDefinition[]{
new AttributeDefinition("DESC", "STRING_TYPE", Multiplicity.OPTIONAL, false, null), new AttributeDefinition("DESC", "STRING_TYPE", Multiplicity.OPTIONAL, false, null),
new AttributeDefinition("DB_LOCATION_URI", "STRING_TYPE", Multiplicity.REQUIRED, false, null), new AttributeDefinition("DB_LOCATION_URI", "STRING_TYPE", Multiplicity.REQUIRED, false, null),
new AttributeDefinition("NAME", "STRING_TYPE", Multiplicity.REQUIRED, false, null), new AttributeDefinition("NAME", "STRING_TYPE", Multiplicity.REQUIRED, false, null),
new AttributeDefinition("OWNER_TYPE", "STRING_TYPE", Multiplicity.OPTIONAL, false, null), new AttributeDefinition("OWNER_TYPE", "STRING_TYPE", Multiplicity.OPTIONAL, false, null),
new AttributeDefinition("OWNER_NAME", "STRING_TYPE", Multiplicity.OPTIONAL, false, null) new AttributeDefinition("OWNER_NAME", "STRING_TYPE", Multiplicity.OPTIONAL, false, null)
} }
); );
HierarchicalTypeDefinition<ClassType> tableClassTypeDef = new HierarchicalTypeDefinition<ClassType>("ClassType",TB_CLASS_TYPE, null, HierarchicalTypeDefinition<ClassType> tableClassTypeDef = new HierarchicalTypeDefinition<ClassType>("ClassType",TB_CLASS_TYPE, null,
new AttributeDefinition[]{ new AttributeDefinition[]{
new AttributeDefinition("CREATE_TIME", "LONG_TYPE", Multiplicity.REQUIRED, false, null), new AttributeDefinition("CREATE_TIME", "LONG_TYPE", Multiplicity.REQUIRED, false, null),
new AttributeDefinition("LAST_ACCESS_TIME", "LONG_TYPE", Multiplicity.REQUIRED, false, null), new AttributeDefinition("LAST_ACCESS_TIME", "LONG_TYPE", Multiplicity.REQUIRED, false, null),
new AttributeDefinition("OWNER", "STRING_TYPE", Multiplicity.REQUIRED, false, null), new AttributeDefinition("OWNER", "STRING_TYPE", Multiplicity.REQUIRED, false, null),
new AttributeDefinition("TBL_NAME", "STRING_TYPE", Multiplicity.REQUIRED, false, null), new AttributeDefinition("TBL_NAME", "STRING_TYPE", Multiplicity.REQUIRED, false, null),
new AttributeDefinition("TBL_TYPE", "STRING_TYPE", Multiplicity.REQUIRED, false, null), new AttributeDefinition("TBL_TYPE", "STRING_TYPE", Multiplicity.REQUIRED, false, null),
new AttributeDefinition("VIEW_EXPANDED_TEXT", "STRING_TYPE", Multiplicity.OPTIONAL, false, null), new AttributeDefinition("VIEW_EXPANDED_TEXT", "STRING_TYPE", Multiplicity.OPTIONAL, false, null),
new AttributeDefinition("VIEW_ORIGINAL_TEXT", "STRING_TYPE", Multiplicity.OPTIONAL, false, null) new AttributeDefinition("VIEW_ORIGINAL_TEXT", "STRING_TYPE", Multiplicity.OPTIONAL, false, null)
} }
); );
HierarchicalTypeDefinition<ClassType> columnClassTypeDef = new HierarchicalTypeDefinition<ClassType>("ClassType",FD_CLASS_TYPE, null, HierarchicalTypeDefinition<ClassType> columnClassTypeDef = new HierarchicalTypeDefinition<ClassType>("ClassType",FD_CLASS_TYPE, null,
new AttributeDefinition[]{ new AttributeDefinition[]{
new AttributeDefinition("COMMENT", "STRING_TYPE", Multiplicity.OPTIONAL, false, null), new AttributeDefinition("COMMENT", "STRING_TYPE", Multiplicity.OPTIONAL, false, null),
new AttributeDefinition("COLUMN_NAME", "STRING_TYPE", Multiplicity.REQUIRED, false, null), new AttributeDefinition("COLUMN_NAME", "STRING_TYPE", Multiplicity.REQUIRED, false, null),
new AttributeDefinition("TYPE_NAME", "STRING_TYPE", Multiplicity.REQUIRED, false, null) new AttributeDefinition("TYPE_NAME", "STRING_TYPE", Multiplicity.REQUIRED, false, null)
} }
); );
}catch(ClassNotFoundException e){ }catch(ClassNotFoundException e){
e.printStackTrace(); e.printStackTrace();
} }
for (HierarchicalTypeDefinition htd : al){ for (HierarchicalTypeDefinition htd : al){
try { try {
ts.defineClassType(htd); ts.defineClassType(htd);
} catch (MetadataException e) { } catch (MetadataException e) {
System.out.println(htd.hierarchicalMetaTypeName + "could not be added to the type system"); System.out.println(htd.hierarchicalMetaTypeName + "could not be added to the type system");
e.printStackTrace(); e.printStackTrace();
} }
} }
return false; return false;
} }
} }
common/src/main/java/org/apache/hadoop/metadata/service/Service.java
...@@ -27,13 +27,6 @@ import java.io.IOException; ...@@ -27,13 +27,6 @@ import java.io.IOException;
public interface Service extends Closeable { public interface Service extends Closeable {
/** /**
* Name of the service.
*
* @return name of the service
*/
String getName();
/**
* Starts the service. This method blocks until the service has completely started. * Starts the service. This method blocks until the service has completely started.
* *
* @throws Exception * @throws Exception
......
common/src/main/java/org/apache/hadoop/metadata/service/ServiceInitializer.java deleted 100644 → 0
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.metadata.service;
import org.apache.commons.configuration.ConfigurationException;
import org.apache.commons.configuration.PropertiesConfiguration;
import org.apache.hadoop.metadata.MetadataException;
import org.apache.hadoop.metadata.util.ReflectionUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Initializer that uses at startup to bring up all the Metadata startup
* services.
*/
public class ServiceInitializer {
private static final Logger LOG = LoggerFactory
.getLogger(ServiceInitializer.class);
private final Services services = Services.get();
// default property file name/path
private static final String DEFAULT_CONFIG_PATH = "application.properties";
// system property referenced by this class to extract user-overriden
// properties file
public static final String PROPERTIES_SYS_PROP = "metadata.properties";
// Path to the properties file (must be on the classpath for
// PropertiesConfiguration to work)
private final String propertyPath;
/**
* Default constructor. Use the metadata.properties System property to
* determine the property file name.
*/
public ServiceInitializer() {
propertyPath = System.getProperty(PROPERTIES_SYS_PROP,
DEFAULT_CONFIG_PATH);
}
/**
* Create a ServiceInitializer, specifying the properties file filename
* explicitly
*
* @param propPath
* the filename of the properties file with the service
* intializer information
*/
public ServiceInitializer(String propPath) {
propertyPath = propPath;
}
/**
* Get the configuration properties for the ServiceInitializer
*
* @return
* @throws ConfigurationException
*/
public PropertiesConfiguration getConfiguration()
throws ConfigurationException {
return new PropertiesConfiguration(propertyPath);
}
/**
* Initialize the services specified by the application.services property
*
* @throws MetadataException
*/
public void initialize() throws MetadataException {
/*
* TODO - determine whether this service model is the right model;
* Inter-service dependencies can wreak havoc using the current model
*/
String[] serviceClassNames;
LOG.info("Loading services using properties file: {}", propertyPath);
try {
PropertiesConfiguration configuration = getConfiguration();
serviceClassNames = configuration
.getStringArray("application.services");
} catch (ConfigurationException e) {
throw new RuntimeException("unable to get server properties");
}
for (String serviceClassName : serviceClassNames) {
serviceClassName = serviceClassName.trim();
if (serviceClassName.isEmpty()) {
continue;
}
Service service = ReflectionUtils
.getInstanceByClassName(serviceClassName);
services.register(service);
LOG.info("Initializing service: {}", serviceClassName);
try {
service.start();
} catch (Throwable t) {
LOG.error("Failed to initialize service {}", serviceClassName,
t);
throw new MetadataException(t);
}
LOG.info("Service initialized: {}", serviceClassName);
}
}
public void destroy() throws MetadataException {
for (Service service : services) {
LOG.info("Destroying service: {}", service.getClass().getName());
try {
service.stop();
} catch (Throwable t) {
LOG.error("Failed to destroy service {}", service.getClass()
.getName(), t);
throw new MetadataException(t);
}
LOG.info("Service destroyed: {}", service.getClass().getName());
}
}
}
common/src/main/java/org/apache/hadoop/metadata/service/Services.java deleted 100644 → 0
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.metadata.service;
import org.apache.commons.configuration.ConfigurationException;
import org.apache.commons.configuration.PropertiesConfiguration;
import org.apache.hadoop.metadata.MetadataException;
import org.apache.hadoop.metadata.util.ReflectionUtils;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.NoSuchElementException;
/**
* Repository of services initialized at startup.
*/
public final class Services implements Iterable<Service> {
private static final Services INSTANCE = new Services();
private Services() {
}
public static Services get() {
return INSTANCE;
}
private final Map<String, Service> services =
new LinkedHashMap<String, Service>();
public synchronized void register(Service service) throws MetadataException {
if (services.containsKey(service.getName())) {
throw new MetadataException("Service " + service.getName() + " already registered");
} else {
services.put(service.getName(), service);
}
}
@SuppressWarnings("unchecked")
public <T extends Service> T getService(String serviceName) {
if (services.containsKey(serviceName)) {
return (T) services.get(serviceName);
} else {
throw new NoSuchElementException(
"Service " + serviceName + " not registered with registry");
}
}
public boolean isRegistered(String serviceName) {
return services.containsKey(serviceName);
}
@Override
public Iterator<Service> iterator() {
return services.values().iterator();
}
public Service init(String serviceName) throws MetadataException {
if (isRegistered(serviceName)) {
throw new MetadataException("Service is already initialized " + serviceName);
}
String serviceClassName;
try {
PropertiesConfiguration configuration =
new PropertiesConfiguration("application.properties");
serviceClassName = configuration.getString(serviceName + ".impl");
} catch (ConfigurationException e) {
throw new MetadataException("unable to get server properties");
}
Service service = ReflectionUtils.getInstanceByClassName(serviceClassName);
register(service);
return service;
}
public void reset() {
services.clear();
}
}
common/src/test/java/org/apache/hadoop/metadata/service/ServiceInitializerTest.java deleted 100644 → 0
package org.apache.hadoop.metadata.service;
import org.testng.Assert;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
/**
* Unit test for the Service Initializer.
*
* Test functionality to allow loading of different property files.
*/
public class ServiceInitializerTest {
private final String propertiesFileName = "test.application.properties";
private ServiceInitializer sinit;
@BeforeClass
public void setUp() throws Exception {
// setup for the test properties file
System.setProperty(ServiceInitializer.PROPERTIES_SYS_PROP,
propertiesFileName);
sinit = new ServiceInitializer();
}
@AfterClass
public void tearDown() throws Exception {
// test destruction of the Services - no exceptions is assumed a success
sinit.destroy();
}
@Test
public void testPropsAreSet() throws Exception {
Assert.assertEquals(
sinit.getConfiguration().getString(
"application.services"),
TestService.NAME);
}
@Test
public void testInitialize() throws Exception {
// test the initialization of the initializer
// no exceptions is assumed a success
sinit.initialize();
}
}
common/src/test/java/org/apache/hadoop/metadata/service/TestService.java deleted 100644 → 0
package org.apache.hadoop.metadata.service;
import java.io.IOException;
public class TestService implements Service {
public static final String NAME = TestService.class.getName();
@Override
public String getName() {
return NAME;
}
@Override
public void start() throws Exception {
}
@Override
public void stop() {
}
@Override
public void close() throws IOException {
}
}
common/src/test/resources/test.application.properties deleted 100644 → 0
application.services=org.apache.hadoop.metadata.service.TestService
\ No newline at end of file
hivehook/src/main/java/com/aetna/hadoop/dgc/hive/HiveLineageBean.java
package com.aetna.hadoop.dgc.hive; package com.aetna.hadoop.dgc.hive;
import java.io.Serializable; import java.io.Serializable;
import java.util.List; import java.util.List;
import java.util.ArrayList; import java.util.ArrayList;
public class HiveLineageBean implements Serializable { public class HiveLineageBean implements Serializable {
/** /**
* *
*/ */
private static final long serialVersionUID = 1L; private static final long serialVersionUID = 1L;
public String queryId; public String queryId;
public String hiveId; public String hiveId;
public String user; public String user;
public String queryStartTime; public String queryStartTime;
public String queryEndTime; public String queryEndTime;
public String query; public String query;
public String tableName; public String tableName;
public String tableLocation; public String tableLocation;
public boolean success; public boolean success;
public boolean failed; public boolean failed;
public String executionEngine; public String executionEngine;
ArrayList<SourceTables> sourceTables; ArrayList<SourceTables> sourceTables;
ArrayList<QueryColumns> queryColumns; ArrayList<QueryColumns> queryColumns;
ArrayList<WhereClause> whereClause; ArrayList<WhereClause> whereClause;
ArrayList<CreateColumns> createColumns; ArrayList<CreateColumns> createColumns;
ArrayList<GroupBy> groupBy; ArrayList<GroupBy> groupBy;
ArrayList<GroupBy> orderBy; ArrayList<GroupBy> orderBy;
public String getQueryId() { public String getQueryId() {
return this.queryId ; return this.queryId ;
} }
public void setQueryId(String queryId) { public void setQueryId(String queryId) {
this.queryId = queryId; this.queryId = queryId;
} }
public String getExecutionEngine() { public String getExecutionEngine() {
return this.executionEngine ; return this.executionEngine ;
} }
public void setExecutionEngine(String executionEngine) { public void setExecutionEngine(String executionEngine) {
this.executionEngine = executionEngine; this.executionEngine = executionEngine;
} }
public String getHiveId() { public String getHiveId() {
return this.hiveId ; return this.hiveId ;
} }
public void setHiveId(String hiveId) { public void setHiveId(String hiveId) {
this.hiveId = hiveId; this.hiveId = hiveId;
} }
public boolean getSuccess() { public boolean getSuccess() {
return this.success ; return this.success ;
} }
public void setSuccess(boolean success) { public void setSuccess(boolean success) {
this.success = success; this.success = success;
} }
public boolean getFailed() { public boolean getFailed() {
return this.failed ; return this.failed ;
} }
public void setFailed(boolean failed) { public void setFailed(boolean failed) {
this.failed = failed; this.failed = failed;
} }
public String getTableName() { public String getTableName() {
return this.tableName ; return this.tableName ;
} }
public void setTableName(String tableName) { public void setTableName(String tableName) {
this.tableName = tableName; this.tableName = tableName;
} }
public String getTableLocation() { public String getTableLocation() {
return this.tableLocation ; return this.tableLocation ;
} }
public void setTableLocation(String tableLocation) { public void setTableLocation(String tableLocation) {
this.tableLocation = tableLocation; this.tableLocation = tableLocation;
} }
public String getUser() { public String getUser() {
return this.user ; return this.user ;
} }
public void setUser(String user) { public void setUser(String user) {
this.user = user; this.user = user;
} }
public String getQueryStartTime() { public String getQueryStartTime() {
return this.queryStartTime ; return this.queryStartTime ;
} }
public void setQueryStartTime(String queryStartTime) { public void setQueryStartTime(String queryStartTime) {
this.queryStartTime = queryStartTime; this.queryStartTime = queryStartTime;
} }
public String getQueryEndTime() { public String getQueryEndTime() {
return this.queryEndTime ; return this.queryEndTime ;
} }
public void setQueryEndTime(String queryEndTime) { public void setQueryEndTime(String queryEndTime) {
this.queryEndTime = queryEndTime; this.queryEndTime = queryEndTime;
} }
public String getQuery() { public String getQuery() {
return this.query ; return this.query ;
} }
public void setQuery(String query) { public void setQuery(String query) {
this.query = query; this.query = query;
} }
public ArrayList<SourceTables> getSourceTables() { public ArrayList<SourceTables> getSourceTables() {
return this.sourceTables ; return this.sourceTables ;
} }
public void setSourceTables(ArrayList<SourceTables> sourceTables) { public void setSourceTables(ArrayList<SourceTables> sourceTables) {
this.sourceTables = sourceTables; this.sourceTables = sourceTables;
} }
public ArrayList<QueryColumns> getQueryColumns() { public ArrayList<QueryColumns> getQueryColumns() {
return this.queryColumns ; return this.queryColumns ;
} }
public void setQueryColumns(ArrayList<QueryColumns> queryColumns) { public void setQueryColumns(ArrayList<QueryColumns> queryColumns) {
this.queryColumns = queryColumns; this.queryColumns = queryColumns;
} }
public ArrayList<WhereClause> getWhereClause() { public ArrayList<WhereClause> getWhereClause() {
return this.whereClause ; return this.whereClause ;
} }
public void setWhereClause(ArrayList<WhereClause> whereClause) { public void setWhereClause(ArrayList<WhereClause> whereClause) {
this.whereClause = whereClause; this.whereClause = whereClause;
} }
public ArrayList<GroupBy> getGroupBy() { public ArrayList<GroupBy> getGroupBy() {
return this.groupBy ; return this.groupBy ;
} }
public void setGroupBy(ArrayList<GroupBy> groupBy) { public void setGroupBy(ArrayList<GroupBy> groupBy) {
this.groupBy = groupBy; this.groupBy = groupBy;
} }
public class SourceTables { public class SourceTables {
public String tableName; public String tableName;
public String tableAlias; public String tableAlias;
public String databaseName; public String databaseName;
public String getTableName() { public String getTableName() {
return this.tableName ; return this.tableName ;
} }
public void setTableName(String tableName) { public void setTableName(String tableName) {
this.tableName = tableName; this.tableName = tableName;
} }
public String getTableAlias() { public String getTableAlias() {
return this.tableAlias ; return this.tableAlias ;
} }
public void setTableAlias(String tableAlias) { public void setTableAlias(String tableAlias) {
this.tableAlias = tableAlias; this.tableAlias = tableAlias;
} }
public String getDatabaseName() { public String getDatabaseName() {
return this.databaseName ; return this.databaseName ;
} }
public void setDatabaseName(String databaseName) { public void setDatabaseName(String databaseName) {
this.databaseName = databaseName; this.databaseName = databaseName;
} }
} }
public class QueryColumns { public class QueryColumns {
public String tbAliasOrName; public String tbAliasOrName;
public String columnName; public String columnName;
public String columnAlias; public String columnAlias;
public String columnFunction; public String columnFunction;
public String getTbAliasOrName() { public String getTbAliasOrName() {
return this.tbAliasOrName ; return this.tbAliasOrName ;
} }
public void setTbAliasOrName(String tbAliasOrName) { public void setTbAliasOrName(String tbAliasOrName) {
this.tbAliasOrName = tbAliasOrName; this.tbAliasOrName = tbAliasOrName;
} }
public String getColumnName() { public String getColumnName() {
return this.columnName ; return this.columnName ;
} }
public void setColumnName(String columnName) { public void setColumnName(String columnName) {
this.columnName = columnName; this.columnName = columnName;
} }
public String getColumnAlias() { public String getColumnAlias() {
return this.columnAlias ; return this.columnAlias ;
} }
public void setColumnAlias(String columnAlias) { public void setColumnAlias(String columnAlias) {
this.columnAlias = columnAlias; this.columnAlias = columnAlias;
} }
public String getColumnFunction() { public String getColumnFunction() {
return this.columnFunction ; return this.columnFunction ;
} }
public void setColumnFunction(String columnFunction) { public void setColumnFunction(String columnFunction) {
this.columnFunction = columnFunction; this.columnFunction = columnFunction;
} }
} }
public class GroupBy { public class GroupBy {
public String tbAliasOrName; public String tbAliasOrName;
public String columnName; public String columnName;
public String getTbAliasOrName() { public String getTbAliasOrName() {
return this.tbAliasOrName ; return this.tbAliasOrName ;
} }
public void setTbAliasOrName(String tbAliasOrName) { public void setTbAliasOrName(String tbAliasOrName) {
this.tbAliasOrName = tbAliasOrName; this.tbAliasOrName = tbAliasOrName;
} }
public String getColumnName() { public String getColumnName() {
return this.columnName ; return this.columnName ;
} }
public void setColumnName(String columnName) { public void setColumnName(String columnName) {
this.columnName = columnName; this.columnName = columnName;
} }
} }
public class WhereClause { public class WhereClause {
public String tbAliasOrName; public String tbAliasOrName;
public String columnCondition; public String columnCondition;
public String columnName; public String columnName;
public String columnOperator; public String columnOperator;
public String columnValue; public String columnValue;
public String getColumnCondition() { public String getColumnCondition() {
return this.columnCondition ; return this.columnCondition ;
} }
public void setColumnCondition(String columnCondition) { public void setColumnCondition(String columnCondition) {
this.columnCondition = columnCondition; this.columnCondition = columnCondition;
} }
public String getTbAliasOrName() { public String getTbAliasOrName() {
return this.tbAliasOrName ; return this.tbAliasOrName ;
} }
public void setTbAliasOrName(String tbAliasOrName) { public void setTbAliasOrName(String tbAliasOrName) {
this.tbAliasOrName = tbAliasOrName; this.tbAliasOrName = tbAliasOrName;
} }
public String getColumnName() { public String getColumnName() {
return this.columnName ; return this.columnName ;
} }
public void setColumnName(String columnName) { public void setColumnName(String columnName) {
this.columnName = columnName; this.columnName = columnName;
} }
public String getColumnOperator() { public String getColumnOperator() {
return this.columnOperator ; return this.columnOperator ;
} }
public void setColumnOperator(String columnOperator) { public void setColumnOperator(String columnOperator) {
this.columnOperator = columnOperator; this.columnOperator = columnOperator;
} }
public String getColumnValue() { public String getColumnValue() {
return this.columnValue ; return this.columnValue ;
} }
public void setColumnValue(String columnValue) { public void setColumnValue(String columnValue) {
this.columnValue = columnValue; this.columnValue = columnValue;
} }
} }
public ArrayList<CreateColumns> getCreateColumns() { public ArrayList<CreateColumns> getCreateColumns() {
return this.createColumns ; return this.createColumns ;
} }
public void setCreateColumns(ArrayList<CreateColumns> createColumns) { public void setCreateColumns(ArrayList<CreateColumns> createColumns) {
this.createColumns = createColumns; this.createColumns = createColumns;
} }
public class CreateColumns { public class CreateColumns {
public String columnName; public String columnName;
public String columnType; public String columnType;
public String getColumnName() { public String getColumnName() {
return this.columnName ; return this.columnName ;
} }
public void setColumnName(String columnName) { public void setColumnName(String columnName) {
this.columnName = columnName; this.columnName = columnName;
} }
public String getColumnType() { public String getColumnType() {
return this.columnType ; return this.columnType ;
} }
public void setColumnType(String columnType) { public void setColumnType(String columnType) {
this.columnType = columnType; this.columnType = columnType;
} }
} }
} }
hivehook/src/main/java/com/aetna/hadoop/dgc/hive/HiveLineageInfo.java
package com.aetna.hadoop.dgc.hive; package com.aetna.hadoop.dgc.hive;
/** /**
* Licensed to the Apache Software Foundation (ASF) under one * Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file * or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information * distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file * regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the * to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance * "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at * with the License. You may obtain a copy of the License at
* *
* http://www.apache.org/licenses/LICENSE-2.0 * http://www.apache.org/licenses/LICENSE-2.0
* *
* Unless required by applicable law or agreed to in writing, software * Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, * distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
import java.io.IOException; import java.io.IOException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.LinkedHashMap; import java.util.LinkedHashMap;
import java.util.Map; import java.util.Map;
import java.util.Stack; import java.util.Stack;
import org.apache.commons.logging.Log; import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory; import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.ql.lib.DefaultGraphWalker; import org.apache.hadoop.hive.ql.lib.DefaultGraphWalker;
import org.apache.hadoop.hive.ql.lib.DefaultRuleDispatcher; import org.apache.hadoop.hive.ql.lib.DefaultRuleDispatcher;
import org.apache.hadoop.hive.ql.lib.Dispatcher; import org.apache.hadoop.hive.ql.lib.Dispatcher;
import org.apache.hadoop.hive.ql.lib.GraphWalker; import org.apache.hadoop.hive.ql.lib.GraphWalker;
import org.apache.hadoop.hive.ql.lib.Node; import org.apache.hadoop.hive.ql.lib.Node;
import org.apache.hadoop.hive.ql.lib.NodeProcessor; import org.apache.hadoop.hive.ql.lib.NodeProcessor;
import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx; import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx;
import org.apache.hadoop.hive.ql.lib.Rule; import org.apache.hadoop.hive.ql.lib.Rule;
import org.apache.hadoop.hive.ql.parse.ASTNode; import org.apache.hadoop.hive.ql.parse.ASTNode;
import org.apache.hadoop.hive.ql.parse.HiveParser; import org.apache.hadoop.hive.ql.parse.HiveParser;
import org.apache.hadoop.hive.ql.parse.ParseDriver; import org.apache.hadoop.hive.ql.parse.ParseDriver;
import org.apache.hadoop.hive.ql.parse.ParseException; import org.apache.hadoop.hive.ql.parse.ParseException;
import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.parse.SemanticException;
import org.apache.log4j.ConsoleAppender; import org.apache.log4j.ConsoleAppender;
import org.apache.log4j.Level; import org.apache.log4j.Level;
import org.apache.log4j.LogManager; import org.apache.log4j.LogManager;
import org.apache.log4j.Logger; import org.apache.log4j.Logger;
import org.apache.log4j.PatternLayout; import org.apache.log4j.PatternLayout;
import com.aetna.hadoop.dgc.hive.HiveLineageBean.CreateColumns; import com.aetna.hadoop.dgc.hive.HiveLineageBean.CreateColumns;
import com.aetna.hadoop.dgc.hive.HiveLineageBean.GroupBy; import com.aetna.hadoop.dgc.hive.HiveLineageBean.GroupBy;
import com.aetna.hadoop.dgc.hive.HiveLineageBean.QueryColumns; import com.aetna.hadoop.dgc.hive.HiveLineageBean.QueryColumns;
import com.aetna.hadoop.dgc.hive.HiveLineageBean.SourceTables; import com.aetna.hadoop.dgc.hive.HiveLineageBean.SourceTables;
import com.aetna.hadoop.dgc.hive.HiveLineageBean.WhereClause; import com.aetna.hadoop.dgc.hive.HiveLineageBean.WhereClause;
import com.google.gson.Gson; import com.google.gson.Gson;
/** /**
* *
* This class prints out the lineage info. It takes sql as input and prints * This class prints out the lineage info. It takes sql as input and prints
* lineage info. Currently this prints only input and output tables for a given * lineage info. Currently this prints only input and output tables for a given
* sql. Later we can expand to add join tables etc. * sql. Later we can expand to add join tables etc.
* *
*/ */
public class HiveLineageInfo implements NodeProcessor { public class HiveLineageInfo implements NodeProcessor {
private final Log LOG = LogFactory.getLog(HiveLineageInfo.class.getName()); private final Log LOG = LogFactory.getLog(HiveLineageInfo.class.getName());
public Map<Integer, String> queryMap; public Map<Integer, String> queryMap;
public Integer counter = 0; public Integer counter = 0;
public HiveLineageBean hlb = new HiveLineageBean();; public HiveLineageBean hlb = new HiveLineageBean();;
public ArrayList<SourceTables> sourceTables; public ArrayList<SourceTables> sourceTables;
public ArrayList<QueryColumns> queryColumns; public ArrayList<QueryColumns> queryColumns;
public ArrayList<GroupBy> groupBy; public ArrayList<GroupBy> groupBy;
public ArrayList<WhereClause> whereClause; public ArrayList<WhereClause> whereClause;
public ArrayList<CreateColumns> createColumns; public ArrayList<CreateColumns> createColumns;
/** /**
* @return Custom HiveLineageBean data to be passed to GSON parsert * @return Custom HiveLineageBean data to be passed to GSON parsert
*/ */
public HiveLineageBean getHLBean() { public HiveLineageBean getHLBean() {
return hlb; return hlb;
} }
/** /**
* Implements the process method for the NodeProcessor interface. * Implements the process method for the NodeProcessor interface.
*/ */
public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx procCtx, Object... nodeOutputs) throws SemanticException { public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx procCtx, Object... nodeOutputs) throws SemanticException {
ASTNode pt = (ASTNode) nd; ASTNode pt = (ASTNode) nd;
/* /*
* Check the 1st-level children and do simple semantic checks: 1) CTLT and * Check the 1st-level children and do simple semantic checks: 1) CTLT and
* CTAS should not coexists. 2) CTLT or CTAS should not coexists with column * CTAS should not coexists. 2) CTLT or CTAS should not coexists with column
* list (target table schema). 3) CTAS does not support partitioning (for * list (target table schema). 3) CTAS does not support partitioning (for
* now). * now).
*/ */
switch (pt.getToken().getType()) { switch (pt.getToken().getType()) {
case HiveParser.TOK_TABREF: case HiveParser.TOK_TABREF:
sourceTables = new ArrayList<SourceTables>(); sourceTables = new ArrayList<SourceTables>();
LOG.debug("From Table Dump: "+pt.dump()); LOG.debug("From Table Dump: "+pt.dump());
fromTableDump(pt); fromTableDump(pt);
this.hlb.setSourceTables(sourceTables); this.hlb.setSourceTables(sourceTables);
break; break;
case HiveParser.TOK_SELECT: case HiveParser.TOK_SELECT:
queryColumns = new ArrayList<QueryColumns>(); queryColumns = new ArrayList<QueryColumns>();
LOG.debug("Column Dump: "+pt.dump()); LOG.debug("Column Dump: "+pt.dump());
columnTableDump(pt); columnTableDump(pt);
this.hlb.setQueryColumns(queryColumns); this.hlb.setQueryColumns(queryColumns);
break; break;
case HiveParser.TOK_WHERE: case HiveParser.TOK_WHERE:
whereClause = new ArrayList<WhereClause>(); whereClause = new ArrayList<WhereClause>();
LOG.debug("WHERE CLAUSE DUMP: "+pt.dump()); LOG.debug("WHERE CLAUSE DUMP: "+pt.dump());
whereDump(pt); whereDump(pt);
this.hlb.setWhereClause(whereClause); this.hlb.setWhereClause(whereClause);
break; break;
case HiveParser.TOK_GROUPBY: case HiveParser.TOK_GROUPBY:
groupBy = new ArrayList<GroupBy>(); groupBy = new ArrayList<GroupBy>();
LOG.debug("GROUPBY CLAUSE DUMP: "+pt.dump()); LOG.debug("GROUPBY CLAUSE DUMP: "+pt.dump());
groupByDump(pt); groupByDump(pt);
this.hlb.setGroupBy(groupBy); this.hlb.setGroupBy(groupBy);
break; break;
case HiveParser.TOK_CREATETABLE: case HiveParser.TOK_CREATETABLE:
createColumns = new ArrayList<CreateColumns>(); createColumns = new ArrayList<CreateColumns>();
LOG.debug("CREATABLE DUMP: "+pt.dump()); LOG.debug("CREATABLE DUMP: "+pt.dump());
createTableDump(pt); createTableDump(pt);
break; break;
} }
return null; return null;
} }
/** /**
* Walks the whereTree called by processWalker * Walks the whereTree called by processWalker
*/ */
public void whereDump(ASTNode nodeIn) { public void whereDump(ASTNode nodeIn) {
counter = 0; counter = 0;
wdump(nodeIn); wdump(nodeIn);
} }
/** /**
* Walks the Where Tree called by whereDump * Walks the Where Tree called by whereDump
*/ */
private void wdump(ASTNode nodeIn) { private void wdump(ASTNode nodeIn) {
boolean parseChild = true; boolean parseChild = true;
if (nodeIn.getType() == HiveParser.TOK_TABLE_OR_COL) { if (nodeIn.getType() == HiveParser.TOK_TABLE_OR_COL) {
WhereClause whreClse = hlb.new WhereClause(); WhereClause whreClse = hlb.new WhereClause();
if (nodeIn.getParent().getText().equalsIgnoreCase(".")) { if (nodeIn.getParent().getText().equalsIgnoreCase(".")) {
ASTNode checkOrAnd = (ASTNode) nodeIn.getParent().getParent().getChild(1).getParent().getParent(); ASTNode checkOrAnd = (ASTNode) nodeIn.getParent().getParent().getChild(1).getParent().getParent();
if (checkOrAnd.getType() == HiveParser.KW_AND || checkOrAnd.getType() == HiveParser.KW_OR) { if (checkOrAnd.getType() == HiveParser.KW_AND || checkOrAnd.getType() == HiveParser.KW_OR) {
LOG.info("WHERE:: "+checkOrAnd.getText()); LOG.info("WHERE:: "+checkOrAnd.getText());
whreClse.setColumnOperator(checkOrAnd.getText()); whreClse.setColumnOperator(checkOrAnd.getText());
} }
LOG.info("Table Alias:: "+nodeIn.getChild(0).getText()); LOG.info("Table Alias:: "+nodeIn.getChild(0).getText());
whreClse.setTbAliasOrName(nodeIn.getChild(0).getText()); whreClse.setTbAliasOrName(nodeIn.getChild(0).getText());
LOG.info("Delimiter:: "+nodeIn.getParent().getText()); LOG.info("Delimiter:: "+nodeIn.getParent().getText());
LOG.info("Column:: "+nodeIn.getParent().getChild(1).getText()); LOG.info("Column:: "+nodeIn.getParent().getChild(1).getText());
whreClse.setColumnName(nodeIn.getParent().getChild(1).getText()); whreClse.setColumnName(nodeIn.getParent().getChild(1).getText());
LOG.info("Column Qualifer:: "+nodeIn.getParent().getParent().getChild(1).getParent().getText()); LOG.info("Column Qualifer:: "+nodeIn.getParent().getParent().getChild(1).getParent().getText());
whreClse.setColumnOperator(nodeIn.getParent().getParent().getChild(1).getParent().getText()); whreClse.setColumnOperator(nodeIn.getParent().getParent().getChild(1).getParent().getText());
LOG.info("Column Value:: "+nodeIn.getParent().getParent().getChild(1).getText()); LOG.info("Column Value:: "+nodeIn.getParent().getParent().getChild(1).getText());
whreClse.setColumnValue(nodeIn.getParent().getParent().getChild(1).getText()); whreClse.setColumnValue(nodeIn.getParent().getParent().getChild(1).getText());
} else { } else {
ASTNode checkOrAnd = (ASTNode) nodeIn.getParent().getParent().getChild(1).getParent(); ASTNode checkOrAnd = (ASTNode) nodeIn.getParent().getParent().getChild(1).getParent();
if (checkOrAnd.getType() == HiveParser.KW_AND || checkOrAnd.getType() == HiveParser.KW_OR) { if (checkOrAnd.getType() == HiveParser.KW_AND || checkOrAnd.getType() == HiveParser.KW_OR) {
LOG.info("WHERE:: "+checkOrAnd.getText()); LOG.info("WHERE:: "+checkOrAnd.getText());
whreClse.setColumnOperator(checkOrAnd.getText()); whreClse.setColumnOperator(checkOrAnd.getText());
} }
LOG.info("Column:: = "+nodeIn.getChild(0).getText()); LOG.info("Column:: = "+nodeIn.getChild(0).getText());
whreClse.setColumnName(nodeIn.getChild(0).getText()); whreClse.setColumnName(nodeIn.getChild(0).getText());
//LOG.info("Delimiter "+nodeIn.getParent().getText()); //LOG.info("Delimiter "+nodeIn.getParent().getText());
LOG.info("Column Qualifer:: "+nodeIn.getParent().getChild(1).getParent().getText()); LOG.info("Column Qualifer:: "+nodeIn.getParent().getChild(1).getParent().getText());
whreClse.setColumnOperator(nodeIn.getParent().getChild(1).getParent().getText()); whreClse.setColumnOperator(nodeIn.getParent().getChild(1).getParent().getText());
LOG.info("Column Value:: "+nodeIn.getParent().getChild(1).getText()); LOG.info("Column Value:: "+nodeIn.getParent().getChild(1).getText());
whreClse.setColumnValue(nodeIn.getParent().getChild(1).getText()); whreClse.setColumnValue(nodeIn.getParent().getChild(1).getText());
} }
whereClause.add(whreClse); whereClause.add(whreClse);
} }
if (parseChild) { if (parseChild) {
int childCount = nodeIn.getChildCount(); int childCount = nodeIn.getChildCount();
if (childCount != 0 ){ if (childCount != 0 ){
for (int numr = 0; numr < childCount; numr++) { for (int numr = 0; numr < childCount; numr++) {
wdump((ASTNode)nodeIn.getChild(numr)); wdump((ASTNode)nodeIn.getChild(numr));
} }
} }
} }
} }
/** /**
* Walks the GroupByTree called by processWalker * Walks the GroupByTree called by processWalker
*/ */
public void groupByDump(ASTNode nodeIn) { public void groupByDump(ASTNode nodeIn) {
counter = 0; counter = 0;
gdump(nodeIn); gdump(nodeIn);
} }
/** /**
* Walks the GroupBy Tree called by groupByDump * Walks the GroupBy Tree called by groupByDump
*/ */
private void gdump(ASTNode nodeIn) { private void gdump(ASTNode nodeIn) {
boolean parseChild = true; boolean parseChild = true;
if (nodeIn.getType() == HiveParser.TOK_TABLE_OR_COL) { if (nodeIn.getType() == HiveParser.TOK_TABLE_OR_COL) {
GroupBy grpBy = hlb.new GroupBy(); GroupBy grpBy = hlb.new GroupBy();
ASTNode parentNode = (ASTNode) nodeIn.getParent(); ASTNode parentNode = (ASTNode) nodeIn.getParent();
if (parentNode.getText().equalsIgnoreCase(".")) { if (parentNode.getText().equalsIgnoreCase(".")) {
LOG.info("GroupBy TableAlias: "+nodeIn.getChild(0).getText()); LOG.info("GroupBy TableAlias: "+nodeIn.getChild(0).getText());
grpBy.setTbAliasOrName(nodeIn.getChild(0).getText()); grpBy.setTbAliasOrName(nodeIn.getChild(0).getText());
LOG.info("GroupBy Column:: "+parentNode.getChild(1).getText()); LOG.info("GroupBy Column:: "+parentNode.getChild(1).getText());
grpBy.setColumnName(parentNode.getChild(1).getText()); grpBy.setColumnName(parentNode.getChild(1).getText());
} else { } else {
LOG.info("GroupBy Column: "+nodeIn.getChild(0).getText()); LOG.info("GroupBy Column: "+nodeIn.getChild(0).getText());
grpBy.setColumnName(nodeIn.getChild(0).getText()); grpBy.setColumnName(nodeIn.getChild(0).getText());
} }
groupBy.add(grpBy); groupBy.add(grpBy);
} }
if (parseChild) { if (parseChild) {
int childCount = nodeIn.getChildCount(); int childCount = nodeIn.getChildCount();
if (childCount != 0 ){ if (childCount != 0 ){
for (int numr = 0; numr < childCount; numr++) { for (int numr = 0; numr < childCount; numr++) {
gdump((ASTNode)nodeIn.getChild(numr)); gdump((ASTNode)nodeIn.getChild(numr));
} }
} }
} }
} }
/** /**
* Walks the CreateTable Tree called by processWalker * Walks the CreateTable Tree called by processWalker
*/ */
public void createTableDump(ASTNode nodeIn) { public void createTableDump(ASTNode nodeIn) {
counter = 0; counter = 0;
if (nodeIn.getFirstChildWithType(HiveParser.TOK_TABNAME) != null) { if (nodeIn.getFirstChildWithType(HiveParser.TOK_TABNAME) != null) {
LOG.info("Create TableName:: "+nodeIn.getFirstChildWithType(HiveParser.TOK_TABNAME).getText()); LOG.info("Create TableName:: "+nodeIn.getFirstChildWithType(HiveParser.TOK_TABNAME).getText());
hlb.setTableName(nodeIn.getFirstChildWithType(HiveParser.TOK_TABNAME).getChild(0).getText()); hlb.setTableName(nodeIn.getFirstChildWithType(HiveParser.TOK_TABNAME).getChild(0).getText());
} }
if (nodeIn.getFirstChildWithType(HiveParser.TOK_TABLELOCATION) != null) { if (nodeIn.getFirstChildWithType(HiveParser.TOK_TABLELOCATION) != null) {
LOG.info("Create Table Location:: "+nodeIn.getFirstChildWithType(HiveParser.TOK_TABLELOCATION).getText()); LOG.info("Create Table Location:: "+nodeIn.getFirstChildWithType(HiveParser.TOK_TABLELOCATION).getText());
hlb.setTableLocation(nodeIn.getFirstChildWithType(HiveParser.TOK_TABLELOCATION).getChild(0).getText()); hlb.setTableLocation(nodeIn.getFirstChildWithType(HiveParser.TOK_TABLELOCATION).getChild(0).getText());
} }
if (nodeIn.getFirstChildWithType(HiveParser.TOK_TABCOLLIST) != null ) { if (nodeIn.getFirstChildWithType(HiveParser.TOK_TABCOLLIST) != null ) {
ctdump((ASTNode)nodeIn.getFirstChildWithType(HiveParser.TOK_TABCOLLIST).getParent()); ctdump((ASTNode)nodeIn.getFirstChildWithType(HiveParser.TOK_TABCOLLIST).getParent());
hlb.setCreateColumns(createColumns); hlb.setCreateColumns(createColumns);
} }
} }
/** /**
* Walks the CreateTable Tree called by createTableDump * Walks the CreateTable Tree called by createTableDump
*/ */
private void ctdump(ASTNode nodeIn) { private void ctdump(ASTNode nodeIn) {
boolean parseChild = true; boolean parseChild = true;
if (nodeIn.getType() == HiveParser.TOK_TABCOL) { if (nodeIn.getType() == HiveParser.TOK_TABCOL) {
CreateColumns crtClmns = hlb.new CreateColumns(); CreateColumns crtClmns = hlb.new CreateColumns();
LOG.info("Create Column Name:: "+nodeIn.getChild(0).getText()); LOG.info("Create Column Name:: "+nodeIn.getChild(0).getText());
crtClmns.setColumnName(nodeIn.getChild(0).getText()); crtClmns.setColumnName(nodeIn.getChild(0).getText());
LOG.info("Create Column Type:: "+nodeIn.getChild(1).getText()); LOG.info("Create Column Type:: "+nodeIn.getChild(1).getText());
crtClmns.setColumnType(nodeIn.getChild(1).getText()); crtClmns.setColumnType(nodeIn.getChild(1).getText());
createColumns.add(crtClmns); createColumns.add(crtClmns);
} }
if (parseChild) { if (parseChild) {
int childCount = nodeIn.getChildCount(); int childCount = nodeIn.getChildCount();
if (childCount != 0 ){ if (childCount != 0 ){
for (int numr = 0; numr < childCount; numr++) { for (int numr = 0; numr < childCount; numr++) {
ctdump((ASTNode)nodeIn.getChild(numr)); ctdump((ASTNode)nodeIn.getChild(numr));
} }
} }
} }
} }
/** /**
* Walks the fromTable Tree called by processWalker * Walks the fromTable Tree called by processWalker
*/ */
public void fromTableDump(ASTNode nodeIn) { public void fromTableDump(ASTNode nodeIn) {
counter = 0; counter = 0;
ftdump(nodeIn); ftdump(nodeIn);
} }
/** /**
* Walks the fromTable Tree called by fromTableDump * Walks the fromTable Tree called by fromTableDump
*/ */
private void ftdump(ASTNode nodeIn) { private void ftdump(ASTNode nodeIn) {
boolean parseChild = true; boolean parseChild = true;
if (nodeIn.getType() == HiveParser.TOK_TABNAME && nodeIn.getParent().getType() == HiveParser.TOK_TABREF) { if (nodeIn.getType() == HiveParser.TOK_TABNAME && nodeIn.getParent().getType() == HiveParser.TOK_TABREF) {
SourceTables hlbSbls = hlb.new SourceTables(); SourceTables hlbSbls = hlb.new SourceTables();
if (nodeIn.getChildCount() == 2) { if (nodeIn.getChildCount() == 2) {
LOG.info("From DBName:: "+nodeIn.getChild(0).getText()); LOG.info("From DBName:: "+nodeIn.getChild(0).getText());
hlbSbls.setDatabaseName(nodeIn.getChild(0).getText()); hlbSbls.setDatabaseName(nodeIn.getChild(0).getText());
LOG.info("From TableName:: "+nodeIn.getChild(1).getText()); LOG.info("From TableName:: "+nodeIn.getChild(1).getText());
hlbSbls.setTableName(nodeIn.getChild(1).getText()); hlbSbls.setTableName(nodeIn.getChild(1).getText());
} else { } else {
LOG.info("From TableName:: "+nodeIn.getChild(0).getText()); LOG.info("From TableName:: "+nodeIn.getChild(0).getText());
hlbSbls.setTableName(nodeIn.getChild(0).getText()); hlbSbls.setTableName(nodeIn.getChild(0).getText());
} }
if (nodeIn.getType() == HiveParser.TOK_TABNAME && nodeIn.getParent().getChild(1) != null) { if (nodeIn.getType() == HiveParser.TOK_TABNAME && nodeIn.getParent().getChild(1) != null) {
LOG.info("From DB/Table Alias:: "+nodeIn.getParent().getChild(1).getText()); LOG.info("From DB/Table Alias:: "+nodeIn.getParent().getChild(1).getText());
hlbSbls.setTableAlias(nodeIn.getParent().getChild(1).getText()); hlbSbls.setTableAlias(nodeIn.getParent().getChild(1).getText());
} }
sourceTables.add(hlbSbls); sourceTables.add(hlbSbls);
} }
if (parseChild) { if (parseChild) {
int childCount = nodeIn.getChildCount(); int childCount = nodeIn.getChildCount();
if (childCount != 0 ){ if (childCount != 0 ){
for (int numr = 0; numr < childCount; numr++) { for (int numr = 0; numr < childCount; numr++) {
ftdump((ASTNode)nodeIn.getChild(numr)); ftdump((ASTNode)nodeIn.getChild(numr));
} }
} }
} }
} }
/** /**
* Walks the column Tree called by processWalker * Walks the column Tree called by processWalker
*/ */
public void columnTableDump(ASTNode nodeIn) { public void columnTableDump(ASTNode nodeIn) {
counter = 0; counter = 0;
clmnTdump(nodeIn); clmnTdump(nodeIn);
} }
/** /**
* Walks the columnDump Tree called by columnTableDump * Walks the columnDump Tree called by columnTableDump
*/ */
private void clmnTdump(ASTNode nodeIn) { private void clmnTdump(ASTNode nodeIn) {
boolean parseChild = true; boolean parseChild = true;
if (nodeIn.getType() == HiveParser.TOK_TABLE_OR_COL && nodeIn.getAncestor(HiveParser.TOK_SELEXPR) != null ) { if (nodeIn.getType() == HiveParser.TOK_TABLE_OR_COL && nodeIn.getAncestor(HiveParser.TOK_SELEXPR) != null ) {
QueryColumns qclmns = hlb.new QueryColumns(); QueryColumns qclmns = hlb.new QueryColumns();
if (nodeIn.getAncestor(HiveParser.TOK_FUNCTION) != null && nodeIn.getAncestor(HiveParser.TOK_SELEXPR) != null) { if (nodeIn.getAncestor(HiveParser.TOK_FUNCTION) != null && nodeIn.getAncestor(HiveParser.TOK_SELEXPR) != null) {
LOG.info("Function Query:: "+nodeIn.getAncestor(HiveParser.TOK_FUNCTION).getChild(0).getText()); LOG.info("Function Query:: "+nodeIn.getAncestor(HiveParser.TOK_FUNCTION).getChild(0).getText());
qclmns.setColumnFunction(nodeIn.getAncestor(HiveParser.TOK_FUNCTION).getChild(0).getText()); qclmns.setColumnFunction(nodeIn.getAncestor(HiveParser.TOK_FUNCTION).getChild(0).getText());
} }
if (nodeIn.getParent().getText().equalsIgnoreCase(".")) { if (nodeIn.getParent().getText().equalsIgnoreCase(".")) {
LOG.info("Table Name/Alias:: "+nodeIn.getChild(0).getText()); LOG.info("Table Name/Alias:: "+nodeIn.getChild(0).getText());
qclmns.setTbAliasOrName(nodeIn.getChild(0).getText()); qclmns.setTbAliasOrName(nodeIn.getChild(0).getText());
LOG.info("Column:: "+nodeIn.getParent().getChild(1).getText()); LOG.info("Column:: "+nodeIn.getParent().getChild(1).getText());
qclmns.setColumnName(nodeIn.getParent().getChild(1).getText()); qclmns.setColumnName(nodeIn.getParent().getChild(1).getText());
if (nodeIn.getAncestor(HiveParser.TOK_SELEXPR).getChild(1) != null) { if (nodeIn.getAncestor(HiveParser.TOK_SELEXPR).getChild(1) != null) {
LOG.info("Column Alias:: "+nodeIn.getAncestor(HiveParser.TOK_SELEXPR).getChild(1).getText()); LOG.info("Column Alias:: "+nodeIn.getAncestor(HiveParser.TOK_SELEXPR).getChild(1).getText());
qclmns.setColumnAlias(nodeIn.getAncestor(HiveParser.TOK_SELEXPR).getChild(1).getText()); qclmns.setColumnAlias(nodeIn.getAncestor(HiveParser.TOK_SELEXPR).getChild(1).getText());
} }
} else { } else {
LOG.info("Column:: "+nodeIn.getChild(0).getText()); LOG.info("Column:: "+nodeIn.getChild(0).getText());
qclmns.setColumnName(nodeIn.getChild(0).getText()); qclmns.setColumnName(nodeIn.getChild(0).getText());
if (nodeIn.getParent().getChild(1) != null) { if (nodeIn.getParent().getChild(1) != null) {
LOG.info("Column Alias:: "+nodeIn.getParent().getChild(1).getText()); LOG.info("Column Alias:: "+nodeIn.getParent().getChild(1).getText());
qclmns.setColumnAlias(nodeIn.getParent().getChild(1).getText()); qclmns.setColumnAlias(nodeIn.getParent().getChild(1).getText());
} }
} }
if (qclmns.getColumnName() != null) { if (qclmns.getColumnName() != null) {
queryColumns.add(qclmns); queryColumns.add(qclmns);
} }
} }
if (parseChild) { if (parseChild) {
int childCount = nodeIn.getChildCount(); int childCount = nodeIn.getChildCount();
if (childCount != 0 ){ if (childCount != 0 ){
for (int numr = 0; numr < childCount; numr++) { for (int numr = 0; numr < childCount; numr++) {
clmnTdump((ASTNode)nodeIn.getChild(numr)); clmnTdump((ASTNode)nodeIn.getChild(numr));
} }
} }
} }
} }
/** /**
* parses given query and gets the lineage info. * parses given query and gets the lineage info.
* *
* @param query * @param query
* @throws ParseException * @throws ParseException
*/ */
public void getLineageInfo(String query) throws ParseException, public void getLineageInfo(String query) throws ParseException,
SemanticException { SemanticException {
/* /*
* Get the AST tree * Get the AST tree
*/ */
ParseDriver pd = new ParseDriver(); ParseDriver pd = new ParseDriver();
ASTNode tree = pd.parse(query); ASTNode tree = pd.parse(query);
LOG.info("DUMP TREE: "+tree.dump()); LOG.info("DUMP TREE: "+tree.dump());
while ((tree.getToken() == null) && (tree.getChildCount() > 0)) { while ((tree.getToken() == null) && (tree.getChildCount() > 0)) {
tree = (ASTNode) tree.getChild(0); tree = (ASTNode) tree.getChild(0);
} }
/* /*
* initialize Event Processor and dispatcher. * initialize Event Processor and dispatcher.
*/ */
// create a walker which walks the tree in a DFS manner while maintaining // create a walker which walks the tree in a DFS manner while maintaining
// the operator stack. The dispatcher // the operator stack. The dispatcher
// generates the plan from the operator tree // generates the plan from the operator tree
Map<Rule, NodeProcessor> rules = new LinkedHashMap<Rule, NodeProcessor>(); Map<Rule, NodeProcessor> rules = new LinkedHashMap<Rule, NodeProcessor>();
// The dispatcher fires the processor corresponding to the closest matching // The dispatcher fires the processor corresponding to the closest matching
// rule and passes the context along // rule and passes the context along
Dispatcher disp = new DefaultRuleDispatcher(this, rules, null); Dispatcher disp = new DefaultRuleDispatcher(this, rules, null);
GraphWalker ogw = new DefaultGraphWalker(disp); GraphWalker ogw = new DefaultGraphWalker(disp);
// Create a list of topop nodes // Create a list of topop nodes
ArrayList<Node> topNodes = new ArrayList<Node>(); ArrayList<Node> topNodes = new ArrayList<Node>();
topNodes.add(tree); topNodes.add(tree);
ogw.startWalking(topNodes, null); ogw.startWalking(topNodes, null);
} }
//Main method to run tests and return json/gson feed from a query //Main method to run tests and return json/gson feed from a query
public static void main(String[] args) throws IOException, ParseException, public static void main(String[] args) throws IOException, ParseException,
SemanticException { SemanticException {
String query = args[0]; String query = args[0];
ConsoleAppender console = new ConsoleAppender(); //create appender ConsoleAppender console = new ConsoleAppender(); //create appender
//configure the appender //configure the appender
String PATTERN = "%d [%p|%c|%C{1}] %m%n"; String PATTERN = "%d [%p|%c|%C{1}] %m%n";
console.setLayout(new PatternLayout(PATTERN)); console.setLayout(new PatternLayout(PATTERN));
console.setThreshold(Level.DEBUG); console.setThreshold(Level.DEBUG);
console.activateOptions(); console.activateOptions();
//add appender to any Logger (here is root) //add appender to any Logger (here is root)
Logger.getRootLogger().addAppender(console); Logger.getRootLogger().addAppender(console);
LogManager.getRootLogger().setLevel(Level.DEBUG); LogManager.getRootLogger().setLevel(Level.DEBUG);
HiveLineageInfo lep = new HiveLineageInfo(); HiveLineageInfo lep = new HiveLineageInfo();
lep.getLineageInfo(query); lep.getLineageInfo(query);
Gson gson = new Gson(); Gson gson = new Gson();
String jsonOut = gson.toJson(lep.getHLBean()); String jsonOut = gson.toJson(lep.getHLBean());
System.out.println("GSON/JSON Generate :: "+jsonOut); System.out.println("GSON/JSON Generate :: "+jsonOut);
} }
} }
hivehook/src/main/java/com/aetna/hadoop/dgc/hive/Hook.java
package com.aetna.hadoop.dgc.hive; package com.aetna.hadoop.dgc.hive;
import com.google.common.util.concurrent.ThreadFactoryBuilder; import com.google.common.util.concurrent.ThreadFactoryBuilder;
import com.google.gson.Gson; import com.google.gson.Gson;
import java.io.DataOutputStream; import java.io.DataOutputStream;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
import java.io.InputStreamReader; import java.io.InputStreamReader;
import java.net.HttpURLConnection; import java.net.HttpURLConnection;
import java.net.URL; import java.net.URL;
import java.net.URLEncoder; import java.net.URLEncoder;
import java.util.Collections; import java.util.Collections;
import java.util.Comparator; import java.util.Comparator;
import java.util.List; import java.util.List;
import java.util.Set; import java.util.Set;
import javax.net.ssl.HttpsURLConnection; import javax.net.ssl.HttpsURLConnection;
import javax.net.ssl.SSLContext; import javax.net.ssl.SSLContext;
import javax.net.ssl.TrustManager; import javax.net.ssl.TrustManager;
import javax.net.ssl.X509TrustManager; import javax.net.ssl.X509TrustManager;
import org.apache.commons.logging.Log; import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory; import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.QueryPlan; import org.apache.hadoop.hive.ql.QueryPlan;
import org.apache.hadoop.hive.ql.exec.ExplainTask; import org.apache.hadoop.hive.ql.exec.ExplainTask;
import org.apache.hadoop.hive.ql.exec.Task; import org.apache.hadoop.hive.ql.exec.Task;
import org.apache.hadoop.hive.ql.exec.Utilities; import org.apache.hadoop.hive.ql.exec.Utilities;
import org.apache.hadoop.hive.ql.exec.tez.TezTask; import org.apache.hadoop.hive.ql.exec.tez.TezTask;
import org.apache.hadoop.hive.ql.hooks.ExecuteWithHookContext; import org.apache.hadoop.hive.ql.hooks.ExecuteWithHookContext;
import org.apache.hadoop.hive.ql.hooks.HookContext; import org.apache.hadoop.hive.ql.hooks.HookContext;
import org.apache.hadoop.hive.ql.hooks.ReadEntity; import org.apache.hadoop.hive.ql.hooks.ReadEntity;
import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.util.StringUtils;
//import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity; //import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
//import org.apache.hadoop.yarn.api.records.timeline.TimelineEvent; //import org.apache.hadoop.yarn.api.records.timeline.TimelineEvent;
//import org.apache.hadoop.yarn.client.api.TimelineClient; //import org.apache.hadoop.yarn.client.api.TimelineClient;
//import org.apache.hadoop.yarn.conf.YarnConfiguration; //import org.apache.hadoop.yarn.conf.YarnConfiguration;
/** /**
* DGC Hook sends query + plan info to DGCCollector Service. To enable (hadoop 2.4 and up) set * DGC Hook sends query + plan info to DGCCollector Service. To enable (hadoop 2.4 and up) set
* hive.exec.pre.hooks/hive.exec.post.hooks/hive.exec.failure.hooks to include this class. * hive.exec.pre.hooks/hive.exec.post.hooks/hive.exec.failure.hooks to include this class.
*/ */
public class Hook implements ExecuteWithHookContext { public class Hook implements ExecuteWithHookContext {
private static final Log LOG = LogFactory.getLog(Hook.class.getName()); private static final Log LOG = LogFactory.getLog(Hook.class.getName());
private HiveLineageBean hlb; private HiveLineageBean hlb;
@Override @Override
public void run(HookContext hookContext) throws Exception { public void run(HookContext hookContext) throws Exception {
long currentTime = System.currentTimeMillis(); long currentTime = System.currentTimeMillis();
String executionEngine = null; String executionEngine = null;
try { try {
QueryPlan plan = hookContext.getQueryPlan(); QueryPlan plan = hookContext.getQueryPlan();
if (plan == null) { if (plan == null) {
return; return;
} }
ExplainTask explain = new ExplainTask(); ExplainTask explain = new ExplainTask();
explain.initialize(hookContext.getConf(), plan, null); explain.initialize(hookContext.getConf(), plan, null);
String queryId = plan.getQueryId(); String queryId = plan.getQueryId();
String queryStartTime = plan.getQueryStartTime().toString(); String queryStartTime = plan.getQueryStartTime().toString();
String user = hookContext.getUgi().getUserName(); String user = hookContext.getUgi().getUserName();
String query = plan.getQueryStr(); String query = plan.getQueryStr();
int numMrJobs = Utilities.getMRTasks(plan.getRootTasks()).size(); int numMrJobs = Utilities.getMRTasks(plan.getRootTasks()).size();
int numTezJobs = Utilities.getTezTasks(plan.getRootTasks()).size(); int numTezJobs = Utilities.getTezTasks(plan.getRootTasks()).size();
String hiveId = explain.getId(); String hiveId = explain.getId();
SessionState sess = SessionState.get(); SessionState sess = SessionState.get();
if (numTezJobs > 0) { if (numTezJobs > 0) {
executionEngine="tez"; executionEngine="tez";
} }
if (numMrJobs > 0) { if (numMrJobs > 0) {
executionEngine="mr"; executionEngine="mr";
} }
hiveId = sess.getSessionId(); hiveId = sess.getSessionId();
switch(hookContext.getHookType()) { switch(hookContext.getHookType()) {
case PRE_EXEC_HOOK: case PRE_EXEC_HOOK:
Set<ReadEntity> db = hookContext.getInputs(); Set<ReadEntity> db = hookContext.getInputs();
for (Object o : db) { for (Object o : db) {
LOG.debug("DB:Table="+o.toString()); LOG.debug("DB:Table="+o.toString());
} }
currentTime = System.currentTimeMillis(); currentTime = System.currentTimeMillis();
HiveLineageInfo lep_pre = new HiveLineageInfo(); HiveLineageInfo lep_pre = new HiveLineageInfo();
lep_pre.getLineageInfo(query); lep_pre.getLineageInfo(query);
hlb=lep_pre.getHLBean(); hlb=lep_pre.getHLBean();
hlb.setQueryEndTime(Long.toString(currentTime)); hlb.setQueryEndTime(Long.toString(currentTime));
hlb.setQueryId(queryId); hlb.setQueryId(queryId);
hlb.setQuery(query); hlb.setQuery(query);
hlb.setUser(user); hlb.setUser(user);
hlb.setHiveId(hiveId); hlb.setHiveId(hiveId);
hlb.setSuccess(false); hlb.setSuccess(false);
if (executionEngine.equalsIgnoreCase("mr")) { if (executionEngine.equalsIgnoreCase("mr")) {
hlb.setExecutionEngine("mapreduce"); hlb.setExecutionEngine("mapreduce");
} }
if (executionEngine.equalsIgnoreCase("tez")) { if (executionEngine.equalsIgnoreCase("tez")) {
hlb.setExecutionEngine("tez"); hlb.setExecutionEngine("tez");
} }
if (executionEngine.equalsIgnoreCase("spark")) { if (executionEngine.equalsIgnoreCase("spark")) {
hlb.setExecutionEngine("spark"); hlb.setExecutionEngine("spark");
} }
hlb.setQueryStartTime(queryStartTime); hlb.setQueryStartTime(queryStartTime);
fireAndForget(hookContext.getConf(), hlb, queryId); fireAndForget(hookContext.getConf(), hlb, queryId);
break; break;
case POST_EXEC_HOOK: case POST_EXEC_HOOK:
currentTime = System.currentTimeMillis(); currentTime = System.currentTimeMillis();
HiveLineageInfo lep_post = new HiveLineageInfo(); HiveLineageInfo lep_post = new HiveLineageInfo();
lep_post.getLineageInfo(query); lep_post.getLineageInfo(query);
hlb=lep_post.getHLBean(); hlb=lep_post.getHLBean();
hlb.setQueryEndTime(Long.toString(currentTime)); hlb.setQueryEndTime(Long.toString(currentTime));
hlb.setQueryId(queryId); hlb.setQueryId(queryId);
hlb.setQuery(query); hlb.setQuery(query);
hlb.setUser(user); hlb.setUser(user);
hlb.setQueryStartTime(queryStartTime); hlb.setQueryStartTime(queryStartTime);
hlb.setSuccess(true); hlb.setSuccess(true);
hlb.setHiveId(hiveId); hlb.setHiveId(hiveId);
if (executionEngine.equalsIgnoreCase("mr")) { if (executionEngine.equalsIgnoreCase("mr")) {
hlb.setExecutionEngine("mapreduce"); hlb.setExecutionEngine("mapreduce");
} }
if (executionEngine.equalsIgnoreCase("tez")) { if (executionEngine.equalsIgnoreCase("tez")) {
hlb.setExecutionEngine("tez"); hlb.setExecutionEngine("tez");
} }
if (executionEngine.equalsIgnoreCase("spark")) { if (executionEngine.equalsIgnoreCase("spark")) {
hlb.setExecutionEngine("spark"); hlb.setExecutionEngine("spark");
} }
fireAndForget(hookContext.getConf(), hlb, queryId); fireAndForget(hookContext.getConf(), hlb, queryId);
break; break;
case ON_FAILURE_HOOK: case ON_FAILURE_HOOK:
HiveLineageInfo lep_failed = new HiveLineageInfo(); HiveLineageInfo lep_failed = new HiveLineageInfo();
lep_failed.getLineageInfo(query); lep_failed.getLineageInfo(query);
hlb=lep_failed.getHLBean(); hlb=lep_failed.getHLBean();
hlb.setQueryEndTime(Long.toString(currentTime)); hlb.setQueryEndTime(Long.toString(currentTime));
hlb.setQueryId(queryId); hlb.setQueryId(queryId);
hlb.setQuery(query); hlb.setQuery(query);
hlb.setUser(user); hlb.setUser(user);
hlb.setQueryStartTime(queryStartTime); hlb.setQueryStartTime(queryStartTime);
hlb.setSuccess(false); hlb.setSuccess(false);
hlb.setFailed(true); hlb.setFailed(true);
hlb.setHiveId(hiveId); hlb.setHiveId(hiveId);
if (executionEngine.equalsIgnoreCase("mr")) { if (executionEngine.equalsIgnoreCase("mr")) {
hlb.setExecutionEngine("mapreduce"); hlb.setExecutionEngine("mapreduce");
} }
if (executionEngine.equalsIgnoreCase("tez")) { if (executionEngine.equalsIgnoreCase("tez")) {
hlb.setExecutionEngine("tez"); hlb.setExecutionEngine("tez");
} }
if (executionEngine.equalsIgnoreCase("spark")) { if (executionEngine.equalsIgnoreCase("spark")) {
hlb.setExecutionEngine("spark"); hlb.setExecutionEngine("spark");
} }
fireAndForget(hookContext.getConf(), hlb, queryId); fireAndForget(hookContext.getConf(), hlb, queryId);
break; break;
default: default:
//ignore //ignore
break; break;
} }
} catch (Exception e) { } catch (Exception e) {
LOG.info("Failed to submit plan to DGC: " + StringUtils.stringifyException(e)); LOG.info("Failed to submit plan to DGC: " + StringUtils.stringifyException(e));
} }
} }
public void fireAndForget(Configuration conf, HiveLineageBean hookData, String queryId) throws Exception { public void fireAndForget(Configuration conf, HiveLineageBean hookData, String queryId) throws Exception {
String postUri = "http://167.69.111.50:20810/HiveHookCollector/HookServlet"; String postUri = "http://167.69.111.50:20810/HiveHookCollector/HookServlet";
if (conf.getTrimmed("aetna.hive.hook") != null) { if (conf.getTrimmed("aetna.hive.hook") != null) {
postUri = conf.getTrimmed("aetna.hive.hook"); postUri = conf.getTrimmed("aetna.hive.hook");
} }
Gson gson = new Gson(); Gson gson = new Gson();
String gsonString = gson.toJson(hookData); String gsonString = gson.toJson(hookData);
LOG.debug("GSON String: "+gsonString); LOG.debug("GSON String: "+gsonString);
String encodedGsonQuery = URLEncoder.encode(gsonString, "UTF-8"); String encodedGsonQuery = URLEncoder.encode(gsonString, "UTF-8");
String encodedQueryId = URLEncoder.encode(queryId, "UTF-8"); String encodedQueryId = URLEncoder.encode(queryId, "UTF-8");
String postData = "hookdata=" + encodedGsonQuery+"&queryid="+encodedQueryId; String postData = "hookdata=" + encodedGsonQuery+"&queryid="+encodedQueryId;
// Create a trust manager that does not validate certificate chains // Create a trust manager that does not validate certificate chains
if (postUri.contains("https:")) { if (postUri.contains("https:")) {
TrustManager[] trustAllCerts = new TrustManager[]{ TrustManager[] trustAllCerts = new TrustManager[]{
new X509TrustManager() { new X509TrustManager() {
public java.security.cert.X509Certificate[] getAcceptedIssuers() { public java.security.cert.X509Certificate[] getAcceptedIssuers() {
return null; return null;
} }
public void checkClientTrusted( public void checkClientTrusted(
java.security.cert.X509Certificate[] certs, String authType) { java.security.cert.X509Certificate[] certs, String authType) {
} }
public void checkServerTrusted( public void checkServerTrusted(
java.security.cert.X509Certificate[] certs, String authType) { java.security.cert.X509Certificate[] certs, String authType) {
} }
} }
}; };
// Install the all-trusting trust manager // Install the all-trusting trust manager
try { try {
SSLContext sc = SSLContext.getInstance("SSL"); SSLContext sc = SSLContext.getInstance("SSL");
sc.init(null, trustAllCerts, new java.security.SecureRandom()); sc.init(null, trustAllCerts, new java.security.SecureRandom());
HttpsURLConnection.setDefaultSSLSocketFactory(sc.getSocketFactory()); HttpsURLConnection.setDefaultSSLSocketFactory(sc.getSocketFactory());
} catch (Exception e) { } catch (Exception e) {
e.printStackTrace(); e.printStackTrace();
} }
} }
URL url = new URL(postUri); URL url = new URL(postUri);
LOG.debug("Post URI: "+postUri); LOG.debug("Post URI: "+postUri);
DataOutputStream wr = null; DataOutputStream wr = null;
//HttpURLConnection urlcon = null; //HttpURLConnection urlcon = null;
if (postUri.contains("https:")) { if (postUri.contains("https:")) {
HttpsURLConnection urlcon = null; HttpsURLConnection urlcon = null;
urlcon = (HttpsURLConnection)url.openConnection(); urlcon = (HttpsURLConnection)url.openConnection();
urlcon.setRequestMethod("POST"); urlcon.setRequestMethod("POST");
urlcon.setRequestProperty("X-Requested-By", "HiveHook"); urlcon.setRequestProperty("X-Requested-By", "HiveHook");
urlcon.setRequestProperty("Content-Type","application/x-www-form-urlencoded"); urlcon.setRequestProperty("Content-Type","application/x-www-form-urlencoded");
urlcon.setUseCaches(false); urlcon.setUseCaches(false);
urlcon.setDoInput(true); urlcon.setDoInput(true);
urlcon.setDoOutput(true); urlcon.setDoOutput(true);
wr = new DataOutputStream (urlcon.getOutputStream()); wr = new DataOutputStream (urlcon.getOutputStream());
LOG.debug("PostString: "+postData); LOG.debug("PostString: "+postData);
//wr.writeBytes(postString.); //wr.writeBytes(postString.);
wr.write(postData.getBytes()); wr.write(postData.getBytes());
wr.flush (); wr.flush ();
wr.close (); wr.close ();
InputStream is = urlcon.getInputStream(); InputStream is = urlcon.getInputStream();
InputStreamReader isr = new InputStreamReader(is); InputStreamReader isr = new InputStreamReader(is);
int numCharsRead; int numCharsRead;
char[] charArray = new char[1024]; char[] charArray = new char[1024];
StringBuffer sb = new StringBuffer(); StringBuffer sb = new StringBuffer();
while ((numCharsRead = isr.read(charArray)) > 0) { while ((numCharsRead = isr.read(charArray)) > 0) {
sb.append(charArray, 0, numCharsRead); sb.append(charArray, 0, numCharsRead);
} }
String result = sb.toString(); String result = sb.toString();
LOG.debug("Post Response: "+result); LOG.debug("Post Response: "+result);
isr.close(); isr.close();
is.close(); is.close();
urlcon.disconnect(); urlcon.disconnect();
} else { } else {
HttpURLConnection urlcon = null; HttpURLConnection urlcon = null;
urlcon = (HttpURLConnection)url.openConnection(); urlcon = (HttpURLConnection)url.openConnection();
urlcon.setRequestMethod("POST"); urlcon.setRequestMethod("POST");
urlcon.setRequestProperty("X-Requested-By", "HiveHook"); urlcon.setRequestProperty("X-Requested-By", "HiveHook");
urlcon.setRequestProperty("Content-Type","application/x-www-form-urlencoded"); urlcon.setRequestProperty("Content-Type","application/x-www-form-urlencoded");
urlcon.setUseCaches(false); urlcon.setUseCaches(false);
urlcon.setDoInput(true); urlcon.setDoInput(true);
urlcon.setDoOutput(true); urlcon.setDoOutput(true);
wr = new DataOutputStream (urlcon.getOutputStream()); wr = new DataOutputStream (urlcon.getOutputStream());
LOG.debug("PostString: "+postData); LOG.debug("PostString: "+postData);
//wr.writeBytes(postString.); //wr.writeBytes(postString.);
wr.write(postData.getBytes()); wr.write(postData.getBytes());
wr.flush (); wr.flush ();
wr.close (); wr.close ();
InputStream is = urlcon.getInputStream(); InputStream is = urlcon.getInputStream();
InputStreamReader isr = new InputStreamReader(is); InputStreamReader isr = new InputStreamReader(is);
int numCharsRead; int numCharsRead;
char[] charArray = new char[1024]; char[] charArray = new char[1024];
StringBuffer sb = new StringBuffer(); StringBuffer sb = new StringBuffer();
while ((numCharsRead = isr.read(charArray)) > 0) { while ((numCharsRead = isr.read(charArray)) > 0) {
sb.append(charArray, 0, numCharsRead); sb.append(charArray, 0, numCharsRead);
} }
String result = sb.toString(); String result = sb.toString();
LOG.debug("Post Response: "+result); LOG.debug("Post Response: "+result);
isr.close(); isr.close();
is.close(); is.close();
urlcon.disconnect(); urlcon.disconnect();
} }
} }
} }
repository/pom.xml
<?xml version="1.0" encoding="UTF-8"?> <?xml version="1.0" encoding="UTF-8"?>
<!-- <!-- ~ Licensed to the Apache Software Foundation (ASF) under one ~ or more
~ Licensed to the Apache Software Foundation (ASF) under one contributor license agreements. See the NOTICE file ~ distributed with this
~ or more contributor license agreements. See the NOTICE file work for additional information ~ regarding copyright ownership. The ASF
~ distributed with this work for additional information licenses this file ~ to you under the Apache License, Version 2.0 (the ~
~ regarding copyright ownership. The ASF licenses this file "License"); you may not use this file except in compliance ~ with the License.
~ to you under the Apache License, Version 2.0 (the You may obtain a copy of the License at ~ ~ http://www.apache.org/licenses/LICENSE-2.0
~ "License"); you may not use this file except in compliance ~ ~ Unless required by applicable law or agreed to in writing, software ~
~ with the License. You may obtain a copy of the License at distributed under the License is distributed on an "AS IS" BASIS, ~ WITHOUT
~ WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ~ See the
~ http://www.apache.org/licenses/LICENSE-2.0 License for the specific language governing permissions and ~ limitations
~ under the License. -->
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS, <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
~ See the License for the specific language governing permissions and
~ limitations under the License. <modelVersion>4.0.0</modelVersion>
--> <parent>
<groupId>org.apache.hadoop.metadata</groupId>
<project xmlns="http://maven.apache.org/POM/4.0.0" <artifactId>metadata-governance</artifactId>
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" <version>0.1-incubating-SNAPSHOT</version>
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> </parent>
<artifactId>metadata-repository</artifactId>
<modelVersion>4.0.0</modelVersion> <description>Apache Metadata Repository Module</description>
<parent> <name>Apache Metadata Repository</name>
<groupId>org.apache.hadoop.metadata</groupId> <packaging>jar</packaging>
<artifactId>metadata-governance</artifactId>
<version>0.1-incubating-SNAPSHOT</version> <profiles>
</parent> <profile>
<artifactId>metadata-repository</artifactId> <id>hadoop-2</id>
<description>Apache Metadata Repository Module</description> <activation>
<name>Apache Metadata Repository</name> <activeByDefault>true</activeByDefault>
<packaging>jar</packaging> </activation>
<dependencies>
<profiles> <dependency>
<profile> <groupId>org.apache.hadoop</groupId>
<id>hadoop-2</id> <artifactId>hadoop-client</artifactId>
<activation> </dependency>
<activeByDefault>true</activeByDefault> <dependency>
</activation> <groupId>org.apache.hadoop</groupId>
<dependencies> <artifactId>hadoop-hdfs</artifactId>
<dependency> </dependency>
<groupId>org.apache.hadoop</groupId> <dependency>
<artifactId>hadoop-client</artifactId> <groupId>org.apache.hadoop</groupId>
</dependency> <artifactId>hadoop-hdfs</artifactId>
<dependency> <classifier>tests</classifier>
<groupId>org.apache.hadoop</groupId> </dependency>
<artifactId>hadoop-hdfs</artifactId> <dependency>
</dependency> <groupId>org.apache.hadoop</groupId>
<dependency> <artifactId>hadoop-common</artifactId>
<groupId>org.apache.hadoop</groupId> <classifier>tests</classifier>
<artifactId>hadoop-hdfs</artifactId> </dependency>
<classifier>tests</classifier> <dependency>
</dependency> <groupId>org.apache.hadoop</groupId>
<dependency> <artifactId>hadoop-common</artifactId>
<groupId>org.apache.hadoop</groupId> </dependency>
<artifactId>hadoop-common</artifactId> </dependencies>
<classifier>tests</classifier> </profile>
</dependency> </profiles>
<dependency>
<groupId>org.apache.hadoop</groupId> <dependencies>
<artifactId>hadoop-common</artifactId> <dependency>
</dependency> <groupId>org.apache.hadoop.metadata</groupId>
</dependencies> <artifactId>metadata-common</artifactId>
</profile> </dependency>
</profiles>
<dependency>
<dependencies>
<dependency>
<groupId>org.apache.hadoop.metadata</groupId>
<artifactId>metadata-common</artifactId>
</dependency>
<dependency>
<groupId>org.apache.hadoop.metadata</groupId> <groupId>org.apache.hadoop.metadata</groupId>
<artifactId>metadata-typesystem</artifactId> <artifactId>metadata-typesystem</artifactId>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.slf4j</groupId> <groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId> <artifactId>slf4j-log4j12</artifactId>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.slf4j</groupId> <groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId> <artifactId>slf4j-api</artifactId>
</dependency> </dependency>
<dependency> <dependency>
<groupId>com.google.inject</groupId> <groupId>com.google.inject</groupId>
<artifactId>guice</artifactId> <artifactId>guice</artifactId>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.codehaus.jettison</groupId> <groupId>com.google.inject.extensions</groupId>
<artifactId>jettison</artifactId> <artifactId>guice-throwingproviders</artifactId>
</dependency> <version>3.0</version>
</dependency>
<dependency>
<groupId>com.googlecode.json-simple</groupId> <dependency>
<artifactId>json-simple</artifactId> <groupId>org.codehaus.jettison</groupId>
</dependency> <artifactId>jettison</artifactId>
</dependency>
<dependency>
<groupId>com.tinkerpop.blueprints</groupId> <dependency>
<artifactId>blueprints-core</artifactId> <groupId>com.googlecode.json-simple</groupId>
</dependency> <artifactId>json-simple</artifactId>
</dependency>
<dependency>
<groupId>com.thinkaurelius.titan</groupId> <dependency>
<artifactId>titan-core</artifactId> <groupId>com.tinkerpop.blueprints</groupId>
</dependency> <artifactId>blueprints-core</artifactId>
</dependency>
<dependency>
<groupId>com.thinkaurelius.titan</groupId> <dependency>
<artifactId>titan-berkeleyje</artifactId> <groupId>com.thinkaurelius.titan</groupId>
</dependency> <artifactId>titan-core</artifactId>
</dependency>
<dependency>
<groupId>com.thinkaurelius.titan</groupId> <dependency>
<artifactId>titan-es</artifactId> <groupId>com.thinkaurelius.titan</groupId>
</dependency> <artifactId>titan-berkeleyje</artifactId>
</dependency>
<dependency>
<groupId>org.testng</groupId> <dependency>
<artifactId>testng</artifactId> <groupId>com.thinkaurelius.titan</groupId>
</dependency> <artifactId>titan-es</artifactId>
</dependency>
<dependency>
<groupId>org.mockito</groupId> <dependency>
<artifactId>mockito-all</artifactId> <groupId>org.testng</groupId>
</dependency> <artifactId>testng</artifactId>
</dependencies> </dependency>
<build> <dependency>
<plugins> <groupId>org.mockito</groupId>
<plugin> <artifactId>mockito-all</artifactId>
<groupId>org.apache.maven.plugins</groupId> </dependency>
<artifactId>maven-compiler-plugin</artifactId> </dependencies>
<configuration>
<source>1.7</source> <build>
<target>1.7</target> <plugins>
</configuration> <plugin>
</plugin> <groupId>org.apache.maven.plugins</groupId>
<plugin> <artifactId>maven-compiler-plugin</artifactId>
<groupId>org.apache.maven.plugins</groupId> <configuration>
<artifactId>maven-jar-plugin</artifactId> <source>1.7</source>
<version>2.4</version> <target>1.7</target>
<configuration> </configuration>
<excludes> </plugin>
<exclude>**/log4j.xml</exclude> <plugin>
</excludes> <groupId>org.apache.maven.plugins</groupId>
</configuration> <artifactId>maven-jar-plugin</artifactId>
</plugin> <version>2.4</version>
</plugins> <configuration>
</build> <excludes>
<exclude>**/log4j.xml</exclude>
</excludes>
</configuration>
</plugin>
</plugins>
</build>
</project> </project>
repository/src/main/java/org/apache/hadoop/metadata/RepositoryMetadataModule.java
...@@ -24,12 +24,49 @@ ...@@ -24,12 +24,49 @@
*/ */
package org.apache.hadoop.metadata; package org.apache.hadoop.metadata;
import org.apache.hadoop.metadata.services.GraphBackedMetadataRepository;
import org.apache.hadoop.metadata.services.GraphProvider;
import org.apache.hadoop.metadata.services.GraphService;
import org.apache.hadoop.metadata.services.GraphServiceConfigurator;
import org.apache.hadoop.metadata.services.MetadataRepository;
import org.apache.hadoop.metadata.services.TitanGraphProvider;
import org.apache.hadoop.metadata.storage.IRepository;
import org.apache.hadoop.metadata.storage.memory.MemRepository;
import com.google.inject.Scopes;
import com.google.inject.throwingproviders.ThrowingProviderBinder;
import com.thinkaurelius.titan.core.TitanGraph;
/** /**
* Guice module for Repository module. * Guice module for Repository module.
*/ */
public class RepositoryMetadataModule extends com.google.inject.AbstractModule { public class RepositoryMetadataModule extends com.google.inject.AbstractModule {
protected void configure() { // Graph Service implementation class
// add configuration logic here private Class<? extends GraphService> graphServiceClass;
} // MetadataRepositoryService implementation class
private Class<? extends MetadataRepository> metadataRepoClass;
public RepositoryMetadataModule() {
GraphServiceConfigurator gsp = new GraphServiceConfigurator();
// get the impl classes for the repo and the graph service
this.graphServiceClass = gsp.getImplClass();
this.metadataRepoClass = GraphBackedMetadataRepository.class;
}
protected void configure() {
// special wiring for Titan Graph
ThrowingProviderBinder.create(binder())
.bind(GraphProvider.class, TitanGraph.class)
.to(TitanGraphProvider.class)
.in(Scopes.SINGLETON);
// allow for dynamic binding of the metadata repo & graph service
// bind the MetadataRepositoryService interface to an implementation
bind(MetadataRepository.class).to(metadataRepoClass);
// bind the GraphService interface to an implementation
bind(GraphService.class).to(graphServiceClass);
}
} }
repository/src/main/java/org/apache/hadoop/metadata/services/DefaultMetadataService.java
...@@ -18,25 +18,31 @@ ...@@ -18,25 +18,31 @@
package org.apache.hadoop.metadata.services; package org.apache.hadoop.metadata.services;
import java.io.IOException;
import java.util.List;
import javax.inject.Inject;
import org.apache.hadoop.metadata.ITypedReferenceableInstance; import org.apache.hadoop.metadata.ITypedReferenceableInstance;
import org.apache.hadoop.metadata.MetadataException; import org.apache.hadoop.metadata.MetadataException;
import org.apache.hadoop.metadata.json.Serialization$; import org.apache.hadoop.metadata.json.Serialization$;
import org.apache.hadoop.metadata.service.Services;
import org.apache.hadoop.metadata.types.TypeSystem; import org.apache.hadoop.metadata.types.TypeSystem;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.List;
public class DefaultMetadataService implements MetadataService { public class DefaultMetadataService implements MetadataService {
private static final Logger LOG = private static final Logger LOG =
LoggerFactory.getLogger(DefaultMetadataService.class); LoggerFactory.getLogger(DefaultMetadataService.class);
public static final String NAME = DefaultMetadataService.class.getSimpleName();
private TypeSystem typeSystem; private final TypeSystem typeSystem;
private MetadataRepository repositoryService; private final MetadataRepository repository;
@Inject
DefaultMetadataService(MetadataRepository repository) throws MetadataException {
this.typeSystem = new TypeSystem();
this.repository = repository;
}
/** /**
* Creates a new type based on the type system to enable adding * Creates a new type based on the type system to enable adding
...@@ -84,7 +90,7 @@ public class DefaultMetadataService implements MetadataService { ...@@ -84,7 +90,7 @@ public class DefaultMetadataService implements MetadataService {
String entityDefinition) throws MetadataException { String entityDefinition) throws MetadataException {
ITypedReferenceableInstance entityInstance = ITypedReferenceableInstance entityInstance =
Serialization$.MODULE$.fromJson(entityDefinition); Serialization$.MODULE$.fromJson(entityDefinition);
return repositoryService.createEntity(entityInstance, entityType); return repository.createEntity(entityInstance, entityType);
} }
/** /**
...@@ -124,35 +130,12 @@ public class DefaultMetadataService implements MetadataService { ...@@ -124,35 +130,12 @@ public class DefaultMetadataService implements MetadataService {
} }
/** /**
* Name of the service.
*
* @return name of the service
*/
@Override
public String getName() {
return NAME;
}
/**
* Starts the service. This method blocks until the service has completely started. * Starts the service. This method blocks until the service has completely started.
* *
* @throws Exception * @throws Exception
*/ */
@Override @Override
public void start() throws Exception { public void start() throws Exception {
LOG.info("Initializing the Metadata service");
if (Services.get().isRegistered(TitanGraphService.NAME)) {
DefaultTypesService typesService = Services.get().getService(DefaultTypesService.NAME);
typeSystem = typesService.getTypeSystem();
} else {
throw new RuntimeException("Types service is not initialized");
}
if (Services.get().isRegistered(TitanGraphService.NAME)) {
repositoryService = Services.get().getService(GraphBackedMetadataRepository.NAME);
} else {
throw new RuntimeException("repository service is not initialized");
}
} }
/** /**
...@@ -160,8 +143,6 @@ public class DefaultMetadataService implements MetadataService { ...@@ -160,8 +143,6 @@ public class DefaultMetadataService implements MetadataService {
*/ */
@Override @Override
public void stop() { public void stop() {
// do nothing
repositoryService = null;
} }
/** /**
......
repository/src/main/java/org/apache/hadoop/metadata/services/DefaultTypesService.java deleted 100644 → 0
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.metadata.services;
import org.apache.hadoop.metadata.types.TypeSystem;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
public class DefaultTypesService implements TypesService {
private static final Logger LOG =
LoggerFactory.getLogger(DefaultTypesService.class);
public static final String NAME = DefaultTypesService.class.getSimpleName();
private TypeSystem typeSystem;
@Override
public TypeSystem getTypeSystem() {
assert typeSystem != null;
return typeSystem;
}
/**
* Name of the service.
*
* @return name of the service
*/
@Override
public String getName() {
return NAME;
}
/**
* Starts the service. This method blocks until the service has completely started.
*
* @throws Exception
*/
@Override
public void start() throws Exception {
LOG.info("Initializing the type system");
typeSystem = new TypeSystem();
}
/**
* Stops the service. This method blocks until the service has completely shut down.
*/
@Override
public void stop() {
}
/**
* A version of stop() that is designed to be usable in Java7 closure
* clauses.
* Implementation classes MUST relay this directly to {@link #stop()}
*
* @throws java.io.IOException never
* @throws RuntimeException on any failure during the stop operation
*/
@Override
public void close() throws IOException {
}
}
repository/src/main/java/org/apache/hadoop/metadata/services/GraphBackedMetadataRepository.java
...@@ -18,16 +18,23 @@ ...@@ -18,16 +18,23 @@
package org.apache.hadoop.metadata.services; package org.apache.hadoop.metadata.services;
import com.tinkerpop.blueprints.Direction; import java.io.IOException;
import com.tinkerpop.blueprints.Edge; import java.util.ArrayList;
import com.tinkerpop.blueprints.TransactionalGraph; import java.util.Collections;
import com.tinkerpop.blueprints.Vertex; import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicInteger;
import javax.inject.Inject;
import org.apache.hadoop.metadata.IReferenceableInstance; import org.apache.hadoop.metadata.IReferenceableInstance;
import org.apache.hadoop.metadata.ITypedInstance; import org.apache.hadoop.metadata.ITypedInstance;
import org.apache.hadoop.metadata.ITypedReferenceableInstance; import org.apache.hadoop.metadata.ITypedReferenceableInstance;
import org.apache.hadoop.metadata.ITypedStruct; import org.apache.hadoop.metadata.ITypedStruct;
import org.apache.hadoop.metadata.MetadataException; import org.apache.hadoop.metadata.MetadataException;
import org.apache.hadoop.metadata.service.Services;
import org.apache.hadoop.metadata.storage.Id; import org.apache.hadoop.metadata.storage.Id;
import org.apache.hadoop.metadata.storage.MapIds; import org.apache.hadoop.metadata.storage.MapIds;
import org.apache.hadoop.metadata.storage.RepositoryException; import org.apache.hadoop.metadata.storage.RepositoryException;
...@@ -41,18 +48,14 @@ import org.apache.hadoop.metadata.types.TypeSystem; ...@@ -41,18 +48,14 @@ import org.apache.hadoop.metadata.types.TypeSystem;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import java.io.IOException; import com.tinkerpop.blueprints.Direction;
import java.util.ArrayList; import com.tinkerpop.blueprints.Edge;
import java.util.Collections; import com.tinkerpop.blueprints.TransactionalGraph;
import java.util.HashMap; import com.tinkerpop.blueprints.Vertex;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicInteger;
/** /**
* An implementation backed by Titan Graph DB. * An implementation backed by a Graph database provided
* as a Graph Service.
*/ */
public class GraphBackedMetadataRepository implements MetadataRepository { public class GraphBackedMetadataRepository implements MetadataRepository {
...@@ -69,19 +72,16 @@ public class GraphBackedMetadataRepository implements MetadataRepository { ...@@ -69,19 +72,16 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
private final AtomicInteger ID_SEQ = new AtomicInteger(0); private final AtomicInteger ID_SEQ = new AtomicInteger(0);
// private ConcurrentHashMap<String, ITypedReferenceableInstance> types; // private ConcurrentHashMap<String, ITypedReferenceableInstance> types;
private ConcurrentHashMap<String, ITypedReferenceableInstance> instances; private final ConcurrentHashMap<String, ITypedReferenceableInstance> instances;
private GraphService graphService; private final GraphService graphService;
private TypeSystem typeSystem; private final TypeSystem typeSystem;
/** @Inject
* Name of the service. GraphBackedMetadataRepository(GraphService graphService) throws MetadataException {
* this.instances = new ConcurrentHashMap<>();
* @return name of the service this.graphService = graphService;
*/ this.typeSystem = new TypeSystem();
@Override
public String getName() {
return NAME;
} }
/** /**
...@@ -91,20 +91,6 @@ public class GraphBackedMetadataRepository implements MetadataRepository { ...@@ -91,20 +91,6 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
*/ */
@Override @Override
public void start() throws Exception { public void start() throws Exception {
if (Services.get().isRegistered(TitanGraphService.NAME)) {
graphService = Services.get().getService(TitanGraphService.NAME);
} else {
throw new RuntimeException("graph service is not initialized");
}
if (Services.get().isRegistered(DefaultTypesService.NAME)) {
DefaultTypesService typesService = Services.get().getService(DefaultTypesService.NAME);
typeSystem = typesService.getTypeSystem();
} else {
throw new RuntimeException("Types service is not initialized");
}
instances = new ConcurrentHashMap<>();
} }
/** /**
...@@ -112,8 +98,6 @@ public class GraphBackedMetadataRepository implements MetadataRepository { ...@@ -112,8 +98,6 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
*/ */
@Override @Override
public void stop() { public void stop() {
// do nothing
graphService = null;
} }
/** /**
...@@ -129,16 +113,12 @@ public class GraphBackedMetadataRepository implements MetadataRepository { ...@@ -129,16 +113,12 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
stop(); stop();
} }
private TransactionalGraph getTransactionalGraph() {
return graphService.getTransactionalGraph();
}
@Override @Override
public String createEntity(IReferenceableInstance entity, public String createEntity(IReferenceableInstance entity,
String entityType) throws RepositoryException { String entityType) throws RepositoryException {
LOG.info("adding entity={} type={}", entity, entityType); LOG.info("adding entity={} type={}", entity, entityType);
final TransactionalGraph transactionalGraph = getTransactionalGraph(); final TransactionalGraph transactionalGraph = graphService.getTransactionalGraph();
try { try {
// todo check if this is a duplicate // todo check if this is a duplicate
......
repository/src/main/java/org/apache/hadoop/metadata/services/GraphProvider.java 0 → 100644
package org.apache.hadoop.metadata.services;
import org.apache.commons.configuration.ConfigurationException;
import com.google.inject.throwingproviders.CheckedProvider;
import com.tinkerpop.blueprints.Graph;
public interface GraphProvider<T extends Graph> extends CheckedProvider<T> {
@Override
T get() throws ConfigurationException;
}
repository/src/main/java/org/apache/hadoop/metadata/services/GraphService.java
...@@ -18,12 +18,13 @@ ...@@ -18,12 +18,13 @@
package org.apache.hadoop.metadata.services; package org.apache.hadoop.metadata.services;
import java.util.Set;
import org.apache.hadoop.metadata.service.Service;
import com.tinkerpop.blueprints.Graph; import com.tinkerpop.blueprints.Graph;
import com.tinkerpop.blueprints.KeyIndexableGraph; import com.tinkerpop.blueprints.KeyIndexableGraph;
import com.tinkerpop.blueprints.TransactionalGraph; import com.tinkerpop.blueprints.TransactionalGraph;
import org.apache.hadoop.metadata.service.Service;
import java.util.Set;
/** /**
* A blueprints based graph service. * A blueprints based graph service.
......
repository/src/main/java/org/apache/hadoop/metadata/services/GraphServiceConfigurator.java 0 → 100644
package org.apache.hadoop.metadata.services;
import com.thinkaurelius.titan.core.TitanGraph;
public class GraphServiceConfigurator extends PropertyBasedConfigurator<GraphService> {
private static final String PROPERTY_NAME = "metadata.graph.impl.class";
private static final String DEFAULT_IMPL_CLASS = TitanGraph.class.getName();
private static final String CONFIG_PATH = "application.properties";
public GraphServiceConfigurator() {
super("metadata.graph.propertyName", "metadata.graph.defaultImplClass",
"metadata.graph.configurationPath", PROPERTY_NAME,
DEFAULT_IMPL_CLASS, CONFIG_PATH);
}
}
repository/src/main/java/org/apache/hadoop/metadata/services/PropertyBasedConfigurator.java 0 → 100644
package org.apache.hadoop.metadata.services;
import java.util.Properties;
import org.apache.commons.configuration.Configuration;
import org.apache.commons.configuration.ConfigurationException;
import org.apache.commons.configuration.PropertiesConfiguration;
public abstract class PropertyBasedConfigurator<T> {
private final String propertyName;
private final String defaultImplClass;
private final String configurationPath;
PropertyBasedConfigurator(String propertyNameProp, String defaultImplClassProp,
String configurationPathProp, String propertyNameDefaultProp,
String defaultImplClassDefaultProp, String configPathDefaultProp) {
Properties props = System.getProperties();
this.propertyName = props.getProperty(propertyNameProp,
propertyNameDefaultProp);
this.defaultImplClass = props.getProperty(defaultImplClassProp,
defaultImplClassDefaultProp);
this.configurationPath = props.getProperty(configurationPathProp,
configPathDefaultProp);
}
PropertyBasedConfigurator(String propertyNameProp, String defaultImplClassProp,
String configurationPathProp) {
Properties props = System.getProperties();
this.propertyName = props.getProperty(propertyNameProp);
this.defaultImplClass = props.getProperty(defaultImplClassProp);
this.configurationPath = props.getProperty(configurationPathProp);
}
public String getPropertyName() {
return propertyName;
}
public String getDefaultImplClass() {
return defaultImplClass;
}
public String getConfigurationPath() {
return configurationPath;
}
public Configuration getConfiguration() {
String path = getConfigurationPath();
Configuration config = null;
try {
config = new PropertiesConfiguration(path);
} catch (ConfigurationException e) {
config = new PropertiesConfiguration();
}
return config;
}
public String getClassName() {
Configuration config = getConfiguration();
String propName = getPropertyName();
String defaultClass = getDefaultImplClass();
return config.getString(propName, defaultClass);
}
@SuppressWarnings("unchecked")
public Class<? extends T> getImplClass() {
String className = getClassName();
Class<? extends T> ret = null;
try {
ret = (Class<? extends T>) PropertyBasedConfigurator.class
.getClassLoader().loadClass(className);
} catch (ClassNotFoundException e) {
throw new RuntimeException(e);
}
return ret;
}
}
repository/src/main/java/org/apache/hadoop/metadata/services/TitanGraphProvider.java 0 → 100644
package org.apache.hadoop.metadata.services;
import javax.inject.Singleton;
import org.apache.commons.configuration.Configuration;
import org.apache.commons.configuration.ConfigurationException;
import org.apache.commons.configuration.PropertiesConfiguration;
import com.thinkaurelius.titan.core.TitanFactory;
import com.thinkaurelius.titan.core.TitanGraph;
public class TitanGraphProvider implements GraphProvider<TitanGraph> {
private static final String SYSTEM_PROP = "";
private static final String DEFAULT_PATH = "graph.properties";
private final String configPath;
public TitanGraphProvider() {
configPath = System.getProperties().getProperty(SYSTEM_PROP,
DEFAULT_PATH);
}
public Configuration getConfiguration() throws ConfigurationException {
return new PropertiesConfiguration(configPath);
}
@Override
@Singleton
public TitanGraph get() throws ConfigurationException {
TitanGraph graph = null;
Configuration config;
try {
config = getConfiguration();
} catch (ConfigurationException e) {
throw new RuntimeException(e);
}
graph = TitanFactory.open(config);
return graph;
}
}
repository/src/main/java/org/apache/hadoop/metadata/services/TitanGraphService.java
...@@ -18,7 +18,22 @@ ...@@ -18,7 +18,22 @@
package org.apache.hadoop.metadata.services; package org.apache.hadoop.metadata.services;
import com.thinkaurelius.titan.core.TitanFactory; import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
import javax.inject.Inject;
import javax.inject.Singleton;
import org.apache.commons.configuration.Configuration;
import org.apache.commons.configuration.ConfigurationException;
import org.apache.commons.configuration.PropertiesConfiguration;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.thinkaurelius.titan.core.TitanGraph; import com.thinkaurelius.titan.core.TitanGraph;
import com.thinkaurelius.titan.core.schema.TitanGraphIndex; import com.thinkaurelius.titan.core.schema.TitanGraphIndex;
import com.thinkaurelius.titan.core.schema.TitanManagement; import com.thinkaurelius.titan.core.schema.TitanManagement;
...@@ -28,119 +43,84 @@ import com.tinkerpop.blueprints.KeyIndexableGraph; ...@@ -28,119 +43,84 @@ import com.tinkerpop.blueprints.KeyIndexableGraph;
import com.tinkerpop.blueprints.TransactionalGraph; import com.tinkerpop.blueprints.TransactionalGraph;
import com.tinkerpop.blueprints.Vertex; import com.tinkerpop.blueprints.Vertex;
import org.apache.commons.configuration.Configuration;
import org.apache.commons.configuration.ConfigurationException;
import org.apache.commons.configuration.PropertiesConfiguration;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
/** /**
* Default implementation for Graph service backed by Titan. * Default implementation for Graph service backed by Titan.
*/ */
@Singleton
public class TitanGraphService implements GraphService { public class TitanGraphService implements GraphService {
private static final Logger LOG = LoggerFactory.getLogger(TitanGraphService.class); private static final Logger LOG = LoggerFactory.getLogger(TitanGraphService.class);
public static final String NAME = TitanGraphService.class.getSimpleName();
/**
/** * Constant for the configuration property that indicates the prefix.
* Constant for the configuration property that indicates the prefix. */
*/ private static final String INDEXER_PREFIX = "metadata.indexer.vertex.";
private static final String METADATA_PREFIX = "metadata.graph."; private static final List<String> acceptedTypes = Arrays.asList("String", "Int", "Long");
private static final String INDEXER_PREFIX = "metadata.indexer.vertex.";
private static final List<String> acceptedTypes = Arrays.asList("String","Int","Long"); private final TitanGraph titanGraph;
private TitanGraph titanGraph; /**
private Set<String> vertexIndexedKeys; * Initialize this service through injection with a custom Provider.
private Set<String> edgeIndexedKeys; *
* @param graph
/** * @throws ConfigurationException
* Name of the service. */
* @Inject
* @return name of the service TitanGraphService(GraphProvider<TitanGraph> graph) throws ConfigurationException {
*/ // TODO reimplement to save the Provider and initialize the graph inside the start() method
@Override this.titanGraph = graph.get();
public String getName() { //start();
return NAME; }
}
/**
/** * Initializes this Service. The starting of Titan is handled by the Provider
* Starts the service. This method blocks until the service has completely started. * @throws ConfigurationException
* */
* @throws Exception @Override
*/ public void start() throws ConfigurationException {
@Override createIndicesForVertexKeys();
public void start() throws Exception { // todo - create Edge Cardinality Constraints
Configuration graphConfig = getConfiguration(); LOG.info("Initialized titanGraph db: {}", titanGraph);
titanGraph = initializeGraphDB(graphConfig);
Set<String> vertexIndexedKeys = getVertexIndexedKeys();
createIndicesForVertexKeys(); LOG.info("Init vertex property keys: {}", vertexIndexedKeys);
// todo - create Edge Cardinality Constraints
LOG.info("Initialized titanGraph db: {}", titanGraph); Set<String> edgeIndexedKeys = getEdgeIndexedKeys();
LOG.info("Init edge property keys: {}", edgeIndexedKeys);
vertexIndexedKeys = getIndexableGraph().getIndexedKeys(Vertex.class); }
LOG.info("Init vertex property keys: {}", vertexIndexedKeys);
private static Configuration getConfiguration(String filename, String prefix)
edgeIndexedKeys = getIndexableGraph().getIndexedKeys(Edge.class); throws ConfigurationException {
LOG.info("Init edge property keys: {}", edgeIndexedKeys); PropertiesConfiguration configProperties = new PropertiesConfiguration(
} filename);
private static Configuration getConfiguration() throws ConfigurationException { Configuration graphConfig = new PropertiesConfiguration();
PropertiesConfiguration configProperties =
new PropertiesConfiguration("application.properties"); final Iterator<String> iterator = configProperties.getKeys();
while (iterator.hasNext()) {
Configuration graphConfig = new PropertiesConfiguration(); String key = iterator.next();
final Iterator<String> iterator = configProperties.getKeys(); if (key.startsWith(prefix)) {
while (iterator.hasNext()) { String value = (String) configProperties.getProperty(key);
String key = iterator.next(); key = key.substring(prefix.length());
if (key.startsWith(METADATA_PREFIX)) { graphConfig.setProperty(key, value);
String value = (String) configProperties.getProperty(key); }
key = key.substring(METADATA_PREFIX.length()); }
graphConfig.setProperty(key, value);
} return graphConfig;
} }
return graphConfig; /**
} * Initializes the indices for the graph.
* @throws ConfigurationException
private static Configuration getConfiguration(String filename, String prefix) throws ConfigurationException { */
PropertiesConfiguration configProperties = // TODO move this functionality to the MetadataRepository?
new PropertiesConfiguration(filename); protected void createIndicesForVertexKeys() throws ConfigurationException {
if (!titanGraph.getIndexedKeys(Vertex.class).isEmpty()) {
Configuration graphConfig = new PropertiesConfiguration(); LOG.info("Indexes already exist for titanGraph");
return;
final Iterator<String> iterator = configProperties.getKeys(); }
while (iterator.hasNext()) {
String key = iterator.next(); LOG.info("Indexes do not exist, Creating indexes for titanGraph using indexer.properties.");
if (key.startsWith(prefix)) {
String value = (String) configProperties.getProperty(key);
key = key.substring(prefix.length());
graphConfig.setProperty(key, value);
}
}
return graphConfig;
}
protected TitanGraph initializeGraphDB(Configuration graphConfig) {
LOG.info("Initializing titanGraph db");
return TitanFactory.open(graphConfig);
}
protected void createIndicesForVertexKeys() throws ConfigurationException {
if (!titanGraph.getIndexedKeys(Vertex.class).isEmpty()) {
LOG.info("Indexes already exist for titanGraph");
return;
}
LOG.info("Indexes do not exist, Creating indexes for titanGraph using indexer.properties.");
TitanManagement mgmt = titanGraph.getManagementSystem(); TitanManagement mgmt = titanGraph.getManagementSystem();
mgmt.buildIndex("mainIndex", Vertex.class).buildMixedIndex("search"); mgmt.buildIndex("mainIndex", Vertex.class).buildMixedIndex("search");
...@@ -154,95 +134,110 @@ public class TitanGraphService implements GraphService { ...@@ -154,95 +134,110 @@ public class TitanGraphService implements GraphService {
if (!indexConfig.isEmpty()) { if (!indexConfig.isEmpty()) {
// Get a list of property names to iterate through... // Get a list of property names to iterate through...
List<String> propList = new ArrayList<String>(); List<String> propList = new ArrayList<String>();
Iterator<String> it = indexConfig.getKeys("property.name"); Iterator<String> it = indexConfig.getKeys("property.name");
while (it.hasNext()) { while (it.hasNext()) {
propList.add(it.next()); propList.add(it.next());
} }
it = propList.iterator(); it = propList.iterator();
while (it.hasNext()) { while (it.hasNext()) {
// Pull the property name and index, so we can register the name and look up the type. // Pull the property name and index, so we can register the name
// and look up the type.
String prop = it.next().toString(); String prop = it.next().toString();
String index = prop.substring(prop.lastIndexOf(".") + 1); String index = prop.substring(prop.lastIndexOf(".") + 1);
String type = null; String type = null;
prop = indexConfig.getProperty(prop).toString(); prop = indexConfig.getProperty(prop).toString();
// Look up the type for the specified property name. // Look up the type for the specified property name.
if (indexConfig.containsKey("property.type." + index)) { if (indexConfig.containsKey("property.type." + index)) {
type = indexConfig.getProperty("property.type." + index).toString(); type = indexConfig.getProperty("property.type." + index)
.toString();
} else { } else {
throw new ConfigurationException("No type specified for property " + index + " in indexer.properties."); throw new ConfigurationException(
"No type specified for property " + index
+ " in indexer.properties.");
} }
// Is the type submitted one of the approved ones? // Is the type submitted one of the approved ones?
if (!acceptedTypes.contains(type)) { if (!acceptedTypes.contains(type)) {
throw new ConfigurationException("The type provided in indexer.properties for property " + prop + " is not supported. Supported types are: " + acceptedTypes.toString()); throw new ConfigurationException(
"The type provided in indexer.properties for property "
+ prop
+ " is not supported. Supported types are: "
+ acceptedTypes.toString());
} }
// Add the key. // Add the key.
LOG.info("Adding property: " + prop + " to index as type: " + type); LOG.info("Adding property: " + prop + " to index as type: "
mgmt.addIndexKey(graphIndex,mgmt.makePropertyKey(prop).dataType(type.getClass()).make()); + type);
mgmt.addIndexKey(graphIndex, mgmt.makePropertyKey(prop)
} .dataType(type.getClass()).make());
}
mgmt.commit(); mgmt.commit();
LOG.info("Index creation complete."); LOG.info("Index creation complete.");
} }
} }
/** /**
* Stops the service. This method blocks until the service has completely shut down. * Stops the service. This method blocks until the service has completely
*/ * shut down.
@Override */
public void stop() { @Override
if (titanGraph != null) { public void stop() {
titanGraph.shutdown(); if (titanGraph != null) {
} titanGraph.shutdown();
} }
}
/**
* A version of stop() that is designed to be usable in Java7 closure /**
* clauses. * A version of stop() that is designed to be usable in Java7 closure
* Implementation classes MUST relay this directly to {@link #stop()} * clauses. Implementation classes MUST relay this directly to
* * {@link #stop()}
* @throws java.io.IOException never *
* @throws RuntimeException on any failure during the stop operation * @throws java.io.IOException
*/ * never
@Override * @throws RuntimeException
public void close() throws IOException { * on any failure during the stop operation
stop(); */
} @Override
public void close() throws IOException {
@Override stop();
public Graph getBlueprintsGraph() { }
return titanGraph;
} @Override
public Graph getBlueprintsGraph() {
@Override return titanGraph;
public KeyIndexableGraph getIndexableGraph() { }
return titanGraph;
} @Override
public KeyIndexableGraph getIndexableGraph() {
@Override return titanGraph;
public TransactionalGraph getTransactionalGraph() { }
return titanGraph;
} @Override
public TransactionalGraph getTransactionalGraph() {
public TitanGraph getTitanGraph() { return titanGraph;
return titanGraph; }
}
public TitanGraph getTitanGraph() {
@Override return titanGraph;
public Set<String> getVertexIndexedKeys() { }
return vertexIndexedKeys;
} @Override
public Set<String> getVertexIndexedKeys() {
@Override // this must use the graph API instead of setting this value as a class member - it can change after creation
public Set<String> getEdgeIndexedKeys() { return getIndexableGraph().getIndexedKeys(Vertex.class);
return edgeIndexedKeys; }
}
@Override
public Set<String> getEdgeIndexedKeys() {
// this must use the graph API instead of setting this value as a class member - it can change after creation
return getIndexableGraph().getIndexedKeys(Edge.class);
}
} }
repository/src/main/java/org/apache/hadoop/metadata/services/TypesService.java deleted 100644 → 0
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.metadata.services;
import org.apache.hadoop.metadata.service.Service;
import org.apache.hadoop.metadata.types.TypeSystem;
public interface TypesService extends Service {
TypeSystem getTypeSystem();
}
repository/src/test/java/org/apache/hadoop/metadata/GuiceEnabledTestBase.java 0 → 100644
package org.apache.hadoop.metadata;
import com.google.inject.Guice;
import com.google.inject.Injector;
import com.google.inject.Module;
public abstract class GuiceEnabledTestBase {
/*
* Guice.createInjector() takes your Modules, and returns a new Injector
* instance. Most applications will call this method exactly once, in their
* main() method.
*/
public final Injector injector;
GuiceEnabledTestBase() {
injector = Guice.createInjector();
}
GuiceEnabledTestBase(Module... modules) {
injector = Guice.createInjector(modules);
}
}
repository/src/test/java/org/apache/hadoop/metadata/RepositoryModuleBaseTest.java 0 → 100644
package org.apache.hadoop.metadata;
public abstract class RepositoryModuleBaseTest extends GuiceEnabledTestBase {
public RepositoryModuleBaseTest() {
super(new RepositoryMetadataModule());
}
}
repository/src/test/java/org/apache/hadoop/metadata/RepositoryServiceLoadingTest.java 0 → 100644
package org.apache.hadoop.metadata;
import junit.framework.Assert;
import org.apache.hadoop.metadata.services.GraphService;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
/**
* Unit test for Guice injector service loading
*/
public class RepositoryServiceLoadingTest extends GuiceEnabledTestBase {
public RepositoryServiceLoadingTest() {
super(new RepositoryMetadataModule());
}
@BeforeClass
public void setUp() throws Exception {
}
@AfterClass
public void tearDown() throws Exception {
}
@Test
public void testGetGraphService() throws Exception {
/*
* Now that we've got the injector, we can build objects.
*/
GraphService gs = injector.getInstance(GraphService.class);
Assert.assertNotNull(gs);
}
}
repository/src/test/java/org/apache/hadoop/metadata/services/GraphBackedMetadataRepositoryTest.java
package org.apache.hadoop.metadata.services; package org.apache.hadoop.metadata.services;
import com.google.common.collect.ImmutableList; import java.util.List;
import com.thinkaurelius.titan.core.TitanGraph;
import com.tinkerpop.blueprints.Direction;
import com.tinkerpop.blueprints.Edge;
import com.tinkerpop.blueprints.Vertex;
import org.apache.hadoop.metadata.ITypedReferenceableInstance; import org.apache.hadoop.metadata.ITypedReferenceableInstance;
import org.apache.hadoop.metadata.MetadataException; import org.apache.hadoop.metadata.MetadataException;
import org.apache.hadoop.metadata.MetadataService;
import org.apache.hadoop.metadata.Referenceable; import org.apache.hadoop.metadata.Referenceable;
import org.apache.hadoop.metadata.service.Services; import org.apache.hadoop.metadata.RepositoryModuleBaseTest;
import org.apache.hadoop.metadata.storage.IRepository;
import org.apache.hadoop.metadata.storage.memory.MemRepository; import org.apache.hadoop.metadata.storage.memory.MemRepository;
import org.apache.hadoop.metadata.types.AttributeDefinition; import org.apache.hadoop.metadata.types.AttributeDefinition;
import org.apache.hadoop.metadata.types.ClassType; import org.apache.hadoop.metadata.types.ClassType;
...@@ -27,59 +24,44 @@ import org.testng.annotations.AfterClass; ...@@ -27,59 +24,44 @@ import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass; import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test; import org.testng.annotations.Test;
import java.util.List; import com.google.common.collect.ImmutableList;
import com.thinkaurelius.titan.core.TitanGraph;
import com.tinkerpop.blueprints.Direction;
import com.tinkerpop.blueprints.Edge;
import com.tinkerpop.blueprints.Vertex;
public class GraphBackedMetadataRepositoryTest { public class GraphBackedMetadataRepositoryTest extends RepositoryModuleBaseTest {
private static final String ENTITY_TYPE = "hive-table"; private static final String ENTITY_TYPE = "hive-table";
private TitanGraphService titanGraphService; private TitanGraphService titanGraphService;
private GraphBackedMetadataRepository repositoryService; private GraphBackedMetadataRepository repositoryService;
protected org.apache.hadoop.metadata.MetadataService ms; private IRepository repo;
private TypeSystem ts;
private String guid; private String guid;
@BeforeClass @BeforeClass
public void setUp() throws Exception { public void setUp() throws Exception {
titanGraphService = new TitanGraphService(); titanGraphService = super.injector.getInstance(TitanGraphService.class);
titanGraphService.start(); titanGraphService.start();
Services.get().register(titanGraphService);
DefaultTypesService typesService = new DefaultTypesService();
typesService.start();
Services.get().register(typesService);
TypeSystem ts = typesService.getTypeSystem();
repositoryService = new GraphBackedMetadataRepository(); repositoryService = super.injector.getInstance(GraphBackedMetadataRepository.class);
repositoryService.start(); repositoryService.start();
Services.get().register(repositoryService);
// todo - only used for types ts = new TypeSystem();
MemRepository mr = new MemRepository(ts); repo = new MemRepository(ts);
ms = new org.apache.hadoop.metadata.MetadataService(mr, ts);
MetadataService.setCurrentService(ms);
defineDeptEmployeeTypes(ts); defineDeptEmployeeTypes(ts);
} }
@AfterClass @AfterClass
public void tearDown() throws Exception { public void tearDown() throws Exception {
Services.get().getService(GraphBackedMetadataRepository.NAME).close();
Services.get().getService(TitanGraphService.NAME).close();
Services.get().reset();
}
@Test
public void testGetName() throws Exception {
Assert.assertEquals(GraphBackedMetadataRepository.NAME,
GraphBackedMetadataRepository.class.getSimpleName());
Assert.assertEquals(repositoryService.getName(), GraphBackedMetadataRepository.NAME);
} }
@Test @Test
public void testSubmitEntity() throws Exception { public void testSubmitEntity() throws Exception {
TypeSystem typeSystem = MetadataService.getCurrentService().getTypeSystem(); Referenceable hrDept = createDeptEg1(ts);
Referenceable hrDept = createDeptEg1(typeSystem); ClassType deptType = ts.getDataType(ClassType.class, "Department");
ClassType deptType = typeSystem.getDataType(ClassType.class, "Department");
ITypedReferenceableInstance hrDept2 = deptType.convert(hrDept, Multiplicity.REQUIRED); ITypedReferenceableInstance hrDept2 = deptType.convert(hrDept, Multiplicity.REQUIRED);
guid = repositoryService.createEntity(hrDept2, ENTITY_TYPE); guid = repositoryService.createEntity(hrDept2, ENTITY_TYPE);
...@@ -118,20 +100,6 @@ public class GraphBackedMetadataRepositoryTest { ...@@ -118,20 +100,6 @@ public class GraphBackedMetadataRepositoryTest {
Assert.assertEquals(entityList.size(), 0); // as this is not implemented yet Assert.assertEquals(entityList.size(), 0); // as this is not implemented yet
} }
@Test(expectedExceptions = RuntimeException.class)
public void testStartWithOutGraphServiceRegistration() throws Exception {
try {
Services.get().reset();
GraphBackedMetadataRepository repositoryService = new
GraphBackedMetadataRepository();
repositoryService.start();
Assert.fail("This should have thrown an exception");
} finally {
Services.get().register(titanGraphService);
Services.get().register(repositoryService);
}
}
/* /*
* Class Hierarchy is: * Class Hierarchy is:
* Department(name : String, employees : Array[Person]) * Department(name : String, employees : Array[Person])
...@@ -184,7 +152,7 @@ public class GraphBackedMetadataRepositoryTest { ...@@ -184,7 +152,7 @@ public class GraphBackedMetadataRepositoryTest {
ts.getDataType(ClassType.class, "Manager") ts.getDataType(ClassType.class, "Manager")
); );
ms.getRepository().defineTypes(types); repo.defineTypes(types);
} }
protected Referenceable createDeptEg1(TypeSystem ts) throws MetadataException { protected Referenceable createDeptEg1(TypeSystem ts) throws MetadataException {
...@@ -228,4 +196,4 @@ public class GraphBackedMetadataRepositoryTest { ...@@ -228,4 +196,4 @@ public class GraphBackedMetadataRepositoryTest {
String name, ImmutableList<String> superTypes, AttributeDefinition... attrDefs) { String name, ImmutableList<String> superTypes, AttributeDefinition... attrDefs) {
return new HierarchicalTypeDefinition(ClassType.class, name, superTypes, attrDefs); return new HierarchicalTypeDefinition(ClassType.class, name, superTypes, attrDefs);
} }
} }
\ No newline at end of file
repository/src/test/java/org/apache/hadoop/metadata/services/TitanGraphServiceTest.java
package org.apache.hadoop.metadata.services; package org.apache.hadoop.metadata.services;
import org.apache.hadoop.metadata.RepositoryModuleBaseTest;
import org.testng.Assert; import org.testng.Assert;
import org.testng.annotations.AfterClass; import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass; import org.testng.annotations.BeforeClass;
...@@ -8,13 +9,14 @@ import org.testng.annotations.Test; ...@@ -8,13 +9,14 @@ import org.testng.annotations.Test;
/** /**
* Unit test for TitanGraphService. * Unit test for TitanGraphService.
*/ */
public class TitanGraphServiceTest { public class TitanGraphServiceTest extends RepositoryModuleBaseTest {
private TitanGraphService titanGraphService; private TitanGraphService titanGraphService;
@BeforeClass @BeforeClass
public void setUp() throws Exception { public void setUp() throws Exception {
titanGraphService = new TitanGraphService(); titanGraphService = super.injector.getInstance(TitanGraphService.class);
//titanGraphService = new TitanGraphService();
titanGraphService.start(); titanGraphService.start();
} }
...@@ -24,12 +26,6 @@ public class TitanGraphServiceTest { ...@@ -24,12 +26,6 @@ public class TitanGraphServiceTest {
} }
@Test @Test
public void testGetName() throws Exception {
Assert.assertEquals(TitanGraphService.NAME, TitanGraphService.class.getSimpleName());
Assert.assertEquals(titanGraphService.getName(), TitanGraphService.NAME);
}
@Test
public void testStart() throws Exception { public void testStart() throws Exception {
Assert.assertNotNull(titanGraphService.getBlueprintsGraph()); Assert.assertNotNull(titanGraphService.getBlueprintsGraph());
} }
...@@ -65,4 +61,4 @@ public class TitanGraphServiceTest { ...@@ -65,4 +61,4 @@ public class TitanGraphServiceTest {
Assert.assertNotNull(titanGraphService.getEdgeIndexedKeys()); Assert.assertNotNull(titanGraphService.getEdgeIndexedKeys());
Assert.assertTrue(titanGraphService.getEdgeIndexedKeys().size() > 0); Assert.assertTrue(titanGraphService.getEdgeIndexedKeys().size() > 0);
} }
} }
\ No newline at end of file
repository/src/test/resources/application.properties
...@@ -16,21 +16,7 @@ ...@@ -16,21 +16,7 @@
# limitations under the License. # limitations under the License.
# #
application.services=org.apache.hadoop.metadata.services.TitanGraphService,\ # GraphService implementation
org.apache.hadoop.metadata.services.GraphBackedMetadataRepositoryService metadata.graph.impl.class=org.apache.hadoop.metadata.services.TitanGraphService
# Graph implementation
#metadata.graph.blueprints.graph=com.thinkaurelius.titan.core.TitanFactory
# Graph Storage
metadata.graph.storage.backend=berkeleyje
metadata.graph.storage.directory=target/data/berkeley
# Graph Search Index
metadata.graph.index.search.backend=elasticsearch
metadata.graph.index.search.directory=target/data/es
metadata.graph.index.search.elasticsearch.client-only=false
metadata.graph.index.search.elasticsearch.local-mode=true
metadata.enableTLS=false metadata.enableTLS=false
repository/src/test/resources/graph.properties 0 → 100644
storage.backend=inmemory
# Graph Search Index
index.search.backend=elasticsearch
index.search.directory=target/data/es
index.search.elasticsearch.client-only=false
index.search.elasticsearch.local-mode=true
\ No newline at end of file
repository/src/test/resources/indexer.properties
# #
# Licensed to the Apache Software Foundation (ASF) under one # Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file # or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information # distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file # regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the # to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance # "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at # with the License. You may obtain a copy of the License at
# #
# http://www.apache.org/licenses/LICENSE-2.0 # http://www.apache.org/licenses/LICENSE-2.0
# #
# Unless required by applicable law or agreed to in writing, software # Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, # distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
# #
# This is formatted as follows: # This is formatted as follows:
# metadata.indexer.vertex.property.name.<index>=<Property Name> # metadata.indexer.vertex.property.name.<index>=<Property Name>
# metadata.indexer.vertex.property.type.<index>=<Data Type> # metadata.indexer.vertex.property.type.<index>=<Data Type>
metadata.indexer.vertex.property.name.0=DESC metadata.indexer.vertex.property.name.0=DESC
metadata.indexer.vertex.property.type.0=String metadata.indexer.vertex.property.type.0=String
metadata.indexer.vertex.property.name.1=DB_LOCATION_URI metadata.indexer.vertex.property.name.1=DB_LOCATION_URI
metadata.indexer.vertex.property.type.1=String metadata.indexer.vertex.property.type.1=String
metadata.indexer.vertex.property.name.2=NAME metadata.indexer.vertex.property.name.2=NAME
metadata.indexer.vertex.property.type.2=String metadata.indexer.vertex.property.type.2=String
metadata.indexer.vertex.property.name.3=OWNER_NAME metadata.indexer.vertex.property.name.3=OWNER_NAME
metadata.indexer.vertex.property.type.3=String metadata.indexer.vertex.property.type.3=String
metadata.indexer.vertex.property.name.4=TBL_NAME metadata.indexer.vertex.property.name.4=TBL_NAME
metadata.indexer.vertex.property.type.4=String metadata.indexer.vertex.property.type.4=String
metadata.indexer.vertex.property.name.5=COMMENT metadata.indexer.vertex.property.name.5=COMMENT
metadata.indexer.vertex.property.type.5=String metadata.indexer.vertex.property.type.5=String
metadata.indexer.vertex.property.name.6=COLUMN_NAME metadata.indexer.vertex.property.name.6=COLUMN_NAME
metadata.indexer.vertex.property.type.6=String metadata.indexer.vertex.property.type.6=String
metadata.indexer.vertex.property.name.7=TYPE_NAME metadata.indexer.vertex.property.name.7=TYPE_NAME
metadata.indexer.vertex.property.type.7=String metadata.indexer.vertex.property.type.7=String
typesystem/src/main/java/org/apache/hadoop/metadata/MetadataService.java
...@@ -22,6 +22,7 @@ package org.apache.hadoop.metadata; ...@@ -22,6 +22,7 @@ package org.apache.hadoop.metadata;
import org.apache.hadoop.metadata.storage.IRepository; import org.apache.hadoop.metadata.storage.IRepository;
import org.apache.hadoop.metadata.types.TypeSystem; import org.apache.hadoop.metadata.types.TypeSystem;
// TODO get rid of this class in favor of Dependency injection
public class MetadataService { public class MetadataService {
final IRepository repo; final IRepository repo;
......
webapp/pom.xml
...@@ -127,6 +127,21 @@ ...@@ -127,6 +127,21 @@
<groupId>org.testng</groupId> <groupId>org.testng</groupId>
<artifactId>testng</artifactId> <artifactId>testng</artifactId>
</dependency> </dependency>
<dependency>
<groupId>com.google.inject.extensions</groupId>
<artifactId>guice-servlet</artifactId>
<version>3.0</version>
</dependency>
<dependency>
<groupId>com.google.code.gson</groupId>
<artifactId>gson</artifactId>
<version>2.3.1</version>
</dependency>
<dependency>
<groupId>com.sun.jersey.contribs</groupId>
<artifactId>jersey-guice</artifactId>
<version>1.18.3</version>
</dependency>
</dependencies> </dependencies>
<build> <build>
...@@ -196,15 +211,15 @@ ...@@ -196,15 +211,15 @@
<artifactId>keytool-maven-plugin</artifactId> <artifactId>keytool-maven-plugin</artifactId>
<executions> <executions>
<execution> <execution>
<id>clean</id>
<phase>generate-resources</phase> <phase>generate-resources</phase>
<id>clean-server</id>
<goals> <goals>
<goal>clean</goal> <goal>clean</goal>
</goals> </goals>
</execution> </execution>
<execution> <execution>
<id>genkey</id>
<phase>generate-resources</phase> <phase>generate-resources</phase>
<id>server</id>
<goals> <goals>
<goal>generateKeyPair</goal> <goal>generateKeyPair</goal>
</goals> </goals>
...@@ -241,6 +256,7 @@ ...@@ -241,6 +256,7 @@
</connector> </connector>
</connectors> </connectors>
<webApp>${project.build.directory}/metadata-webapp-${project.version}</webApp> <webApp>${project.build.directory}/metadata-webapp-${project.version}</webApp>
<contextPath>/</contextPath>
<useTestClasspath>true</useTestClasspath> <useTestClasspath>true</useTestClasspath>
<systemProperties> <systemProperties>
<systemProperty> <systemProperty>
......
webapp/src/main/java/org/apache/hadoop/metadata/web/listeners/ApplicationStartupListener.java deleted 100644 → 0
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.metadata.web.listeners;
import org.apache.commons.configuration.ConfigurationException;
import org.apache.commons.configuration.PropertiesConfiguration;
import org.apache.hadoop.metadata.MetadataException;
import org.apache.hadoop.metadata.service.ServiceInitializer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.servlet.ServletContextEvent;
import javax.servlet.ServletContextListener;
/**
* Listener for bootstrapping Services and configuration properties.
*/
public class ApplicationStartupListener implements ServletContextListener {
private static final Logger LOG = LoggerFactory.getLogger(ApplicationStartupListener.class);
private final ServiceInitializer startupServices = new ServiceInitializer();
@Override
public void contextInitialized(ServletContextEvent sce) {
try {
startupServices.initialize();
showStartupInfo();
} catch (MetadataException e) {
throw new RuntimeException("Error starting services", e);
}
}
private void showStartupInfo() {
StringBuilder buffer = new StringBuilder();
buffer.append("\n############################################");
buffer.append("\n Metadata Server (STARTED) ");
buffer.append("\n############################################");
try {
PropertiesConfiguration configuration = new PropertiesConfiguration("application.properties");
buffer.append(configuration.toString());
} catch (ConfigurationException e) {
buffer.append("*** Unable to get build info ***").append(e.getMessage());
}
LOG.info(buffer.toString());
}
@Override
public void contextDestroyed(ServletContextEvent sce) {
try {
startupServices.destroy();
} catch (MetadataException e) {
LOG.warn("Error destroying services", e);
}
StringBuilder buffer = new StringBuilder();
buffer.append("\n############################################");
buffer.append("\n Metadata Server (SHUTDOWN) ");
buffer.append("\n############################################");
LOG.info(buffer.toString());
}
}
\ No newline at end of file
webapp/src/main/java/org/apache/hadoop/metadata/web/listeners/GuiceServletConfig.java 0 → 100644
package org.apache.hadoop.metadata.web.listeners;
import java.util.HashMap;
import java.util.Map;
import org.apache.hadoop.metadata.RepositoryMetadataModule;
import org.apache.hadoop.metadata.services.GraphService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.inject.Guice;
import com.google.inject.Injector;
import com.google.inject.servlet.GuiceServletContextListener;
import com.sun.jersey.api.core.PackagesResourceConfig;
import com.sun.jersey.guice.JerseyServletModule;
import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
public class GuiceServletConfig extends GuiceServletContextListener {
private static final Logger LOG = LoggerFactory
.getLogger(GuiceServletConfig.class);
private static final String GUICE_CTX_PARAM = "guice.packages";
@Override
protected Injector getInjector() {
LOG.info("Loading Guice modules");
/*
* More information on this can be found here:
* https://jersey.java.net/nonav/apidocs/1.11/contribs/jersey-guice/com/sun/jersey/guice/spi/container/servlet/package-summary.html
*/
Injector injector = Guice.createInjector(
new RepositoryMetadataModule(),
new JerseyServletModule() {
@Override
protected void configureServlets() {
String packages = getServletContext().getInitParameter(GUICE_CTX_PARAM);
LOG.info("Jersey loading from packages: " + packages);
Map<String, String> params = new HashMap<String, String>();
params.put(PackagesResourceConfig.PROPERTY_PACKAGES, packages);
serve("/api/metadata/*").with(GuiceContainer.class, params);
}
});
LOG.info("Guice modules loaded");
LOG.info("Bootstrapping services");
// get the Graph Service
GraphService graphService = injector.getInstance(GraphService.class);
try {
// start/init the service
graphService.start();
} catch (Exception e) {
throw new RuntimeException(e);
}
LOG.info(String.format("Loaded Service: %s", graphService.getClass().getName()));
LOG.info("Services bootstrapped successfully");
return injector;
}
}
webapp/src/main/java/org/apache/hadoop/metadata/web/resources/AdminResource.java
...@@ -23,6 +23,7 @@ import org.apache.hadoop.metadata.web.util.Servlets; ...@@ -23,6 +23,7 @@ import org.apache.hadoop.metadata.web.util.Servlets;
import org.codehaus.jettison.json.JSONException; import org.codehaus.jettison.json.JSONException;
import org.codehaus.jettison.json.JSONObject; import org.codehaus.jettison.json.JSONObject;
import javax.inject.Singleton;
import javax.ws.rs.GET; import javax.ws.rs.GET;
import javax.ws.rs.Path; import javax.ws.rs.Path;
import javax.ws.rs.Produces; import javax.ws.rs.Produces;
...@@ -34,6 +35,7 @@ import javax.ws.rs.core.Response; ...@@ -34,6 +35,7 @@ import javax.ws.rs.core.Response;
* Jersey Resource for admin operations. * Jersey Resource for admin operations.
*/ */
@Path("admin") @Path("admin")
@Singleton
public class AdminResource { public class AdminResource {
@GET @GET
......
webapp/src/main/java/org/apache/hadoop/metadata/web/resources/EntityResource.java
...@@ -18,19 +18,11 @@ ...@@ -18,19 +18,11 @@
package org.apache.hadoop.metadata.web.resources; package org.apache.hadoop.metadata.web.resources;
import com.google.common.base.Preconditions; import java.io.IOException;
import org.apache.commons.io.IOUtils; import java.io.StringWriter;
import org.apache.hadoop.metadata.MetadataException;
import org.apache.hadoop.metadata.service.Services;
import org.apache.hadoop.metadata.services.DefaultMetadataService;
import org.apache.hadoop.metadata.services.MetadataService;
import org.apache.hadoop.metadata.web.util.Servlets;
import org.codehaus.jettison.json.JSONObject;
import org.json.simple.JSONValue;
import org.json.simple.parser.ParseException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.inject.Inject;
import javax.inject.Singleton;
import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletRequest;
import javax.ws.rs.Consumes; import javax.ws.rs.Consumes;
import javax.ws.rs.DELETE; import javax.ws.rs.DELETE;
...@@ -45,8 +37,18 @@ import javax.ws.rs.WebApplicationException; ...@@ -45,8 +37,18 @@ import javax.ws.rs.WebApplicationException;
import javax.ws.rs.core.Context; import javax.ws.rs.core.Context;
import javax.ws.rs.core.MediaType; import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response; import javax.ws.rs.core.Response;
import java.io.IOException;
import java.io.StringWriter; import org.apache.commons.io.IOUtils;
import org.apache.hadoop.metadata.MetadataException;
import org.apache.hadoop.metadata.services.MetadataService;
import org.apache.hadoop.metadata.web.util.Servlets;
import org.codehaus.jettison.json.JSONObject;
import org.json.simple.JSONValue;
import org.json.simple.parser.ParseException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.base.Preconditions;
/** /**
* Entity management operations as REST API. * Entity management operations as REST API.
...@@ -55,17 +57,22 @@ import java.io.StringWriter; ...@@ -55,17 +57,22 @@ import java.io.StringWriter;
* of the Type they correspond with. * of the Type they correspond with.
*/ */
@Path("entities") @Path("entities")
@Singleton
public class EntityResource { public class EntityResource {
private static final Logger LOG = LoggerFactory.getLogger(EntityResource.class); private static final Logger LOG = LoggerFactory.getLogger(EntityResource.class);
private MetadataService metadataService; private final MetadataService metadataService;
public EntityResource() { /**
metadataService = Services.get().getService(DefaultMetadataService.NAME); * Created by the Guice ServletModule and injected with the
if (metadataService == null) { * configured MetadataService.
throw new RuntimeException("graph service is not initialized"); *
} * @param metadataService
*/
@Inject
public EntityResource(MetadataService metadataService) {
this.metadataService = metadataService;
} }
@POST @POST
......
webapp/src/main/java/org/apache/hadoop/metadata/web/resources/MetadataDiscoveryResource.java
...@@ -18,6 +18,7 @@ ...@@ -18,6 +18,7 @@
package org.apache.hadoop.metadata.web.resources; package org.apache.hadoop.metadata.web.resources;
import javax.inject.Singleton;
import javax.ws.rs.Path; import javax.ws.rs.Path;
/** /**
...@@ -30,5 +31,6 @@ import javax.ws.rs.Path; ...@@ -30,5 +31,6 @@ import javax.ws.rs.Path;
* 'search': find entities generated by Hive processes or that were generated by Sqoop, etc. * 'search': find entities generated by Hive processes or that were generated by Sqoop, etc.
*/ */
@Path("discovery") @Path("discovery")
@Singleton
public class MetadataDiscoveryResource { public class MetadataDiscoveryResource {
} }
webapp/src/main/java/org/apache/hadoop/metadata/web/resources/RexsterGraphResource.java
...@@ -18,25 +18,12 @@ ...@@ -18,25 +18,12 @@
package org.apache.hadoop.metadata.web.resources; package org.apache.hadoop.metadata.web.resources;
import com.tinkerpop.blueprints.Direction; import java.util.HashMap;
import com.tinkerpop.blueprints.Edge; import java.util.Map;
import com.tinkerpop.blueprints.Element; import java.util.Set;
import com.tinkerpop.blueprints.Graph;
import com.tinkerpop.blueprints.Vertex;
import com.tinkerpop.blueprints.VertexQuery;
import com.tinkerpop.blueprints.util.io.graphson.GraphSONMode;
import com.tinkerpop.blueprints.util.io.graphson.GraphSONUtility;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.metadata.service.Services;
import org.apache.hadoop.metadata.services.GraphService;
import org.apache.hadoop.metadata.services.TitanGraphService;
import org.apache.hadoop.metadata.web.util.Servlets;
import org.codehaus.jettison.json.JSONArray;
import org.codehaus.jettison.json.JSONException;
import org.codehaus.jettison.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.inject.Inject;
import javax.inject.Singleton;
import javax.ws.rs.DefaultValue; import javax.ws.rs.DefaultValue;
import javax.ws.rs.GET; import javax.ws.rs.GET;
import javax.ws.rs.Path; import javax.ws.rs.Path;
...@@ -46,9 +33,24 @@ import javax.ws.rs.QueryParam; ...@@ -46,9 +33,24 @@ import javax.ws.rs.QueryParam;
import javax.ws.rs.WebApplicationException; import javax.ws.rs.WebApplicationException;
import javax.ws.rs.core.MediaType; import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response; import javax.ws.rs.core.Response;
import java.util.HashMap;
import java.util.Map; import org.apache.commons.lang.StringUtils;
import java.util.Set; import org.apache.hadoop.metadata.services.GraphService;
import org.apache.hadoop.metadata.web.util.Servlets;
import org.codehaus.jettison.json.JSONArray;
import org.codehaus.jettison.json.JSONException;
import org.codehaus.jettison.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.tinkerpop.blueprints.Direction;
import com.tinkerpop.blueprints.Edge;
import com.tinkerpop.blueprints.Element;
import com.tinkerpop.blueprints.Graph;
import com.tinkerpop.blueprints.Vertex;
import com.tinkerpop.blueprints.VertexQuery;
import com.tinkerpop.blueprints.util.io.graphson.GraphSONMode;
import com.tinkerpop.blueprints.util.io.graphson.GraphSONUtility;
/** /**
* Jersey Resource for lineage metadata operations. * Jersey Resource for lineage metadata operations.
...@@ -59,22 +61,25 @@ import java.util.Set; ...@@ -59,22 +61,25 @@ import java.util.Set;
* for accessing the backend graph. * for accessing the backend graph.
*/ */
@Path("graph") @Path("graph")
@Singleton
public class RexsterGraphResource { public class RexsterGraphResource {
private static final Logger LOG = LoggerFactory.getLogger(RexsterGraphResource.class); private static final Logger LOG = LoggerFactory.getLogger(RexsterGraphResource.class);
public static final String RESULTS = "results"; public static final String RESULTS = "results";
public static final String TOTAL_SIZE = "totalSize"; public static final String TOTAL_SIZE = "totalSize";
private GraphService graphService; private final GraphService graphService;
public RexsterGraphResource() { @Inject
graphService = Services.get().getService(TitanGraphService.NAME); public RexsterGraphResource(GraphService graphService) {
this.graphService = graphService;
/*graphService = Services.get().getService(TitanGraphService.NAME);
if (graphService == null) { if (graphService == null) {
throw new WebApplicationException(Response throw new WebApplicationException(Response
.status(Response.Status.INTERNAL_SERVER_ERROR) .status(Response.Status.INTERNAL_SERVER_ERROR)
.tag("graph service is not initialized") .tag("graph service is not initialized")
.build()); .build());
} }*/
} }
protected Graph getGraph() { protected Graph getGraph() {
......
webapp/src/main/java/org/apache/hadoop/metadata/web/resources/TypesResource.java
...@@ -18,6 +18,7 @@ ...@@ -18,6 +18,7 @@
package org.apache.hadoop.metadata.web.resources; package org.apache.hadoop.metadata.web.resources;
import javax.inject.Singleton;
import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletRequest;
import javax.ws.rs.*; import javax.ws.rs.*;
import javax.ws.rs.core.Context; import javax.ws.rs.core.Context;
...@@ -31,6 +32,7 @@ import javax.ws.rs.core.Response; ...@@ -31,6 +32,7 @@ import javax.ws.rs.core.Response;
* e.g. a Hive table * e.g. a Hive table
*/ */
@Path("types") @Path("types")
@Singleton
public class TypesResource { public class TypesResource {
@POST @POST
......
webapp/src/main/webapp/WEB-INF/web.xml
<?xml version="1.0" encoding="UTF-8"?> <?xml version="1.0" encoding="UTF-8"?>
<!-- <!-- ~ Licensed to the Apache Software Foundation (ASF) under one ~ or more
~ Licensed to the Apache Software Foundation (ASF) under one contributor license agreements. See the NOTICE file ~ distributed with this
~ or more contributor license agreements. See the NOTICE file work for additional information ~ regarding copyright ownership. The ASF
~ distributed with this work for additional information licenses this file ~ to you under the Apache License, Version 2.0 (the ~
~ regarding copyright ownership. The ASF licenses this file "License"); you may not use this file except in compliance ~ with the License.
~ to you under the Apache License, Version 2.0 (the You may obtain a copy of the License at ~ ~ http://www.apache.org/licenses/LICENSE-2.0
~ "License"); you may not use this file except in compliance ~ ~ Unless required by applicable law or agreed to in writing, software ~
~ with the License. You may obtain a copy of the License at distributed under the License is distributed on an "AS IS" BASIS, ~ WITHOUT
~ WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ~ See the
~ http://www.apache.org/licenses/LICENSE-2.0 License for the specific language governing permissions and ~ limitations
~ under the License. -->
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<!DOCTYPE web-app PUBLIC "-//Sun Microsystems, Inc.//DTD Web Application 2.3//EN" <!DOCTYPE web-app PUBLIC "-//Sun Microsystems, Inc.//DTD Web Application 2.3//EN"
"http://java.sun.com/dtd/web-app_2_3.dtd"> "http://java.sun.com/dtd/web-app_2_3.dtd">
<web-app> <web-app>
<display-name>Apache Metadata Placeholder</display-name>
<display-name>Apache Falcon Placeholder</display-name> <description>Apache Metadata Placeholder</description>
<description>Apache Falcon Placeholder</description>
<context-param>
<filter> <param-name>guice.packages</param-name>
<filter-name>audit</filter-name> <param-value>org.apache.hadoop.metadata.web.resources,org.apache.hadoop.metadata.web.params</param-value>
<filter-class>org.apache.hadoop.metadata.web.filters.AuditFilter</filter-class> </context-param>
</filter>
<!--
<filter> More information can be found here:
<filter-name>authentication</filter-name>
<filter-class>org.apache.hadoop.metadata.web.filters.AuthenticationFilter</filter-class> https://jersey.java.net/nonav/apidocs/1.11/contribs/jersey-guice/com/sun/jersey/guice/spi/container/servlet/package-summary.html
</filter> -->
<filter-mapping> <filter>
<filter-name>audit</filter-name> <filter-name>guiceFilter</filter-name>
<servlet-name>MetadataRESTApi</servlet-name> <filter-class>com.google.inject.servlet.GuiceFilter</filter-class>
</filter-mapping> </filter>
<filter-mapping> <filter-mapping>
<filter-name>authentication</filter-name> <filter-name>guiceFilter</filter-name>
<servlet-name>MetadataRESTApi</servlet-name> <url-pattern>/*</url-pattern>
</filter-mapping> </filter-mapping>
<listener> <listener>
<listener-class>org.apache.hadoop.metadata.web.listeners.ApplicationStartupListener</listener-class> <listener-class>org.apache.hadoop.metadata.web.listeners.ApplicationStartupListener</listener-class>
</listener> </listener>
<listener>
<servlet> <listener-class>org.apache.hadoop.metadata.web.listeners.GuiceServletConfig</listener-class>
<servlet-name>MetadataRESTApi</servlet-name> </listener>
<servlet-class>com.sun.jersey.spi.container.servlet.ServletContainer</servlet-class>
<init-param>
<param-name>com.sun.jersey.config.property.resourceConfigClass</param-name>
<param-value>com.sun.jersey.api.core.PackagesResourceConfig</param-value>
</init-param>
<init-param>
<param-name>com.sun.jersey.config.property.packages</param-name>
<param-value>
org.apache.hadoop.metadata.web.resources,org.apache.hadoop.metadata.web.params
</param-value>
</init-param>
<load-on-startup>1</load-on-startup>
</servlet>
<servlet-mapping>
<servlet-name>MetadataRESTApi</servlet-name>
<url-pattern>/api/metadata/*</url-pattern>
</servlet-mapping>
</web-app> </web-app>
webapp/src/test/java/org/apache/hadoop/metadata/GraphRepositoryServiceIT.java deleted 100644 → 0
package org.apache.hadoop.metadata;
import org.apache.hadoop.metadata.service.Services;
import org.apache.hadoop.metadata.services.GraphBackedMetadataRepository;
import org.apache.hadoop.metadata.services.TitanGraphService;
import org.json.simple.JSONValue;
import org.testng.Assert;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
import java.util.HashMap;
import java.util.Map;
public class GraphRepositoryServiceIT {
private static final String ENTITY_NAME = "clicks-table";
private static final String ENTITY_TYPE = "hive-table";
private static final String DATABASE_NAME = "ads";
private static final String TABLE_NAME = "clicks-table";
@BeforeClass
public void setUp() throws Exception {
TitanGraphService titanGraphService = new TitanGraphService();
titanGraphService.start();
Services.get().register(titanGraphService);
GraphBackedMetadataRepository repositoryService
= new GraphBackedMetadataRepository();
repositoryService.start();
Services.get().register(repositoryService);
}
@AfterClass
public void tearDown() throws Exception {
Services.get().getService(GraphBackedMetadataRepository.NAME).close();
Services.get().getService(TitanGraphService.NAME).close();
Services.get().reset();
}
/*
@Test
public void testRepository() throws Exception {
GraphBackedMetadataRepositoryService repositoryService =
Services.get().getService(GraphBackedMetadataRepositoryService.NAME);
String entityStream = getTestEntityJSON();
String guid = repositoryService.createEntity(entityStream, ENTITY_TYPE);
Assert.assertNotNull(guid);
String entity = repositoryService.getEntityDefinition(ENTITY_NAME, ENTITY_TYPE);
@SuppressWarnings("unchecked")
Map<String, String> entityProperties =
(Map<String, String>) JSONValue.parseWithException(entity);
Assert.assertEquals(entityProperties.get("guid"), guid);
Assert.assertEquals(entityProperties.get("entityName"), ENTITY_NAME);
Assert.assertEquals(entityProperties.get("entityType"), ENTITY_TYPE);
Assert.assertEquals(entityProperties.get("database"), DATABASE_NAME);
Assert.assertEquals(entityProperties.get("table"), TABLE_NAME);
}
private String getTestEntityJSON() {
Map<String, String> props = new HashMap<>();
props.put("entityName", ENTITY_NAME);
props.put("entityType", ENTITY_TYPE);
props.put("database", DATABASE_NAME);
props.put("table", TABLE_NAME);
return JSONValue.toJSONString(props);
}
*/
}
webapp/src/test/java/org/apache/hadoop/metadata/TitanGraphServiceIT.java deleted 100644 → 0
package org.apache.hadoop.metadata;
import com.tinkerpop.blueprints.GraphQuery;
import com.tinkerpop.blueprints.TransactionalGraph;
import com.tinkerpop.blueprints.Vertex;
import org.apache.hadoop.metadata.services.TitanGraphService;
import org.apache.hadoop.metadata.util.GraphUtils;
import org.testng.annotations.Test;
import java.util.Iterator;
import java.util.UUID;
/**
* End to end graph put/get test.
*/
public class TitanGraphServiceIT {
@Test
public void testTitanGraph() throws Exception {
TitanGraphService titanGraphService = new TitanGraphService();
titanGraphService.start();
try {
String guid = UUID.randomUUID().toString();
final TransactionalGraph graph = titanGraphService.getTransactionalGraph();
System.out.println("graph = " + graph);
System.out.println("graph.getVertices() = " + graph.getVertices());
Vertex entityVertex = null;
try {
graph.rollback();
entityVertex = graph.addVertex(null);
entityVertex.setProperty("guid", guid);
entityVertex.setProperty("entityName", "entityName");
entityVertex.setProperty("entityType", "entityType");
} catch (Exception e) {
graph.rollback();
e.printStackTrace();
} finally {
graph.commit();
}
System.out.println("vertex = " + GraphUtils.vertexString(entityVertex));
GraphQuery query = graph.query()
.has("entityName", "entityName")
.has("entityType", "entityType");
Iterator<Vertex> results = query.vertices().iterator();
if (results.hasNext()) {
Vertex vertexFromQuery = results.next();
System.out.println("vertex = " + GraphUtils.vertexString(vertexFromQuery));
}
} finally {
Thread.sleep(1000);
titanGraphService.stop();
}
}
}
webapp/src/test/java/org/apache/hadoop/metadata/web/resources/EntityJerseyResourceIT.java
...@@ -18,18 +18,22 @@ ...@@ -18,18 +18,22 @@
package org.apache.hadoop.metadata.web.resources; package org.apache.hadoop.metadata.web.resources;
import com.sun.jersey.api.client.ClientResponse; import java.util.HashMap;
import com.sun.jersey.api.client.WebResource; import java.util.Map;
import org.json.simple.JSONValue; import java.util.UUID;
import org.testng.Assert;
import org.testng.annotations.Test;
import javax.ws.rs.HttpMethod; import javax.ws.rs.HttpMethod;
import javax.ws.rs.core.MediaType; import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response; import javax.ws.rs.core.Response;
import java.util.HashMap;
import java.util.Map; import org.json.simple.JSONValue;
import java.util.UUID; import org.testng.Assert;
import org.testng.annotations.Test;
import com.google.gson.JsonElement;
import com.google.gson.JsonParser;
import com.sun.jersey.api.client.ClientResponse;
import com.sun.jersey.api.client.WebResource;
/** /**
* Integration tests for Entity Jersey Resource. * Integration tests for Entity Jersey Resource.
...@@ -44,6 +48,7 @@ public class EntityJerseyResourceIT extends BaseResourceIT { ...@@ -44,6 +48,7 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
@Test @Test
public void testSubmitEntity() { public void testSubmitEntity() {
String entityStream = getTestEntityJSON(); String entityStream = getTestEntityJSON();
JsonParser parser = new JsonParser();
WebResource resource = service WebResource resource = service
.path("api/metadata/entities/submit") .path("api/metadata/entities/submit")
...@@ -54,11 +59,15 @@ public class EntityJerseyResourceIT extends BaseResourceIT { ...@@ -54,11 +59,15 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
.type(MediaType.APPLICATION_JSON) .type(MediaType.APPLICATION_JSON)
.method(HttpMethod.POST, ClientResponse.class, entityStream); .method(HttpMethod.POST, ClientResponse.class, entityStream);
Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode()); Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode());
String response = clientResponse.getEntity(String.class); String response = clientResponse.getEntity(String.class);
Assert.assertNotNull(response); Assert.assertNotNull(response);
JsonElement elem = parser.parse(response);
String guid = elem.getAsJsonObject().get("GUID").getAsString();
try { try {
Assert.assertNotNull(UUID.fromString(response)); Assert.assertNotNull(UUID.fromString(guid));
} catch (IllegalArgumentException e) { } catch (IllegalArgumentException e) {
Assert.fail("Response is not a guid, " + response); Assert.fail("Response is not a guid, " + response);
} }
......
webapp/src/main/resources/application.properties → webapp/src/test/resources/application.properties
...@@ -16,40 +16,7 @@ ...@@ -16,40 +16,7 @@
# limitations under the License. # limitations under the License.
# #
application.services=org.apache.hadoop.metadata.services.TitanGraphService,\ # GraphService implementation
org.apache.hadoop.metadata.services.GraphBackedMetadataRepositoryService,\ metadata.graph.impl.class=org.apache.hadoop.metadata.services.TitanGraphService
org.apache.hadoop.metadata.services.DefaultMetadataService
######### Implementation classes #########
## DO NOT MODIFY UNLESS SURE ABOUT CHANGE ##
metadata.GraphService.impl=org.apache.hadoop.metadata.services.TitanGraphService
metadata.MetadataRepositoryService.impl=org.apache.hadoop.metadata.services.GraphBackedMetadataRepositoryService
######### Implementation classes #########
######### Graph Database Configs #########
# Graph implementation
#metadata.graph.blueprints.graph=com.thinkaurelius.titan.core.TitanFactory
# Graph Storage
metadata.graph.storage.backend=berkeleyje
metadata.graph.storage.directory=target/data/berkeley
# Graph Search Index
metadata.graph.index.search.backend=elasticsearch
metadata.graph.index.search.directory=target/data/es
metadata.graph.index.search.elasticsearch.client-only=false
metadata.graph.index.search.elasticsearch.local-mode=true
######### Graph Database Configs #########
######### Security Properties #########
# SSL config
metadata.enableTLS=false metadata.enableTLS=false
######### Security Properties #########
webapp/src/test/resources/graph.properties 0 → 100644
storage.backend=inmemory
# Graph Search Index
index.search.backend=elasticsearch
index.search.directory=target/data/es
index.search.elasticsearch.client-only=false
index.search.elasticsearch.local-mode=true
\ No newline at end of file
webapp/src/test/resources/indexer.properties 0 → 100644
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# This is formatted as follows:
# metadata.indexer.vertex.property.name.<index>=<Property Name>
# metadata.indexer.vertex.property.type.<index>=<Data Type>
metadata.indexer.vertex.property.name.0=DESC
metadata.indexer.vertex.property.type.0=String
metadata.indexer.vertex.property.name.1=DB_LOCATION_URI
metadata.indexer.vertex.property.type.1=String
metadata.indexer.vertex.property.name.2=NAME
metadata.indexer.vertex.property.type.2=String
metadata.indexer.vertex.property.name.3=OWNER_NAME
metadata.indexer.vertex.property.type.3=String
metadata.indexer.vertex.property.name.4=TBL_NAME
metadata.indexer.vertex.property.type.4=String
metadata.indexer.vertex.property.name.5=COMMENT
metadata.indexer.vertex.property.type.5=String
metadata.indexer.vertex.property.name.6=COLUMN_NAME
metadata.indexer.vertex.property.type.6=String
metadata.indexer.vertex.property.name.7=TYPE_NAME
metadata.indexer.vertex.property.type.7=String
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment