Skip to content

A
atlas
Commit a15b6ef1 by Shwetha GS
Options

full text search + other minor fixes

parent 5324e905
Showing with 309 additions and 76 deletions
.gitignore
......@@ -34,3 +34,4 @@ maven-eclipse.xml
#log files
logs
*.log
test-output
addons/hive-bridge/src/main/java/org/apache/hadoop/metadata/hive/bridge/HiveMetaStoreBridge.java
......@@ -62,7 +62,7 @@ public class HiveMetaStoreBridge {
}
}
public static final String DGI_URL_PROPERTY = "hive.dgi.url";
public static final String DGI_URL_PROPERTY = "hive.hook.dgi.url";
private static final Logger LOG = LoggerFactory.getLogger(HiveMetaStoreBridge.class);
......
addons/hive-bridge/src/main/java/org/apache/hadoop/metadata/hive/hook/HiveHook.java
......@@ -78,6 +78,8 @@ import java.util.Map;
import java.util.Set;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
/**
......@@ -91,6 +93,14 @@ public class HiveHook implements ExecuteWithHookContext, HiveSemanticAnalyzerHoo
private static final int WAIT_TIME = 3;
private static ExecutorService executor;
private static final String MIN_THREADS = "hive.hook.dgi.minThreads";
private static final String MAX_THREADS = "hive.hook.dgi.maxThreads";
private static final String KEEP_ALIVE_TIME = "hive.hook.dgi.keepAliveTime";
private static final int minThreadsDefault = 5;
private static final int maxThreadsDefault = 5;
private static final long keepAliveTimeDefault = 10;
static {
// anything shared should be initialized here and destroyed in the
// shutdown hook The hook contract is weird in that it creates a
......@@ -99,7 +109,14 @@ public class HiveHook implements ExecuteWithHookContext, HiveSemanticAnalyzerHoo
// initialize the async facility to process hook calls. We don't
// want to do this inline since it adds plenty of overhead for the
// query.
executor = Executors.newSingleThreadExecutor(
HiveConf hiveConf = new HiveConf();
int minThreads = hiveConf.getInt(MIN_THREADS, minThreadsDefault);
int maxThreads = hiveConf.getInt(MAX_THREADS, maxThreadsDefault);
long keepAliveTime = hiveConf.getLong(KEEP_ALIVE_TIME, keepAliveTimeDefault);
executor = new ThreadPoolExecutor(minThreads, maxThreads,
keepAliveTime, TimeUnit.MILLISECONDS,
new LinkedBlockingQueue<Runnable>(),
new ThreadFactoryBuilder()
.setDaemon(true)
.setNameFormat("DGI Logger %d")
......@@ -199,8 +216,16 @@ public class HiveHook implements ExecuteWithHookContext, HiveSemanticAnalyzerHoo
}
private void registerCTAS(HiveMetaStoreBridge dgiBridge, HookContext hookContext, HiveConf conf) throws Exception {
LOG.debug("Registering CTAS");
Set<ReadEntity> inputs = hookContext.getInputs();
Set<WriteEntity> outputs = hookContext.getOutputs();
//Even explain CTAS has operation name as CREATETABLE_AS_SELECT
if (inputs.isEmpty() && outputs.isEmpty()) {
LOG.info("Explain statement. Skipping...");
}
String user = hookContext.getUserName();
HiveOperation operation = HiveOperation.valueOf(hookContext.getOperationName());
String queryId = null;
......@@ -214,7 +239,6 @@ public class HiveHook implements ExecuteWithHookContext, HiveSemanticAnalyzerHoo
queryStartTime = plan.getQueryStartTime();
}
Referenceable processReferenceable = new Referenceable(HiveDataTypes.HIVE_PROCESS.getName());
processReferenceable.set("processName", operation.getOperationName());
processReferenceable.set("startTime", queryStartTime);
......@@ -311,7 +335,7 @@ public class HiveHook implements ExecuteWithHookContext, HiveSemanticAnalyzerHoo
ExplainTask explain = new ExplainTask();
explain.initialize(conf, queryPlan, null);
org.json.JSONObject explainPlan = explain.getJSONLogicalPlan(null, ew);
org.json.JSONObject explainPlan = explain.getJSONPlan(null, ew);
return explainPlan.toString();
}
......
addons/hive-bridge/src/site/twiki/Bridge-Hive.twiki
......@@ -20,12 +20,13 @@ Hive metadata can be modelled in DGI using its Type System. The default modellin
---++ Importing Hive Metadata
org.apache.hadoop.metadata.hive.bridge.HiveMetaStoreBridge imports the hive metadata into DGI using the typesystem defined in org.apache.hadoop.metadata.hive.model.HiveDataModelGenerator. import-hive.sh command can be used to facilitate this. Set-up the following configs in <dgi package>/conf/hive-site.xml:
org.apache.hadoop.metadata.hive.bridge.HiveMetaStoreBridge imports the hive metadata into DGI using the typesystem defined in org.apache.hadoop.metadata.hive.model.HiveDataModelGenerator. import-hive.sh command can be used to facilitate this.
Set-up the following configs in <dgi package>/conf/hive-site.xml:
* Hive metastore configuration - Refer [[https://cwiki.apache.org/confluence/display/Hive/AdminManual+MetastoreAdmin][Hive Metastore Configuration documentation]]
* DGI endpoint - Add the following property with the DGI endpoint for your set-up
<verbatim>
<property>
<name>hive.dgi.url</name>
<name>hive.hook.dgi.url</name>
<value>http://localhost:21000/</value>
</property>
</verbatim>
......@@ -34,13 +35,26 @@ Usage: <dgi package>/bin/import-hive.sh
---++ Hive Hook
Hive supports listeners on hive command execution using hive hooks. This can be used to add/update/remove entities in DGI. Follow the following instructions in your hive set-up
* Add org.apache.hadoop.metadata.hive.hook.HiveHook as post execution hook in hive-ste.xml
Hive supports listeners on hive command execution using hive hooks. This is used to add/update/remove entities in DGI using the model defined in org.apache.hadoop.metadata.hive.model.HiveDataModelGenerator.
The hook submits the request to a thread pool executor to avoid blocking the command execution. Follow the these instructions in your hive set-up to add hive hook for DGI:
* Add org.apache.hadoop.metadata.hive.hook.HiveHook as post execution hook in hive-site.xml
<verbatim>
<property>
<name>hive.exec.post.hooks</name>
<value>org.apache.hadoop.metadata.hive.hook.HiveHook</value>
</property>
</verbatim>
* Add the following property in hive-ste.xml with the DGI endpoint for your set-up
<verbatim>
<property>
<name>hive.hook.dgi.url</name>
<value>http://localhost:21000/</value>
</property>
</verbatim>
* Add 'export HIVE_AUX_JARS_PATH=<dgi package>/hook/hive' in hive-env.sh
* Restart hive-server2
The following properties in hive-site.xml control the thread pool details:
* hive.hook.dgi.minThreads - core number of threads. default 5
* hive.hook.dgi.maxThreads - maximum number of threads. default 5
* hive.hook.dgi.keepAliveTime - keep alive time in msecs. default 10
addons/hive-bridge/src/test/java/org/apache/hadoop/metadata/hive/hook/HiveHookIT.java
......@@ -69,6 +69,7 @@ public class HiveHookIT {
hiveConf.setBoolVar(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY, false);
hiveConf.setVar(HiveConf.ConfVars.METASTOREWAREHOUSE, System.getProperty("user.dir") + "/target/metastore");
hiveConf.set(HiveMetaStoreBridge.DGI_URL_PROPERTY, DGI_URL);
hiveConf.set("javax.jdo.option.ConnectionURL", "jdbc:derby:./target/metastore_db;create=true");
hiveConf.set("debug", "true");
return hiveConf;
}
......
addons/hive-bridge/src/test/resources/application.properties
......@@ -19,11 +19,11 @@
######### Graph Database Configs #########
# Graph Storage
metadata.graph.storage.backend=berkeleyje
metadata.graph.storage.directory=./data/berkeley
metadata.graph.storage.directory=./target/data/berkeley
# Graph Search Index
metadata.graph.index.search.backend=elasticsearch
metadata.graph.index.search.directory=./data/es
metadata.graph.index.search.directory=./target/data/es
metadata.graph.index.search.elasticsearch.client-only=false
metadata.graph.index.search.elasticsearch.local-mode=true
......
repository/src/main/java/org/apache/hadoop/metadata/RepositoryMetadataModule.java
......@@ -29,7 +29,7 @@ import org.apache.hadoop.metadata.repository.graph.GraphBackedMetadataRepository
import org.apache.hadoop.metadata.repository.graph.GraphBackedSearchIndexer;
import org.apache.hadoop.metadata.repository.graph.GraphProvider;
import org.apache.hadoop.metadata.repository.graph.TitanGraphProvider;
import org.apache.hadoop.metadata.repository.typestore.GraphTypeStore;
import org.apache.hadoop.metadata.repository.typestore.GraphBackedTypeStore;
import org.apache.hadoop.metadata.repository.typestore.ITypeStore;
import org.apache.hadoop.metadata.services.DefaultMetadataService;
import org.apache.hadoop.metadata.services.MetadataService;
......@@ -55,7 +55,7 @@ public class RepositoryMetadataModule extends com.google.inject.AbstractModule {
// get the impl classes for the repo and the graph service
// this.graphServiceClass = gsp.getImplClass();
this.metadataRepoClass = GraphBackedMetadataRepository.class;
this.typeStore = GraphTypeStore.class;
this.typeStore = GraphBackedTypeStore.class;
this.metadataService = DefaultMetadataService.class;
this.discoveryService = GraphBackedDiscoveryService.class;
this.searchIndexer = GraphBackedSearchIndexer.class;
......
repository/src/main/java/org/apache/hadoop/metadata/discovery/DiscoveryService.java
......@@ -27,6 +27,11 @@ import java.util.Map;
public interface DiscoveryService {
/**
* Full text search
*/
String searchByFullText(String query) throws DiscoveryException;
/**
* Search using query DSL.
*
* @param dslQuery query in DSL format.
......
repository/src/main/java/org/apache/hadoop/metadata/discovery/graph/GraphBackedDiscoveryService.java
......@@ -18,9 +18,15 @@
package org.apache.hadoop.metadata.discovery.graph;
import com.google.gson.JsonArray;
import com.google.gson.JsonObject;
import com.thinkaurelius.titan.core.TitanGraph;
import com.thinkaurelius.titan.core.TitanGraphQuery;
import com.thinkaurelius.titan.core.TitanProperty;
import com.thinkaurelius.titan.core.TitanVertex;
import com.thinkaurelius.titan.core.attribute.Text;
import com.tinkerpop.blueprints.Vertex;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.metadata.discovery.DiscoveryException;
import org.apache.hadoop.metadata.discovery.DiscoveryService;
import org.apache.hadoop.metadata.query.Expressions;
......@@ -30,8 +36,12 @@ import org.apache.hadoop.metadata.query.GremlinQueryResult;
import org.apache.hadoop.metadata.query.GremlinTranslator;
import org.apache.hadoop.metadata.query.QueryParser;
import org.apache.hadoop.metadata.query.QueryProcessor;
import org.apache.hadoop.metadata.repository.Constants;
import org.apache.hadoop.metadata.repository.MetadataRepository;
import org.apache.hadoop.metadata.repository.graph.GraphProvider;
import org.apache.hadoop.metadata.typesystem.Referenceable;
import org.apache.hadoop.metadata.typesystem.json.InstanceSerialization;
import org.apache.hadoop.metadata.typesystem.types.TypeSystem;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import scala.util.Either;
......@@ -44,8 +54,11 @@ import javax.script.ScriptEngineManager;
import javax.script.ScriptException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
/**
* Graph backed implementation of Search.
......@@ -64,6 +77,23 @@ public class GraphBackedDiscoveryService implements DiscoveryService {
this.graphPersistenceStrategy = new DefaultGraphPersistenceStrategy(metadataRepository);
}
@Override
public String searchByFullText(String query) throws DiscoveryException {
Iterator iterator = titanGraph.query().has(Constants.ENTITY_TEXT_PROPERTY_KEY, Text.CONTAINS, query).vertices().iterator();
JsonArray results = new JsonArray();
while (iterator.hasNext()) {
Vertex vertex = (Vertex) iterator.next();
JsonObject row = new JsonObject();
row.addProperty("guid", vertex.<String>getProperty(Constants.GUID_PROPERTY_KEY));
row.addProperty("typeName", vertex.<String>getProperty(Constants.ENTITY_TYPE_PROPERTY_KEY));
results.add(row);
}
JsonObject response = new JsonObject();
response.addProperty("query", query);
response.add("results", results);
return response.toString();
}
/**
* Search using query DSL.
*
......@@ -118,47 +148,48 @@ public class GraphBackedDiscoveryService implements DiscoveryService {
try {
Object o = engine.eval(gremlinQuery, bindings);
if (!(o instanceof List)) {
throw new DiscoveryException(
String.format("Cannot process gremlin result %s", o.toString()));
}
return extractResult(o);
} catch (ScriptException se) {
throw new DiscoveryException(se);
}
}
List l = (List) o;
List<Map<String, String>> result = new ArrayList<>();
for (Object r : l) {
Map<String, String> oRow = new HashMap<>();
if (r instanceof Map) {
@SuppressWarnings("unchecked")
Map<Object, Object> iRow = (Map) r;
for (Map.Entry e : iRow.entrySet()) {
Object k = e.getKey();
Object v = e.getValue();
oRow.put(k.toString(), v.toString());
}
} else if (r instanceof TitanVertex) {
Iterable<TitanProperty> ps = ((TitanVertex) r).getProperties();
for (TitanProperty tP : ps) {
String pName = tP.getPropertyKey().getName();
Object pValue = ((TitanVertex) r).getProperty(pName);
if (pValue != null) {
oRow.put(pName, pValue.toString());
}
}
private List<Map<String, String>> extractResult(Object o) throws DiscoveryException {
if (!(o instanceof List)) {
throw new DiscoveryException(String.format("Cannot process result %s", o.toString()));
}
} else if (r instanceof String) {
oRow.put("", r.toString());
} else {
throw new DiscoveryException(
String.format("Cannot process gremlin result %s", o.toString()));
List l = (List) o;
List<Map<String, String>> result = new ArrayList<>();
for (Object r : l) {
Map<String, String> oRow = new HashMap<>();
if (r instanceof Map) {
@SuppressWarnings("unchecked")
Map<Object, Object> iRow = (Map) r;
for (Map.Entry e : iRow.entrySet()) {
Object k = e.getKey();
Object v = e.getValue();
oRow.put(k.toString(), v.toString());
}
} else if (r instanceof TitanVertex) {
Iterable<TitanProperty> ps = ((TitanVertex) r).getProperties();
for (TitanProperty tP : ps) {
String pName = tP.getPropertyKey().getName();
Object pValue = ((TitanVertex) r).getProperty(pName);
if (pValue != null) {
oRow.put(pName, pValue.toString());
}
}
result.add(oRow);
} else if (r instanceof String) {
oRow.put("", r.toString());
} else {
throw new DiscoveryException(String.format("Cannot process result %s", o.toString()));
}
return result;
} catch (ScriptException se) {
throw new DiscoveryException(se);
result.add(oRow);
}
return result;
}
}
repository/src/main/java/org/apache/hadoop/metadata/repository/Constants.java
......@@ -32,6 +32,8 @@ public final class Constants {
public static final String ENTITY_TYPE_PROPERTY_KEY = "typeName";
public static final String ENTITY_TYPE_INDEX = "type_index";
public static final String ENTITY_TEXT_PROPERTY_KEY = "entityText";
/**
* Properties for type store graph
*/
......
repository/src/main/java/org/apache/hadoop/metadata/repository/graph/GraphBackedMetadataRepository.java
......@@ -36,9 +36,12 @@ import org.apache.hadoop.metadata.typesystem.ITypedReferenceableInstance;
import org.apache.hadoop.metadata.typesystem.ITypedStruct;
import org.apache.hadoop.metadata.typesystem.persistence.Id;
import org.apache.hadoop.metadata.typesystem.persistence.MapIds;
import org.apache.hadoop.metadata.typesystem.persistence.StructInstance;
import org.apache.hadoop.metadata.typesystem.types.AttributeInfo;
import org.apache.hadoop.metadata.typesystem.types.ClassType;
import org.apache.hadoop.metadata.typesystem.types.DataTypes;
import org.apache.hadoop.metadata.typesystem.types.EnumType;
import org.apache.hadoop.metadata.typesystem.types.EnumValue;
import org.apache.hadoop.metadata.typesystem.types.IDataType;
import org.apache.hadoop.metadata.typesystem.types.Multiplicity;
import org.apache.hadoop.metadata.typesystem.types.ObjectGraphWalker;
......@@ -67,6 +70,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
private static final Logger LOG =
LoggerFactory.getLogger(GraphBackedMetadataRepository.class);
private static final String FULL_TEXT_DELIMITER = " ";
private final AtomicInteger ID_SEQ = new AtomicInteger(0);
......@@ -422,7 +426,71 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
List<ITypedReferenceableInstance> newTypedInstances = discoverInstances(entityProcessor);
entityProcessor.createVerticesForClassTypes(newTypedInstances);
return addDiscoveredInstances(typedInstance, entityProcessor, newTypedInstances);
String guid = addDiscoveredInstances(typedInstance, entityProcessor, newTypedInstances);
addFullTextProperty(entityProcessor, newTypedInstances);
return guid;
}
private void addFullTextProperty(EntityProcessor entityProcessor, List<ITypedReferenceableInstance> newTypedInstances) throws MetadataException {
for (ITypedReferenceableInstance typedInstance : newTypedInstances) { // Traverse
Id id = typedInstance.getId();
Vertex instanceVertex = entityProcessor.idToVertexMap.get(id);
String fullText = getFullText(instanceVertex, true);
instanceVertex.setProperty(Constants.ENTITY_TEXT_PROPERTY_KEY, fullText);
}
}
private String getFullText(Vertex instanceVertex, boolean followReferences) throws MetadataException {
String guid = instanceVertex.getProperty(Constants.GUID_PROPERTY_KEY);
ITypedReferenceableInstance typedReference = graphToInstanceMapper.mapGraphToTypedInstance(guid, instanceVertex);
return getFullText(typedReference, followReferences);
}
private String getFullText(ITypedInstance typedInstance, boolean followReferences) throws MetadataException {
StringBuilder fullText = new StringBuilder();
for (AttributeInfo attributeInfo : typedInstance.fieldMapping().fields.values()) {
Object attrValue = typedInstance.get(attributeInfo.name);
if (attrValue == null) {
continue;
}
String attrFullText = null;
switch(attributeInfo.dataType().getTypeCategory()) {
case PRIMITIVE:
attrFullText = String.valueOf(attrValue);
break;
case ENUM:
attrFullText = ((EnumValue)attrValue).value;
break;
case ARRAY:
break;
case MAP:
break;
case CLASS:
if (followReferences) {
String refGuid = ((ITypedReferenceableInstance) attrValue).getId()._getId();
Vertex refVertex = getVertexForGUID(refGuid);
attrFullText = getFullText(refVertex, false);
}
break;
case STRUCT:
case TRAIT:
if (followReferences) {
attrFullText = getFullText((ITypedInstance) attrValue, false);
}
break;
}
if (attrFullText != null) {
fullText = fullText.append(FULL_TEXT_DELIMITER).append(attributeInfo.name)
.append(FULL_TEXT_DELIMITER).append(attrFullText);
}
}
return fullText.toString();
}
/**
......@@ -518,6 +586,9 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
IDataType dataType) throws MetadataException {
LOG.debug("mapping attributeInfo {}", attributeInfo);
final String propertyName = typedInstance.getTypeName() + "." + attributeInfo.name;
if (typedInstance.get(attributeInfo.name) == null) {
return;
}
switch (dataType.getTypeCategory()) {
case PRIMITIVE:
......@@ -824,8 +895,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
AttributeInfo attributeInfo) throws MetadataException {
LOG.debug("mapping attributeInfo = " + attributeInfo);
final IDataType dataType = attributeInfo.dataType();
final String vertexPropertyName =
typedInstance.getTypeName() + "." + attributeInfo.name;
final String vertexPropertyName = typedInstance.getTypeName() + "." + attributeInfo.name;
switch (dataType.getTypeCategory()) {
case PRIMITIVE:
......@@ -833,6 +903,10 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
break; // add only if vertex has this attribute
case ENUM:
if (instanceVertex.getProperty(vertexPropertyName) == null) {
return;
}
typedInstance.setInt(attributeInfo.name,
instanceVertex.<Integer>getProperty(vertexPropertyName));
break;
......@@ -1060,8 +1134,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
ITypedInstance typedInstance,
AttributeInfo attributeInfo) throws MetadataException {
LOG.debug("Adding primitive {} from vertex {}", attributeInfo, instanceVertex);
final String vertexPropertyName = typedInstance.getTypeName() + "." +
attributeInfo.name;
final String vertexPropertyName = typedInstance.getTypeName() + "." + attributeInfo.name;
if (instanceVertex.getProperty(vertexPropertyName) == null) {
return;
}
......
repository/src/main/java/org/apache/hadoop/metadata/repository/graph/GraphBackedSearchIndexer.java
......@@ -95,9 +95,30 @@ public class GraphBackedSearchIndexer implements SearchIndexer {
createCompositeAndMixedIndex(Constants.TRAIT_NAMES_INDEX,
Constants.TRAIT_NAMES_PROPERTY_KEY, String.class, false, Cardinality.SET);
//Index for full text search
createVertexMixedIndex(Constants.ENTITY_TEXT_PROPERTY_KEY, String.class, Cardinality.SINGLE);
//Indexes for graph backed type system store
createTypeStoreIndexes();
LOG.info("Index creation for global keys complete.");
}
private void createTypeStoreIndexes() {
//Create unique index on typeName
createCompositeIndex(Constants.TYPENAME_PROPERTY_KEY, Constants.TYPENAME_PROPERTY_KEY, String.class,
true, Cardinality.SINGLE);
//Create index on vertex type + typeName
//todo doesn't work, review
TitanManagement management = titanGraph.getManagementSystem();
PropertyKey vertexType = management.makePropertyKey(Constants.VERTEX_TYPE_PROPERTY_KEY).dataType(String.class).make();
PropertyKey typeName = management.getPropertyKey(Constants.TYPENAME_PROPERTY_KEY);
management.buildIndex("byTypeName", Vertex.class).addKey(vertexType).addKey(typeName).buildCompositeIndex();
management.commit();
LOG.debug("Created composite index on {} and {}", Constants.VERTEX_TYPE_PROPERTY_KEY, Constants.TYPENAME_PROPERTY_KEY);
}
/**
* This is upon adding a new type to Store.
*
......
repository/src/main/java/org/apache/hadoop/metadata/repository/typestore/GraphTypeStore.java → repository/src/main/java/org/apache/hadoop/metadata/repository/typestore/GraphBackedTypeStore.java
......@@ -52,7 +52,8 @@ import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
public class GraphTypeStore implements ITypeStore {
public class GraphBackedTypeStore implements ITypeStore {
public static final String VERTEX_TYPE = "typeSystem";
private static final String PROPERTY_PREFIX = "type.";
public static final String SUPERTYPE_EDGE_LABEL = PROPERTY_PREFIX + ".supertype";
public static final String SUBTYPE_EDGE_LABEL = PROPERTY_PREFIX + ".subtype";
......@@ -60,12 +61,12 @@ public class GraphTypeStore implements ITypeStore {
private static final ImmutableList META_PROPERTIES = ImmutableList.of(Constants.VERTEX_TYPE_PROPERTY_KEY,
Constants.TYPE_CATEGORY_PROPERTY_KEY, Constants.TYPENAME_PROPERTY_KEY);
private static Logger LOG = LoggerFactory.getLogger(GraphTypeStore.class);
private static Logger LOG = LoggerFactory.getLogger(GraphBackedTypeStore.class);
private final TitanGraph titanGraph;
@Inject
public GraphTypeStore(GraphProvider<TitanGraph> graphProvider) {
public GraphBackedTypeStore(GraphProvider<TitanGraph> graphProvider) {
titanGraph = graphProvider.get();
}
......@@ -82,7 +83,7 @@ public class GraphTypeStore implements ITypeStore {
for (String typeName : typeNames) {
if (!coreTypes.contains(typeName)) {
IDataType dataType = typeSystem.getDataType(IDataType.class, typeName);
LOG.debug("Adding {}.{} to type store", dataType.getTypeCategory(), dataType.getName());
LOG.debug("Processing {}.{} in type store", dataType.getTypeCategory(), dataType.getName());
switch (dataType.getTypeCategory()) {
case ENUM:
storeInGraph((EnumType)dataType);
......@@ -207,7 +208,7 @@ public class GraphTypeStore implements ITypeStore {
@Override
public TypesDef restore() throws MetadataException {
//Get all vertices for type system
Iterator vertices = titanGraph.query().has(Constants.VERTEX_TYPE_PROPERTY_KEY, PROPERTY_PREFIX).vertices().iterator();
Iterator vertices = titanGraph.query().has(Constants.VERTEX_TYPE_PROPERTY_KEY, VERTEX_TYPE).vertices().iterator();
ImmutableList.Builder<EnumTypeDefinition> enums = ImmutableList.builder();
ImmutableList.Builder<StructTypeDefinition> structs = ImmutableList.builder();
......@@ -294,10 +295,9 @@ public class GraphTypeStore implements ITypeStore {
* @return vertex
*/
private Vertex findVertex(DataTypes.TypeCategory category, String typeName) {
LOG.debug("Finding vertex for ({} - {}), ({} - {})", Constants.TYPE_CATEGORY_PROPERTY_KEY, category,
Constants.TYPENAME_PROPERTY_KEY, typeName);
LOG.debug("Finding vertex for {}.{}", category, typeName);
Iterator results = titanGraph.query().has(Constants.VERTEX_TYPE_PROPERTY_KEY, PROPERTY_PREFIX)
Iterator results = titanGraph.query().has(Constants.VERTEX_TYPE_PROPERTY_KEY, VERTEX_TYPE)
.has(Constants.TYPENAME_PROPERTY_KEY, typeName).vertices().iterator();
Vertex vertex = null;
if (results != null && results.hasNext()) {
......@@ -312,7 +312,7 @@ public class GraphTypeStore implements ITypeStore {
if (vertex == null) {
LOG.debug("Adding vertex {}{}", PROPERTY_PREFIX, typeName);
vertex = titanGraph.addVertex(null);
vertex.setProperty(Constants.VERTEX_TYPE_PROPERTY_KEY, PROPERTY_PREFIX); //Mark as type vertex
vertex.setProperty(Constants.VERTEX_TYPE_PROPERTY_KEY, VERTEX_TYPE); //Mark as type vertex
vertex.setProperty(Constants.TYPE_CATEGORY_PROPERTY_KEY, category);
vertex.setProperty(Constants.TYPENAME_PROPERTY_KEY, typeName);
}
......
repository/src/test/java/org/apache/hadoop/metadata/TestUtils.java
......@@ -37,6 +37,7 @@ import org.testng.Assert;
import static org.apache.hadoop.metadata.typesystem.types.utils.TypesUtil.createClassTypeDef;
import static org.apache.hadoop.metadata.typesystem.types.utils.TypesUtil.createOptionalAttrDef;
import static org.apache.hadoop.metadata.typesystem.types.utils.TypesUtil.createRequiredAttrDef;
import static org.apache.hadoop.metadata.typesystem.types.utils.TypesUtil.createStructTypeDef;
import static org.apache.hadoop.metadata.typesystem.types.utils.TypesUtil.createTraitTypeDef;
/**
......@@ -61,10 +62,13 @@ public final class TestUtils {
new EnumTypeDefinition("OrgLevel", new EnumValue("L1", 1), new EnumValue("L2", 2));
ts.defineEnumType(orgLevelEnum);
StructTypeDefinition addressDetails = createStructTypeDef("Address",
createRequiredAttrDef("street", DataTypes.STRING_TYPE),
createRequiredAttrDef("city", DataTypes.STRING_TYPE));
HierarchicalTypeDefinition<ClassType> deptTypeDef =
createClassTypeDef("Department", ImmutableList.<String>of(),
createRequiredAttrDef("name", DataTypes.STRING_TYPE),
createOptionalAttrDef("orgLevel", ts.getDataType(EnumType.class, "OrgLevel")),
new AttributeDefinition("employees",
String.format("array<%s>", "Person"), Multiplicity.COLLECTION, true,
"department")
......@@ -73,6 +77,8 @@ public final class TestUtils {
HierarchicalTypeDefinition<ClassType> personTypeDef = createClassTypeDef("Person",
ImmutableList.<String>of(),
createRequiredAttrDef("name", DataTypes.STRING_TYPE),
createOptionalAttrDef("orgLevel", ts.getDataType(EnumType.class, "OrgLevel")),
createOptionalAttrDef("address", "Address"),
new AttributeDefinition("department",
"Department", Multiplicity.REQUIRED, false, "employees"),
new AttributeDefinition("manager",
......@@ -92,7 +98,7 @@ public final class TestUtils {
createRequiredAttrDef("level", DataTypes.INT_TYPE)
);
ts.defineTypes(ImmutableList.<StructTypeDefinition>of(),
ts.defineTypes(ImmutableList.of(addressDetails),
ImmutableList.of(securityClearanceTypeDef),
ImmutableList.of(deptTypeDef, personTypeDef, managerTypeDef));
}
......@@ -101,12 +107,21 @@ public final class TestUtils {
Referenceable hrDept = new Referenceable("Department");
Referenceable john = new Referenceable("Person");
Referenceable jane = new Referenceable("Manager", "SecurityClearance");
Referenceable johnAddr = new Referenceable("Address");
Referenceable janeAddr = new Referenceable("Address");
hrDept.set("name", "hr");
john.set("name", "John");
john.set("department", hrDept);
johnAddr.set("street", "Stewart Drive");
johnAddr.set("city", "Sunnyvale");
john.set("address", johnAddr);
jane.set("name", "Jane");
jane.set("department", hrDept);
janeAddr.set("street", "Great Americal Parkway");
janeAddr.set("city", "Santa Clara");
jane.set("address", janeAddr);
john.set("manager", jane);
......
repository/src/test/java/org/apache/hadoop/metadata/discovery/GraphBackedDiscoveryServiceTest.java
......@@ -291,4 +291,32 @@ public class GraphBackedDiscoveryServiceTest {
Assert.assertNotEquals(name, "null");
}
}
@Test
public void testFullTextSearch() throws Exception {
//person in hr department whose name is john
String response = discoveryService.searchByFullText("john hr");
Assert.assertNotNull(response);
JSONObject jsonResponse = new JSONObject(response);
JSONArray results = jsonResponse.getJSONArray("results");
Assert.assertEquals(results.length(), 1);
JSONObject row = (JSONObject) results.get(0);
Assert.assertEquals(row.get("typeName"), "Person");
//person in hr department who lives in santa clara
response = discoveryService.searchByFullText("hr santa clara");
Assert.assertNotNull(response);
jsonResponse = new JSONObject(response);
results = jsonResponse.getJSONArray("results");
Assert.assertEquals(results.length(), 1);
row = (JSONObject) results.get(0);
Assert.assertEquals(row.get("typeName"), "Manager");
//search for hr should return - hr department and its 2 employess
response = discoveryService.searchByFullText("hr");
Assert.assertNotNull(response);
jsonResponse = new JSONObject(response);
results = jsonResponse.getJSONArray("results");
Assert.assertEquals(results.length(), 3);
}
}
\ No newline at end of file
repository/src/test/java/org/apache/hadoop/metadata/repository/memory/EnumTest.java
......@@ -195,7 +195,7 @@ public class EnumTest extends BaseTest {
"\tj : \t1\n" +
"\tk : \t1\n" +
"\tenum3 : \tCOMMITTED\n" +
"\tl : \t2014-12-10\n" +
"\tl : \t2014-12-11\n" +
"\tm : \t[1, 1]\n" +
"\tn : \t[1.1, 1.1]\n" +
"\to : \t{b=2.0, a=1.0}\n" +
......@@ -227,7 +227,7 @@ public class EnumTest extends BaseTest {
"\tj : \t1\n" +
"\tk : \t1\n" +
"\tenum3 : \tCOMMITTED\n" +
"\tl : \t2014-12-10\n" +
"\tl : \t2014-12-11\n" +
"\tm : \t[1, 1]\n" +
"\tn : \t[1.1, 1.1]\n" +
"\to : \t{b=2.0, a=1.0}\n" +
......@@ -264,7 +264,7 @@ public class EnumTest extends BaseTest {
"\tj : \t1\n" +
"\tk : \t1\n" +
"\tenum3 : \tCOMMITTED\n" +
"\tl : \t2014-12-10\n" +
"\tl : \t2014-12-11\n" +
"\tm : \t[1, 1]\n" +
"\tn : \t[1.1, 1.1]\n" +
"\to : \t{b=2.0, a=1.0}\n" +
......
repository/src/test/java/org/apache/hadoop/metadata/repository/memory/StructTest.java
......@@ -58,7 +58,7 @@ public class StructTest extends BaseTest {
"\ti : \t1.0\n" +
"\tj : \t1\n" +
"\tk : \t1\n" +
"\tl : \t2014-12-10\n" +
"\tl : \t2014-12-11\n" +
"\tm : \t[1, 1]\n" +
"\tn : \t[1.1, 1.1]\n" +
"\to : \t{b=2.0, a=1.0}\n" +
......@@ -101,7 +101,7 @@ public class StructTest extends BaseTest {
"\ti : \t1.0\n" +
"\tj : \t1\n" +
"\tk : \t1\n" +
"\tl : \t2014-12-10\n" +
"\tl : \t2014-12-11\n" +
"\tm : \t[1, 1]\n" +
"\tn : \t[1.100000000000000088817841970012523233890533447265625, 1" +
".100000000000000088817841970012523233890533447265625]\n" +
......
repository/src/test/java/org/apache/hadoop/metadata/repository/typestore/GraphTypeStoreTest.java → repository/src/test/java/org/apache/hadoop/metadata/repository/typestore/GraphBackedTypeStoreTest.java
......@@ -34,6 +34,7 @@ import org.apache.hadoop.metadata.typesystem.types.ClassType;
import org.apache.hadoop.metadata.typesystem.types.DataTypes;
import org.apache.hadoop.metadata.typesystem.types.EnumTypeDefinition;
import org.apache.hadoop.metadata.typesystem.types.HierarchicalTypeDefinition;
import org.apache.hadoop.metadata.typesystem.types.StructTypeDefinition;
import org.apache.hadoop.metadata.typesystem.types.TraitType;
import org.apache.hadoop.metadata.typesystem.types.TypeSystem;
import org.testng.annotations.BeforeClass;
......@@ -44,7 +45,7 @@ import javax.inject.Inject;
import java.util.List;
@Guice(modules = RepositoryMetadataModule.class)
public class GraphTypeStoreTest {
public class GraphBackedTypeStoreTest {
@Inject
private GraphProvider<TitanGraph> graphProvider;
......@@ -85,7 +86,9 @@ public class GraphTypeStoreTest {
Assert.assertEquals(1, enumTypes.size());
//validate class
Assert.assertTrue(types.structTypesAsJavaList().isEmpty());
List<StructTypeDefinition> structTypes = types.structTypesAsJavaList();
Assert.assertEquals(1, structTypes.size());
List<HierarchicalTypeDefinition<ClassType>> classTypes = types.classTypesAsJavaList();
Assert.assertEquals(3, classTypes.size());
for (HierarchicalTypeDefinition<ClassType> classType : classTypes) {
......
repository/src/test/resources/application.properties
......@@ -22,7 +22,7 @@ metadata.graph.storage.backend=inmemory
# Graph Search Index
metadata.graph.index.search.backend=elasticsearch
metadata.graph.index.search.directory=./data/es
metadata.graph.index.search.directory=./target/data/es
metadata.graph.index.search.elasticsearch.client-only=false
metadata.graph.index.search.elasticsearch.local-mode=true
......
src/conf/application.properties
......@@ -19,11 +19,11 @@
######### Graph Database Configs #########
# Graph Storage
metadata.graph.storage.backend=berkeleyje
metadata.graph.storage.directory=./data/berkeley
metadata.graph.storage.directory=./target/data/berkeley
# Graph Search Index
metadata.graph.index.search.backend=elasticsearch
metadata.graph.index.search.directory=./data/es
metadata.graph.index.search.directory=./target/data/es
metadata.graph.index.search.elasticsearch.client-only=false
metadata.graph.index.search.elasticsearch.local-mode=true
......
typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/TypeSystem.java
......@@ -35,13 +35,22 @@ import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TimeZone;
import java.util.concurrent.ConcurrentHashMap;
@Singleton
@InterfaceAudience.Private
public class TypeSystem {
private static final TypeSystem INSTANCE = new TypeSystem();
public static SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd");
public static ThreadLocal<DateFormat> dateFormat = new ThreadLocal() {
@Override
public DateFormat initialValue() {
DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd");
dateFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
return dateFormat;
}
};
private Map<String, IDataType> types;
private IdType idType;
......@@ -277,7 +286,7 @@ public class TypeSystem {
}
public DateFormat getDateFormat() {
return dateFormat;
return dateFormat.get();
}
public boolean allowNullsInCollections() {
......
typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/utils/TypesUtil.java
......@@ -26,6 +26,7 @@ import org.apache.hadoop.metadata.typesystem.types.EnumValue;
import org.apache.hadoop.metadata.typesystem.types.HierarchicalTypeDefinition;
import org.apache.hadoop.metadata.typesystem.types.IDataType;
import org.apache.hadoop.metadata.typesystem.types.Multiplicity;
import org.apache.hadoop.metadata.typesystem.types.StructTypeDefinition;
import org.apache.hadoop.metadata.typesystem.types.TraitType;
/**
......@@ -73,6 +74,10 @@ public class TypesUtil {
return new HierarchicalTypeDefinition<>(TraitType.class, name, superTypes, attrDefs);
}
public static StructTypeDefinition createStructTypeDef(String name, AttributeDefinition... attrDefs) {
return new StructTypeDefinition(name, attrDefs);
}
public static HierarchicalTypeDefinition<ClassType> createClassTypeDef(
String name, ImmutableList<String> superTypes, AttributeDefinition... attrDefs) {
return new HierarchicalTypeDefinition<>(ClassType.class, name, superTypes, attrDefs);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment