Skip to content

A
atlas
Commit d1335fc9 by A195882
Options

Updated Indexer code - indexes to be created are loaded from a property

file (indexer.properties).
parent c1995379
Showing with 104 additions and 41 deletions
repository/src/main/java/org/apache/hadoop/metadata/services/TitanGraphService.java
......@@ -18,9 +18,9 @@
package org.apache.hadoop.metadata.services;
import com.thinkaurelius.titan.core.PropertyKey;
import com.thinkaurelius.titan.core.TitanFactory;
import com.thinkaurelius.titan.core.TitanGraph;
import com.thinkaurelius.titan.core.schema.TitanGraphIndex;
import com.thinkaurelius.titan.core.schema.TitanManagement;
import com.tinkerpop.blueprints.Edge;
import com.tinkerpop.blueprints.Graph;
......@@ -35,7 +35,10 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
/**
......@@ -50,6 +53,8 @@ public class TitanGraphService implements GraphService {
* Constant for the configuration property that indicates the prefix.
*/
private static final String METADATA_PREFIX = "metadata.graph.";
private static final String INDEXER_PREFIX = "metadata.indexer.vertex.";
private static final List<String> acceptedTypes = Arrays.asList("String","Int","Long");
private TitanGraph titanGraph;
private Set<String> vertexIndexedKeys;
......@@ -104,70 +109,88 @@ public class TitanGraphService implements GraphService {
return graphConfig;
}
private static Configuration getConfiguration(String filename, String prefix) throws ConfigurationException {
PropertiesConfiguration configProperties =
new PropertiesConfiguration(filename);
Configuration graphConfig = new PropertiesConfiguration();
final Iterator<String> iterator = configProperties.getKeys();
while (iterator.hasNext()) {
String key = iterator.next();
if (key.startsWith(prefix)) {
String value = (String) configProperties.getProperty(key);
key = key.substring(prefix.length());
graphConfig.setProperty(key, value);
}
}
return graphConfig;
}
protected TitanGraph initializeGraphDB(Configuration graphConfig) {
LOG.info("Initializing titanGraph db");
return TitanFactory.open(graphConfig);
}
protected void createIndicesForVertexKeys() {
protected void createIndicesForVertexKeys() throws ConfigurationException {
if (!titanGraph.getIndexedKeys(Vertex.class).isEmpty()) {
LOG.info("Indexes already exist for titanGraph");
return;
}
LOG.info("Indexes does not exist, Creating indexes for titanGraph");
// todo - add index for vertex and edge property keys
// Titan index backend does not support Mixed Index - must use a separate backend for that.
// Using composite for now. Literal matches only. Using Global Titan index.
TitanManagement mgmt = titanGraph.getManagementSystem();
// This is the first run try-it-out index setup for property keys.
// These were pulled from the Hive bridge. Edge indices to come.
PropertyKey desc = mgmt.makePropertyKey("DESC").dataType(String.class).make();
mgmt.buildIndex("byDesc",Vertex.class).addKey(desc).buildCompositeIndex();
PropertyKey dbLoc = mgmt.makePropertyKey("DB_LOCATION_URI").dataType(String.class).make();
mgmt.buildIndex("byDbloc",Vertex.class).addKey(dbLoc).buildCompositeIndex();
PropertyKey name = mgmt.makePropertyKey("NAME").dataType(String.class).make();
mgmt.buildIndex("byName",Vertex.class).addKey(name).buildCompositeIndex();
LOG.info("Indexes do not exist, Creating indexes for titanGraph using indexer.properties.");
PropertyKey tableName = mgmt.makePropertyKey("TBL_NAME").dataType(String.class).make();
mgmt.buildIndex("byTableName",Vertex.class).addKey(tableName).buildCompositeIndex();
Configuration indexConfig = getConfiguration("indexer.properties", INDEXER_PREFIX);
PropertyKey tableType = mgmt.makePropertyKey("TBL_TYPE").dataType(String.class).make();
mgmt.buildIndex("byTableType",Vertex.class).addKey(tableType).buildCompositeIndex();
TitanManagement mgmt = titanGraph.getManagementSystem();
mgmt.buildIndex("mainIndex", Vertex.class).buildMixedIndex("search");
TitanGraphIndex graphIndex = mgmt.getGraphIndex("mainIndex");
PropertyKey createTime = mgmt.makePropertyKey("CREATE_TIME").dataType(Long.class).make();
mgmt.buildIndex("byCreateTime",Vertex.class).addKey(createTime).buildCompositeIndex();
// Properties are formatted: prop_name:type;prop_name:type
// E.g. Name:String;Date:Long
if (!indexConfig.isEmpty()) {
PropertyKey colName = mgmt.makePropertyKey("COLUMN_NAME").dataType(String.class).make();
mgmt.buildIndex("byColName",Vertex.class).addKey(colName).buildCompositeIndex();
// Get a list of property names to iterate through...
List<String> propList = new ArrayList<String>();
PropertyKey typeName = mgmt.makePropertyKey("TYPE_NAME").dataType(String.class).make();
mgmt.buildIndex("byTypeName",Vertex.class).addKey(typeName).buildCompositeIndex();
Iterator<String> it = indexConfig.getKeys("property.name");
while (it.hasNext()) {
propList.add(it.next());
}
/* More attributes from the Hive bridge.
it = propList.iterator();
while (it.hasNext()) {
PropertyKey ownerName = mgmt.makePropertyKey("OWNER_NAME").dataType(String.class).make();
mgmt.buildIndex("byOwnerName",Vertex.class).addKey(ownerName).buildCompositeIndex();
// Pull the property name and index, so we can register the name and look up the type.
String prop = it.next().toString();
String index = prop.substring(prop.lastIndexOf(".") + 1);
String type = null;
prop = indexConfig.getProperty(prop).toString();
PropertyKey lastAccess = mgmt.makePropertyKey("LAST_ACCESS_TIME").dataType(Long.class).make();
mgmt.buildIndex("byLastAccess",Vertex.class).addKey(lastAccess).buildCompositeIndex();
// Look up the type for the specified property name.
if (indexConfig.containsKey("property.type." + index)) {
type = indexConfig.getProperty("property.type." + index).toString();
} else {
throw new ConfigurationException("No type specified for property " + index + " in indexer.properties.");
}
PropertyKey viewExpandedText = mgmt.makePropertyKey("VIEW_EXPANDED_TEXT").dataType(String.class).make();
mgmt.buildIndex("byExpandedText",Vertex.class).addKey(viewExpandedText).buildCompositeIndex();
// Is the type submitted one of the approved ones?
if (!acceptedTypes.contains(type)) {
throw new ConfigurationException("The type provided in indexer.properties for property " + prop + " is not supported. Supported types are: " + acceptedTypes.toString());
}
PropertyKey viewOrigText= mgmt.makePropertyKey("VIEW_ORIGINAL_TEXT").dataType(String.class).make();
mgmt.buildIndex("byOrigText",Vertex.class).addKey(viewOrigText).buildCompositeIndex();
// Add the key.
LOG.info("Adding property: " + prop + " to index as type: " + type);
mgmt.addIndexKey(graphIndex,mgmt.makePropertyKey(prop).dataType(type.getClass()).make());
PropertyKey comment = mgmt.makePropertyKey("COMMENT").dataType(Integer.class).make();
mgmt.buildIndex("byComment",Vertex.class).addKey(comment).buildCompositeIndex();
*/
}
mgmt.commit();
LOG.info("Index creation complete.");
}
}
/**
......
repository/src/test/resources/indexer.properties 0 → 100644
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# This is formatted as follows:
# metadata.indexer.vertex.property.name.<index>=<Property Name>
# metadata.indexer.vertex.property.type.<index>=<Data Type>
metadata.indexer.vertex.property.name.0=DESC
metadata.indexer.vertex.property.type.0=String
metadata.indexer.vertex.property.name.1=DB_LOCATION_URI
metadata.indexer.vertex.property.type.1=String
metadata.indexer.vertex.property.name.2=NAME
metadata.indexer.vertex.property.type.2=String
metadata.indexer.vertex.property.name.3=OWNER_NAME
metadata.indexer.vertex.property.type.3=String
metadata.indexer.vertex.property.name.4=TBL_NAME
metadata.indexer.vertex.property.type.4=String
metadata.indexer.vertex.property.name.5=COMMENT
metadata.indexer.vertex.property.type.5=String
metadata.indexer.vertex.property.name.6=COLUMN_NAME
metadata.indexer.vertex.property.type.6=String
metadata.indexer.vertex.property.name.7=TYPE_NAME
metadata.indexer.vertex.property.type.7=String
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment