Skip to content

A
atlas
Commit 7da5e03f by Venkatesh Seetharam
Options

ISSUE-39 Add typed instance list end to end. Contributed by Venkatesh Seetharam

parent be7c4cb6
Showing with 141 additions and 48 deletions
repository/src/main/java/org/apache/hadoop/metadata/repository/graph/GraphBackedMetadataRepository.java
......@@ -23,6 +23,7 @@ import com.thinkaurelius.titan.core.TitanVertex;
import com.tinkerpop.blueprints.Direction;
import com.tinkerpop.blueprints.Edge;
import com.tinkerpop.blueprints.Graph;
import com.tinkerpop.blueprints.GraphQuery;
import com.tinkerpop.blueprints.TransactionalGraph;
import com.tinkerpop.blueprints.Vertex;
import org.apache.commons.collections.map.HashedMap;
......@@ -58,6 +59,7 @@ import java.math.BigInteger;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicInteger;
......@@ -160,6 +162,27 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
}
}
@Override
public List<String> getEntityList(String entityType) throws RepositoryException {
LOG.info("Retrieving entity list for type={}", entityType);
GraphQuery query = graphService.getBlueprintsGraph().query()
.has(Constants.ENTITY_TYPE_PROPERTY_KEY, entityType);
Iterator<Vertex> results = query.vertices().iterator();
if (!results.hasNext()) {
return Collections.emptyList();
}
ArrayList<String> entityList = new ArrayList<>();
while (results.hasNext()) {
Vertex vertex = results.next();
entityList.add(vertex.<String>getProperty(Constants.GUID_PROPERTY_KEY));
}
return entityList;
}
@Override
public List<Map<String,String>> rawSearch(String gremlinQuery) throws RepositoryException {
ScriptEngineManager manager = new ScriptEngineManager();
ScriptEngine engine = manager.getEngineByName("gremlin-groovy");
......@@ -209,12 +232,6 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
}
}
@Override
public List<String> getEntityList(String entityType) throws RepositoryException {
LOG.info("Retrieving entity list for type={}", entityType);
return Collections.emptyList();
}
private final class EntityProcessor implements ObjectGraphWalker.NodeProcessor {
public final Map<Id, Id> idToNewIdMap;
......@@ -328,7 +345,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
throws MetadataException {
String typedInstanceGUID = null;
for (ITypedReferenceableInstance typedInstance : newTypedInstances) { // Traverse over newInstances
LOG.debug("Adding typed instance {}", typedInstance);
LOG.debug("Adding typed instance {}", typedInstance.getTypeName());
Id id = typedInstance.getId();
if (id == null) { // oops
......@@ -364,7 +381,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
Map<String, AttributeInfo> fields,
Map<Id, Vertex> idToVertexMap) throws MetadataException {
LOG.debug("Mapping instance {} to vertex {} for fields {}",
typedInstance, instanceVertex, fields);
typedInstance.getTypeName(), instanceVertex, fields);
for (AttributeInfo attributeInfo : fields.values()) {
LOG.debug("mapping attributeInfo {}", attributeInfo);
final IDataType dataType = attributeInfo.dataType();
......@@ -611,7 +628,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
private void mapVertexToStructInstance(Vertex instanceVertex,
ITypedInstance typedInstance,
AttributeInfo attributeInfo) throws MetadataException {
LOG.debug("mapping vertex {} to struct {}", typedInstance, attributeInfo.name);
LOG.debug("mapping vertex {} to struct {}", instanceVertex, attributeInfo.name);
StructType structType = typeSystem.getDataType(
StructType.class, attributeInfo.dataType().getName());
ITypedStruct structInstance = structType.createInstance();
......
repository/src/main/java/org/apache/hadoop/metadata/services/DefaultMetadataService.java
......@@ -27,11 +27,11 @@ import org.apache.hadoop.metadata.json.TypesSerialization;
import org.apache.hadoop.metadata.listener.TypedInstanceChangeListener;
import org.apache.hadoop.metadata.listener.TypesChangeListener;
import org.apache.hadoop.metadata.repository.MetadataRepository;
import org.apache.hadoop.metadata.storage.RepositoryException;
import org.apache.hadoop.metadata.types.IDataType;
import org.apache.hadoop.metadata.types.TypeSystem;
import org.codehaus.jettison.json.JSONException;
import org.codehaus.jettison.json.JSONObject;
import org.json.simple.JSONValue;
import org.json.simple.parser.ParseException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
......@@ -73,7 +73,7 @@ public class DefaultMetadataService implements MetadataService {
public JSONObject createType(String typeName,
String typeDefinition) throws MetadataException {
try {
validate(typeName, typeDefinition);
validateTypeDoesNotExist(typeName, typeDefinition);
TypesDef typesDef = TypesSerialization.fromJson(typeDefinition);
Map<String, IDataType> typesAdded = typeSystem.defineTypes(typesDef);
......@@ -86,20 +86,16 @@ public class DefaultMetadataService implements MetadataService {
}
return response;
} catch (ParseException e) {
LOG.error("Unable to parse JSON for type {}", typeName, e);
throw new MetadataException("validation failed for: " + typeName);
} catch (JSONException e) {
LOG.error("Unable to persist type {}", typeName, e);
throw new MetadataException("Unable to create response for: " + typeName);
}
}
private void validate(String typeName,
String typeDefinition) throws ParseException, MetadataException {
private void validateTypeDoesNotExist(String typeName,
String typeDefinition) throws MetadataException {
Preconditions.checkNotNull(typeName, "type name cannot be null");
Preconditions.checkNotNull(typeDefinition, "type definition cannot be null");
JSONValue.parseWithException(typeDefinition);
// verify if the type already exists
String existingTypeDefinition = null;
......@@ -110,7 +106,7 @@ public class DefaultMetadataService implements MetadataService {
}
if (existingTypeDefinition != null) {
throw new MetadataException("type is already defined for : " + typeName);
throw new RepositoryException("type is already defined for : " + typeName);
}
}
......@@ -178,8 +174,9 @@ public class DefaultMetadataService implements MetadataService {
*/
@Override
public String getEntityDefinition(String guid) throws MetadataException {
final ITypedReferenceableInstance instance =
repository.getEntityDefinition(guid);
Preconditions.checkNotNull(guid, "guid cannot be null");
final ITypedReferenceableInstance instance = repository.getEntityDefinition(guid);
return instance == null
? null
: Serialization$.MODULE$.toJson(instance);
......@@ -192,8 +189,26 @@ public class DefaultMetadataService implements MetadataService {
* @return list of entity names for the given type in the repository
*/
@Override
public List<String> getEntityNamesList(String entityType) throws MetadataException {
throw new UnsupportedOperationException();
public List<String> getEntityList(String entityType) throws MetadataException {
validateTypeExists(entityType);
return repository.getEntityList(entityType);
}
private void validateTypeExists(String entityType) throws MetadataException {
Preconditions.checkNotNull(entityType, "entity type cannot be null");
// verify if the type exists
String existingTypeDefinition = null;
try {
existingTypeDefinition = getTypeDefinition(entityType);
} catch (MetadataException ignore) {
// do nothing
}
if (existingTypeDefinition == null) {
throw new RepositoryException("type is not defined for : " + entityType);
}
}
private void onAdd(Map<String, IDataType> typesAdded) throws MetadataException {
......
repository/src/main/java/org/apache/hadoop/metadata/services/MetadataService.java
......@@ -78,5 +78,5 @@ public interface MetadataService extends Service {
* @param entityType type
* @return list of entity names for the given type in the repository
*/
List<String> getEntityNamesList(String entityType) throws MetadataException;
List<String> getEntityList(String entityType) throws MetadataException;
}
webapp/src/main/java/org/apache/hadoop/metadata/web/resources/EntityResource.java
......@@ -19,8 +19,11 @@
package org.apache.hadoop.metadata.web.resources;
import com.google.common.base.Preconditions;
import org.apache.hadoop.metadata.MetadataException;
import org.apache.hadoop.metadata.services.MetadataService;
import org.apache.hadoop.metadata.web.util.Servlets;
import org.codehaus.jettison.json.JSONArray;
import org.codehaus.jettison.json.JSONException;
import org.codehaus.jettison.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
......@@ -40,6 +43,7 @@ import javax.ws.rs.WebApplicationException;
import javax.ws.rs.core.Context;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import java.util.List;
/**
* Entity management operations as REST API.
......@@ -120,19 +124,27 @@ public class EntityResource {
}
@GET
@Path("definition/{entityType}/{entityName}")
@Produces(MediaType.APPLICATION_JSON)
public Response getEntityDefinition(@PathParam("entityType") String entityType,
@PathParam("entityName") String entityName) {
return Response.status(Response.Status.SERVICE_UNAVAILABLE).build();
}
@GET
@Path("list/{entityType}")
@Produces(MediaType.APPLICATION_JSON)
public Response getEntityList(@PathParam("entityType") String entityType,
@DefaultValue("0") @QueryParam("offset") Integer offset,
@QueryParam("numResults") Integer resultsPerPage) {
return Response.ok().build();
try {
final List<String> entityList = metadataService.getEntityList(entityType);
JSONObject response = new JSONObject();
response.put("requestId", Thread.currentThread().getName());
response.put("list", new JSONArray(entityList));
return Response.ok(response).build();
} catch (MetadataException e) {
LOG.error("Unable to get entity list for type {}", entityType, e);
throw new WebApplicationException(
Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST));
} catch (JSONException e) {
LOG.error("Unable to get entity list for type {}", entityType, e);
throw new WebApplicationException(
Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR));
}
}
}
webapp/src/main/java/org/apache/hadoop/metadata/web/resources/TypesResource.java
......@@ -124,7 +124,7 @@ public class TypesResource {
return Response.ok(response).build();
} catch (Exception e) {
LOG.error("Unable to persist entity object", e);
LOG.error("Unable to get types list", e);
throw new WebApplicationException(
Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST));
}
......
webapp/src/test/java/org/apache/hadoop/metadata/web/resources/EntityJerseyResourceIT.java
......@@ -33,6 +33,8 @@ import org.apache.hadoop.metadata.types.HierarchicalTypeDefinition;
import org.apache.hadoop.metadata.types.Multiplicity;
import org.apache.hadoop.metadata.types.StructTypeDefinition;
import org.apache.hadoop.metadata.types.TraitType;
import org.codehaus.jettison.json.JSONArray;
import org.codehaus.jettison.json.JSONException;
import org.codehaus.jettison.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
......@@ -140,7 +142,7 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
*/
@Test
public void testGetInvalidEntityDefinition() {
public void testGetInvalidEntityDefinition() throws Exception {
WebResource resource = service
.path("api/metadata/entities/definition")
.path("blah");
......@@ -150,12 +152,13 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
.type(MediaType.APPLICATION_JSON)
.method(HttpMethod.GET, ClientResponse.class);
Assert.assertEquals(clientResponse.getStatus(), Response.Status.NOT_FOUND.getStatusCode());
String response = clientResponse.getEntity(String.class);
System.out.println("response = " + response);
String responseAsString = clientResponse.getEntity(String.class);
Assert.assertNotNull(responseAsString);
}
@Test (dependsOnMethods = "testSubmitEntity", enabled = false)
public void testGetEntityList() {
@Test (dependsOnMethods = "testSubmitEntity")
public void testGetEntityList() throws Exception {
ClientResponse clientResponse = service
.path("api/metadata/entities/list/")
.path(TABLE_TYPE)
......@@ -163,20 +166,63 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
.type(MediaType.APPLICATION_JSON)
.method(HttpMethod.GET, ClientResponse.class);
Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode());
String response = clientResponse.getEntity(String.class);
System.out.println("response = " + response);
String responseAsString = clientResponse.getEntity(String.class);
Assert.assertNotNull(responseAsString);
JSONObject response = new JSONObject(responseAsString);
Assert.assertNotNull(response.get("requestId"));
final JSONArray list = response.getJSONArray("list");
Assert.assertNotNull(list);
Assert.assertEquals(list.length(), 1);
}
@Test (enabled = false) // todo: enable this later
public void testGetEntityListForBadEntityType() {
@Test
public void testGetEntityListForBadEntityType() throws Exception {
ClientResponse clientResponse = service
.path("api/metadata/entities/list/blah")
.accept(MediaType.APPLICATION_JSON)
.type(MediaType.APPLICATION_JSON)
.method(HttpMethod.GET, ClientResponse.class);
Assert.assertEquals(clientResponse.getStatus(), Response.Status.NOT_FOUND.getStatusCode());
String response = clientResponse.getEntity(String.class);
System.out.println("response = " + response);
Assert.assertEquals(clientResponse.getStatus(), Response.Status.BAD_REQUEST.getStatusCode());
String responseAsString = clientResponse.getEntity(String.class);
Assert.assertNotNull(responseAsString);
}
@Test
public void testGetEntityListForNoInstances() throws Exception {
addNewType();
ClientResponse clientResponse = service
.path("api/metadata/entities/list/test")
.accept(MediaType.APPLICATION_JSON)
.type(MediaType.APPLICATION_JSON)
.method(HttpMethod.GET, ClientResponse.class);
Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode());
String responseAsString = clientResponse.getEntity(String.class);
Assert.assertNotNull(responseAsString);
JSONObject response = new JSONObject(responseAsString);
Assert.assertNotNull(response.get("requestId"));
final JSONArray list = response.getJSONArray("list");
Assert.assertEquals(list.length(), 0);
}
private void addNewType() throws Exception {
HierarchicalTypeDefinition<ClassType> testTypeDefinition =
createClassTypeDef("test",
ImmutableList.<String>of(),
createRequiredAttrDef("name", DataTypes.STRING_TYPE),
createRequiredAttrDef("description", DataTypes.STRING_TYPE));
typeSystem.defineClassType(testTypeDefinition);
String typesAsJSON = TypesSerialization.toJson(typeSystem,
Arrays.asList(new String[]{"test"}));
sumbitType(typesAsJSON, "test");
}
private void createHiveTypes() throws Exception {
......@@ -220,10 +266,13 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
private void submitTypes() throws Exception {
String typesAsJSON = TypesSerialization.toJson(typeSystem,
Arrays.asList(new String[]{DATABASE_TYPE, TABLE_TYPE, "serdeType", "classification"}));
sumbitType(typesAsJSON, TABLE_TYPE);
}
private void sumbitType(String typesAsJSON, String type) throws JSONException {
WebResource resource = service
.path("api/metadata/types/submit")
.path(TABLE_TYPE);
.path(type);
ClientResponse clientResponse = resource
.accept(MediaType.APPLICATION_JSON)
......@@ -235,12 +284,12 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
Assert.assertNotNull(responseAsString);
JSONObject response = new JSONObject(responseAsString);
Assert.assertEquals(response.get("typeName"), TABLE_TYPE);
Assert.assertEquals(response.get("typeName"), type);
Assert.assertNotNull(response.get("types"));
Assert.assertNotNull(response.get("requestId"));
}
protected ITypedReferenceableInstance createHiveTableInstance() throws Exception {
private ITypedReferenceableInstance createHiveTableInstance() throws Exception {
Referenceable databaseInstance = new Referenceable(DATABASE_TYPE);
databaseInstance.set("name", DATABASE_NAME);
databaseInstance.set("description", "foo database");
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment