Commit 386d8d38 by Venkatesh Seetharam

ISSUE-35 Add typed instance put/get end to end. Contributed by Venkatesh Seetharam

parent 8285e3b9
......@@ -83,8 +83,10 @@ public class DefaultMetadataService implements MetadataService {
return response;
} catch (ParseException e) {
LOG.error("Unable to parse JSON for type {}", typeName, e);
throw new MetadataException("validation failed for: " + typeName);
} catch (JSONException e) {
LOG.error("Unable to persist type {}", typeName, e);
throw new MetadataException("Unable to create response for: " + typeName);
}
}
......@@ -147,6 +149,7 @@ public class DefaultMetadataService implements MetadataService {
Serialization$.MODULE$.fromJson(entityDefinition);
return repository.createEntity(entityInstance, entityType);
} catch (ParseException e) {
LOG.error("Unable to parse JSON {} for type {}", entityDefinition, entityType, e);
throw new MetadataException("validation failed for: " + entityType);
}
}
......@@ -154,7 +157,9 @@ public class DefaultMetadataService implements MetadataService {
private void validateEntity(String entity, String entityType) throws ParseException {
Preconditions.checkNotNull(entity, "entity cannot be null");
Preconditions.checkNotNull(entityType, "entity type cannot be null");
JSONValue.parseWithException(entity);
// todo: this is failing for instances but not types
// JSONValue.parseWithException(entity);
}
/**
......@@ -167,21 +172,9 @@ public class DefaultMetadataService implements MetadataService {
public String getEntityDefinition(String guid) throws MetadataException {
final ITypedReferenceableInstance instance =
repository.getEntityDefinition(guid);
return Serialization$.MODULE$.toJson(instance);
}
/**
* Return the definition for the given entity name and type.
*
* @param entityName name
* @param entityType type
* @return entity definition as JSON
*/
@Override
public String getEntityDefinition(String entityName,
String entityType) throws MetadataException {
throw new UnsupportedOperationException();
return instance == null
? null
: Serialization$.MODULE$.toJson(instance);
}
/**
......
......@@ -73,15 +73,6 @@ public interface MetadataService extends Service {
String getEntityDefinition(String guid) throws MetadataException;
/**
* Return the definition for the given entity name and type.
*
* @param entityName name
* @param entityType type
* @return entity definition as JSON
*/
String getEntityDefinition(String entityName, String entityType) throws MetadataException;
/**
* Return the list of entity names for the given type in the repository.
*
* @param entityType type
......
......@@ -23,9 +23,45 @@ public final class GraphUtils {
private static final Logger LOG = LoggerFactory.getLogger(GraphUtils.class);
private static final String GUID_PROPERTY_KEY = "guid";
private static final String TIMESTAMP_PROPERTY_KEY = "timestamp";
private GraphUtils() {
}
public static Edge addEdge(Vertex fromVertex, Vertex toVertex, String edgeLabel) {
return addEdge(fromVertex, toVertex, edgeLabel, null);
}
public static Edge addEdge(Vertex fromVertex, Vertex toVertex,
String edgeLabel, String timestamp) {
Edge edge = findEdge(fromVertex, toVertex, edgeLabel);
Edge edgeToVertex = edge != null ? edge : fromVertex.addEdge(edgeLabel, toVertex);
if (timestamp != null) {
edgeToVertex.setProperty(TIMESTAMP_PROPERTY_KEY, timestamp);
}
return edgeToVertex;
}
public static Edge findEdge(Vertex fromVertex, Vertex toVertex, String edgeLabel) {
return findEdge(fromVertex, toVertex.getProperty(GUID_PROPERTY_KEY), edgeLabel);
}
public static Edge findEdge(Vertex fromVertex, Object toVertexName, String edgeLabel) {
Edge edgeToFind = null;
for (Edge edge : fromVertex.getEdges(Direction.OUT, edgeLabel)) {
if (edge.getVertex(Direction.IN).getProperty(
GUID_PROPERTY_KEY).equals(toVertexName)) {
edgeToFind = edge;
break;
}
}
return edgeToFind;
}
public static Vertex findVertex(Graph blueprintsGraph,
String guid) {
LOG.debug("Finding vertex for: guid={}", guid);
......
......@@ -67,22 +67,23 @@ public class EntityResource {
}
@POST
@Path("submit/{entityType}")
@Path("submit/{typeName}")
@Consumes(MediaType.APPLICATION_JSON)
@Produces(MediaType.APPLICATION_JSON)
public Response submit(@Context HttpServletRequest request,
@PathParam("entityType") final String entityType) {
@PathParam("typeName") final String typeName) {
try {
final String entity = Servlets.getRequestPayload(request);
System.out.println("entity = " + entity);
LOG.debug("submitting entity {} ", entity);
final String guid = metadataService.createEntity(entity, entityType);
final String guid = metadataService.createEntity(typeName, entity);
JSONObject response = new JSONObject();
response.put("GUID", guid);
response.put("requestId", Thread.currentThread().getName());
return Response.ok(response).build();
} catch (Exception e) {
LOG.error("Unable to persist instance for type {}", typeName, e);
throw new WebApplicationException(
Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST));
}
......@@ -109,8 +110,7 @@ public class EntityResource {
return Response.status(status).entity(response).build();
} catch (Exception e) {
LOG.error("Action failed: {}\nError: {}",
Response.Status.INTERNAL_SERVER_ERROR, e.getMessage());
LOG.error("Unable to get instance definition for GUID {}", guid, e);
throw new WebApplicationException(e, Response
.status(Response.Status.INTERNAL_SERVER_ERROR)
.entity(e.getMessage())
......
......@@ -10,16 +10,21 @@ import org.apache.hadoop.metadata.types.HierarchicalTypeDefinition;
import org.apache.hadoop.metadata.types.IDataType;
import org.apache.hadoop.metadata.types.Multiplicity;
import org.apache.hadoop.metadata.types.TraitType;
import org.apache.hadoop.metadata.types.TypeSystem;
import org.testng.annotations.BeforeClass;
import javax.ws.rs.core.UriBuilder;
public class BaseResourceIT {
public abstract class BaseResourceIT {
protected TypeSystem typeSystem;
protected WebResource service;
@BeforeClass
public void setUp() throws Exception {
typeSystem = TypeSystem.getInstance();
typeSystem.reset();
String baseUrl = "http://localhost:21000/";
DefaultClientConfig config = new DefaultClientConfig();
......
......@@ -18,54 +18,78 @@
package org.apache.hadoop.metadata.web.resources;
import java.util.HashMap;
import java.util.Map;
import java.util.UUID;
import com.google.common.collect.ImmutableList;
import com.sun.jersey.api.client.ClientResponse;
import com.sun.jersey.api.client.WebResource;
import org.apache.hadoop.metadata.ITypedReferenceableInstance;
import org.apache.hadoop.metadata.Referenceable;
import org.apache.hadoop.metadata.Struct;
import org.apache.hadoop.metadata.json.Serialization$;
import org.apache.hadoop.metadata.json.TypesSerialization;
import org.apache.hadoop.metadata.types.AttributeDefinition;
import org.apache.hadoop.metadata.types.ClassType;
import org.apache.hadoop.metadata.types.DataTypes;
import org.apache.hadoop.metadata.types.HierarchicalTypeDefinition;
import org.apache.hadoop.metadata.types.Multiplicity;
import org.apache.hadoop.metadata.types.StructTypeDefinition;
import org.apache.hadoop.metadata.types.TraitType;
import org.codehaus.jettison.json.JSONObject;
import org.testng.Assert;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
import javax.ws.rs.HttpMethod;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import org.json.simple.JSONValue;
import org.testng.Assert;
import org.testng.annotations.Test;
import com.google.gson.JsonElement;
import com.google.gson.JsonParser;
import com.sun.jersey.api.client.ClientResponse;
import com.sun.jersey.api.client.WebResource;
import java.util.ArrayList;
import java.util.List;
import java.util.UUID;
/**
* Integration tests for Entity Jersey Resource.
*/
@Test (enabled = false)
public class EntityJerseyResourceIT extends BaseResourceIT {
private static final String ENTITY_NAME = "clicks-table";
private static final String ENTITY_TYPE = "hive-table";
private static final String DATABASE_NAME = "ads";
private static final String TABLE_NAME = "clicks-table";
private static final String DATABASE_TYPE = "hive_database";
private static final String DATABASE_NAME = "foo";
private static final String TABLE_TYPE = "hive_table";
private static final String TABLE_NAME = "bar";
private static final String TRAIT_TYPE = "hive_fetl";
private String guid;
@BeforeClass
public void setUp() throws Exception {
super.setUp();
List<HierarchicalTypeDefinition> typeDefinitions = createHiveTypes();
submitTypes(typeDefinitions);
}
@Test
public void testSubmitEntity() throws Exception {
ITypedReferenceableInstance tableInstance = createHiveTableInstance();
@Test (enabled = false)
public void testSubmitEntity() {
String entityStream = getTestEntityJSON();
JsonParser parser = new JsonParser();
String instanceAsJSON = Serialization$.MODULE$.toJson(tableInstance);
WebResource resource = service
.path("api/metadata/entities/submit")
.path(ENTITY_TYPE);
.path(TABLE_TYPE);
ClientResponse clientResponse = resource
.accept(MediaType.APPLICATION_JSON)
.type(MediaType.APPLICATION_JSON)
.method(HttpMethod.POST, ClientResponse.class, entityStream);
.method(HttpMethod.POST, ClientResponse.class, instanceAsJSON);
Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode());
String response = clientResponse.getEntity(String.class);
Assert.assertNotNull(response);
String responseAsString = clientResponse.getEntity(String.class);
Assert.assertNotNull(responseAsString);
JSONObject response = new JSONObject(responseAsString);
Assert.assertNotNull(response.get("requestId"));
JsonElement elem = parser.parse(response);
String guid = elem.getAsJsonObject().get("GUID").getAsString();
guid = response.get("GUID").toString();
Assert.assertNotNull(guid);
try {
Assert.assertNotNull(UUID.fromString(guid));
......@@ -74,12 +98,11 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
}
}
@Test (dependsOnMethods = "testSubmitEntity", enabled = false)
@Test (dependsOnMethods = "testSubmitEntity")
public void testGetEntityDefinition() {
WebResource resource = service
.path("api/metadata/entities/definition")
.path(ENTITY_TYPE)
.path(ENTITY_NAME);
.path(guid);
ClientResponse clientResponse = resource
.accept(MediaType.APPLICATION_JSON)
......@@ -90,20 +113,10 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
System.out.println("response = " + response);
}
private static String getTestEntityJSON() {
Map<String, String> props = new HashMap<>();
props.put("entityName", ENTITY_NAME);
props.put("entityType", ENTITY_TYPE);
props.put("database", DATABASE_NAME);
props.put("table", TABLE_NAME);
return JSONValue.toJSONString(props);
}
@Test (enabled = false)
@Test
public void testGetInvalidEntityDefinition() {
WebResource resource = service
.path("api/metadata/entities/definition")
.path(ENTITY_TYPE)
.path("blah");
ClientResponse clientResponse = resource
......@@ -119,7 +132,7 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
public void testGetEntityList() {
ClientResponse clientResponse = service
.path("api/metadata/entities/list/")
.path(ENTITY_TYPE)
.path(TABLE_TYPE)
.accept(MediaType.APPLICATION_JSON)
.type(MediaType.APPLICATION_JSON)
.method(HttpMethod.GET, ClientResponse.class);
......@@ -139,4 +152,81 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
String response = clientResponse.getEntity(String.class);
System.out.println("response = " + response);
}
private List<HierarchicalTypeDefinition> createHiveTypes() throws Exception {
ArrayList<HierarchicalTypeDefinition> typeDefinitions = new ArrayList<>();
HierarchicalTypeDefinition<ClassType> databaseTypeDefinition =
createClassTypeDef(DATABASE_TYPE,
ImmutableList.<String>of(),
createRequiredAttrDef("name", DataTypes.STRING_TYPE),
createRequiredAttrDef("description", DataTypes.STRING_TYPE));
typeDefinitions.add(databaseTypeDefinition);
HierarchicalTypeDefinition<ClassType> tableTypeDefinition =
createClassTypeDef(TABLE_TYPE,
ImmutableList.<String>of(),
createRequiredAttrDef("name", DataTypes.STRING_TYPE),
createRequiredAttrDef("description", DataTypes.STRING_TYPE),
createRequiredAttrDef("type", DataTypes.STRING_TYPE),
new AttributeDefinition(DATABASE_TYPE,
DATABASE_TYPE, Multiplicity.REQUIRED, true, DATABASE_TYPE));
typeDefinitions.add(tableTypeDefinition);
HierarchicalTypeDefinition<TraitType> fetlTypeDefinition =
createTraitTypeDef(TRAIT_TYPE,
ImmutableList.<String>of(),
createRequiredAttrDef("level", DataTypes.INT_TYPE));
typeDefinitions.add(fetlTypeDefinition);
typeSystem.defineTypes(
ImmutableList.<StructTypeDefinition>of(),
ImmutableList.of(fetlTypeDefinition),
ImmutableList.of(databaseTypeDefinition, tableTypeDefinition));
return typeDefinitions;
}
private void submitTypes(List<HierarchicalTypeDefinition> typeDefinitions) throws Exception {
for (HierarchicalTypeDefinition typeDefinition : typeDefinitions) {
String typesAsJSON = TypesSerialization.toJson(
typeSystem, typeDefinition.typeName);
WebResource resource = service
.path("api/metadata/types/submit")
.path(typeDefinition.typeName);
ClientResponse clientResponse = resource
.accept(MediaType.APPLICATION_JSON)
.type(MediaType.APPLICATION_JSON)
.method(HttpMethod.POST, ClientResponse.class, typesAsJSON);
Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode());
String responseAsString = clientResponse.getEntity(String.class);
Assert.assertNotNull(responseAsString);
JSONObject response = new JSONObject(responseAsString);
Assert.assertEquals(response.get("typeName"), typeDefinition.typeName);
Assert.assertNotNull(response.get("types"));
Assert.assertNotNull(response.get("requestId"));
}
}
protected ITypedReferenceableInstance createHiveTableInstance() throws Exception {
Referenceable databaseInstance = new Referenceable(DATABASE_TYPE);
databaseInstance.set("name", DATABASE_NAME);
databaseInstance.set("description", "foo database");
Referenceable tableInstance = new Referenceable(TABLE_TYPE, TRAIT_TYPE);
tableInstance.set("name", TABLE_NAME);
tableInstance.set("description", "bar table");
tableInstance.set("type", "managed");
tableInstance.set(DATABASE_TYPE, databaseInstance);
Struct traitInstance = (Struct) tableInstance.getTrait(TRAIT_TYPE);
traitInstance.set("level", 1);
ClassType tableType = typeSystem.getDataType(ClassType.class, TABLE_TYPE);
return tableType.convert(tableInstance, Multiplicity.REQUIRED);
}
}
......@@ -21,7 +21,6 @@ package org.apache.hadoop.metadata.web.resources;
import com.google.common.collect.ImmutableList;
import com.sun.jersey.api.client.ClientResponse;
import com.sun.jersey.api.client.WebResource;
import org.apache.hadoop.metadata.MetadataException;
import org.apache.hadoop.metadata.json.TypesSerialization;
import org.apache.hadoop.metadata.types.AttributeDefinition;
import org.apache.hadoop.metadata.types.ClassType;
......@@ -30,7 +29,6 @@ import org.apache.hadoop.metadata.types.HierarchicalTypeDefinition;
import org.apache.hadoop.metadata.types.Multiplicity;
import org.apache.hadoop.metadata.types.StructTypeDefinition;
import org.apache.hadoop.metadata.types.TraitType;
import org.apache.hadoop.metadata.types.TypeSystem;
import org.codehaus.jettison.json.JSONArray;
import org.codehaus.jettison.json.JSONObject;
import org.testng.Assert;
......@@ -49,14 +47,12 @@ import java.util.List;
*/
public class TypesJerseyResourceIT extends BaseResourceIT {
private TypeSystem typeSystem;
private List<HierarchicalTypeDefinition> typeDefinitions;
@BeforeClass
public void setUp() throws Exception {
super.setUp();
typeSystem = TypeSystem.getInstance();
typeDefinitions = createHiveTypes();
}
......@@ -151,11 +147,12 @@ public class TypesJerseyResourceIT extends BaseResourceIT {
Assert.assertNotNull(list);
}
private List<HierarchicalTypeDefinition> createHiveTypes() throws MetadataException {
private List<HierarchicalTypeDefinition> createHiveTypes() throws Exception {
ArrayList<HierarchicalTypeDefinition> typeDefinitions = new ArrayList<>();
HierarchicalTypeDefinition<ClassType> databaseTypeDefinition =
createClassTypeDef("database", ImmutableList.<String>of(),
createClassTypeDef("database",
ImmutableList.<String>of(),
createRequiredAttrDef("name", DataTypes.STRING_TYPE),
createRequiredAttrDef("description", DataTypes.STRING_TYPE));
typeDefinitions.add(databaseTypeDefinition);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment