Merge branch 'master' into dal

client/src/main/java/org/apache/hadoop/metadata/MetadataServiceClient.java

@@ -82,7 +82,7 @@ public class MetadataServiceClient {
         PropertiesConfiguration clientConfig = null;
         try {
             clientConfig = getClientProperties();
-            if (clientConfig.getBoolean(TLS_ENABLED)  || clientConfig.getString("metadata.http.authentication.type") != null) {
+            if (clientConfig.getBoolean(TLS_ENABLED, false)) {
                 // create an SSL properties configuration if one doesn't exist.  SSLFactory expects a file, so forced to create a
                 // configuration object, persist it, then subsequently pass in an empty configuration to SSLFactory
                 SecureClientUtils.persistSSLClientConfiguration(clientConfig);

repository/src/main/java/org/apache/hadoop/metadata/discovery/graph/GraphBackedDiscoveryService.java

@@ -133,12 +133,12 @@ public class GraphBackedDiscoveryService implements DiscoveryService {
             if (either.isRight()) {
                 Expressions.Expression expression = either.right().get();
                 return evaluate(expression);
+            } else {
+                throw new DiscoveryException("Invalid expression : " + dslQuery + ". " + either.left());
             }
         } catch (Exception e) { // unable to catch ExpressionException
             throw new DiscoveryException("Invalid expression : " + dslQuery, e);
         }
-
-        throw new DiscoveryException("Invalid expression : " + dslQuery);
     }
 
     public GremlinQueryResult evaluate(Expressions.Expression expression) {

repository/src/main/java/org/apache/hadoop/metadata/services/DefaultMetadataService.java

@@ -53,6 +53,7 @@ import org.slf4j.LoggerFactory;
 
 import javax.inject.Inject;
 import javax.inject.Singleton;
+import java.util.HashMap;
 import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Map;
@@ -83,8 +84,8 @@ public class DefaultMetadataService implements MetadataService {
         this.typeSystem = TypeSystem.getInstance();
         this.repository = repository;
 
-        restoreTypeSystem();
         registerListener(searchIndexer);
+        restoreTypeSystem();
     }
 
     private void restoreTypeSystem() {
@@ -113,17 +114,18 @@ public class DefaultMetadataService implements MetadataService {
             return; // this is already registered
         }
 
+        Map<String, IDataType> superTypes = new HashMap();
         HierarchicalTypeDefinition<ClassType> superTypeDefinition =
                 TypesUtil.createClassTypeDef(MetadataServiceClient.INFRASTRUCTURE_SUPER_TYPE,
-                        ImmutableList.<String>of(),
-                        NAME_ATTRIBUTE, DESCRIPTION_ATTRIBUTE);
-        typeSystem.defineClassType(superTypeDefinition);
+                        ImmutableList.<String>of(), NAME_ATTRIBUTE, DESCRIPTION_ATTRIBUTE);
+        superTypes.put(MetadataServiceClient.INFRASTRUCTURE_SUPER_TYPE, typeSystem.defineClassType
+                (superTypeDefinition));
 
         superTypeDefinition =
                 TypesUtil.createClassTypeDef(MetadataServiceClient.DATA_SET_SUPER_TYPE,
                         ImmutableList.<String>of(),
                         NAME_ATTRIBUTE, DESCRIPTION_ATTRIBUTE);
-        typeSystem.defineClassType(superTypeDefinition);
+        superTypes.put(MetadataServiceClient.DATA_SET_SUPER_TYPE, typeSystem.defineClassType(superTypeDefinition));
 
         superTypeDefinition =
                 TypesUtil.createClassTypeDef(MetadataServiceClient.PROCESS_SUPER_TYPE,
@@ -136,7 +138,8 @@ public class DefaultMetadataService implements MetadataService {
                                 DataTypes.arrayTypeName(MetadataServiceClient.DATA_SET_SUPER_TYPE),
                                 new Multiplicity(0, Integer.MAX_VALUE, false), false, null)
                 );
-        typeSystem.defineClassType(superTypeDefinition);
+        superTypes.put(MetadataServiceClient.PROCESS_SUPER_TYPE, typeSystem.defineClassType(superTypeDefinition));
+        onTypesAddedToRepo(superTypes);
     }
 
     /**

typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/HierarchicalType.java

@@ -70,7 +70,7 @@ public abstract class HierarchicalType<ST extends HierarchicalType, T> extends A
         this.fieldMapping = null;
         this.numFields = numFields;
         this.superTypes = superTypes;
-        this.immediateAttrs = null;
+        this.immediateAttrs = ImmutableList.of();
         this.attributeNameToType = null;
     }
 
@@ -86,7 +86,7 @@ public abstract class HierarchicalType<ST extends HierarchicalType, T> extends A
         this.attributeNameToType = p.right;
         this.numFields = this.fieldMapping.fields.size();
         this.superTypes = superTypes == null ? ImmutableList.<String>of() : superTypes;
-        this.immediateAttrs = ImmutableList.<AttributeInfo>copyOf(fields);
+        this.immediateAttrs = ImmutableList.copyOf(fields);
     }
 
     @Override
@@ -184,7 +184,7 @@ public abstract class HierarchicalType<ST extends HierarchicalType, T> extends A
                     (ST) typeSystem.getDataType(superTypeClass, currentPath.typeName);
 
             ImmutableList<AttributeInfo> superTypeFields = superType == this ?
-                    ImmutableList.<AttributeInfo>copyOf(fields) : superType.immediateAttrs;
+                    ImmutableList.copyOf(fields) : superType.immediateAttrs;
 
             Set<String> immediateFields = new HashSet<String>();
 

webapp/src/main/java/org/apache/hadoop/metadata/web/resources/EntityResource.java

@@ -85,8 +85,8 @@ public class EntityResource {
      * Submits an entity definition (instance) corresponding to a given type.
      */
     @POST
-    @Consumes(MediaType.APPLICATION_JSON)
-    @Produces(MediaType.APPLICATION_JSON)
+    @Consumes(Servlets.JSON_MEDIA_TYPE)
+    @Produces(Servlets.JSON_MEDIA_TYPE)
     public Response submit(@Context HttpServletRequest request) {
         try {
             final String entity = Servlets.getRequestPayload(request);
@@ -127,7 +127,7 @@ public class EntityResource {
      */
     @GET
     @Path("{guid}")
-    @Produces(MediaType.APPLICATION_JSON)
+    @Produces(Servlets.JSON_MEDIA_TYPE)
     public Response getEntityDefinition(@PathParam("guid") String guid) {
         try {
             LOG.debug("Fetching entity definition for guid={} ", guid);
@@ -165,7 +165,7 @@ public class EntityResource {
      * @param entityType name of a type which is unique
      */
     @GET
-    @Produces(MediaType.APPLICATION_JSON)
+    @Produces(Servlets.JSON_MEDIA_TYPE)
     public Response getEntityListByType(@QueryParam("type") String entityType) {
         try {
             Preconditions.checkNotNull(entityType, "Entity type cannot be null");
@@ -204,7 +204,8 @@ public class EntityResource {
      */
     @PUT
     @Path("{guid}")
-    @Produces(MediaType.APPLICATION_JSON)
+    @Consumes(Servlets.JSON_MEDIA_TYPE)
+    @Produces(Servlets.JSON_MEDIA_TYPE)
     public Response update(@PathParam("guid") String guid,
                            @QueryParam("property") String property,
                            @QueryParam("value") String value) {
@@ -237,7 +238,7 @@ public class EntityResource {
      */
     @GET
     @Path("{guid}/traits")
-    @Produces(MediaType.APPLICATION_JSON)
+    @Produces(Servlets.JSON_MEDIA_TYPE)
     public Response getTraitNames(@PathParam("guid") String guid) {
         try {
             LOG.debug("Fetching trait names for entity={}", guid);
@@ -268,8 +269,8 @@ public class EntityResource {
      */
     @POST
     @Path("{guid}/traits")
-    @Consumes(MediaType.APPLICATION_JSON)
-    @Produces(MediaType.APPLICATION_JSON)
+    @Consumes(Servlets.JSON_MEDIA_TYPE)
+    @Produces(Servlets.JSON_MEDIA_TYPE)
     public Response addTrait(@Context HttpServletRequest request,
                              @PathParam("guid") String guid) {
         try {
@@ -304,8 +305,8 @@ public class EntityResource {
      */
     @DELETE
     @Path("{guid}/traits/{traitName}")
-    @Consumes(MediaType.APPLICATION_JSON)
-    @Produces(MediaType.APPLICATION_JSON)
+    @Consumes(Servlets.JSON_MEDIA_TYPE)
+    @Produces(Servlets.JSON_MEDIA_TYPE)
     public Response deleteTrait(@Context HttpServletRequest request,
                                 @PathParam("guid") String guid,
                                 @PathParam(TRAIT_NAME) String traitName) {

webapp/src/main/java/org/apache/hadoop/metadata/web/resources/HiveLineageResource.java

@@ -65,8 +65,8 @@ public class HiveLineageResource {
      */
     @GET
     @Path("table/{tableName}/inputs/graph")
-    @Consumes(MediaType.APPLICATION_JSON)
-    @Produces(MediaType.APPLICATION_JSON)
+    @Consumes(Servlets.JSON_MEDIA_TYPE)
+    @Produces(Servlets.JSON_MEDIA_TYPE)
     public Response inputsGraph(@Context HttpServletRequest request,
                                 @PathParam("tableName") String tableName) {
         LOG.info("Fetching lineage inputs graph for tableName={}", tableName);
@@ -103,8 +103,8 @@ public class HiveLineageResource {
      */
     @GET
     @Path("table/{tableName}/outputs/graph")
-    @Consumes(MediaType.APPLICATION_JSON)
-    @Produces(MediaType.APPLICATION_JSON)
+    @Consumes(Servlets.JSON_MEDIA_TYPE)
+    @Produces(Servlets.JSON_MEDIA_TYPE)
     public Response outputsGraph(@Context HttpServletRequest request,
                                  @PathParam("tableName") String tableName) {
         LOG.info("Fetching lineage outputs graph for tableName={}", tableName);
@@ -141,8 +141,8 @@ public class HiveLineageResource {
      */
     @GET
     @Path("table/{tableName}/schema")
-    @Consumes(MediaType.APPLICATION_JSON)
-    @Produces(MediaType.APPLICATION_JSON)
+    @Consumes(Servlets.JSON_MEDIA_TYPE)
+    @Produces(Servlets.JSON_MEDIA_TYPE)
     public Response schema(@Context HttpServletRequest request,
                            @PathParam("tableName") String tableName) {
         LOG.info("Fetching schema for tableName={}", tableName);

webapp/src/main/java/org/apache/hadoop/metadata/web/resources/MetadataDiscoveryResource.java

@@ -32,6 +32,7 @@ import org.slf4j.LoggerFactory;
 
 import javax.inject.Inject;
 import javax.inject.Singleton;
+import javax.ws.rs.Consumes;
 import javax.ws.rs.GET;
 import javax.ws.rs.Path;
 import javax.ws.rs.Produces;
@@ -75,7 +76,8 @@ public class MetadataDiscoveryResource {
      */
     @GET
     @Path("search")
-    @Produces(MediaType.APPLICATION_JSON)
+    @Consumes(Servlets.JSON_MEDIA_TYPE)
+    @Produces(Servlets.JSON_MEDIA_TYPE)
     public Response search(@QueryParam("query") String query) {
         JSONObject response;
         try {   // fall back to dsl
@@ -125,7 +127,8 @@ public class MetadataDiscoveryResource {
      */
     @GET
     @Path("search/dsl")
-    @Produces(MediaType.APPLICATION_JSON)
+    @Consumes(Servlets.JSON_MEDIA_TYPE)
+    @Produces(Servlets.JSON_MEDIA_TYPE)
     public Response searchUsingQueryDSL(@QueryParam("query") String dslQuery) {
         try {
             ParamChecker.notEmpty(dslQuery, "dslQuery cannot be null");
@@ -156,7 +159,8 @@ public class MetadataDiscoveryResource {
      */
     @GET
     @Path("search/gremlin")
-    @Produces(MediaType.APPLICATION_JSON)
+    @Consumes(Servlets.JSON_MEDIA_TYPE)
+    @Produces(Servlets.JSON_MEDIA_TYPE)
     public Response searchUsingGremlinQuery(@QueryParam("query") String gremlinQuery) {
         try {
             ParamChecker.notEmpty(gremlinQuery, "gremlinQuery cannot be null or empty");
@@ -196,7 +200,8 @@ public class MetadataDiscoveryResource {
      */
     @GET
     @Path("search/fulltext")
-    @Produces(MediaType.APPLICATION_JSON)
+    @Consumes(Servlets.JSON_MEDIA_TYPE)
+    @Produces(Servlets.JSON_MEDIA_TYPE)
     public Response searchUsingFullText(@QueryParam("query") String query) {
         try {
             ParamChecker.notEmpty(query, "query cannot be null or empty");

webapp/src/main/java/org/apache/hadoop/metadata/web/resources/RexsterGraphResource.java

@@ -115,7 +115,7 @@ public class RexsterGraphResource {
      */
     @GET
     @Path("/vertices/{id}")
-    @Produces({MediaType.APPLICATION_JSON})
+    @Produces({Servlets.JSON_MEDIA_TYPE})
     public Response getVertex(@PathParam("id") final String vertexId) {
         LOG.info("Get vertex for vertexId= {}", vertexId);
         validateInputs("Invalid argument: vertex id passed is null or empty.", vertexId);
@@ -152,7 +152,7 @@ public class RexsterGraphResource {
      */
     @GET
     @Path("/vertices/properties/{id}")
-    @Produces({MediaType.APPLICATION_JSON})
+    @Produces({Servlets.JSON_MEDIA_TYPE})
     public Response getVertexProperties(@PathParam("id") final String vertexId,
                                         @DefaultValue("false") @QueryParam("relationships")
                                         final String relationships) {
@@ -192,7 +192,7 @@ public class RexsterGraphResource {
      */
     @GET
     @Path("/vertices")
-    @Produces({MediaType.APPLICATION_JSON})
+    @Produces({Servlets.JSON_MEDIA_TYPE})
     public Response getVertices(@QueryParam("key") final String key,
                                 @QueryParam("value") final String value) {
         LOG.info("Get vertices for property key= {}, value= {}", key, value);
@@ -216,7 +216,7 @@ public class RexsterGraphResource {
      */
     @GET
     @Path("vertices/{id}/{direction}")
-    @Produces({MediaType.APPLICATION_JSON})
+    @Produces({Servlets.JSON_MEDIA_TYPE})
     public Response getVertexEdges(@PathParam("id") String vertexId,
                                    @PathParam("direction") String direction) {
         LOG.info("Get vertex edges for vertexId= {}, direction= {}", vertexId, direction);
@@ -288,7 +288,7 @@ public class RexsterGraphResource {
      */
     @GET
     @Path("/edges/{id}")
-    @Produces({MediaType.APPLICATION_JSON})
+    @Produces({Servlets.JSON_MEDIA_TYPE})
     public Response getEdge(@PathParam("id") final String edgeId) {
         LOG.info("Get vertex for edgeId= {}", edgeId);
         validateInputs("Invalid argument: edge id passed is null or empty.", edgeId);

webapp/src/main/java/org/apache/hadoop/metadata/web/resources/TypesResource.java

@@ -78,8 +78,8 @@ public class TypesResource {
      * domain. Could represent things like Hive Database, Hive Table, etc.
      */
     @POST
-    @Consumes(MediaType.APPLICATION_JSON)
-    @Produces(MediaType.APPLICATION_JSON)
+    @Consumes(Servlets.JSON_MEDIA_TYPE)
+    @Produces(Servlets.JSON_MEDIA_TYPE)
     public Response submit(@Context HttpServletRequest request) {
         try {
             final String typeDefinition = Servlets.getRequestPayload(request);
@@ -119,7 +119,7 @@ public class TypesResource {
      */
     @GET
     @Path("{typeName}")
-    @Produces(MediaType.APPLICATION_JSON)
+    @Produces(Servlets.JSON_MEDIA_TYPE)
     public Response getDefinition(@Context HttpServletRequest request,
                                   @PathParam("typeName") String typeName) {
         try {
@@ -155,7 +155,7 @@ public class TypesResource {
      * @return entity names response payload as json
      */
     @GET
-    @Produces(MediaType.APPLICATION_JSON)
+    @Produces(Servlets.JSON_MEDIA_TYPE)
     public Response getTypesByFilter(@Context HttpServletRequest request,
                                      @DefaultValue(TYPE_ALL) @QueryParam("type") String type) {
         try {

webapp/src/main/java/org/apache/hadoop/metadata/web/util/Servlets.java

@@ -48,6 +48,7 @@ public final class Servlets {
         /* singleton */
     }
 
+    public static final String JSON_MEDIA_TYPE = MediaType.APPLICATION_JSON + "; charset=UTF-8";
     /**
      * Returns the user of the given request.
      *