ATLAS-1312: Update QuickStart to use the v2 APIs for types and entities creation

client/src/main/java/org/apache/atlas/AtlasBaseClient.java

@@ -294,7 +294,6 @@ public abstract class AtlasBaseClient {
 
             if (clientResponse.getStatus() == api.getExpectedStatus().getStatusCode()) {
                 if (null == responseType) {
-                    LOG.warn("No response type specified, returning null");
                     return null;
                 }
                 try {

client/src/main/java/org/apache/atlas/AtlasLineageClientV2.java

+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.sun.jersey.api.client.WebResource;
+import com.sun.jersey.core.util.MultivaluedMapImpl;
+import org.apache.atlas.model.lineage.AtlasLineageInfo;
+import org.apache.atlas.model.lineage.AtlasLineageInfo.LineageDirection;
+import org.apache.commons.configuration.Configuration;
+import org.apache.hadoop.security.UserGroupInformation;
+
+import javax.ws.rs.HttpMethod;
+import javax.ws.rs.core.MultivaluedMap;
+import javax.ws.rs.core.Response;
+
+public class AtlasLineageClientV2 extends AtlasBaseClient {
+
+    private static final String  LINEAGE_URI  = BASE_URI + "v2/lineage/%s/";
+    private static final APIInfo LINEAGE_INFO = new APIInfo(LINEAGE_URI, HttpMethod.GET, Response.Status.OK);
+
+    public AtlasLineageClientV2(String[] baseUrl, String[] basicAuthUserNamePassword) {
+        super(baseUrl, basicAuthUserNamePassword);
+    }
+
+    public AtlasLineageClientV2(String... baseUrls) throws AtlasException {
+        super(baseUrls);
+    }
+
+    public AtlasLineageClientV2(UserGroupInformation ugi, String doAsUser, String... baseUrls) {
+        super(ugi, doAsUser, baseUrls);
+    }
+
+    protected AtlasLineageClientV2() {
+        super();
+    }
+
+    @VisibleForTesting
+    AtlasLineageClientV2(WebResource service, Configuration configuration) {
+        super(service, configuration);
+    }
+
+    public AtlasLineageInfo getLineageInfo(final String guid, final LineageDirection direction, final int depth) throws AtlasServiceException {
+        MultivaluedMap<String, String> queryParams = new MultivaluedMapImpl();
+        queryParams.add("direction", direction.toString());
+        queryParams.add("depth", String.valueOf(depth));
+
+        return callAPI(formatPathForPathParams(LINEAGE_INFO, guid), AtlasLineageInfo.class, queryParams);
+    }
+}
\ No newline at end of file

distro/src/bin/quick_start.py

@@ -27,16 +27,16 @@ def main():
     jvm_opts_list = cmdline.setup_jvm_opts_list(conf_dir, 'quick_start.log')
     atlas_classpath = cmdline.get_atlas_classpath(conf_dir)
 
-    process = mc.java("org.apache.atlas.examples.QuickStart", sys.argv[1:], atlas_classpath, jvm_opts_list)
+    process = mc.java("org.apache.atlas.examples.QuickStartV2", sys.argv[1:], atlas_classpath, jvm_opts_list)
     return process.wait()
 
 if __name__ == '__main__':
     try:
         returncode = main()
         if returncode == 0:
-            print "Example data added to Apache Atlas Server!!!\n"
+            print "Sample data added to Apache Atlas Server.\n"
         else:
-            print "No data was added to the Apache Atlas Server.\n"
+            print "No sample data added to Apache Atlas Server.\n"
     except Exception as e:
         print "Exception: %s " % str(e)
         returncode = -1

distro/src/bin/quick_start_v1.py

+#!/usr/bin/env python
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import os
+import sys
+
+import atlas_config as mc
+import atlas_client_cmdline as cmdline
+
+def main():
+
+    conf_dir = cmdline.setup_conf_dir()
+    jvm_opts_list = cmdline.setup_jvm_opts_list(conf_dir, 'quick_start_v1.log')
+    atlas_classpath = cmdline.get_atlas_classpath(conf_dir)
+
+    process = mc.java("org.apache.atlas.examples.QuickStart", sys.argv[1:], atlas_classpath, jvm_opts_list)
+    return process.wait()
+
+if __name__ == '__main__':
+    try:
+        returncode = main()
+        if returncode == 0:
+            print "Example data added to Apache Atlas Server!!!\n"
+        else:
+            print "No data was added to the Apache Atlas Server.\n"
+    except Exception as e:
+        print "Exception: %s " % str(e)
+        returncode = -1
+
+    sys.exit(returncode)

intg/src/main/java/org/apache/atlas/type/AtlasTypeUtil.java

@@ -17,6 +17,7 @@
  */
 package org.apache.atlas.type;
 
+import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
 
 import org.apache.atlas.AtlasErrorCode;
@@ -35,6 +36,7 @@ import org.apache.commons.collections.CollectionUtils;
 import org.apache.commons.lang.StringUtils;
 
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.LinkedList;
 import java.util.List;
@@ -50,6 +52,8 @@ import static org.apache.atlas.model.typedef.AtlasBaseTypeDef.ATLAS_TYPE_ARRAY_S
 import static org.apache.atlas.model.typedef.AtlasBaseTypeDef.ATLAS_TYPE_MAP_KEY_VAL_SEP;
 import static org.apache.atlas.model.typedef.AtlasBaseTypeDef.ATLAS_TYPE_MAP_PREFIX;
 import static org.apache.atlas.model.typedef.AtlasBaseTypeDef.ATLAS_TYPE_MAP_SUFFIX;
+import static org.apache.atlas.model.typedef.AtlasStructDef.AtlasConstraintDef.CONSTRAINT_PARAM_REF_ATTRIBUTE;
+import static org.apache.atlas.model.typedef.AtlasStructDef.AtlasConstraintDef.CONSTRAINT_TYPE_MAPPED_FROM_REF;
 
 /**
  * Utility methods for AtlasType/AtlasTypeDef.
@@ -170,6 +174,41 @@ public class AtlasTypeUtil {
             Collections.<AtlasStructDef.AtlasConstraintDef>emptyList());
     }
 
+    public static AtlasAttributeDef createListRequiredAttrDef(String name, String dataType) {
+        return new AtlasAttributeDef(name, dataType, false,
+                Cardinality.LIST, 1, Integer.MAX_VALUE,
+                false, true,
+                Collections.<AtlasStructDef.AtlasConstraintDef>emptyList());
+    }
+
+    public static AtlasAttributeDef createOptionalListAttrDef(String name, String dataType) {
+        return new AtlasAttributeDef(name, dataType, true,
+                Cardinality.LIST, 1, Integer.MAX_VALUE,
+                false, true,
+                Collections.<AtlasStructDef.AtlasConstraintDef>emptyList());
+    }
+
+    public static AtlasAttributeDef createRequiredListAttrDefWithConstraint(String name, String dataType, String type, Map param) {
+        AtlasAttributeDef ret = AtlasTypeUtil.createListRequiredAttrDef(name, dataType);
+        ret.addConstraint(new AtlasStructDef.AtlasConstraintDef(type, param));
+
+        return ret;
+    }
+
+    public static AtlasAttributeDef createRequiredAttrDefWithConstraint(String name, String typeName, String type, Map param) {
+        AtlasAttributeDef ret = AtlasTypeUtil.createRequiredAttrDef(name, typeName);
+        ret.addConstraint(new AtlasStructDef.AtlasConstraintDef(type, param));
+
+        return ret;
+    }
+
+    public static AtlasAttributeDef createOptionalAttrDefWithConstraint(String name, String typeName, String type, Map param) {
+        AtlasAttributeDef ret = AtlasTypeUtil.createOptionalAttrDef(name, typeName);
+        ret.addConstraint(new AtlasStructDef.AtlasConstraintDef(type, param));
+
+        return ret;
+    }
+
     public static AtlasAttributeDef createUniqueRequiredAttrDef(String name, AtlasType dataType) {
         return new AtlasAttributeDef(name, dataType.getTypeName(), false,
             Cardinality.SINGLE, 1, 1,
@@ -204,7 +243,7 @@ public class AtlasTypeUtil {
     }
 
     public static AtlasClassificationDef createTraitTypeDef(String name, String description, String version, ImmutableSet<String> superTypes, AtlasAttributeDef... attrDefs) {
-        return new AtlasClassificationDef(name, description, "1.0", Arrays.asList(attrDefs), superTypes);
+        return new AtlasClassificationDef(name, description, version, Arrays.asList(attrDefs), superTypes);
     }
 
     public static AtlasStructDef createStructTypeDef(String name, AtlasAttributeDef... attrDefs) {
@@ -227,7 +266,7 @@ public class AtlasTypeUtil {
 
     public static AtlasEntityDef createClassTypeDef(String name, String description, String version,
         ImmutableSet<String> superTypes, AtlasAttributeDef... attrDefs) {
-        return new AtlasEntityDef(name, description, "1.0", Arrays.asList(attrDefs), superTypes);
+        return new AtlasEntityDef(name, description, version, Arrays.asList(attrDefs), superTypes);
     }
 
     public static AtlasTypesDef getTypesDef(List<AtlasEnumDef> enums,

release-log.txt

@@ -9,6 +9,7 @@ ATLAS-1060 Add composite indexes for exact match performance improvements for al
 ATLAS-1127 Modify creation and modification timestamps to Date instead of Long(sumasai)
 
 ALL CHANGES:
+ATLAS-1312 Update QuickStart to use the v2 APIs for types and entities creation (sarath.kum4r@gmail.com via mneethiraj)
 ATLAS-1498 added unit-tests to validate handling of array/map/struct attributes in entity create/update (sumasai via mneethiraj)
 ATLAS-1114 Performance improvements for create/update entity (jnhagelb)
 ATLAS-1403 Perf and stability improvements to DSL search and lineage query execution (sarath.kum4r@gmail.com via mneethiraj)

webapp/src/main/java/org/apache/atlas/examples/QuickStart.java

@@ -108,7 +108,7 @@ public class QuickStart {
         Configuration configuration = ApplicationProperties.get();
         String[] urls = configuration.getStringArray(ATLAS_REST_ADDRESS);
         if (urls == null || urls.length == 0) {
-            System.out.println("Usage: quick_start.py <atlas endpoint of format <http/https>://<atlas-fqdn>:<atlas port> like http://localhost:21000>");
+            System.out.println("Usage: quick_start_v1.py <atlas endpoint of format <http/https>://<atlas-fqdn>:<atlas port> like http://localhost:21000>");
             System.exit(-1);
         }
 

webapp/src/test/java/org/apache/atlas/examples/QuickStartV2IT.java

+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.examples;
+
+import org.apache.atlas.AtlasClient;
+import org.apache.atlas.AtlasServiceException;
+import org.apache.atlas.model.instance.AtlasClassification;
+import org.apache.atlas.model.instance.AtlasEntity;
+import org.apache.atlas.model.instance.AtlasEntityHeader;
+import org.apache.atlas.model.lineage.AtlasLineageInfo;
+import org.apache.atlas.model.lineage.AtlasLineageInfo.LineageDirection;
+import org.apache.atlas.model.lineage.AtlasLineageInfo.LineageRelation;
+import org.apache.atlas.web.resources.BaseResourceIT;
+import org.codehaus.jettison.json.JSONException;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertNotNull;
+import static org.testng.AssertJUnit.assertTrue;
+
+public class QuickStartV2IT extends BaseResourceIT {
+
+    @BeforeClass
+    public void runQuickStart() throws Exception {
+        super.setUp();
+        QuickStartV2.runQuickstart(new String[]{}, new String[]{"admin", "admin"});
+    }
+
+    @Test
+    public void testDBIsAdded() throws Exception {
+        AtlasEntity db = getDB(QuickStartV2.SALES_DB);
+        Map<String, Object> dbAttributes = db.getAttributes();
+        assertEquals(QuickStartV2.SALES_DB, dbAttributes.get("name"));
+        assertEquals("sales database", dbAttributes.get("description"));
+    }
+
+    private AtlasEntity getDB(String dbName) throws AtlasServiceException, JSONException {
+        AtlasEntity dbEntity = entitiesClientV2.getEntityByAttribute(QuickStartV2.DATABASE_TYPE, "name", dbName);
+        return dbEntity;
+    }
+
+    @Test
+    public void testTablesAreAdded() throws AtlasServiceException, JSONException {
+        AtlasEntity table = getTable(QuickStart.SALES_FACT_TABLE);
+        verifySimpleTableAttributes(table);
+
+        verifyDBIsLinkedToTable(table);
+
+        verifyColumnsAreAddedToTable(table);
+
+        verifyTrait(table);
+    }
+
+    private AtlasEntity getTable(String tableName) throws AtlasServiceException {
+        AtlasEntity tableEntity = entitiesClientV2.getEntityByAttribute(QuickStartV2.TABLE_TYPE, AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, tableName);
+        return tableEntity;
+    }
+
+    private AtlasEntity getProcess(String processName) throws AtlasServiceException {
+        AtlasEntity processEntity = entitiesClientV2.getEntityByAttribute(QuickStartV2.LOAD_PROCESS_TYPE, AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, processName);
+        return processEntity;
+    }
+
+    private void verifyTrait(AtlasEntity table) throws AtlasServiceException {
+        AtlasClassification.AtlasClassifications classfications = entitiesClientV2.getClassifications(table.getGuid());
+        List<AtlasClassification> traits = classfications.getList();
+        assertNotNull(traits.get(0).getTypeName());
+    }
+
+    private void verifyColumnsAreAddedToTable(AtlasEntity table) throws JSONException {
+        Map<String, Object> tableAttributes = table.getAttributes();
+        List<AtlasEntity> columns = (List<AtlasEntity>) tableAttributes.get("columns");
+        assertEquals(4, columns.size());
+
+        Map<String, Object> column = (Map) columns.get(0);
+        Map<String, Object> columnAttributes = (Map) column.get("attributes");
+        assertEquals(QuickStartV2.TIME_ID_COLUMN, columnAttributes.get("name"));
+        assertEquals("int", columnAttributes.get("dataType"));
+    }
+
+    private void verifyDBIsLinkedToTable(AtlasEntity table) throws AtlasServiceException, JSONException {
+        AtlasEntity db = getDB(QuickStartV2.SALES_DB);
+        Map<String, Object> tableAttributes = table.getAttributes();
+        Map dbFromTable = (Map) tableAttributes.get("db");
+        assertEquals(db.getGuid(), dbFromTable.get("guid"));
+    }
+
+    private void verifySimpleTableAttributes(AtlasEntity table) throws JSONException {
+        Map<String, Object> tableAttributes = table.getAttributes();
+        assertEquals(QuickStartV2.SALES_FACT_TABLE, tableAttributes.get("name"));
+        assertEquals("sales fact table", tableAttributes.get("description"));
+    }
+
+    @Test
+    public void testProcessIsAdded() throws AtlasServiceException, JSONException {
+        AtlasEntity loadProcess = entitiesClientV2.getEntityByAttribute(QuickStartV2.LOAD_PROCESS_TYPE, AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME,
+                QuickStartV2.LOAD_SALES_DAILY_PROCESS);
+
+        Map loadProcessAttribs = loadProcess.getAttributes();
+        assertEquals(QuickStartV2.LOAD_SALES_DAILY_PROCESS, loadProcessAttribs.get(AtlasClient.NAME));
+        assertEquals("hive query for daily summary", loadProcessAttribs.get("description"));
+
+        List inputs = (List) loadProcessAttribs.get("inputs");
+        List outputs = (List) loadProcessAttribs.get("outputs");
+        assertEquals(2, inputs.size());
+
+        String salesFactTableId = getTableId(QuickStartV2.SALES_FACT_TABLE);
+        String timeDimTableId = getTableId(QuickStartV2.TIME_DIM_TABLE);
+        String salesFactDailyMVId = getTableId(QuickStartV2.SALES_FACT_DAILY_MV_TABLE);
+
+        assertEquals(salesFactTableId, ((Map) inputs.get(0)).get("guid"));
+        assertEquals(timeDimTableId, ((Map) inputs.get(1)).get("guid"));
+        assertEquals(salesFactDailyMVId, ((Map) outputs.get(0)).get("guid"));
+    }
+
+    private String getTableId(String tableName) throws AtlasServiceException {
+        return getTable(tableName).getGuid();
+    }
+
+    private String getProcessId(String processName) throws AtlasServiceException {
+        return getProcess(processName).getGuid();
+    }
+
+    @Test
+    public void testLineageIsMaintained() throws AtlasServiceException, JSONException {
+        String salesFactTableId      = getTableId(QuickStartV2.SALES_FACT_TABLE);
+        String timeDimTableId        = getTableId(QuickStartV2.TIME_DIM_TABLE);
+        String salesFactDailyMVId    = getTableId(QuickStartV2.SALES_FACT_DAILY_MV_TABLE);
+        String salesFactMonthlyMvId  = getTableId(QuickStartV2.SALES_FACT_MONTHLY_MV_TABLE);
+        String salesDailyProcessId   = getProcessId(QuickStartV2.LOAD_SALES_DAILY_PROCESS);
+        String salesMonthlyProcessId = getProcessId(QuickStartV2.LOAD_SALES_MONTHLY_PROCESS);
+
+        AtlasLineageInfo inputLineage = lineageClientV2.getLineageInfo(salesFactDailyMVId, LineageDirection.BOTH, 0);
+        List<LineageRelation> relations = new ArrayList<>(inputLineage.getRelations());
+        Map<String, AtlasEntityHeader> entityMap = inputLineage.getGuidEntityMap();
+
+        assertEquals(relations.size(), 5);
+        assertEquals(entityMap.size(), 6);
+
+        assertTrue(entityMap.containsKey(salesFactTableId));
+        assertTrue(entityMap.containsKey(timeDimTableId));
+        assertTrue(entityMap.containsKey(salesFactDailyMVId));
+        assertTrue(entityMap.containsKey(salesDailyProcessId));
+        assertTrue(entityMap.containsKey(salesFactMonthlyMvId));
+        assertTrue(entityMap.containsKey(salesMonthlyProcessId));
+    }
+
+    @Test
+    public void testViewIsAdded() throws AtlasServiceException, JSONException {
+        AtlasEntity view = entitiesClientV2.getEntityByAttribute(QuickStartV2.VIEW_TYPE, AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, QuickStartV2.PRODUCT_DIM_VIEW);
+        Map<String, Object> viewAttributes = view.getAttributes();
+        assertEquals(QuickStartV2.PRODUCT_DIM_VIEW, viewAttributes.get(AtlasClient.NAME));
+
+        String productDimId = getTable(QuickStartV2.PRODUCT_DIM_TABLE).getGuid();
+        List inputTables = (List) viewAttributes.get("inputTables");
+        Map inputTablesMap = (Map) inputTables.get(0);
+        assertEquals(productDimId, inputTablesMap.get("guid"));
+    }
+}

webapp/src/test/java/org/apache/atlas/web/resources/BaseResourceIT.java

@@ -26,6 +26,7 @@ import org.apache.atlas.ApplicationProperties;
 import org.apache.atlas.AtlasClient;
 import org.apache.atlas.AtlasDiscoveryClientV2;
 import org.apache.atlas.AtlasEntitiesClientV2;
+import org.apache.atlas.AtlasLineageClientV2;
 import org.apache.atlas.AtlasServiceException;
 import org.apache.atlas.AtlasTypedefClientV2;
 import org.apache.atlas.model.instance.AtlasClassification;
@@ -87,6 +88,7 @@ public abstract class BaseResourceIT {
     protected AtlasTypedefClientV2 typedefClientV2;
     protected AtlasEntitiesClientV2 entitiesClientV2;
     protected AtlasDiscoveryClientV2 discoveryClientV2;
+    protected AtlasLineageClientV2 lineageClientV2;
 
     public static final Logger LOG = LoggerFactory.getLogger(BaseResourceIT.class);
     protected static final int MAX_WAIT_TIME = 60000;
@@ -113,11 +115,13 @@ public abstract class BaseResourceIT {
             typedefClientV2 = new AtlasTypedefClientV2(atlasUrls, new String[]{"admin", "admin"});
             entitiesClientV2 = new AtlasEntitiesClientV2(atlasUrls, new String[]{"admin", "admin"});
             discoveryClientV2 = new AtlasDiscoveryClientV2(atlasUrls, new String[]{"admin", "admin"});
+            lineageClientV2 = new AtlasLineageClientV2(atlasUrls, new String[]{"admin", "admin"});
         } else {
             atlasClientV1 = new AtlasClient(atlasUrls);
             typedefClientV2 = new AtlasTypedefClientV2(atlasUrls);
             entitiesClientV2 = new AtlasEntitiesClientV2(atlasUrls);
             discoveryClientV2 = new AtlasDiscoveryClientV2(atlasUrls);
+            lineageClientV2 = new AtlasLineageClientV2(atlasUrls);
         }
     }