ATLAS-415 Hive import fails when importing a table that is already imported…

addons/hive-bridge/pom.xml

@@ -127,6 +127,11 @@
         </dependency>
 
         <dependency>
+            <groupId>org.mockito</groupId>
+            <artifactId>mockito-all</artifactId>
+        </dependency>
+
+        <dependency>
             <groupId>org.eclipse.jetty</groupId>
             <artifactId>jetty-server</artifactId>
             <scope>test</scope>

addons/hive-bridge/src/test/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridgeTest.java

+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.hive.bridge;
+
+import org.apache.atlas.AtlasClient;
+import org.apache.atlas.AtlasServiceException;
+import org.apache.atlas.hive.model.HiveDataTypes;
+import org.apache.atlas.typesystem.Referenceable;
+import org.apache.hadoop.hive.metastore.TableType;
+import org.apache.hadoop.hive.metastore.api.Database;
+import org.apache.hadoop.hive.ql.metadata.Hive;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.metadata.Partition;
+import org.apache.hadoop.hive.ql.metadata.Table;
+import org.apache.hadoop.mapred.TextInputFormat;
+import org.codehaus.jettison.json.JSONArray;
+import org.codehaus.jettison.json.JSONException;
+import org.codehaus.jettison.json.JSONObject;
+import org.mockito.ArgumentMatcher;
+import org.mockito.Mock;
+import org.mockito.MockitoAnnotations;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+import scala.actors.threadpool.Arrays;
+
+import java.util.List;
+
+import static org.mockito.Mockito.argThat;
+import static org.mockito.Mockito.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+public class HiveMetaStoreBridgeTest {
+
+    private static final String TEST_DB_NAME = "default";
+    public static final String CLUSTER_NAME = "primary";
+    public static final String TEST_TABLE_NAME = "test_table";
+
+    @Mock
+    private Hive hiveClient;
+
+    @Mock
+    private AtlasClient atlasClient;
+
+    @BeforeMethod
+    public void initializeMocks() {
+        MockitoAnnotations.initMocks(this);
+    }
+
+    @Test
+    public void testImportThatUpdatesRegisteredDatabase() throws Exception {
+        // setup database
+        when(hiveClient.getAllDatabases()).thenReturn(Arrays.asList(new String[]{TEST_DB_NAME}));
+        String description = "This is a default database";
+        when(hiveClient.getDatabase(TEST_DB_NAME)).thenReturn(
+                new Database(TEST_DB_NAME, description, "/user/hive/default", null));
+        when(hiveClient.getAllTables(TEST_DB_NAME)).thenReturn(Arrays.asList(new String[]{}));
+
+        returnExistingDatabase(TEST_DB_NAME, atlasClient, CLUSTER_NAME);
+
+        HiveMetaStoreBridge bridge = new HiveMetaStoreBridge(CLUSTER_NAME, hiveClient, atlasClient);
+        bridge.importHiveMetadata();
+
+        // verify update is called
+        verify(atlasClient).updateEntity(eq("72e06b34-9151-4023-aa9d-b82103a50e76"),
+                (Referenceable) argThat(
+                        new MatchesReferenceableProperty(HiveMetaStoreBridge.DESCRIPTION_ATTR, description)));
+    }
+
+    @Test
+    public void testImportThatUpdatesRegisteredTable() throws Exception {
+        setupDB(hiveClient, TEST_DB_NAME);
+
+        setupTable(hiveClient, TEST_DB_NAME, TEST_TABLE_NAME);
+
+        returnExistingDatabase(TEST_DB_NAME, atlasClient, CLUSTER_NAME);
+
+        // return existing table
+        when(atlasClient.searchByDSL(HiveMetaStoreBridge.getTableDSLQuery(CLUSTER_NAME, TEST_DB_NAME, TEST_TABLE_NAME,
+                HiveDataTypes.HIVE_TABLE.getName()))).thenReturn(
+                getEntityReference("82e06b34-9151-4023-aa9d-b82103a50e77"));
+        when(atlasClient.getEntity("82e06b34-9151-4023-aa9d-b82103a50e77")).thenReturn(createTableReference());
+
+        HiveMetaStoreBridge bridge = new HiveMetaStoreBridge(CLUSTER_NAME, hiveClient, atlasClient);
+        bridge.importHiveMetadata();
+
+        // verify update is called on table
+        verify(atlasClient).updateEntity(eq("82e06b34-9151-4023-aa9d-b82103a50e77"),
+                (Referenceable) argThat(new MatchesReferenceableProperty(HiveMetaStoreBridge.TABLE_TYPE_ATTR,
+                        TableType.EXTERNAL_TABLE.name())));
+    }
+
+    private void returnExistingDatabase(String databaseName, AtlasClient atlasClient, String clusterName)
+            throws AtlasServiceException, JSONException {
+        when(atlasClient.searchByDSL(HiveMetaStoreBridge.getDatabaseDSLQuery(clusterName, databaseName,
+                HiveDataTypes.HIVE_DB.getName()))).thenReturn(
+                getEntityReference("72e06b34-9151-4023-aa9d-b82103a50e76"));
+    }
+
+    private Table setupTable(Hive hiveClient, String databaseName, String tableName) throws HiveException {
+        when(hiveClient.getAllTables(databaseName)).thenReturn(Arrays.asList(new String[]{tableName}));
+        Table testTable = createTestTable(databaseName, tableName);
+        when(hiveClient.getTable(databaseName, tableName)).thenReturn(testTable);
+        return testTable;
+    }
+
+    private void setupDB(Hive hiveClient, String databaseName) throws HiveException {
+        when(hiveClient.getAllDatabases()).thenReturn(Arrays.asList(new String[]{databaseName}));
+        when(hiveClient.getDatabase(databaseName)).thenReturn(
+                new Database(databaseName, "Default database", "/user/hive/default", null));
+    }
+
+    @Test
+    public void testImportThatUpdatesRegisteredPartition() throws Exception {
+        setupDB(hiveClient, TEST_DB_NAME);
+        Table hiveTable = setupTable(hiveClient, TEST_DB_NAME, TEST_TABLE_NAME);
+
+        returnExistingDatabase(TEST_DB_NAME, atlasClient, CLUSTER_NAME);
+
+        when(atlasClient.searchByDSL(HiveMetaStoreBridge.getTableDSLQuery(CLUSTER_NAME, TEST_DB_NAME,
+                TEST_TABLE_NAME,
+                HiveDataTypes.HIVE_TABLE.getName()))).thenReturn(
+                getEntityReference("82e06b34-9151-4023-aa9d-b82103a50e77"));
+        when(atlasClient.getEntity("82e06b34-9151-4023-aa9d-b82103a50e77")).thenReturn(createTableReference());
+
+        Partition partition = mock(Partition.class);
+        when(partition.getTable()).thenReturn(hiveTable);
+        List partitionValues = Arrays.asList(new String[]{"name", "location"});
+        when(partition.getValues()).thenReturn(partitionValues);
+        int lastAccessTime = 1234512345;
+        when(partition.getLastAccessTime()).thenReturn(lastAccessTime);
+
+        when(hiveClient.getPartitions(hiveTable)).thenReturn(Arrays.asList(new Partition[]{partition}));
+
+        when(atlasClient.searchByGremlin(
+                HiveMetaStoreBridge.getPartitionGremlinQuery(
+                        HiveMetaStoreBridge.joinPartitionValues(partitionValues),
+                        HiveMetaStoreBridge.getTableQualifiedName(CLUSTER_NAME, TEST_DB_NAME, TEST_TABLE_NAME)))).
+                thenReturn(getPartitionReference("9ae06b34-9151-3043-aa9d-b82103a50e99"));
+
+        HiveMetaStoreBridge bridge = new HiveMetaStoreBridge(CLUSTER_NAME, hiveClient, atlasClient);
+        bridge.importHiveMetadata();
+
+        verify(atlasClient).updateEntity(eq("9ae06b34-9151-3043-aa9d-b82103a50e99"),
+                (Referenceable) argThat(new MatchesReferenceableProperty(HiveMetaStoreBridge.LAST_ACCESS_TIME_ATTR,
+                        new Integer(lastAccessTime))));
+    }
+
+    private JSONArray getPartitionReference(String id) throws JSONException {
+        JSONObject resultEntry = new JSONObject();
+        resultEntry.put(HiveMetaStoreBridge.SEARCH_ENTRY_GUID_ATTR, id);
+        JSONArray results = new JSONArray();
+        results.put(resultEntry);
+        return results;
+    }
+
+    private JSONArray getEntityReference(String id) throws JSONException {
+        return new JSONArray(String.format("[{\"$id$\":{\"id\":\"%s\"}}]", id));
+    }
+
+    private Referenceable createTableReference() {
+        Referenceable tableReference = new Referenceable(HiveDataTypes.HIVE_TABLE.getName());
+        Referenceable sdReference = new Referenceable(HiveDataTypes.HIVE_STORAGEDESC.getName());
+        tableReference.set("sd", sdReference);
+        return tableReference;
+    }
+
+    private Table createTestTable(String databaseName, String tableName) throws HiveException {
+        Table table = new Table(databaseName, tableName);
+        table.setInputFormatClass(TextInputFormat.class);
+        table.setTableType(TableType.EXTERNAL_TABLE);
+        return table;
+    }
+
+    private class MatchesReferenceableProperty extends ArgumentMatcher<Object> {
+        private final String attrName;
+        private final Object attrValue;
+
+        public MatchesReferenceableProperty(String attrName, Object attrValue) {
+            this.attrName = attrName;
+            this.attrValue = attrValue;
+        }
+
+        @Override
+        public boolean matches(Object o) {
+            return attrValue.equals(((Referenceable) o).get(attrName));
+        }
+    }
+}

pom.xml

@@ -1171,7 +1171,7 @@
                 <groupId>org.mockito</groupId>
                 <artifactId>mockito-all</artifactId>
                 <version>1.8.5</version>
-                <scope>provided</scope>
+                <scope>test</scope>
             </dependency>
 
             <dependency>

release-log.txt

@@ -7,6 +7,7 @@ ATLAS-409 Atlas will not import avro tables with schema read from a file (dosset
 ATLAS-379 Create sqoop and falcon metadata addons (venkatnrangan,bvellanki,sowmyaramesh via shwethags)
 
 ALL CHANGES:
+ATLAS-415 Hive import fails when importing a table that is already imported without StorageDescriptor information (yhemanth via shwethags)
 ATLAS-450 quick_start fails on cygwin (dkantor via shwethags)
 ATLAS-451 Doc: Fix few broken links due to Wiki words in Atlas documentation (ssainath via shwethags)
 ATLAS-439 Investigate apache build failure - EntityJerseyResourceIT.testEntityDeduping (shwethags)