ATLAS-2769: Atlas start just after the upgrade fails with…

repository/src/main/java/org/apache/atlas/repository/graph/AtlasGraphProvider.java

@@ -19,10 +19,14 @@
 package org.apache.atlas.repository.graph;
 
 import com.google.common.annotations.VisibleForTesting;
+import org.apache.atlas.ApplicationProperties;
+import org.apache.atlas.AtlasException;
 import org.apache.atlas.repository.RepositoryException;
 import org.apache.atlas.repository.graphdb.AtlasGraph;
 import org.apache.atlas.repository.graphdb.GraphDatabase;
 import org.apache.atlas.util.AtlasRepositoryConfiguration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
 
@@ -35,6 +39,14 @@ public class AtlasGraphProvider implements IAtlasGraphProvider {
 
     private static volatile GraphDatabase<?,?> graphDb_;
 
+    private static final Logger  LOG                              = LoggerFactory.getLogger(AtlasGraphProvider.class);
+    private static final Integer MAX_RETRY_COUNT                  = getMaxRetryCount();
+    private static final Long    RETRY_SLEEP_TIME_MS              = getRetrySleepTime();
+    private static final String  GRAPH_REPOSITORY_MAX_RETRIES     = "atlas.graph.repository.max.retries";
+    private static final String  GRAPH_REPOSITORY_RETRY_SLEEPTIME = "atlas.graph.repository.retry.sleeptime.ms";
+
+    private static org.apache.commons.configuration.Configuration APPLICATION_PROPERTIES = null;
+
     public static <V, E> AtlasGraph<V, E> getGraphInstance() {
         GraphDatabase<?,?> db = getGraphDatabase();      
         AtlasGraph<?, ?> graph = db.getGraph();
@@ -67,7 +79,59 @@ public class AtlasGraphProvider implements IAtlasGraphProvider {
 
     @Override
     @Bean(destroyMethod = "")
-    public AtlasGraph get() throws RepositoryException {
+    public AtlasGraph get() throws RepositoryException{
+        try {
+            return getGraphInstance();
+        } catch (Exception ex) {
+            LOG.info("Failed to obtain graph instance, retrying " + MAX_RETRY_COUNT + " times, error: " + ex);
+
+            return retry();
+        }
+    }
+
+    private AtlasGraph retry() throws RepositoryException {
+        int retryCounter = 0;
+
+        while (retryCounter < MAX_RETRY_COUNT) {
+            try {
+                // Retry after 30 sec to get graph instance
+                Thread.sleep(RETRY_SLEEP_TIME_MS);
+
                 return getGraphInstance();
+            } catch (Exception ex) {
+                retryCounter++;
+
+                LOG.info("Failed to obtain graph instance on retry " + retryCounter + " of " + MAX_RETRY_COUNT + " error: " + ex);
+
+                if (retryCounter >= MAX_RETRY_COUNT) {
+                    LOG.info("Max retries exceeded.");
+                    break;
+                }
+            }
+        }
+
+        throw new RepositoryException("Max retries exceeded. Failed to obtain graph instance after " + MAX_RETRY_COUNT + " retries");
+    }
+
+    private static Integer getMaxRetryCount() {
+        initApplicationProperties();
+
+        return (APPLICATION_PROPERTIES == null) ? 3 : APPLICATION_PROPERTIES.getInt(GRAPH_REPOSITORY_MAX_RETRIES, 3);
+    }
+
+    private static Long getRetrySleepTime() {
+        initApplicationProperties();
+
+        return (APPLICATION_PROPERTIES == null) ? 30000 : APPLICATION_PROPERTIES.getLong(GRAPH_REPOSITORY_RETRY_SLEEPTIME, 30000);
+    }
+
+    private static void initApplicationProperties() {
+        if (APPLICATION_PROPERTIES == null) {
+            try {
+                APPLICATION_PROPERTIES = ApplicationProperties.get();
+            } catch (AtlasException ex) {
+                // ignore
+            }
+        }
     }
 }