ATLAS-1436: (Part 3) ignoreCache and query tuning for MetricsService

repository/src/main/java/org/apache/atlas/services/MetricsService.java

@@ -87,8 +87,8 @@ public class MetricsService {
     }
 
     @SuppressWarnings("unchecked")
-    public AtlasMetrics getMetrics() {
-        if (!isCacheValid()) {
+    public AtlasMetrics getMetrics(boolean ignoreCache) {
+        if (ignoreCache || !isCacheValid()) {
             AtlasMetrics metrics = new AtlasMetrics();
 
             for (MetricQuery metricQuery : MetricQuery.values()) {
@@ -96,18 +96,7 @@ public class MetricsService {
                     if (LOG.isDebugEnabled()) {
                         LOG.debug("Executing query: {}", metricQuery);
                     }
-
-                    if (metricQuery == MetricQuery.ENTITIES_PER_TYPE) {
-                        Collection<String> entityDefNames = atlasTypeRegistry.getAllEntityDefNames();
-
-                        for (String entityDefName : entityDefNames) {
-                            String formattedQuery = String.format(metricQuery.query, entityDefName);
-
-                            executeGremlinQuery(metrics, metricQuery.type, entityDefName, formattedQuery);
-                        }
-                    } else {
-                        executeGremlinQuery(metrics, metricQuery.type, metricQuery.name, metricQuery.query);
-                    }
+                    executeGremlinQuery(metrics, metricQuery.type, metricQuery.name, metricQuery.query);
                 } catch (ScriptException e) {
                     LOG.error("Gremlin execution failed for metric {}", metricQuery, e);
                 }
@@ -130,8 +119,10 @@ public class MetricsService {
         if (result instanceof Number) {
             metrics.addData(type, name, ((Number) result).intValue());
         } else if (result instanceof List) {
-            for (Map resultMap : (List<Map>) result) {
-                metrics.addData(type, (String) resultMap.get("key"), ((Number) resultMap.get("value")).intValue());
+            for (Map<String, Number> resultMap : (List<Map<String, Number>>) result) {
+                for (Map.Entry<String, Number> entry : resultMap.entrySet()) {
+                    metrics.addData(type, entry.getKey(), entry.getValue().intValue());
+                }
             }
         } else {
             String returnClassName = result != null ? result.getClass().getSimpleName() : "null";
@@ -176,10 +167,10 @@ public class MetricsService {
         TAGS_COUNT(GENERAL, METRIC_TAG_COUNT, "g.V().has('__type', 'typeSystem').filter({it.'__type.category'.name() == 'TRAIT'}).count()"),
         DELETED_ENTITY_COUNT(GENERAL, METRIC_ENTITY_DELETED, "g.V().has('__superTypeNames', T.in, ['Referenceable']).has('__status', 'DELETED').count()"),
 
-        ENTITIES_PER_TYPE(ENTITY, METRIC_TYPE_ENTITIES, "g.V().has('__typeName', T.in, ['%s']).count()"),
+        ENTITIES_PER_TYPE(ENTITY, METRIC_TYPE_ENTITIES, "g.V().has('__typeName', T.in, g.V().has('__type', 'typeSystem').filter({it.'__type.category'.name() != 'TRAIT'}).'__type.name'.toSet()).groupCount{it.'__typeName'}.cap.toList()"),
         TAGGED_ENTITIES(ENTITY, METRIC_TAGGED_ENTITIES, "g.V().has('__superTypeNames', T.in, ['Referenceable']).has('__traitNames').count()"),
 
-        TAGS_PER_ENTITY(TAG, METRIC_TAGS_PER_ENTITY, "g.V().has('__superTypeNames', T.in, ['Referenceable']).has('__traitNames').transform{[ key: it.'Referenceable.qualifiedName', value: it.'__traitNames'.size()]}.dedup().toList()"),
+        ENTITIES_WITH_SPECIFIC_TAG(TAG, METRIC_TAGS_PER_ENTITY, "g.V().has('__typeName', T.in, g.V().has('__type', 'typeSystem').filter{it.'__type.category'.name() == 'TRAIT'}.'__type.name'.toSet()).groupCount{it.'__typeName'}.cap.toList()"),
         ;
 
         private final String type;

repository/src/test/java/org/apache/atlas/services/MetricsServiceTest.java

-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
 package org.apache.atlas.services;
 
-import org.apache.atlas.AtlasException;
 import org.apache.atlas.model.metrics.AtlasMetrics;
 import org.apache.atlas.repository.graphdb.AtlasGraph;
 import org.apache.atlas.type.AtlasTypeRegistry;
@@ -50,37 +32,20 @@ public class MetricsServiceTest {
     private Number mockCount = 10;
 
     @BeforeClass
-    public void init() throws ScriptException, AtlasException {
-        Map<String, Object> aMockMap = new HashMap<>();
-        Map<String, Object> bMockMap = new HashMap<>();
-        Map<String, Object> cMockMap = new HashMap<>();
-        aMockMap.put("key", "a");
-        aMockMap.put("value", 1);
-
-        bMockMap.put("key", "b");
-        bMockMap.put("value", 2);
-
-        cMockMap.put("key", "c");
-        cMockMap.put("value", 3);
-        mockMapList.add(aMockMap);
-        mockMapList.add(bMockMap);
-        mockMapList.add(cMockMap);
+    public void init() throws ScriptException {
+        Map<String, Object> mockMap = new HashMap<>();
+        mockMap.put("a", 1);
+        mockMap.put("b", 2);
+        mockMap.put("c", 3);
+        mockMapList.add(mockMap);
 
         when(mockConfig.getInt(anyString(), anyInt())).thenReturn(5);
-        when(mockConfig.getString(anyString(), anyString()))
-                // we have seven count queries so stubbing 7 counts
-                .thenReturn("dummyTestQuery.count()")
-                .thenReturn("dummyTestQuery.count()")
-                .thenReturn("dummyTestQuery.count()")
-                .thenReturn("dummyTestQuery.count()")
-                .thenReturn("dummyTestQuery.count()")
-                .thenReturn("dummyTestQuery.count()")
-                .thenReturn("dummyTestQuery.count()")
-                // The last query is a map
-                .thenReturn("dummyTestQuery");
         assertEquals(mockConfig.getInt("test", 1), 5);
+        when(mockConfig.getString(anyString(), anyString()))
+                .thenReturn("count()", "count()", "count()", "count()", "count()", "toList()", "count()", "toList()");
         when(mockTypeRegistry.getAllEntityDefNames()).thenReturn(Arrays.asList("a", "b", "c"));
         setupMockGraph();
+
         metricsService = new MetricsService(mockConfig, mockGraph, mockTypeRegistry);
     }
 
@@ -101,19 +66,19 @@ public class MetricsServiceTest {
     @Test
     public void testGetMetrics() throws InterruptedException, ScriptException {
         assertNotNull(metricsService);
-        AtlasMetrics metrics = metricsService.getMetrics();
+        AtlasMetrics metrics = metricsService.getMetrics(false);
         assertNotNull(metrics);
         Number aCount = metrics.getMetric("entity", "a");
         assertNotNull(aCount);
-        assertEquals(aCount, 10);
+        assertEquals(aCount, 1);
 
         Number bCount = metrics.getMetric("entity", "b");
         assertNotNull(bCount);
-        assertEquals(bCount, 10);
+        assertEquals(bCount, 2);
 
         Number cCount = metrics.getMetric("entity", "c");
         assertNotNull(cCount);
-        assertEquals(cCount, 10);
+        assertEquals(cCount, 3);
 
         Number aTags = metrics.getMetric("tag", "a");
         assertNotNull(aTags);
@@ -130,12 +95,12 @@ public class MetricsServiceTest {
         verify(mockGraph, atLeastOnce()).executeGremlinScript(anyString(), anyBoolean());
 
         // Subsequent call within the cache timeout window
-        metricsService.getMetrics();
+        metricsService.getMetrics(false);
         verifyZeroInteractions(mockGraph);
 
         // Now test the cache refresh
         Thread.sleep(6000);
-        metricsService.getMetrics();
+        metricsService.getMetrics(true);
         verify(mockGraph, atLeastOnce()).executeGremlinScript(anyString(), anyBoolean());
     }
 }
\ No newline at end of file

webapp/src/main/java/org/apache/atlas/web/resources/AdminResource.java

@@ -254,12 +254,12 @@ public class AdminResource {
     @GET
     @Path("metrics")
     @Produces(Servlets.JSON_MEDIA_TYPE)
-    public AtlasMetrics getMetrics() {
+    public AtlasMetrics getMetrics(@QueryParam("ignoreCache") boolean ignoreCache) {
         if (LOG.isDebugEnabled()) {
             LOG.debug("==> AdminResource.getMetrics()");
         }
 
-        AtlasMetrics metrics = metricsService.getMetrics();
+        AtlasMetrics metrics = metricsService.getMetrics(ignoreCache);
 
         if (LOG.isDebugEnabled()) {
             LOG.debug("<== AdminResource.getMetrics()");