Skip to content

A
atlas
Commit 9db4d261 by apoorvnaik Committed by Madhan Neethiraj
Options

ATLAS-1436: (Part 3) ignoreCache and query tuning for MetricsService

Signed-off-by: 's avatarMadhan Neethiraj <madhan@apache.org>
parent ce20d6f5
Showing with 27 additions and 70 deletions
repository/src/main/java/org/apache/atlas/services/MetricsService.java
......@@ -87,8 +87,8 @@ public class MetricsService {
}
@SuppressWarnings("unchecked")
public AtlasMetrics getMetrics() {
if (!isCacheValid()) {
public AtlasMetrics getMetrics(boolean ignoreCache) {
if (ignoreCache || !isCacheValid()) {
AtlasMetrics metrics = new AtlasMetrics();
for (MetricQuery metricQuery : MetricQuery.values()) {
......@@ -96,18 +96,7 @@ public class MetricsService {
if (LOG.isDebugEnabled()) {
LOG.debug("Executing query: {}", metricQuery);
}
if (metricQuery == MetricQuery.ENTITIES_PER_TYPE) {
Collection<String> entityDefNames = atlasTypeRegistry.getAllEntityDefNames();
for (String entityDefName : entityDefNames) {
String formattedQuery = String.format(metricQuery.query, entityDefName);
executeGremlinQuery(metrics, metricQuery.type, entityDefName, formattedQuery);
}
} else {
executeGremlinQuery(metrics, metricQuery.type, metricQuery.name, metricQuery.query);
}
executeGremlinQuery(metrics, metricQuery.type, metricQuery.name, metricQuery.query);
} catch (ScriptException e) {
LOG.error("Gremlin execution failed for metric {}", metricQuery, e);
}
......@@ -130,8 +119,10 @@ public class MetricsService {
if (result instanceof Number) {
metrics.addData(type, name, ((Number) result).intValue());
} else if (result instanceof List) {
for (Map resultMap : (List<Map>) result) {
metrics.addData(type, (String) resultMap.get("key"), ((Number) resultMap.get("value")).intValue());
for (Map<String, Number> resultMap : (List<Map<String, Number>>) result) {
for (Map.Entry<String, Number> entry : resultMap.entrySet()) {
metrics.addData(type, entry.getKey(), entry.getValue().intValue());
}
}
} else {
String returnClassName = result != null ? result.getClass().getSimpleName() : "null";
......@@ -176,10 +167,10 @@ public class MetricsService {
TAGS_COUNT(GENERAL, METRIC_TAG_COUNT, "g.V().has('__type', 'typeSystem').filter({it.'__type.category'.name() == 'TRAIT'}).count()"),
DELETED_ENTITY_COUNT(GENERAL, METRIC_ENTITY_DELETED, "g.V().has('__superTypeNames', T.in, ['Referenceable']).has('__status', 'DELETED').count()"),
ENTITIES_PER_TYPE(ENTITY, METRIC_TYPE_ENTITIES, "g.V().has('__typeName', T.in, ['%s']).count()"),
ENTITIES_PER_TYPE(ENTITY, METRIC_TYPE_ENTITIES, "g.V().has('__typeName', T.in, g.V().has('__type', 'typeSystem').filter({it.'__type.category'.name() != 'TRAIT'}).'__type.name'.toSet()).groupCount{it.'__typeName'}.cap.toList()"),
TAGGED_ENTITIES(ENTITY, METRIC_TAGGED_ENTITIES, "g.V().has('__superTypeNames', T.in, ['Referenceable']).has('__traitNames').count()"),
TAGS_PER_ENTITY(TAG, METRIC_TAGS_PER_ENTITY, "g.V().has('__superTypeNames', T.in, ['Referenceable']).has('__traitNames').transform{[ key: it.'Referenceable.qualifiedName', value: it.'__traitNames'.size()]}.dedup().toList()"),
ENTITIES_WITH_SPECIFIC_TAG(TAG, METRIC_TAGS_PER_ENTITY, "g.V().has('__typeName', T.in, g.V().has('__type', 'typeSystem').filter{it.'__type.category'.name() == 'TRAIT'}.'__type.name'.toSet()).groupCount{it.'__typeName'}.cap.toList()"),
;
private final String type;
......
repository/src/test/java/org/apache/atlas/services/MetricsServiceTest.java
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.services;
import org.apache.atlas.AtlasException;
import org.apache.atlas.model.metrics.AtlasMetrics;
import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.type.AtlasTypeRegistry;
......@@ -50,37 +32,20 @@ public class MetricsServiceTest {
private Number mockCount = 10;
@BeforeClass
public void init() throws ScriptException, AtlasException {
Map<String, Object> aMockMap = new HashMap<>();
Map<String, Object> bMockMap = new HashMap<>();
Map<String, Object> cMockMap = new HashMap<>();
aMockMap.put("key", "a");
aMockMap.put("value", 1);
bMockMap.put("key", "b");
bMockMap.put("value", 2);
cMockMap.put("key", "c");
cMockMap.put("value", 3);
mockMapList.add(aMockMap);
mockMapList.add(bMockMap);
mockMapList.add(cMockMap);
public void init() throws ScriptException {
Map<String, Object> mockMap = new HashMap<>();
mockMap.put("a", 1);
mockMap.put("b", 2);
mockMap.put("c", 3);
mockMapList.add(mockMap);
when(mockConfig.getInt(anyString(), anyInt())).thenReturn(5);
when(mockConfig.getString(anyString(), anyString()))
// we have seven count queries so stubbing 7 counts
.thenReturn("dummyTestQuery.count()")
.thenReturn("dummyTestQuery.count()")
.thenReturn("dummyTestQuery.count()")
.thenReturn("dummyTestQuery.count()")
.thenReturn("dummyTestQuery.count()")
.thenReturn("dummyTestQuery.count()")
.thenReturn("dummyTestQuery.count()")
// The last query is a map
.thenReturn("dummyTestQuery");
assertEquals(mockConfig.getInt("test", 1), 5);
when(mockConfig.getString(anyString(), anyString()))
.thenReturn("count()", "count()", "count()", "count()", "count()", "toList()", "count()", "toList()");
when(mockTypeRegistry.getAllEntityDefNames()).thenReturn(Arrays.asList("a", "b", "c"));
setupMockGraph();
metricsService = new MetricsService(mockConfig, mockGraph, mockTypeRegistry);
}
......@@ -101,19 +66,19 @@ public class MetricsServiceTest {
@Test
public void testGetMetrics() throws InterruptedException, ScriptException {
assertNotNull(metricsService);
AtlasMetrics metrics = metricsService.getMetrics();
AtlasMetrics metrics = metricsService.getMetrics(false);
assertNotNull(metrics);
Number aCount = metrics.getMetric("entity", "a");
assertNotNull(aCount);
assertEquals(aCount, 10);
assertEquals(aCount, 1);
Number bCount = metrics.getMetric("entity", "b");
assertNotNull(bCount);
assertEquals(bCount, 10);
assertEquals(bCount, 2);
Number cCount = metrics.getMetric("entity", "c");
assertNotNull(cCount);
assertEquals(cCount, 10);
assertEquals(cCount, 3);
Number aTags = metrics.getMetric("tag", "a");
assertNotNull(aTags);
......@@ -130,12 +95,12 @@ public class MetricsServiceTest {
verify(mockGraph, atLeastOnce()).executeGremlinScript(anyString(), anyBoolean());
// Subsequent call within the cache timeout window
metricsService.getMetrics();
metricsService.getMetrics(false);
verifyZeroInteractions(mockGraph);
// Now test the cache refresh
Thread.sleep(6000);
metricsService.getMetrics();
metricsService.getMetrics(true);
verify(mockGraph, atLeastOnce()).executeGremlinScript(anyString(), anyBoolean());
}
}
\ No newline at end of file
webapp/src/main/java/org/apache/atlas/web/resources/AdminResource.java
......@@ -254,12 +254,12 @@ public class AdminResource {
@GET
@Path("metrics")
@Produces(Servlets.JSON_MEDIA_TYPE)
public AtlasMetrics getMetrics() {
public AtlasMetrics getMetrics(@QueryParam("ignoreCache") boolean ignoreCache) {
if (LOG.isDebugEnabled()) {
LOG.debug("==> AdminResource.getMetrics()");
}
AtlasMetrics metrics = metricsService.getMetrics();
AtlasMetrics metrics = metricsService.getMetrics(ignoreCache);
if (LOG.isDebugEnabled()) {
LOG.debug("<== AdminResource.getMetrics()");
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment