Skip to content

A
atlas
Commit ee99930f by Nikhil Bonte Committed by Madhan Neethiraj
Options

ATLAS-3052: removed caching of metrics query results

Change-Id: Ifeabb13acb387781a72d00146d4acd4ea9a279d8 Signed-off-by: 's avatarMadhan Neethiraj <madhan@apache.org>
parent a0e4cd1a
Showing with 57 additions and 83 deletions
repository/src/main/java/org/apache/atlas/services/MetricsService.java
...@@ -59,108 +59,95 @@ public class MetricsService { ...@@ -59,108 +59,95 @@ public class MetricsService {
protected static final String METRIC_TAG_COUNT = TAG + "Count"; protected static final String METRIC_TAG_COUNT = TAG + "Count";
protected static final String METRIC_ENTITIES_PER_TAG = TAG + "Entities"; protected static final String METRIC_ENTITIES_PER_TAG = TAG + "Entities";
public static final String METRIC_QUERY_CACHE_TTL = "atlas.metric.query.cache.ttlInSecs";
public static final String METRIC_QUERY_GREMLIN_TYPES_BATCH_SIZE = "atlas.metric.query.gremlin.typesBatchSize";
public static final int DEFAULT_CACHE_TTL_IN_SECS = 900;
public static final String METRIC_COLLECTION_TIME = "collectionTime"; public static final String METRIC_COLLECTION_TIME = "collectionTime";
private final AtlasGraph atlasGraph; private final AtlasGraph atlasGraph;
private final AtlasTypeRegistry typeRegistry; private final AtlasTypeRegistry typeRegistry;
private final int cacheTTLInSecs;
private final String indexSearchPrefix = AtlasGraphUtilsV2.getIndexSearchPrefix(); private final String indexSearchPrefix = AtlasGraphUtilsV2.getIndexSearchPrefix();
private AtlasMetrics cachedMetrics = null;
private long cacheExpirationTime = 0;
@Inject @Inject
public MetricsService(final Configuration configuration, final AtlasGraph graph, final AtlasTypeRegistry typeRegistry) { public MetricsService(final AtlasGraph graph, final AtlasTypeRegistry typeRegistry) {
this.atlasGraph = graph; this.atlasGraph = graph;
this.cacheTTLInSecs = configuration != null ? configuration.getInt(METRIC_QUERY_CACHE_TTL, DEFAULT_CACHE_TTL_IN_SECS) : DEFAULT_CACHE_TTL_IN_SECS;
this.typeRegistry = typeRegistry; this.typeRegistry = typeRegistry;
} }
@SuppressWarnings("unchecked") @SuppressWarnings("unchecked")
public AtlasMetrics getMetrics(boolean ignoreCache) { public AtlasMetrics getMetrics() {
if (ignoreCache || !isCacheValid()) { AtlasMetrics metrics = new AtlasMetrics();
AtlasMetrics metrics = new AtlasMetrics();
metrics.addMetric(GENERAL, METRIC_TYPE_COUNT, getAllTypesCount()); metrics.addMetric(GENERAL, METRIC_TYPE_COUNT, getAllTypesCount());
metrics.addMetric(GENERAL, METRIC_TAG_COUNT, getAllTagsCount()); metrics.addMetric(GENERAL, METRIC_TAG_COUNT, getAllTagsCount());
Map<String, Long> activeCountMap = new HashMap<>(); Map<String, Long> activeCountMap = new HashMap<>();
Map<String, Long> deletedCountMap = new HashMap<>(); Map<String, Long> deletedCountMap = new HashMap<>();
// metrics for classifications // metrics for classifications
Collection<String> classificationDefNames = typeRegistry.getAllClassificationDefNames(); Collection<String> classificationDefNames = typeRegistry.getAllClassificationDefNames();
if (classificationDefNames != null) { if (classificationDefNames != null) {
for (String classificationDefName : classificationDefNames) { for (String classificationDefName : classificationDefNames) {
activeCountMap.put(classificationDefName, getTypeCount(classificationDefName, ACTIVE)); activeCountMap.put(classificationDefName, getTypeCount(classificationDefName, ACTIVE));
}
} }
}
// metrics for entities // metrics for entities
Collection<String> entityDefNames = typeRegistry.getAllEntityDefNames(); Collection<String> entityDefNames = typeRegistry.getAllEntityDefNames();
if (entityDefNames != null) { if (entityDefNames != null) {
for (String entityDefName : entityDefNames) { for (String entityDefName : entityDefNames) {
activeCountMap.put(entityDefName, getTypeCount(entityDefName, ACTIVE)); activeCountMap.put(entityDefName, getTypeCount(entityDefName, ACTIVE));
deletedCountMap.put(entityDefName, getTypeCount(entityDefName, DELETED)); deletedCountMap.put(entityDefName, getTypeCount(entityDefName, DELETED));
}
} }
}
Map<String, Long> activeEntityCount = new HashMap<>(); Map<String, Long> activeEntityCount = new HashMap<>();
Map<String, Long> deletedEntityCount = new HashMap<>(); Map<String, Long> deletedEntityCount = new HashMap<>();
long unusedTypeCount = 0; long unusedTypeCount = 0;
long totalEntities = 0; long totalEntities = 0;
for (String entityDefName : typeRegistry.getAllEntityDefNames()) { for (String entityDefName : typeRegistry.getAllEntityDefNames()) {
Long activeCount = activeCountMap.get(entityDefName); Long activeCount = activeCountMap.get(entityDefName);
Long deletedCount = deletedCountMap.get(entityDefName); Long deletedCount = deletedCountMap.get(entityDefName);
if (activeCount > 0) { if (activeCount > 0) {
activeEntityCount.put(entityDefName, activeCount); activeEntityCount.put(entityDefName, activeCount);
totalEntities += activeCount.longValue(); totalEntities += activeCount.longValue();
} }
if (deletedCount > 0) { if (deletedCount > 0) {
deletedEntityCount.put(entityDefName, deletedCount); deletedEntityCount.put(entityDefName, deletedCount);
totalEntities += deletedCount.longValue(); totalEntities += deletedCount.longValue();
} }
if (activeCount == 0 && deletedCount == 0) { if (activeCount == 0 && deletedCount == 0) {
unusedTypeCount++; unusedTypeCount++;
}
} }
}
metrics.addMetric(GENERAL, METRIC_TYPE_UNUSED_COUNT, unusedTypeCount); metrics.addMetric(GENERAL, METRIC_TYPE_UNUSED_COUNT, unusedTypeCount);
metrics.addMetric(GENERAL, METRIC_ENTITY_COUNT, totalEntities); metrics.addMetric(GENERAL, METRIC_ENTITY_COUNT, totalEntities);
metrics.addMetric(ENTITY, METRIC_ENTITY_ACTIVE, activeEntityCount); metrics.addMetric(ENTITY, METRIC_ENTITY_ACTIVE, activeEntityCount);
metrics.addMetric(ENTITY, METRIC_ENTITY_DELETED, deletedEntityCount); metrics.addMetric(ENTITY, METRIC_ENTITY_DELETED, deletedEntityCount);
Map<String, Long> taggedEntityCount = new HashMap<>(); Map<String, Long> taggedEntityCount = new HashMap<>();
for (String classificationName : typeRegistry.getAllClassificationDefNames()) { for (String classificationName : typeRegistry.getAllClassificationDefNames()) {
Long count = activeCountMap.get(classificationName); Long count = activeCountMap.get(classificationName);
if (count > 0) { if (count > 0) {
taggedEntityCount.put(classificationName, count); taggedEntityCount.put(classificationName, count);
}
} }
}
metrics.addMetric(TAG, METRIC_ENTITIES_PER_TAG, taggedEntityCount); metrics.addMetric(TAG, METRIC_ENTITIES_PER_TAG, taggedEntityCount);
// Miscellaneous metrics
long collectionTime = System.currentTimeMillis();
metrics.addMetric(GENERAL, METRIC_COLLECTION_TIME, collectionTime); // Miscellaneous metrics
long collectionTime = System.currentTimeMillis();
this.cachedMetrics = metrics; metrics.addMetric(GENERAL, METRIC_COLLECTION_TIME, collectionTime);
this.cacheExpirationTime = (collectionTime + cacheTTLInSecs * 1000);
}
return cachedMetrics; return metrics;
} }
private Long getTypeCount(String typeName, Status status) { private Long getTypeCount(String typeName, Status status) {
...@@ -183,16 +170,4 @@ public class MetricsService { ...@@ -183,16 +170,4 @@ public class MetricsService {
return CollectionUtils.isNotEmpty(allTagNames) ? allTagNames.size() : 0; return CollectionUtils.isNotEmpty(allTagNames) ? allTagNames.size() : 0;
} }
private boolean isCacheValid() {
boolean valid = cachedMetrics != null && System.currentTimeMillis() < cacheExpirationTime;
if (LOG.isDebugEnabled()) {
LOG.debug("cachedMetrics: {}", cachedMetrics != null);
LOG.debug("cacheExpirationTime: {}", cacheExpirationTime);
LOG.debug("valid: {}", valid);
}
return valid;
}
} }
\ No newline at end of file
repository/src/test/java/org/apache/atlas/services/MetricsServiceTest.java
...@@ -124,7 +124,7 @@ public class MetricsServiceTest { ...@@ -124,7 +124,7 @@ public class MetricsServiceTest {
@Test @Test
public void testGetMetrics() { public void testGetMetrics() {
AtlasMetrics metrics = metricsService.getMetrics(true); AtlasMetrics metrics = metricsService.getMetrics();
assertNotNull(metrics); assertNotNull(metrics);
......
webapp/src/main/java/org/apache/atlas/web/resources/AdminResource.java
...@@ -319,12 +319,12 @@ public class AdminResource { ...@@ -319,12 +319,12 @@ public class AdminResource {
@GET @GET
@Path("metrics") @Path("metrics")
@Produces(Servlets.JSON_MEDIA_TYPE) @Produces(Servlets.JSON_MEDIA_TYPE)
public AtlasMetrics getMetrics(@QueryParam("ignoreCache") boolean ignoreCache) { public AtlasMetrics getMetrics() {
if (LOG.isDebugEnabled()) { if (LOG.isDebugEnabled()) {
LOG.debug("==> AdminResource.getMetrics()"); LOG.debug("==> AdminResource.getMetrics()");
} }
AtlasMetrics metrics = metricsService.getMetrics(ignoreCache); AtlasMetrics metrics = metricsService.getMetrics();
if (LOG.isDebugEnabled()) { if (LOG.isDebugEnabled()) {
LOG.debug("<== AdminResource.getMetrics()"); LOG.debug("<== AdminResource.getMetrics()");
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment