diff --git a/DBAccess/src/main/java/gr/uoa/di/madgik/statstool/domain/cache/CacheEntry.java b/DBAccess/src/main/java/gr/uoa/di/madgik/statstool/domain/cache/CacheEntry.java index af32d06..12bcb2b 100644 --- a/DBAccess/src/main/java/gr/uoa/di/madgik/statstool/domain/cache/CacheEntry.java +++ b/DBAccess/src/main/java/gr/uoa/di/madgik/statstool/domain/cache/CacheEntry.java @@ -16,12 +16,23 @@ public class CacheEntry { private int sessionHits = 0; private boolean pinned = false; + private String profile; + + private long execTime; + public CacheEntry(String key, QueryWithParameters query, Result result) { this.key = key; this.query = query; this.result = result; } + public CacheEntry(String key, QueryWithParameters query, Result result, long execTime) { + this.key = key; + this.query = query; + this.result = result; + this.execTime = execTime; + } + public String getKey() { return key; } @@ -94,6 +105,22 @@ public void setPinned(boolean pinned) { this.pinned = pinned; } + public long getExecTime() { + return execTime; + } + + public void setExecTime(long execTime) { + this.execTime = execTime; + } + + public String getProfile() { + return profile; + } + + public void setProfile(String profile) { + this.profile = profile; + } + @Override public String toString() { return "CacheEntry{" + @@ -106,6 +133,8 @@ public String toString() { ", totalHits=" + totalHits + ", sessionHits=" + sessionHits + ", pinned=" + pinned + + ", execTime= " + execTime + + ", profile= " + profile + '}'; } } \ No newline at end of file diff --git a/DBAccess/src/main/java/gr/uoa/di/madgik/statstool/repositories/StatsCache.java b/DBAccess/src/main/java/gr/uoa/di/madgik/statstool/repositories/StatsCache.java index 202e9a9..9df8a2a 100644 --- a/DBAccess/src/main/java/gr/uoa/di/madgik/statstool/repositories/StatsCache.java +++ b/DBAccess/src/main/java/gr/uoa/di/madgik/statstool/repositories/StatsCache.java @@ -23,7 +23,7 @@ static String getCacheKey(QueryWithParameters query) throws Exception { Result get(String key) throws Exception; - String save(QueryWithParameters fullSqlQuery, Result result) throws Exception; + void save(QueryWithParameters fullSqlQuery, Result result, long execTime) throws Exception; void storeEntry(CacheEntry entry) throws Exception; diff --git a/DBAccess/src/main/java/gr/uoa/di/madgik/statstool/repositories/StatsDBRepository.java b/DBAccess/src/main/java/gr/uoa/di/madgik/statstool/repositories/StatsDBRepository.java index 7fd02f4..d0222c2 100644 --- a/DBAccess/src/main/java/gr/uoa/di/madgik/statstool/repositories/StatsDBRepository.java +++ b/DBAccess/src/main/java/gr/uoa/di/madgik/statstool/repositories/StatsDBRepository.java @@ -51,7 +51,9 @@ public void postInit() { "updated timestamp default now() not null, " + "total_hits int default 0 not null," + "session_hits int default 0 not null," + - "pinned boolean default false not null)"); + "pinned boolean default false not null," + + "exectime long default 0 not null," + + "profile varchar(100) not null)"); jdbcTemplate.execute("create index key_idx on cache_entry(key)"); } @@ -100,7 +102,7 @@ public void dropCache() throws Exception { } @Override - public String save(QueryWithParameters fullSqlQuery, Result result) throws Exception { + public void save(QueryWithParameters fullSqlQuery, Result result, long execTime) throws Exception { DatasourceContext.setContext(CACHE_DB_NAME); try { String key = StatsCache.getCacheKey(fullSqlQuery); @@ -110,10 +112,11 @@ public String save(QueryWithParameters fullSqlQuery, Result result) throws Excep else { CacheEntry entry = new CacheEntry(key, fullSqlQuery, result); + entry.setExecTime(execTime); + entry.setProfile(fullSqlQuery.getDbId()); + storeEntry(entry); } - - return key; } catch (Exception e) { throw new RedisException(e); } @@ -122,8 +125,7 @@ public String save(QueryWithParameters fullSqlQuery, Result result) throws Excep @Override public void storeEntry(CacheEntry entry) throws Exception { DatasourceContext.setContext(CACHE_DB_NAME); -// String query = "insert into cache_entry (key, result, shadow, query, created, updated, total_hits, session_hits, pinned) values (?, ?, ?, ?, ?, ?, ?, ?, ?)"; - String query = "merge into cache_entry as t using (values(?, ?, ?, ?, ?, ?, ?, ?, ?)) as vals(key, result, shadow, query, created, updated, total, session, pinned) on t.key=vals.key when matched then update set t.result=vals.result, t.shadow=vals.shadow, t.query=vals.query, t.created=vals.updated, t.updated=vals.updated, t.total_hits=vals.total, t.session_hits=vals.session, t.pinned=vals.pinned when not matched then insert values vals.key, vals.result, vals.shadow, vals.query, vals.created, vals.updated, vals.total, vals.session, vals.pinned;"; + String query = "merge into cache_entry as t using (values(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)) as vals(key, result, shadow, query, created, updated, total, session, pinned, exectime, profile) on t.key=vals.key when matched then update set t.result=vals.result, t.shadow=vals.shadow, t.query=vals.query, t.created=vals.updated, t.updated=vals.updated, t.total_hits=vals.total, t.session_hits=vals.session, t.pinned=vals.pinned when not matched then insert values vals.key, vals.result, vals.shadow, vals.query, vals.created, vals.updated, vals.total, vals.session, vals.pinned, vals.exectime, vals.profile;"; log.debug("Storing entry " + entry); @@ -139,7 +141,9 @@ public void storeEntry(CacheEntry entry) throws Exception { Timestamp.from(entry.getUpdated().toInstant()), entry.getTotalHits(), entry.getSessionHits(), - entry.isPinned() + entry.isPinned(), + entry.getExecTime(), + entry.getProfile() ); } @@ -173,6 +177,8 @@ public List getEntries() { entry.setTotalHits(rs.getInt("total_hits")); entry.setSessionHits(rs.getInt("session_hits")); entry.setPinned(rs.getBoolean("pinned")); + entry.setExecTime(rs.getLong("exectime")); + entry.setProfile(rs.getString("profile")); } catch (IOException e) { log.error("Error reading entry", e); } @@ -199,7 +205,7 @@ public Map stats() { stats.put("total", jdbcTemplate.queryForObject("select count(*) from cache_entry",new Object[] {}, Integer.class)); stats.put("with_shadow", jdbcTemplate.queryForObject("select count(*) from cache_entry where shadow is not null and shadow != ''",new Object[] {}, Integer.class)); - stats.put("total.top10", jdbcTemplate.query("select * from cache_entry where key not in ('SHADOW_STATS_NUMBERS', 'STATS_NUMBERS') order by total_hits limit 10", (rs, rowNum) -> { + stats.put("total.top10", jdbcTemplate.query("select * from cache_entry where key not in ('SHADOW_STATS_NUMBERS', 'STATS_NUMBERS') order by total_hits desc limit 10", (rs, rowNum) -> { CacheEntry entry = null; try { @@ -222,7 +228,7 @@ public Map stats() { return entry; })); - stats.put("session.top10", jdbcTemplate.query("select * from cache_entry where key not in ('SHADOW_STATS_NUMBERS', 'STATS_NUMBERS') order by session_hits limit 10", (rs, rowNum) -> { + stats.put("session.top10", jdbcTemplate.query("select * from cache_entry where key not in ('SHADOW_STATS_NUMBERS', 'STATS_NUMBERS') order by session_hits desc limit 10", (rs, rowNum) -> { CacheEntry entry = null; try { @@ -245,6 +251,30 @@ public Map stats() { return entry; })); + stats.put("total.heavy10", jdbcTemplate.query("select * from cache_entry where key not in ('SHADOW_STATS_NUMBERS', 'STATS_NUMBERS') order by exectime desc limit 10", (rs, rowNum) -> { + CacheEntry entry = null; + + try { + QueryWithParameters query = new ObjectMapper().readValue(rs.getString("query"), QueryWithParameters.class); + String key = rs.getString("key"); + + entry = new CacheEntry(key, query, null); + + if (rs.getTimestamp("created") != null) + entry.setCreated(new Date(rs.getTimestamp("created").getTime())); + if (rs.getTimestamp("updated") != null) + entry.setUpdated(new Date(rs.getTimestamp("updated").getTime())); + + entry.setTotalHits(rs.getInt("total_hits")); + entry.setSessionHits(rs.getInt("session_hits")); + entry.setPinned(rs.getBoolean("pinned")); + } catch (IOException e) { + log.error("Error reading entry", e); + } + + return entry; + })); + return stats; } } diff --git a/DBAccess/src/main/java/gr/uoa/di/madgik/statstool/repositories/StatsRedisRepository.java b/DBAccess/src/main/java/gr/uoa/di/madgik/statstool/repositories/StatsRedisRepository.java index 2497ae5..461aeb4 100644 --- a/DBAccess/src/main/java/gr/uoa/di/madgik/statstool/repositories/StatsRedisRepository.java +++ b/DBAccess/src/main/java/gr/uoa/di/madgik/statstool/repositories/StatsRedisRepository.java @@ -71,17 +71,19 @@ public Result get(String key) throws RedisException { } @Override - public String save(QueryWithParameters fullSqlQuery, Result result) throws RedisException { + public void save(QueryWithParameters fullSqlQuery, Result result, long execTime) throws RedisException { try { String key = StatsCache.getCacheKey(fullSqlQuery); + CacheEntry entry = new CacheEntry(key, fullSqlQuery, result); + + entry.setExecTime(execTime); + entry.setProfile(fullSqlQuery.getDbId()); if (!enableCache) log.debug("Cache is not enabled. Noop!"); else - storeEntry(new CacheEntry(key, fullSqlQuery, result)); - - return key; + storeEntry(entry); } catch (Exception e) { throw new RedisException(e); } diff --git a/DBAccess/src/main/java/gr/uoa/di/madgik/statstool/services/CacheServiceImpl.java b/DBAccess/src/main/java/gr/uoa/di/madgik/statstool/services/CacheServiceImpl.java index cf0965c..d80bb98 100644 --- a/DBAccess/src/main/java/gr/uoa/di/madgik/statstool/services/CacheServiceImpl.java +++ b/DBAccess/src/main/java/gr/uoa/di/madgik/statstool/services/CacheServiceImpl.java @@ -1,6 +1,7 @@ package gr.uoa.di.madgik.statstool.services; import com.fasterxml.jackson.core.JsonProcessingException; +import gr.uoa.di.madgik.statstool.domain.Result; import gr.uoa.di.madgik.statstool.domain.cache.CacheEntry; import gr.uoa.di.madgik.statstool.repositories.StatsCache; import gr.uoa.di.madgik.statstool.repositories.StatsRepository; @@ -76,7 +77,12 @@ private void doUpdateCache() { i.getAndIncrement(); log.debug(i.get() + ". Updating entry " + entry.getKey() + "(" + entry.getQuery().getDbId() + ") with query " + entry.getQuery()); - entry.setShadowResult(statsRepository.executeQuery(entry.getQuery().getQuery(), entry.getQuery().getParameters(), entry.getQuery().getDbId().replace("public", "shadow"))); + long start = new Date().getTime(); + Result shadow = statsRepository.executeQuery(entry.getQuery().getQuery(), entry.getQuery().getParameters(), entry.getQuery().getDbId().replace("public", "shadow")); + long execTime = new Date().getTime() - start; + + entry.setShadowResult(shadow); + entry.setExecTime(execTime); } else { log.info("time or # of queries limits exceeded. Invalidating entry " + entry.getKey()); diff --git a/DBAccess/src/main/java/gr/uoa/di/madgik/statstool/services/StatsServiceImpl.java b/DBAccess/src/main/java/gr/uoa/di/madgik/statstool/services/StatsServiceImpl.java index b90b879..2630f07 100644 --- a/DBAccess/src/main/java/gr/uoa/di/madgik/statstool/services/StatsServiceImpl.java +++ b/DBAccess/src/main/java/gr/uoa/di/madgik/statstool/services/StatsServiceImpl.java @@ -9,6 +9,7 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Date; import java.util.List; import org.apache.logging.log4j.Logger; @@ -76,9 +77,12 @@ public List query(List queryList, String orderBy) throws StatsSer log.debug("Key " + cacheKey + " in cache! Returning: " + result); } else { log.debug("result for key " + cacheKey + " not in cache. Querying db!"); + long start = new Date().getTime(); result = statsRepository.executeQuery(querySql, parameters, profile); log.debug("result: " + result); - statsCache.save(new QueryWithParameters(querySql, parameters, profile), result); + long execTime = new Date().getTime() - start; + + statsCache.save(new QueryWithParameters(querySql, parameters, profile), result, execTime); } } else { log.debug("Cache disabled for query.");