Skip to content

Commit

Permalink
added execution time and profile in cache
Browse files Browse the repository at this point in the history
  • Loading branch information
@antleb
antleb committed Feb 17, 2024
1 parent 2f054f9 commit 5dde48f
Show file tree
Hide file tree
Showing 6 changed files with 87 additions and 16 deletions.
29 changes: 29 additions & 0 deletions DBAccess/src/main/java/gr/uoa/di/madgik/statstool/domain/cache/CacheEntry.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,12 +16,23 @@ public class CacheEntry {
private int sessionHits = 0;
private boolean pinned = false;

private String profile;

private long execTime;

public CacheEntry(String key, QueryWithParameters query, Result result) {
this.key = key;
this.query = query;
this.result = result;
}

public CacheEntry(String key, QueryWithParameters query, Result result, long execTime) {
this.key = key;
this.query = query;
this.result = result;
this.execTime = execTime;
}

public String getKey() {
return key;
}
Expand Down Expand Up @@ -94,6 +105,22 @@ public void setPinned(boolean pinned) {
this.pinned = pinned;
}

public long getExecTime() {
return execTime;
}

public void setExecTime(long execTime) {
this.execTime = execTime;
}

public String getProfile() {
return profile;
}

public void setProfile(String profile) {
this.profile = profile;
}

@Override
public String toString() {
return "CacheEntry{" +
Expand All @@ -106,6 +133,8 @@ public String toString() {
", totalHits=" + totalHits +
", sessionHits=" + sessionHits +
", pinned=" + pinned +
", execTime= " + execTime +
", profile= " + profile +
'}';
}
}
2 changes: 1 addition & 1 deletion DBAccess/src/main/java/gr/uoa/di/madgik/statstool/repositories/StatsCache.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ static String getCacheKey(QueryWithParameters query) throws Exception {

Result get(String key) throws Exception;

String save(QueryWithParameters fullSqlQuery, Result result) throws Exception;
void save(QueryWithParameters fullSqlQuery, Result result, long execTime) throws Exception;

void storeEntry(CacheEntry entry) throws Exception;

Expand Down
48 changes: 39 additions & 9 deletions DBAccess/src/main/java/gr/uoa/di/madgik/statstool/repositories/StatsDBRepository.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,9 @@ public void postInit() {
"updated timestamp default now() not null, " +
"total_hits int default 0 not null," +
"session_hits int default 0 not null," +
"pinned boolean default false not null)");
"pinned boolean default false not null," +
"exectime long default 0 not null," +
"profile varchar(100) not null)");

jdbcTemplate.execute("create index key_idx on cache_entry(key)");
}
Expand Down Expand Up @@ -100,7 +102,7 @@ public void dropCache() throws Exception {
}

@Override
public String save(QueryWithParameters fullSqlQuery, Result result) throws Exception {
public void save(QueryWithParameters fullSqlQuery, Result result, long execTime) throws Exception {
DatasourceContext.setContext(CACHE_DB_NAME);
try {
String key = StatsCache.getCacheKey(fullSqlQuery);
Expand All @@ -110,10 +112,11 @@ public String save(QueryWithParameters fullSqlQuery, Result result) throws Excep
else {
CacheEntry entry = new CacheEntry(key, fullSqlQuery, result);

entry.setExecTime(execTime);
entry.setProfile(fullSqlQuery.getDbId());

storeEntry(entry);
}

return key;
} catch (Exception e) {
throw new RedisException(e);
}
Expand All @@ -122,8 +125,7 @@ public String save(QueryWithParameters fullSqlQuery, Result result) throws Excep
@Override
public void storeEntry(CacheEntry entry) throws Exception {
DatasourceContext.setContext(CACHE_DB_NAME);
// String query = "insert into cache_entry (key, result, shadow, query, created, updated, total_hits, session_hits, pinned) values (?, ?, ?, ?, ?, ?, ?, ?, ?)";
String query = "merge into cache_entry as t using (values(?, ?, ?, ?, ?, ?, ?, ?, ?)) as vals(key, result, shadow, query, created, updated, total, session, pinned) on t.key=vals.key when matched then update set t.result=vals.result, t.shadow=vals.shadow, t.query=vals.query, t.created=vals.updated, t.updated=vals.updated, t.total_hits=vals.total, t.session_hits=vals.session, t.pinned=vals.pinned when not matched then insert values vals.key, vals.result, vals.shadow, vals.query, vals.created, vals.updated, vals.total, vals.session, vals.pinned;";
String query = "merge into cache_entry as t using (values(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)) as vals(key, result, shadow, query, created, updated, total, session, pinned, exectime, profile) on t.key=vals.key when matched then update set t.result=vals.result, t.shadow=vals.shadow, t.query=vals.query, t.created=vals.updated, t.updated=vals.updated, t.total_hits=vals.total, t.session_hits=vals.session, t.pinned=vals.pinned when not matched then insert values vals.key, vals.result, vals.shadow, vals.query, vals.created, vals.updated, vals.total, vals.session, vals.pinned, vals.exectime, vals.profile;";

log.debug("Storing entry " + entry);

Expand All @@ -139,7 +141,9 @@ public void storeEntry(CacheEntry entry) throws Exception {
Timestamp.from(entry.getUpdated().toInstant()),
entry.getTotalHits(),
entry.getSessionHits(),
entry.isPinned()
entry.isPinned(),
entry.getExecTime(),
entry.getProfile()
);
}

Expand Down Expand Up @@ -173,6 +177,8 @@ public List<CacheEntry> getEntries() {
entry.setTotalHits(rs.getInt("total_hits"));
entry.setSessionHits(rs.getInt("session_hits"));
entry.setPinned(rs.getBoolean("pinned"));
entry.setExecTime(rs.getLong("exectime"));
entry.setProfile(rs.getString("profile"));
} catch (IOException e) {
log.error("Error reading entry", e);
}
Expand All @@ -199,7 +205,7 @@ public Map<String, Object> stats() {

stats.put("total", jdbcTemplate.queryForObject("select count(*) from cache_entry",new Object[] {}, Integer.class));
stats.put("with_shadow", jdbcTemplate.queryForObject("select count(*) from cache_entry where shadow is not null and shadow != ''",new Object[] {}, Integer.class));
stats.put("total.top10", jdbcTemplate.query("select * from cache_entry where key not in ('SHADOW_STATS_NUMBERS', 'STATS_NUMBERS') order by total_hits limit 10", (rs, rowNum) -> {
stats.put("total.top10", jdbcTemplate.query("select * from cache_entry where key not in ('SHADOW_STATS_NUMBERS', 'STATS_NUMBERS') order by total_hits desc limit 10", (rs, rowNum) -> {
CacheEntry entry = null;

try {
Expand All @@ -222,7 +228,7 @@ public Map<String, Object> stats() {

return entry;
}));
stats.put("session.top10", jdbcTemplate.query("select * from cache_entry where key not in ('SHADOW_STATS_NUMBERS', 'STATS_NUMBERS') order by session_hits limit 10", (rs, rowNum) -> {
stats.put("session.top10", jdbcTemplate.query("select * from cache_entry where key not in ('SHADOW_STATS_NUMBERS', 'STATS_NUMBERS') order by session_hits desc limit 10", (rs, rowNum) -> {
CacheEntry entry = null;

try {
Expand All @@ -245,6 +251,30 @@ public Map<String, Object> stats() {

return entry;
}));
stats.put("total.heavy10", jdbcTemplate.query("select * from cache_entry where key not in ('SHADOW_STATS_NUMBERS', 'STATS_NUMBERS') order by exectime desc limit 10", (rs, rowNum) -> {
CacheEntry entry = null;

try {
QueryWithParameters query = new ObjectMapper().readValue(rs.getString("query"), QueryWithParameters.class);
String key = rs.getString("key");

entry = new CacheEntry(key, query, null);

if (rs.getTimestamp("created") != null)
entry.setCreated(new Date(rs.getTimestamp("created").getTime()));
if (rs.getTimestamp("updated") != null)
entry.setUpdated(new Date(rs.getTimestamp("updated").getTime()));

entry.setTotalHits(rs.getInt("total_hits"));
entry.setSessionHits(rs.getInt("session_hits"));
entry.setPinned(rs.getBoolean("pinned"));
} catch (IOException e) {
log.error("Error reading entry", e);
}

return entry;
}));

return stats;
}
}
10 changes: 6 additions & 4 deletions DBAccess/src/main/java/gr/uoa/di/madgik/statstool/repositories/StatsRedisRepository.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,17 +71,19 @@ public Result get(String key) throws RedisException {
}

@Override
public String save(QueryWithParameters fullSqlQuery, Result result) throws RedisException {
public void save(QueryWithParameters fullSqlQuery, Result result, long execTime) throws RedisException {

try {
String key = StatsCache.getCacheKey(fullSqlQuery);
CacheEntry entry = new CacheEntry(key, fullSqlQuery, result);

entry.setExecTime(execTime);
entry.setProfile(fullSqlQuery.getDbId());

if (!enableCache)
log.debug("Cache is not enabled. Noop!");
else
storeEntry(new CacheEntry(key, fullSqlQuery, result));

return key;
storeEntry(entry);
} catch (Exception e) {
throw new RedisException(e);
}
Expand Down
8 changes: 7 additions & 1 deletion DBAccess/src/main/java/gr/uoa/di/madgik/statstool/services/CacheServiceImpl.java
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package gr.uoa.di.madgik.statstool.services;

import com.fasterxml.jackson.core.JsonProcessingException;
import gr.uoa.di.madgik.statstool.domain.Result;
import gr.uoa.di.madgik.statstool.domain.cache.CacheEntry;
import gr.uoa.di.madgik.statstool.repositories.StatsCache;
import gr.uoa.di.madgik.statstool.repositories.StatsRepository;
Expand Down Expand Up @@ -76,7 +77,12 @@ private void doUpdateCache() {
i.getAndIncrement();
log.debug(i.get() + ". Updating entry " + entry.getKey() + "(" + entry.getQuery().getDbId() + ") with query " + entry.getQuery());

entry.setShadowResult(statsRepository.executeQuery(entry.getQuery().getQuery(), entry.getQuery().getParameters(), entry.getQuery().getDbId().replace("public", "shadow")));
long start = new Date().getTime();
Result shadow = statsRepository.executeQuery(entry.getQuery().getQuery(), entry.getQuery().getParameters(), entry.getQuery().getDbId().replace("public", "shadow"));
long execTime = new Date().getTime() - start;

entry.setShadowResult(shadow);
entry.setExecTime(execTime);
} else {
log.info("time or # of queries limits exceeded. Invalidating entry " + entry.getKey());

Expand Down
6 changes: 5 additions & 1 deletion DBAccess/src/main/java/gr/uoa/di/madgik/statstool/services/StatsServiceImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@

import java.io.IOException;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;

import org.apache.logging.log4j.Logger;
Expand Down Expand Up @@ -76,9 +77,12 @@ public List<Result> query(List<Query> queryList, String orderBy) throws StatsSer
log.debug("Key " + cacheKey + " in cache! Returning: " + result);
} else {
log.debug("result for key " + cacheKey + " not in cache. Querying db!");
long start = new Date().getTime();
result = statsRepository.executeQuery(querySql, parameters, profile);
log.debug("result: " + result);
statsCache.save(new QueryWithParameters(querySql, parameters, profile), result);
long execTime = new Date().getTime() - start;

statsCache.save(new QueryWithParameters(querySql, parameters, profile), result, execTime);
}
} else {
log.debug("Cache disabled for query.");
Expand Down

0 comments on commit 5dde48f

Please sign in to comment.