diff --git a/project.clj b/project.clj index ec68331..e4645b1 100644 --- a/project.clj +++ b/project.clj @@ -3,10 +3,10 @@ :dependencies [;; clj [org.clojure/clojure "1.9.0"] [bk/ring-gzip "0.3.0"] - [buddy/buddy-auth "2.1.0"] - [buddy/buddy-core "1.5.0"] + [buddy/buddy-auth "2.2.0"] + [buddy/buddy-core "1.6.0"] [clj-http "3.10.0"] - [clj-time "0.15.1"] + [clj-time "0.15.2"] [com.syncron.amazonaws/simba-athena-jdbc-driver "2.0.2"] [com.draines/postal "2.0.3"] [compojure "1.6.1" :exclusions [ring/ring-core]] @@ -24,16 +24,16 @@ :exclusions [com.fasterxml.jackson.core/jackson-core]] [net.sf.uadetector/uadetector-resources "2014.10"] [org.clojure/data.json "0.2.6"] - [org.clojure/java.jdbc "0.7.9"] - [org.clojure/tools.logging "0.5.0-alpha.1"] - [org.postgresql/postgresql "42.2.6"] - [org.slf4j/slf4j-log4j12 "1.7.26" + [org.clojure/java.jdbc "0.7.10"] + [org.clojure/tools.logging "0.5.0"] + [org.postgresql/postgresql "42.2.8"] + [org.slf4j/slf4j-log4j12 "1.7.28" :exclusions [org.clojure/tools.reader org.slf4j/slf4j-api]] [ring "1.7.1"] [ring/ring-anti-forgery "1.3.0"] [ring/ring-defaults "0.3.2" ] - [ring/ring-json "0.4.0"] + [ring/ring-json "0.5.0"] ;; cljs [org.clojure/clojurescript "1.10.520" @@ -46,8 +46,8 @@ [cljs-log "0.2.3"] [com.andrewmcveigh/cljs-time "0.5.2"] [day8.re-frame/http-fx "0.1.6"] - [day8.re-frame/tracing-stubs "0.5.1"] - [re-frame "0.10.7" + [day8.re-frame/tracing-stubs "0.5.3"] + [re-frame "0.10.9" :exclusions [com.google.code.findbugs/jsr305]] [re-frame-datatable "0.6.0"] [reagent "0.8.1"] @@ -66,9 +66,9 @@ :profiles {:dev {:env {:log-level "1"} ;; cljs/log debug+ :dependencies [[binaryage/devtools "0.9.10"] [com.bhauman/cljs-test-display "0.1.1"] - [day8.re-frame/re-frame-10x "0.4.1" + [day8.re-frame/re-frame-10x "0.4.3" :exclusions [com.google.code.findbugs/jsr305]] - [day8.re-frame/tracing "0.5.1"] + [day8.re-frame/tracing "0.5.3"] [javax.servlet/servlet-api "2.5"] [ring/ring-mock "0.4.0" :exclusions [cheshire]]] diff --git a/resources/athena/asset_type_athena.sql b/resources/athena/asset_type_athena.sql index 0472dea..c15d3af 100644 --- a/resources/athena/asset_type_athena.sql +++ b/resources/athena/asset_type_athena.sql @@ -27,8 +27,8 @@ with now_ts as ( cast(portfolio.quantity as decimal(10,4)) as quantity, cast(portfolio.cost_per_share as decimal(6,2)) as cost_per_share from - dw.portfolio_dim portfolio - join dw.markets_dim markets on markets.ticker = portfolio.ticker + datalake.portfolio portfolio + join datalake.markets markets on markets.ticker = portfolio.ticker where portfolio.dataset = ( select datasource from datasource ) and user = ( select _user from _user ) @@ -36,29 +36,34 @@ with now_ts as ( 1,2,3,4,5,6,7,8 ), max_known_date as ( select - max(cast(date as date)) max_known_date + max(date) max_known_date from ( - select date, dataset, count(*) - from dw.equities_fact + select + date, + dataset, + count(*) + from + datalake.equities where - s3uploaddate >= cast((select beginning_of_year from beginning_of_year ) as date) - and s3uploaddate <> cast((select now from now) as date) + date >= (select beginning_of_year from beginning_of_year ) + and date <> (select now from now) and ticker in ( select distinct ticker from portfolio ) - and dataset <> 'ALPHA-VANTAGE' group by 1,2 having count(*) > 30 ) src ), fx as ( select - currency, cast(rate as decimal(24,14)) rate + currency, + rate from - dw.currency_fact + datalake.currency where currency = 'GBP' - and ( s3uploaddate = cast((select today from date ) as date) - or s3uploaddate = cast((select yesterday from date ) as date )) - order by date desc + and ( date = (select today from date ) + or date = (select yesterday from date ) ) + order by + date desc limit 1 ), fx_backup as ( select @@ -67,20 +72,21 @@ with now_ts as ( select coalesce(fx.currency,fx_backup.currency) currency, coalesce(fx.rate ,fx_backup.rate) rate - from fx + from + fx right join fx_backup on fx.currency = fx_backup.currency ), equities as ( select ticker, - cast(date as date) as date, - avg(case when ticker = 'LON:FCH' then try_cast(close as decimal(10,2)) * (select rate from fx_with_backup where currency = 'GBP') / 100 else try_cast(close as decimal(10,2)) end) as close + date, + avg(case when ticker = 'LON:FCH' then close * (select rate from fx_with_backup where currency = 'GBP') / 100 else close end) as close from - dw.equities_fact equities + datalake.equities equities where - s3uploaddate = cast((select today from date) as date) - or s3uploaddate = cast((select yesterday from date) as date) - or s3uploaddate = cast((select max_known_date from max_known_date) as date) - or s3uploaddate = cast((select beginning_of_year from beginning_of_year) as date) + date = (select today from date) + or date = (select yesterday from date) + or date = (select max_known_date from max_known_date) + or date = (select beginning_of_year from beginning_of_year) group by 1,2 ), today as ( @@ -174,7 +180,8 @@ with now_ts as ( today full outer join yesterday on today.ticker = yesterday.ticker full outer join ytd on yesterday.ticker = ytd.ticker - order by today.market_value desc + order by + today.market_value desc ), detail_with_backup as ( select coalesce(detail.asset_type, backup.asset_type) asset_type, @@ -228,7 +235,8 @@ with now_ts as ( where ticker <> '' group by 1 - order by market_value desc + order by + market_value desc ), _union as ( select * from benchmark union all diff --git a/resources/athena/capitalization_athena.sql b/resources/athena/capitalization_athena.sql index 2d40a0c..87f6f44 100644 --- a/resources/athena/capitalization_athena.sql +++ b/resources/athena/capitalization_athena.sql @@ -27,8 +27,8 @@ with now_ts as ( cast(portfolio.quantity as decimal(10,4)) as quantity, cast(portfolio.cost_per_share as decimal(6,2)) as cost_per_share from - dw.portfolio_dim portfolio - join dw.markets_dim markets on markets.ticker = portfolio.ticker + datalake.portfolio portfolio + join datalake.markets markets on markets.ticker = portfolio.ticker where portfolio.dataset = ( select datasource from datasource ) and user = ( select _user from _user ) @@ -36,29 +36,35 @@ with now_ts as ( 1,2,3,4,5,6,7,8 ), max_known_date as ( select - max(cast(date as date)) max_known_date + select + max(date) max_known_date from ( - select date, dataset, count(*) - from dw.equities_fact + select + date, + dataset, + count(*) + from + datalake.equities where - s3uploaddate >= cast((select beginning_of_year from beginning_of_year ) as date) - and s3uploaddate <> cast((select now from now) as date) + date >= (select beginning_of_year from beginning_of_year ) + and date <> (select now from now) and ticker in ( select distinct ticker from portfolio ) - and dataset <> 'ALPHA-VANTAGE' group by 1,2 having count(*) > 30 ) src ), fx as ( select - currency, cast(rate as decimal(24,14)) rate + currency, + rate from - dw.currency_fact + datalake.currency where currency = 'GBP' - and ( s3uploaddate = cast((select today from date ) as date) - or s3uploaddate = cast((select yesterday from date ) as date )) - order by date desc + and ( date = (select today from date ) + or date = (select yesterday from date ) ) + order by + date desc limit 1 ), fx_backup as ( select @@ -67,20 +73,21 @@ with now_ts as ( select coalesce(fx.currency,fx_backup.currency) currency, coalesce(fx.rate ,fx_backup.rate) rate - from fx + from + fx right join fx_backup on fx.currency = fx_backup.currency ), equities as ( select ticker, - cast(date as date) as date, - avg(case when ticker = 'LON:FCH' then try_cast(close as decimal(10,2)) * (select rate from fx_with_backup where currency = 'GBP') / 100 else try_cast(close as decimal(10,2)) end) as close + date, + avg(case when ticker = 'LON:FCH' then close * (select rate from fx_with_backup where currency = 'GBP') / 100 else close end) as close from - dw.equities_fact equities + datalake.equities equities where - s3uploaddate = cast((select today from date) as date) - or s3uploaddate = cast((select yesterday from date) as date) - or s3uploaddate = cast((select max_known_date from max_known_date) as date) - or s3uploaddate = cast((select beginning_of_year from beginning_of_year) as date) + date = (select today from date) + or date = (select yesterday from date) + or date = (select max_known_date from max_known_date) + or date = (select beginning_of_year from beginning_of_year) group by 1,2 ), today as ( @@ -174,7 +181,8 @@ with now_ts as ( today full outer join yesterday on today.ticker = yesterday.ticker full outer join ytd on yesterday.ticker = ytd.ticker - order by today.market_value desc + order by + today.market_value desc ), detail_with_backup as ( select coalesce(detail.asset_type, backup.asset_type) asset_type, @@ -228,7 +236,8 @@ with now_ts as ( where ticker <> '' group by 1 - order by market_value desc + order by + market_value desc ), _union as ( select * from benchmark union all diff --git a/resources/athena/dashboard.sql b/resources/athena/dashboard.sql index 9247510..8ede14e 100644 --- a/resources/athena/dashboard.sql +++ b/resources/athena/dashboard.sql @@ -5,16 +5,18 @@ with date as ( dataset, ticker, currency, - cast(date as date) as date, - cast(rate as decimal(24,14)) as rate, - cast(high as decimal(24,14)) as high, - cast(low as decimal(24,14)) as low + date, + rate, + high, + low from - dw.currency_fact currency + datalake.currency ), data as ( select row_number() over (partition by currency order by date desc) as rn, - currency, date, rate + currency, + date, + rate from _currency where diff --git a/resources/athena/investment_style_athena.sql b/resources/athena/investment_style_athena.sql index bd74f98..6547970 100644 --- a/resources/athena/investment_style_athena.sql +++ b/resources/athena/investment_style_athena.sql @@ -27,8 +27,8 @@ with now_ts as ( cast(portfolio.quantity as decimal(10,4)) as quantity, cast(portfolio.cost_per_share as decimal(6,2)) as cost_per_share from - dw.portfolio_dim portfolio - join dw.markets_dim markets on markets.ticker = portfolio.ticker + datalake.portfolio portfolio + join datalake.markets markets on markets.ticker = portfolio.ticker where portfolio.dataset = ( select datasource from datasource ) and user = ( select _user from _user ) @@ -36,29 +36,34 @@ with now_ts as ( 1,2,3,4,5,6,7,8 ), max_known_date as ( select - max(cast(date as date)) max_known_date + max(date) max_known_date from ( - select date, dataset, count(*) - from dw.equities_fact + select + date, + dataset, + count(*) + from + datalake.equities where - s3uploaddate >= cast((select beginning_of_year from beginning_of_year ) as date) - and s3uploaddate <> cast((select now from now) as date) + date >= (select beginning_of_year from beginning_of_year ) + and date <> (select now from now) and ticker in ( select distinct ticker from portfolio ) - and dataset <> 'ALPHA-VANTAGE' group by 1,2 having count(*) > 30 ) src ), fx as ( select - currency, cast(rate as decimal(24,14)) rate + currency, + rate from - dw.currency_fact + datalake.currency where currency = 'GBP' - and ( s3uploaddate = cast((select today from date ) as date) - or s3uploaddate = cast((select yesterday from date ) as date )) - order by date desc + and ( date = (select today from date ) + or date = (select yesterday from date ) ) + order by + date desc limit 1 ), fx_backup as ( select @@ -67,20 +72,21 @@ with now_ts as ( select coalesce(fx.currency,fx_backup.currency) currency, coalesce(fx.rate ,fx_backup.rate) rate - from fx + from + fx right join fx_backup on fx.currency = fx_backup.currency ), equities as ( select ticker, - cast(date as date) as date, - avg(case when ticker = 'LON:FCH' then try_cast(close as decimal(10,2)) * (select rate from fx_with_backup where currency = 'GBP') / 100 else try_cast(close as decimal(10,2)) end) as close + date, + avg(case when ticker = 'LON:FCH' then close * (select rate from fx_with_backup where currency = 'GBP') / 100 else close end) as close from - dw.equities_fact equities + datalake.equities equities where - s3uploaddate = cast((select today from date) as date) - or s3uploaddate = cast((select yesterday from date) as date) - or s3uploaddate = cast((select max_known_date from max_known_date) as date) - or s3uploaddate = cast((select beginning_of_year from beginning_of_year) as date) + date = (select today from date) + or date = (select yesterday from date) + or date = (select max_known_date from max_known_date) + or date = (select beginning_of_year from beginning_of_year) group by 1,2 ), today as ( @@ -174,7 +180,8 @@ with now_ts as ( today full outer join yesterday on today.ticker = yesterday.ticker full outer join ytd on yesterday.ticker = ytd.ticker - order by today.market_value desc + order by + today.market_value desc ), detail_with_backup as ( select coalesce(detail.asset_type, backup.asset_type) asset_type, @@ -228,7 +235,8 @@ with now_ts as ( where ticker <> '' group by 1 - order by market_value desc + order by + market_value desc ), _union as ( select * from benchmark union all diff --git a/resources/athena/latest.sql b/resources/athena/latest.sql index 6859f3a..b64806f 100644 --- a/resources/athena/latest.sql +++ b/resources/athena/latest.sql @@ -1,5 +1,5 @@ select * -from dw.:table +from datalake.:table order by date desc limit 10 ; diff --git a/resources/athena/location_athena.sql b/resources/athena/location_athena.sql index 688348e..ae47df7 100644 --- a/resources/athena/location_athena.sql +++ b/resources/athena/location_athena.sql @@ -27,8 +27,8 @@ with now_ts as ( cast(portfolio.quantity as decimal(10,4)) as quantity, cast(portfolio.cost_per_share as decimal(6,2)) as cost_per_share from - dw.portfolio_dim portfolio - join dw.markets_dim markets on markets.ticker = portfolio.ticker + datalake.portfolio portfolio + join datalake.markets markets on markets.ticker = portfolio.ticker where portfolio.dataset = ( select datasource from datasource ) and user = ( select _user from _user ) @@ -36,29 +36,34 @@ with now_ts as ( 1,2,3,4,5,6,7,8 ), max_known_date as ( select - max(cast(date as date)) max_known_date + max(date) max_known_date from ( - select date, dataset, count(*) - from dw.equities_fact + select + date, + dataset, + count(*) + from + datalake.equities where - s3uploaddate >= cast((select beginning_of_year from beginning_of_year ) as date) - and s3uploaddate <> cast((select now from now) as date) + date >= (select beginning_of_year from beginning_of_year ) + and date <> (select now from now) and ticker in ( select distinct ticker from portfolio ) - and dataset <> 'ALPHA-VANTAGE' group by 1,2 having count(*) > 30 ) src ), fx as ( select - currency, cast(rate as decimal(24,14)) rate + currency, + rate from - dw.currency_fact + datalake.currency where currency = 'GBP' - and ( s3uploaddate = cast((select today from date ) as date) - or s3uploaddate = cast((select yesterday from date ) as date )) - order by date desc + and ( date = (select today from date ) + or date = (select yesterday from date ) ) + order by + date desc limit 1 ), fx_backup as ( select @@ -67,20 +72,21 @@ with now_ts as ( select coalesce(fx.currency,fx_backup.currency) currency, coalesce(fx.rate ,fx_backup.rate) rate - from fx + from + fx right join fx_backup on fx.currency = fx_backup.currency ), equities as ( select ticker, - cast(date as date) as date, - avg(case when ticker = 'LON:FCH' then try_cast(close as decimal(10,2)) * (select rate from fx_with_backup where currency = 'GBP') / 100 else try_cast(close as decimal(10,2)) end) as close + date, + avg(case when ticker = 'LON:FCH' then close * (select rate from fx_with_backup where currency = 'GBP') / 100 else close end) as close from - dw.equities_fact equities + datalake.equities equities where - s3uploaddate = cast((select today from date) as date) - or s3uploaddate = cast((select yesterday from date) as date) - or s3uploaddate = cast((select max_known_date from max_known_date) as date) - or s3uploaddate = cast((select beginning_of_year from beginning_of_year) as date) + date = (select today from date) + or date = (select yesterday from date) + or date = (select max_known_date from max_known_date) + or date = (select beginning_of_year from beginning_of_year) group by 1,2 ), today as ( @@ -174,7 +180,8 @@ with now_ts as ( today full outer join yesterday on today.ticker = yesterday.ticker full outer join ytd on yesterday.ticker = ytd.ticker - order by today.market_value desc + order by + today.market_value desc ), detail_with_backup as ( select coalesce(detail.asset_type, backup.asset_type) asset_type, @@ -228,7 +235,8 @@ with now_ts as ( where ticker <> '' group by 1 - order by market_value desc + order by + market_value desc ), _union as ( select * from benchmark union all diff --git a/resources/athena/portfolio_athena.sql b/resources/athena/portfolio_athena.sql index 75e2084..cc82c55 100644 --- a/resources/athena/portfolio_athena.sql +++ b/resources/athena/portfolio_athena.sql @@ -27,36 +27,43 @@ with now_ts as ( cast(portfolio.quantity as decimal(10,4)) as quantity, cast(portfolio.cost_per_share as decimal(6,2)) as cost_per_share from - dw.portfolio_dim portfolio - join dw.markets_dim markets on markets.ticker = portfolio.ticker + datalake.portfolio portfolio + join datalake.markets markets on markets.ticker = portfolio.ticker where - portfolio.dataset = 'ALPHA-VANTAGE' + portfolio.dataset = ( select datasource from datasource ) and user = ( select _user from _user ) group by 1,2,3,4,5,6,7,8 ), max_known_date as ( select - max(cast(date as date)) max_known_date + max(date) max_known_date from ( - select date, dataset, count(*) - from dw.equities_fact - where dataset <> 'ALPHA-VANTAGE' - and ticker in ( select distinct ticker from dw.portfolio_dim where dataset = ( select datasource from datasource ) ) - and s3uploaddate <> (select now from now) + select + date, + dataset, + count(*) + from + datalake.equities + where + date >= (select beginning_of_year from beginning_of_year ) + and date <> (select now from now) + and ticker in ( select distinct ticker from portfolio ) group by 1,2 having count(*) > 30 ) src ), fx as ( select - currency, cast(rate as decimal(24,14)) rate + currency, + rate from - dw.currency_fact + datalake.currency where currency = 'GBP' - and ( s3uploaddate = cast((select today from date ) as date) - or s3uploaddate = cast((select yesterday from date ) as date )) - order by date desc + and ( date = (select today from date ) + or date = (select yesterday from date ) ) + order by + date desc limit 1 ), fx_backup as ( select @@ -65,20 +72,21 @@ with now_ts as ( select coalesce(fx.currency,fx_backup.currency) currency, coalesce(fx.rate ,fx_backup.rate) rate - from fx + from + fx right join fx_backup on fx.currency = fx_backup.currency ), equities as ( select ticker, - cast(date as date) as date, - avg(case when ticker = 'LON:FCH' then try_cast(close as decimal(10,2)) * (select rate from fx_with_backup where currency = 'GBP') / 100 else try_cast(close as decimal(10,2)) end) as close + date, + avg(case when ticker = 'LON:FCH' then close * (select rate from fx_with_backup where currency = 'GBP') / 100 else close end) as close from - dw.equities_fact equities + datalake.equities equities where - s3uploaddate = cast((select today from date) as date) - or s3uploaddate = cast((select yesterday from date) as date) - or s3uploaddate = cast((select max_known_date from max_known_date) as date) - or s3uploaddate = cast((select beginning_of_year from beginning_of_year) as date) + date = (select today from date) + or date = (select yesterday from date) + or date = (select max_known_date from max_known_date) + or date = (select beginning_of_year from beginning_of_year) group by 1,2 ), today as ( @@ -172,7 +180,8 @@ with now_ts as ( today full outer join yesterday on today.ticker = yesterday.ticker full outer join ytd on yesterday.ticker = ytd.ticker - order by today.market_value desc + order by + today.market_value desc ), detail_with_backup as ( select coalesce(detail.asset_type, backup.asset_type) asset_type, @@ -214,7 +223,7 @@ with now_ts as ( detail_with_backup where ticker = 'VTSAX' group by - 1 + 1,2 ), results as ( select ticker, @@ -229,7 +238,8 @@ with now_ts as ( where ticker <> '' group by 1,2 - order by market_value desc + order by + market_value desc ), _union as ( select * from summary union all diff --git a/resources/athena/quote.sql b/resources/athena/quote.sql index a6b6fce..8cc1eb9 100644 --- a/resources/athena/quote.sql +++ b/resources/athena/quote.sql @@ -1,7 +1,7 @@ select * from - dw.:table + datalake.:table where ticker = ':ticker' and date = ':date'