Merge pull request #41 from skilbjo/orc

Increment libs per lein ancient
skilbjo · Oct 6, 2019 · 74a219f · 74a219f
2 parents 1be9e2d + ddfdfac
commit 74a219f
Show file tree

Hide file tree

Showing 9 changed files with 182 additions and 137 deletions.
diff --git a/project.clj b/project.clj
@@ -3,10 +3,10 @@
   :dependencies [;; clj
                  [org.clojure/clojure "1.9.0"]
                  [bk/ring-gzip "0.3.0"]
-                 [buddy/buddy-auth "2.1.0"]
-                 [buddy/buddy-core "1.5.0"]
+                 [buddy/buddy-auth "2.2.0"]
+                 [buddy/buddy-core "1.6.0"]
                  [clj-http "3.10.0"]
-                 [clj-time "0.15.1"]
+                 [clj-time "0.15.2"]
                  [com.syncron.amazonaws/simba-athena-jdbc-driver "2.0.2"]
                  [com.draines/postal "2.0.3"]
                  [compojure "1.6.1" :exclusions [ring/ring-core]]
@@ -24,16 +24,16 @@
                   :exclusions [com.fasterxml.jackson.core/jackson-core]]
                  [net.sf.uadetector/uadetector-resources "2014.10"]
                  [org.clojure/data.json "0.2.6"]
-                 [org.clojure/java.jdbc "0.7.9"]
-                 [org.clojure/tools.logging "0.5.0-alpha.1"]
-                 [org.postgresql/postgresql "42.2.6"]
-                 [org.slf4j/slf4j-log4j12 "1.7.26"
+                 [org.clojure/java.jdbc "0.7.10"]
+                 [org.clojure/tools.logging "0.5.0"]
+                 [org.postgresql/postgresql "42.2.8"]
+                 [org.slf4j/slf4j-log4j12 "1.7.28"
                   :exclusions [org.clojure/tools.reader
                                org.slf4j/slf4j-api]]
                  [ring "1.7.1"]
                  [ring/ring-anti-forgery "1.3.0"]
                  [ring/ring-defaults "0.3.2" ]
-                 [ring/ring-json "0.4.0"]
+                 [ring/ring-json "0.5.0"]
 
                  ;; cljs
                  [org.clojure/clojurescript "1.10.520"
@@ -46,8 +46,8 @@
                  [cljs-log "0.2.3"]
                  [com.andrewmcveigh/cljs-time "0.5.2"]
                  [day8.re-frame/http-fx "0.1.6"]
-                 [day8.re-frame/tracing-stubs "0.5.1"]
-                 [re-frame "0.10.7"
+                 [day8.re-frame/tracing-stubs "0.5.3"]
+                 [re-frame "0.10.9"
                   :exclusions [com.google.code.findbugs/jsr305]]
                  [re-frame-datatable "0.6.0"]
                  [reagent "0.8.1"]
@@ -66,9 +66,9 @@
   :profiles {:dev {:env {:log-level "1"} ;; cljs/log debug+
                    :dependencies [[binaryage/devtools "0.9.10"]
                                   [com.bhauman/cljs-test-display "0.1.1"]
-                                  [day8.re-frame/re-frame-10x "0.4.1"
+                                  [day8.re-frame/re-frame-10x "0.4.3"
                                    :exclusions [com.google.code.findbugs/jsr305]]
-                                  [day8.re-frame/tracing "0.5.1"]
+                                  [day8.re-frame/tracing "0.5.3"]
                                   [javax.servlet/servlet-api "2.5"]
                                   [ring/ring-mock "0.4.0"
                                    :exclusions [cheshire]]]

diff --git a/resources/athena/asset_type_athena.sql b/resources/athena/asset_type_athena.sql
@@ -27,38 +27,43 @@ with now_ts as (
     cast(portfolio.quantity as decimal(10,4))      as quantity,
     cast(portfolio.cost_per_share as decimal(6,2)) as cost_per_share
   from
-    dw.portfolio_dim portfolio
-    join dw.markets_dim markets on markets.ticker = portfolio.ticker
+    datalake.portfolio portfolio
+    join datalake.markets markets on markets.ticker = portfolio.ticker
   where
     portfolio.dataset = ( select datasource from datasource )
     and user = ( select _user from _user )
   group by
     1,2,3,4,5,6,7,8
 ), max_known_date as (
   select
-    max(cast(date as date)) max_known_date
+    max(date) max_known_date
   from (
-    select date, dataset, count(*)
-    from dw.equities_fact
+    select
+      date,
+      dataset,
+      count(*)
+    from
+      datalake.equities
     where
-      s3uploaddate >= cast((select beginning_of_year from beginning_of_year ) as date)
-      and s3uploaddate <> cast((select now from now) as date)
+      date >= (select beginning_of_year from beginning_of_year )
+      and date <> (select now from now)
       and ticker in ( select distinct ticker from portfolio )
-      and dataset <> 'ALPHA-VANTAGE'
     group by
       1,2
     having count(*) > 30
    ) src
 ), fx as (
   select
-    currency, cast(rate as decimal(24,14)) rate
+    currency,
+    rate
   from
-    dw.currency_fact
+    datalake.currency
   where
     currency = 'GBP'
-    and ( s3uploaddate = cast((select today from date ) as date)
-    or    s3uploaddate = cast((select yesterday from date ) as date ))
-  order by date desc
+    and ( date = (select today from date )
+    or    date = (select yesterday from date ) )
+  order by
+    date desc
   limit 1
 ), fx_backup as (
   select
@@ -67,20 +72,21 @@ with now_ts as (
   select
     coalesce(fx.currency,fx_backup.currency) currency,
     coalesce(fx.rate    ,fx_backup.rate)     rate
-  from fx
+  from
+    fx
     right join fx_backup on fx.currency = fx_backup.currency
 ), equities as (
   select
     ticker,
-    cast(date as date)                     as date,
-    avg(case when ticker = 'LON:FCH' then try_cast(close as decimal(10,2)) * (select rate from fx_with_backup where currency = 'GBP') / 100 else try_cast(close as decimal(10,2)) end) as close
+    date,
+    avg(case when ticker = 'LON:FCH' then close * (select rate from fx_with_backup where currency = 'GBP') / 100 else close end) as close
   from
-    dw.equities_fact equities
+    datalake.equities equities
   where
-    s3uploaddate    = cast((select today from date) as date)
-    or s3uploaddate = cast((select yesterday from date) as date)
-    or s3uploaddate = cast((select max_known_date from max_known_date) as date)
-    or s3uploaddate = cast((select beginning_of_year from beginning_of_year) as date)
+    date    = (select today from date)
+    or date = (select yesterday from date)
+    or date = (select max_known_date from max_known_date)
+    or date = (select beginning_of_year from beginning_of_year)
   group by
     1,2
 ), today as (
@@ -174,7 +180,8 @@ with now_ts as (
     today
     full outer join yesterday on today.ticker = yesterday.ticker
     full outer join ytd on yesterday.ticker = ytd.ticker
-  order by today.market_value desc
+  order by
+    today.market_value desc
 ), detail_with_backup as (
   select
     coalesce(detail.asset_type,       backup.asset_type) asset_type,
@@ -228,7 +235,8 @@ with now_ts as (
   where ticker <> ''
   group by
     1
-  order by market_value desc
+  order by
+    market_value desc
 ), _union as (
   select * from benchmark
   union all

diff --git a/resources/athena/capitalization_athena.sql b/resources/athena/capitalization_athena.sql
@@ -27,38 +27,44 @@ with now_ts as (
     cast(portfolio.quantity as decimal(10,4))      as quantity,
     cast(portfolio.cost_per_share as decimal(6,2)) as cost_per_share
   from
-    dw.portfolio_dim portfolio
-    join dw.markets_dim markets on markets.ticker = portfolio.ticker
+    datalake.portfolio portfolio
+    join datalake.markets markets on markets.ticker = portfolio.ticker
   where
     portfolio.dataset = ( select datasource from datasource )
     and user = ( select _user from _user )
   group by
     1,2,3,4,5,6,7,8
 ), max_known_date as (
   select
-    max(cast(date as date)) max_known_date
+  select
+    max(date) max_known_date
   from (
-    select date, dataset, count(*)
-    from dw.equities_fact
+    select
+      date,
+      dataset,
+      count(*)
+    from
+      datalake.equities
     where
-      s3uploaddate >= cast((select beginning_of_year from beginning_of_year ) as date)
-      and s3uploaddate <> cast((select now from now) as date)
+      date >= (select beginning_of_year from beginning_of_year )
+      and date <> (select now from now)
       and ticker in ( select distinct ticker from portfolio )
-      and dataset <> 'ALPHA-VANTAGE'
     group by
       1,2
     having count(*) > 30
    ) src
 ), fx as (
   select
-    currency, cast(rate as decimal(24,14)) rate
+    currency,
+    rate
   from
-    dw.currency_fact
+    datalake.currency
   where
     currency = 'GBP'
-    and ( s3uploaddate = cast((select today from date ) as date)
-    or    s3uploaddate = cast((select yesterday from date ) as date ))
-  order by date desc
+    and ( date = (select today from date )
+    or    date = (select yesterday from date ) )
+  order by
+    date desc
   limit 1
 ), fx_backup as (
   select
@@ -67,20 +73,21 @@ with now_ts as (
   select
     coalesce(fx.currency,fx_backup.currency) currency,
     coalesce(fx.rate    ,fx_backup.rate)     rate
-  from fx
+  from
+    fx
     right join fx_backup on fx.currency = fx_backup.currency
 ), equities as (
   select
     ticker,
-    cast(date as date)                     as date,
-    avg(case when ticker = 'LON:FCH' then try_cast(close as decimal(10,2)) * (select rate from fx_with_backup where currency = 'GBP') / 100 else try_cast(close as decimal(10,2)) end) as close
+    date,
+    avg(case when ticker = 'LON:FCH' then close * (select rate from fx_with_backup where currency = 'GBP') / 100 else close end) as close
   from
-    dw.equities_fact equities
+    datalake.equities equities
   where
-    s3uploaddate    = cast((select today from date) as date)
-    or s3uploaddate = cast((select yesterday from date) as date)
-    or s3uploaddate = cast((select max_known_date from max_known_date) as date)
-    or s3uploaddate = cast((select beginning_of_year from beginning_of_year) as date)
+    date    = (select today from date)
+    or date = (select yesterday from date)
+    or date = (select max_known_date from max_known_date)
+    or date = (select beginning_of_year from beginning_of_year)
   group by
     1,2
 ), today as (
@@ -174,7 +181,8 @@ with now_ts as (
     today
     full outer join yesterday on today.ticker = yesterday.ticker
     full outer join ytd on yesterday.ticker = ytd.ticker
-  order by today.market_value desc
+  order by
+    today.market_value desc
 ), detail_with_backup as (
   select
     coalesce(detail.asset_type,       backup.asset_type) asset_type,
@@ -228,7 +236,8 @@ with now_ts as (
   where ticker <> ''
   group by
     1
-  order by market_value desc
+  order by
+    market_value desc
 ), _union as (
   select * from benchmark
   union all

diff --git a/resources/athena/dashboard.sql b/resources/athena/dashboard.sql
@@ -5,16 +5,18 @@ with date as (
     dataset,
     ticker,
     currency,
-    cast(date as date)               as date,
-    cast(rate as decimal(24,14))     as rate,
-    cast(high as decimal(24,14)) as high,
-    cast(low as decimal(24,14))  as low
+    date,
+    rate,
+    high,
+    low
   from
-    dw.currency_fact currency
+    datalake.currency
 ), data as (
   select
     row_number() over (partition by currency order by date desc) as rn,
-    currency, date, rate
+    currency,
+    date,
+    rate
   from
     _currency
   where