From bfc5a1261453e0cd49eb85b493ff2ff5be546c89 Mon Sep 17 00:00:00 2001 From: Tim Loyer Date: Thu, 25 Feb 2021 15:30:14 -0500 Subject: [PATCH] Initial commit for vaultpy package --- .idea/.gitignore | 8 + .idea/inspectionProfiles/Project_Default.xml | 271 ++++++++++++++++++ .../inspectionProfiles/profiles_settings.xml | 6 + .idea/misc.xml | 4 + .idea/modules.xml | 8 + .idea/vaultpy.iml | 8 + LICENSE | 1 + README.md | 24 ++ build/lib/vault/__init__.py | 85 ++++++ dist/vaultpy-0.0.1-py3-none-any.whl | Bin 0 -> 2633 bytes dist/vaultpy-0.0.1.tar.gz | Bin 0 -> 2246 bytes pyproject.toml | 6 + setup.cfg | 13 + vault/__init__.py | 85 ++++++ vaultpy.egg-info/PKG-INFO | 21 ++ vaultpy.egg-info/SOURCES.txt | 8 + vaultpy.egg-info/dependency_links.txt | 1 + vaultpy.egg-info/top_level.txt | 1 + 18 files changed, 550 insertions(+) create mode 100644 .idea/.gitignore create mode 100644 .idea/inspectionProfiles/Project_Default.xml create mode 100644 .idea/inspectionProfiles/profiles_settings.xml create mode 100644 .idea/misc.xml create mode 100644 .idea/modules.xml create mode 100644 .idea/vaultpy.iml create mode 100644 LICENSE create mode 100644 README.md create mode 100644 build/lib/vault/__init__.py create mode 100644 dist/vaultpy-0.0.1-py3-none-any.whl create mode 100644 dist/vaultpy-0.0.1.tar.gz create mode 100644 pyproject.toml create mode 100644 setup.cfg create mode 100644 vault/__init__.py create mode 100644 vaultpy.egg-info/PKG-INFO create mode 100644 vaultpy.egg-info/SOURCES.txt create mode 100644 vaultpy.egg-info/dependency_links.txt create mode 100644 vaultpy.egg-info/top_level.txt diff --git a/.idea/.gitignore b/.idea/.gitignore new file mode 100644 index 0000000..73f69e0 --- /dev/null +++ b/.idea/.gitignore @@ -0,0 +1,8 @@ +# Default ignored files +/shelf/ +/workspace.xml +# Datasource local storage ignored files +/dataSources/ +/dataSources.local.xml +# Editor-based HTTP Client requests +/httpRequests/ diff --git a/.idea/inspectionProfiles/Project_Default.xml b/.idea/inspectionProfiles/Project_Default.xml new file mode 100644 index 0000000..b936b6c --- /dev/null +++ b/.idea/inspectionProfiles/Project_Default.xml @@ -0,0 +1,271 @@ + + + + \ No newline at end of file diff --git a/.idea/inspectionProfiles/profiles_settings.xml b/.idea/inspectionProfiles/profiles_settings.xml new file mode 100644 index 0000000..105ce2d --- /dev/null +++ b/.idea/inspectionProfiles/profiles_settings.xml @@ -0,0 +1,6 @@ + + + + \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml new file mode 100644 index 0000000..a2e120d --- /dev/null +++ b/.idea/misc.xml @@ -0,0 +1,4 @@ + + + + \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml new file mode 100644 index 0000000..8d25e4b --- /dev/null +++ b/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/.idea/vaultpy.iml b/.idea/vaultpy.iml new file mode 100644 index 0000000..d0876a7 --- /dev/null +++ b/.idea/vaultpy.iml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..3c453c0 --- /dev/null +++ b/LICENSE @@ -0,0 +1 @@ +Copyright (c) 2021, DirectEmployers Association - https://directemployers.org diff --git a/README.md b/README.md new file mode 100644 index 0000000..1b38767 --- /dev/null +++ b/README.md @@ -0,0 +1,24 @@ +# Vaultpy +A module to parse injected [Vault](https://www.vaultproject.io/) secrets and track their usage with Datadog. + +## Requirements +- Local Datadog agent +- Environment variables to access it +- Set [container annotations](https://www.vaultproject.io/docs/platform/k8s/injector/annotations) to inject Vault secrets in K8s + +## Setup +For production, use the `VAULT_SECRETS_PATH` environment variable to set the path to the secrets that are injected by the Vault Agent in Kubernetes. + +For local development, a `de_secrets.py` can be used to load secrets in a format not unlike Django settings. + +## Usage +Import `vault.secrets` and then access the loaded secrets using by accessing dynamic properties loaded into the `secrets` object (i.e. `secrets.FOO`). + +Example of usage in a settings file: +```python +from vault import secrets + +FOO = secrets.FOO +BAR = getattr(secrets, "BAR", "") +BAZ = getattr(secrets, "BAZ") +``` diff --git a/build/lib/vault/__init__.py b/build/lib/vault/__init__.py new file mode 100644 index 0000000..97b561e --- /dev/null +++ b/build/lib/vault/__init__.py @@ -0,0 +1,85 @@ +import logging +from base64 import b64decode +from importlib import import_module +from json import loads +from os import environ +from typing import Dict + +from datadog import statsd + +logger = logging.getLogger(__name__) + + +def _load_de_secrets() -> Dict: + """ + Imports de_secrets module and returns a dictionary of its attributes. + """ + de_secrets = import_module("de_secrets") + return {k: getattr(de_secrets, k) for k in dir(de_secrets) if not k.startswith("_")} + + +def _load_vault_secrets() -> Dict: + """ + Load Vault injected secrets file located at VAULT_SECRETS_PATH, then perform + base 64 decode followed by JSON decode on file contents. This function + should not be called anywhere except within this module! + """ + with open(environ["VAULT_SECRETS_PATH"]) as file: + contents = file.read() + + json_secrets = b64decode(contents) + return loads(json_secrets) + + +def _get_secrets() -> Dict: + """ + Get secrets from de_secrets.py in local dev, or from Vault injected secrets file + located at path in VAULT_SECRETS_PATH. Performs base 64 decode followed by JSON + decode on file contents. + """ + if not environ.get("USE_VAULT"): + # Use dev secrets when available. + return _load_de_secrets() + + return _load_vault_secrets() + + +class VaultSecretsWrapper: + """ + Provide access to secrets as attributes and send secret-usage analytics to Datadog. + """ + + def __init__(self, secrets: Dict): + self._keys = secrets.keys() + self._env = environ.get("DD_ENV") + + for key, value in secrets.items(): + statsd.increment( + "vault.secrets.access_count", + value=1, + tags=[f"env:{self._env}", f"secret_key:{key}"], + ) + setattr(self, key, value) + + def __getattribute__(self, key: str): + """ + Override the default getattribute method so that we can track secret key + usage with Datadog. Non-secret attributes are passed on to the default method. + """ + if key not in ["_keys", "_env"] and key in self._keys: + try: + statsd.increment( + "vault.secrets.access_count", + value=1, + tags=[f"env:{self._env}", f"secret_key:{key}"], + ) + return super().__getattribute__(key) + except AttributeError as error: + logger.error(f"Requested secret could not be loaded: {key}") + raise error + + return super().__getattribute__(key) + + +secrets = VaultSecretsWrapper(_get_secrets()) +__all__ = ("secrets",) diff --git a/dist/vaultpy-0.0.1-py3-none-any.whl b/dist/vaultpy-0.0.1-py3-none-any.whl new file mode 100644 index 0000000000000000000000000000000000000000..331a48045b7fef699533e9d09e4e0b0a9e7cc09e GIT binary patch literal 2633 zcmaKu3pAAZ8^>QpE;B?L8|7NaWG)P9sK|ZHU}hM~rABUWzT&w0;zKIi$p&-eR1KO!C^00jU*7#O(Y zU{f38Ix-0Y0820c?AZ7kdOE}>7)7B_eW}3|3erC;+uq5Whk;&PEW)s8#Y#GG(Rqm} zqk$`xL_LqrPzBtMSj7HRjEz}RvRL*yEkU{Eb`=}WW8`pk^=`ile;w4|Tg0Y27W5ev zCgWIg`6*)x1`)zJJuk#nP~@g4&AYv>QnO=GZ^;AQ1sw4#ckRB|o2s(vJH;!H2p70x z${rrgNyx#I`jw?|o(_3>amB5vXba~8=byvidX%6GuQ)b1mB9d9sj>y`6ko^ywJC3m zKH2*~=I-jOZR^fDhQzqR(C^912lAX#N}(F_j~#nr%WxUUHO$RFw3>WHw9f2FEm zDr=&rV0pTR z6VqH+_Jr-=;;{-NLdEyYlqn$;9HsP;DAI7>ep?WuwSM*4a1Y`na&5V`;$&x9+#pBm z(H`V!j!tjYJfB=5a1}Zgb+r4j-uP;dNG3247yYH#|6bM{YE!IxVG$Q z00Ay`=2y|m$$ZP}?(LI}`Y9phNNSCCYB7ZoeC{FdK*HLS9klT>g{ze73E<_miKQ`5 zGH>p*AXetx{Bta=+P7RH{8|-iO3|OG~nBMcv|mLF{f#iReSiuCoT@VMm$u@%BQI z!^QVUr~K@Wf42vGxo`W=9?^f~Yx>MW={@_^!I$RHSm#81oP1{_7zWSI8w=5ilbK{f zL0)%t9y(nN9Ev0RQ@Y;J#p{1S?t6qn{W`wFWRF(7m=QH-8?BqZa`-?)O8t8|!C9@} z8&4$#DRIv*K-T8=za*RMCX)Ew*~ECZ`@y-74xIJW-EgP0c}A?u^8CZXj5Fw{*y#T4 zQOi5m!c-G^zYkPQ*t>x8X+;APEB5qN+3Fn?0{zAEE)p_hOh2K*$64E^Ga};k} ziTs`LjNkb8#iJgh?#E4)cI@5-wu`L3l!KvXpexIBq?X$+9US_wH@-^u+>9=F{FST` zsEFJm@Z*ETn>Emeawlxk+Ui2=N;dDf?JZ&~$_2+gtTK1I6^_5iNBk!gQ0WdfE?Ood z*o`nCY?Q*^q2M2e&_HS+HIeSrpkM^m*V7MWX<=+?ZEG6Tj>?5MTXNJ$9oSy723iB% zP4aJouN#aUh@NA z4`F;(k4fSK05m@UC~h%?l_}ZK#E@)wlT>XD(v*0!Dy5zsIUdfZ9GiU1L|IZ&$>drz zKM7N-qN!`}Xfbf;m(trpWN5|R^gHQt28ww$*PFkSPM#}_2nthGZFc2zvO5lSoYR_a z(HGC~_O|nZNJlJqoK(8T$aFQ*cni`wEfQx976~!IWjc#MZ;k*3T$9{nM}K+| zj)Y5DM%a>}uu-n@N`<6b7FT6NH)Dt(Pf>Msgq>gHuI(i(Ih3M3x^AUc(q~wZFxZup zVC~3?lsCPHjm!t~#e7VcOmz>jVKiR!)NIRC{n~KfOu~F<;E^ZP-IY%@R}%Kda|r7_ z52qGJC58jbl+j2mYq}A*g>em@C{sP(cYWza7O4enyNg7Pheh5?@2eS9TB3ozUdK>1 zE7Eym4WCMGi*+2wnwnZdI9RJTt{5J1puLq-YNOGuH5y20!y|C;CRB+H7S{wvUPTGz zkvcRwv<`M@bxwerIVkP|yjM&Ofop5BEU{689k6-@wy-R>QOnitZENA+d#qYn6XTmz z60K@@QFvnf^io|rTnTAbw>7E^pG(Y zxd{>XYDy(&Guu(jd`s~=S-t@Zb2LMGTFOXgJ*Q6BDy&g9n-lFY2gG66X=~XD;asSd zCfQzWiS-}(*^z@%3(#EFpcUpxz>#1dMn+2=eU}%W>N6ruq4zq-dQ0lZd6vY82vBW9 zx{3OY_i9So-t5x8WUlN5v60cAm%jRa85H3;X@oH=nok#Wnc)R>DY%pazYIl3#JO{Bh1OhG5jy9606X`fc(f@u!QnDr6O)hBayKZJ4>WHLmGDhf2(%S>-IW(AdHwJAyWg_^UdQdN>Hoo?@pMQ!j%S_Hgbhg{8`iQ*3yF0xOA?wdhUADOJe! zFBV7^lF#Z@B3O_!DF{tNB6Awt5qZm^oXiBBGBS_kEjfm{Lq2t^k333w79fO$v(kC1$=$KEu`)z9a1$1F4F zYKRl@cJo;*DJNb2@#YQr#Qr@4Iue#jv21Sv?E#I;HwmChWuP-QGL&|F75oXN%IJ zOP<8L^8a3Mz5d_p9rV!u?*gvJvnUQ*Vj(0;ZmhgW~`#Z z(>lLpEViv1%e+1YZ_mK(A6CaZy3bFTq&8X<^#6GN^Z)4ox1ImI^YHxVRr-Hzyq#b& z-D&>c@2=~ASL;9e{~h@Mq&NV&u~M2a@Zjaq%Dp-DffjS@L-oN5Jg&}KOp~wATA<^p z!C~^|pF}Yml4V6}V~8J^6A51~GDUa&NMe>Nik@*2WnFWPR@_(_ ztQD3h^di!5Xl43T#IFQb=6uAuX49!3r4xQ*Njmj)zy!Ve0X9V~%l)xc4fOx$|MC6@ z`hVm9^;7#t=>PNX{~ZkaF5drpHTfUbfB$#(f35!8edl5Q_m$g!A41Yd`o5Dbc9H+} zK;75$zwZt(|Jw;fNyc+YVm_Tl>C~F!JRxH$Sieh(u5rH`vVe!obeh&U8kha%$2ZBG z8S|Hb3~Y&+hQc&+Q8qGjs%x4d|AMK@uHz_>mgxwKv*4l!HcE=nvea6cN<@oAJ5wf4 zbX&vsRTAs_P0NC8b3%MIJ0yL6kxMq3r1f4;F*J2-+t$Co=@|rBjVzN>m3+eUEKea3 zB_X^CIUdaygilDMl2a-3XgrflI5mtFOow$AHSATN-PD841mw%zkN_f-pfq9q(LRq!0&*n zMk>kYqpOok|6=sv)9CWTKR>$sxUaJFlw>SdR|7RdB}W7V$%qfo5XXED?~fPcUl(Vm zWjn}XHDkNUFCD zum4daP?rpN#?nS1f7kXV9rjI=P($!=dF8QOT2WSTa)_21AY`gf343L)m+I0eU#jh~ zR)r?#~a|Lt2_qo$DEVl$K)H|(p6k*{aBn=AIK`yEUXwOssM$r`<cjP*+?*k$hFEsu~1sI|NHhmbXmd zil#JP$SBY+*6#W$`~~+w9117g5GdL93cEtKq)qM#ld&9yPWW*Aaq z_VKYlI{j=H%u%523|s7zdm7JFMQ({Rk}LtFtC=-sZIYcR4a_~BMy*%Vq`B;rSPaYj zfX`A{oF!I4&wA)>c1SuEhu0GumOuQmr01)>PbRj(t#~l}0{?uqZ|d(i>q{;StYPbl z)@GIUg%#HOxZDCY0Z^P*cu-1&g0p*;=SnQV4k|QS>#B%I!sIOnY2Yvflx?mISz44f z4H0UNCBiLrPIAiARxz|HXkY^~5M5A0&^XC!OE9xKTh|O42s3Iegsr}|wIBggZN**t zMoX|7pyi+xw7Lk$d@pzwHPquK_blOtKlI<7M|L4{J z`)=m|>;G8)NBh6y`oFgSSpP@+U)cW_`40h}|NZ~K^ZHo--v!|J|M>kse*cg8KgNF; z|6%-x@!wkfcX4+0>BH#4kq`2R#ebmw>-qm+;NtgxJAu{x`1-e>*78#jYvWRo_02TF zdQ5;eJ42SSG-PS8@Z%`G6Wd2gp7}AmXYuz*Ap-h;^#ADp5uV}yw^`^H_y6!%*Z)q( z9eC*fcLAvC=>O6GqyI<$kN$s)|9_F(>uLYLrvDwc*F*omyZuLfM?gS8KtMo1KtMo1 UKtMo1_}jvN09=`qlK^-C0L#dl!T=42", + "wheel" +] +build-backend = "setuptools.build_meta" diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..00b77bf --- /dev/null +++ b/setup.cfg @@ -0,0 +1,13 @@ +[metadata] +name = vaultpy +version = 0.0.1 +author = Tim Loyer +author_email = tloyer@apps.directemployers.org +description = A module to parse injected Vault secrets and track their usage with Datadog. +long_description = file: README.md +long_description_content_type = text/markdown +url = https://github.com/DirectEmployers/vaultpy + +[options] +packages = find: +python_requires = >=3.6 diff --git a/vault/__init__.py b/vault/__init__.py new file mode 100644 index 0000000..97b561e --- /dev/null +++ b/vault/__init__.py @@ -0,0 +1,85 @@ +import logging +from base64 import b64decode +from importlib import import_module +from json import loads +from os import environ +from typing import Dict + +from datadog import statsd + +logger = logging.getLogger(__name__) + + +def _load_de_secrets() -> Dict: + """ + Imports de_secrets module and returns a dictionary of its attributes. + """ + de_secrets = import_module("de_secrets") + return {k: getattr(de_secrets, k) for k in dir(de_secrets) if not k.startswith("_")} + + +def _load_vault_secrets() -> Dict: + """ + Load Vault injected secrets file located at VAULT_SECRETS_PATH, then perform + base 64 decode followed by JSON decode on file contents. This function + should not be called anywhere except within this module! + """ + with open(environ["VAULT_SECRETS_PATH"]) as file: + contents = file.read() + + json_secrets = b64decode(contents) + return loads(json_secrets) + + +def _get_secrets() -> Dict: + """ + Get secrets from de_secrets.py in local dev, or from Vault injected secrets file + located at path in VAULT_SECRETS_PATH. Performs base 64 decode followed by JSON + decode on file contents. + """ + if not environ.get("USE_VAULT"): + # Use dev secrets when available. + return _load_de_secrets() + + return _load_vault_secrets() + + +class VaultSecretsWrapper: + """ + Provide access to secrets as attributes and send secret-usage analytics to Datadog. + """ + + def __init__(self, secrets: Dict): + self._keys = secrets.keys() + self._env = environ.get("DD_ENV") + + for key, value in secrets.items(): + statsd.increment( + "vault.secrets.access_count", + value=1, + tags=[f"env:{self._env}", f"secret_key:{key}"], + ) + setattr(self, key, value) + + def __getattribute__(self, key: str): + """ + Override the default getattribute method so that we can track secret key + usage with Datadog. Non-secret attributes are passed on to the default method. + """ + if key not in ["_keys", "_env"] and key in self._keys: + try: + statsd.increment( + "vault.secrets.access_count", + value=1, + tags=[f"env:{self._env}", f"secret_key:{key}"], + ) + return super().__getattribute__(key) + except AttributeError as error: + logger.error(f"Requested secret could not be loaded: {key}") + raise error + + return super().__getattribute__(key) + + +secrets = VaultSecretsWrapper(_get_secrets()) +__all__ = ("secrets",) diff --git a/vaultpy.egg-info/PKG-INFO b/vaultpy.egg-info/PKG-INFO new file mode 100644 index 0000000..ec41023 --- /dev/null +++ b/vaultpy.egg-info/PKG-INFO @@ -0,0 +1,21 @@ +Metadata-Version: 2.1 +Name: vaultpy +Version: 0.0.1 +Summary: A module to parse injected Vault secrets and track their usage with Datadog. +Home-page: https://github.com/DirectEmployers/vaultpy +Author: Tim Loyer +Author-email: tloyer@apps.directemployers.org +License: UNKNOWN +Description: # Vaultpy + A module to parse injected [Vault](https://www.vaultproject.io/) secrets and track their usage with Datadog. + + ## Requirements + - Local Datadog agent + - Environment variables to access it + + ## Usage + + +Platform: UNKNOWN +Requires-Python: >=3.6 +Description-Content-Type: text/markdown diff --git a/vaultpy.egg-info/SOURCES.txt b/vaultpy.egg-info/SOURCES.txt new file mode 100644 index 0000000..0c4a161 --- /dev/null +++ b/vaultpy.egg-info/SOURCES.txt @@ -0,0 +1,8 @@ +README.md +pyproject.toml +setup.cfg +vault/__init__.py +vaultpy.egg-info/PKG-INFO +vaultpy.egg-info/SOURCES.txt +vaultpy.egg-info/dependency_links.txt +vaultpy.egg-info/top_level.txt \ No newline at end of file diff --git a/vaultpy.egg-info/dependency_links.txt b/vaultpy.egg-info/dependency_links.txt new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/vaultpy.egg-info/dependency_links.txt @@ -0,0 +1 @@ + diff --git a/vaultpy.egg-info/top_level.txt b/vaultpy.egg-info/top_level.txt new file mode 100644 index 0000000..4c0870e --- /dev/null +++ b/vaultpy.egg-info/top_level.txt @@ -0,0 +1 @@ +vault