diff --git a/.DS_Store b/.DS_Store index bb203052..2a939538 100644 Binary files a/.DS_Store and b/.DS_Store differ diff --git a/.gitignore b/.gitignore index b0582f99..4fbab003 100644 --- a/.gitignore +++ b/.gitignore @@ -16,4 +16,5 @@ build/ node_modules datafog_debug.log sotu_2023.txt -/examples/* \ No newline at end of file +/examples/* +.DS_Store \ No newline at end of file diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 00000000..d4bb2cbb --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = . +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/_build/doctrees/environment.pickle b/docs/_build/doctrees/environment.pickle new file mode 100644 index 00000000..f0ed44bd Binary files /dev/null and b/docs/_build/doctrees/environment.pickle differ diff --git a/docs/_build/doctrees/index.doctree b/docs/_build/doctrees/index.doctree new file mode 100644 index 00000000..4008b3e3 Binary files /dev/null and b/docs/_build/doctrees/index.doctree differ diff --git a/docs/_build/html/.buildinfo b/docs/_build/html/.buildinfo new file mode 100644 index 00000000..f13fb290 --- /dev/null +++ b/docs/_build/html/.buildinfo @@ -0,0 +1,4 @@ +# Sphinx build info version 1 +# This file hashes the configuration used when building these files. When it is not found, a full rebuild will be done. +config: ba395bb2b2bc9431a75de7a41966a748 +tags: 645f666f9bcd5a90fca523b33c5a78b7 diff --git a/docs/_build/html/_sources/index.rst.txt b/docs/_build/html/_sources/index.rst.txt new file mode 100644 index 00000000..9971b10c --- /dev/null +++ b/docs/_build/html/_sources/index.rst.txt @@ -0,0 +1,20 @@ +.. DataFog documentation master file, created by + sphinx-quickstart on Tue Mar 12 16:44:44 2024. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Welcome to DataFog's documentation! +=================================== + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + + + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/docs/_build/html/_static/alabaster.css b/docs/_build/html/_static/alabaster.css new file mode 100644 index 00000000..47e87f9e --- /dev/null +++ b/docs/_build/html/_static/alabaster.css @@ -0,0 +1,734 @@ +@import url("basic.css"); + +/* -- page layout ----------------------------------------------------------- */ + +body { + font-family: Georgia, serif; + font-size: 17px; + background-color: #fff; + color: #000; + margin: 0; + padding: 0; +} + +div.document { + width: 940px; + margin: 30px auto 0 auto; +} + +div.documentwrapper { + float: left; + width: 100%; +} + +div.bodywrapper { + margin: 0 0 0 220px; +} + +div.sphinxsidebar { + width: 220px; + font-size: 14px; + line-height: 1.5; +} + +hr { + border: 1px solid #b1b4b6; +} + +div.body { + background-color: #fff; + color: #3e4349; + padding: 0 30px 0 30px; +} + +div.body > .section { + text-align: left; +} + +div.footer { + width: 940px; + margin: 20px auto 30px auto; + font-size: 14px; + color: #888; + text-align: right; +} + +div.footer a { + color: #888; +} + +p.caption { + font-family: inherit; + font-size: inherit; +} + +div.relations { + display: none; +} + +div.sphinxsidebar { + max-height: 100%; + overflow-y: auto; +} + +div.sphinxsidebar a { + color: #444; + text-decoration: none; + border-bottom: 1px dotted #999; +} + +div.sphinxsidebar a:hover { + border-bottom: 1px solid #999; +} + +div.sphinxsidebarwrapper { + padding: 18px 10px; +} + +div.sphinxsidebarwrapper p.logo { + padding: 0; + margin: -10px 0 0 0px; + text-align: center; +} + +div.sphinxsidebarwrapper h1.logo { + margin-top: -10px; + text-align: center; + margin-bottom: 5px; + text-align: left; +} + +div.sphinxsidebarwrapper h1.logo-name { + margin-top: 0px; +} + +div.sphinxsidebarwrapper p.blurb { + margin-top: 0; + font-style: normal; +} + +div.sphinxsidebar h3, +div.sphinxsidebar h4 { + font-family: Georgia, serif; + color: #444; + font-size: 24px; + font-weight: normal; + margin: 0 0 5px 0; + padding: 0; +} + +div.sphinxsidebar h4 { + font-size: 20px; +} + +div.sphinxsidebar h3 a { + color: #444; +} + +div.sphinxsidebar p.logo a, +div.sphinxsidebar h3 a, +div.sphinxsidebar p.logo a:hover, +div.sphinxsidebar h3 a:hover { + border: none; +} + +div.sphinxsidebar p { + color: #555; + margin: 10px 0; +} + +div.sphinxsidebar ul { + margin: 10px 0; + padding: 0; + color: #000; +} + +div.sphinxsidebar ul li.toctree-l1 > a { + font-size: 120%; +} + +div.sphinxsidebar ul li.toctree-l2 > a { + font-size: 110%; +} + +div.sphinxsidebar input { + border: 1px solid #ccc; + font-family: Georgia, serif; + font-size: 1em; +} + +div.sphinxsidebar #searchbox input[type="text"] { + width: 160px; +} + +div.sphinxsidebar .search > div { + display: table-cell; +} + +div.sphinxsidebar hr { + border: none; + height: 1px; + color: #aaa; + background: #aaa; + + text-align: left; + margin-left: 0; + width: 50%; +} + +div.sphinxsidebar .badge { + border-bottom: none; +} + +div.sphinxsidebar .badge:hover { + border-bottom: none; +} + +/* To address an issue with donation coming after search */ +div.sphinxsidebar h3.donation { + margin-top: 10px; +} + +/* -- body styles ----------------------------------------------------------- */ + +a { + color: #004b6b; + text-decoration: underline; +} + +a:hover { + color: #6d4100; + text-decoration: underline; +} + +div.body h1, +div.body h2, +div.body h3, +div.body h4, +div.body h5, +div.body h6 { + font-family: Georgia, serif; + font-weight: normal; + margin: 30px 0px 10px 0px; + padding: 0; +} + +div.body h1 { + margin-top: 0; + padding-top: 0; + font-size: 240%; +} +div.body h2 { + font-size: 180%; +} +div.body h3 { + font-size: 150%; +} +div.body h4 { + font-size: 130%; +} +div.body h5 { + font-size: 100%; +} +div.body h6 { + font-size: 100%; +} + +a.headerlink { + color: #ddd; + padding: 0 4px; + text-decoration: none; +} + +a.headerlink:hover { + color: #444; + background: #eaeaea; +} + +div.body p, +div.body dd, +div.body li { + line-height: 1.4em; +} + +div.admonition { + margin: 20px 0px; + padding: 10px 30px; + background-color: #eee; + border: 1px solid #ccc; +} + +div.admonition tt.xref, +div.admonition code.xref, +div.admonition a tt { + background-color: #fbfbfb; + border-bottom: 1px solid #fafafa; +} + +div.admonition p.admonition-title { + font-family: Georgia, serif; + font-weight: normal; + font-size: 24px; + margin: 0 0 10px 0; + padding: 0; + line-height: 1; +} + +div.admonition p.last { + margin-bottom: 0; +} + +div.highlight { + background-color: #fff; +} + +dt:target, +.highlight { + background: #faf3e8; +} + +div.warning { + background-color: #fcc; + border: 1px solid #faa; +} + +div.danger { + background-color: #fcc; + border: 1px solid #faa; + -moz-box-shadow: 2px 2px 4px #d52c2c; + -webkit-box-shadow: 2px 2px 4px #d52c2c; + box-shadow: 2px 2px 4px #d52c2c; +} + +div.error { + background-color: #fcc; + border: 1px solid #faa; + -moz-box-shadow: 2px 2px 4px #d52c2c; + -webkit-box-shadow: 2px 2px 4px #d52c2c; + box-shadow: 2px 2px 4px #d52c2c; +} + +div.caution { + background-color: #fcc; + border: 1px solid #faa; +} + +div.attention { + background-color: #fcc; + border: 1px solid #faa; +} + +div.important { + background-color: #eee; + border: 1px solid #ccc; +} + +div.note { + background-color: #eee; + border: 1px solid #ccc; +} + +div.tip { + background-color: #eee; + border: 1px solid #ccc; +} + +div.hint { + background-color: #eee; + border: 1px solid #ccc; +} + +div.seealso { + background-color: #eee; + border: 1px solid #ccc; +} + +div.topic { + background-color: #eee; +} + +p.admonition-title { + display: inline; +} + +p.admonition-title:after { + content: ":"; +} + +pre, +tt, +code { + font-family: "Consolas", "Menlo", "DejaVu Sans Mono", + "Bitstream Vera Sans Mono", monospace; + font-size: 0.9em; +} + +.hll { + background-color: #ffc; + margin: 0 -12px; + padding: 0 12px; + display: block; +} + +img.screenshot { +} + +tt.descname, +tt.descclassname, +code.descname, +code.descclassname { + font-size: 0.95em; +} + +tt.descname, +code.descname { + padding-right: 0.08em; +} + +img.screenshot { + -moz-box-shadow: 2px 2px 4px #eee; + -webkit-box-shadow: 2px 2px 4px #eee; + box-shadow: 2px 2px 4px #eee; +} + +table.docutils { + border: 1px solid #888; + -moz-box-shadow: 2px 2px 4px #eee; + -webkit-box-shadow: 2px 2px 4px #eee; + box-shadow: 2px 2px 4px #eee; +} + +table.docutils td, +table.docutils th { + border: 1px solid #888; + padding: 0.25em 0.7em; +} + +table.field-list, +table.footnote { + border: none; + -moz-box-shadow: none; + -webkit-box-shadow: none; + box-shadow: none; +} + +table.footnote { + margin: 15px 0; + width: 100%; + border: 1px solid #eee; + background: #fdfdfd; + font-size: 0.9em; +} + +table.footnote + table.footnote { + margin-top: -15px; + border-top: none; +} + +table.field-list th { + padding: 0 0.8em 0 0; +} + +table.field-list td { + padding: 0; +} + +table.field-list p { + margin-bottom: 0.8em; +} + +/* Cloned from + * https://github.com/sphinx-doc/sphinx/commit/ef60dbfce09286b20b7385333d63a60321784e68 + */ +.field-name { + -moz-hyphens: manual; + -ms-hyphens: manual; + -webkit-hyphens: manual; + hyphens: manual; +} + +table.footnote td.label { + width: 0.1px; + padding: 0.3em 0 0.3em 0.5em; +} + +table.footnote td { + padding: 0.3em 0.5em; +} + +dl { + margin-left: 0; + margin-right: 0; + margin-top: 0; + padding: 0; +} + +dl dd { + margin-left: 30px; +} + +blockquote { + margin: 0 0 0 30px; + padding: 0; +} + +ul, +ol { + /* Matches the 30px from the narrow-screen "li > ul" selector below */ + margin: 10px 0 10px 30px; + padding: 0; +} + +pre { + background: #eee; + padding: 7px 30px; + margin: 15px 0px; + line-height: 1.3em; +} + +div.viewcode-block:target { + background: #ffd; +} + +dl pre, +blockquote pre, +li pre { + margin-left: 0; + padding-left: 30px; +} + +tt, +code { + background-color: #ecf0f3; + color: #222; + /* padding: 1px 2px; */ +} + +tt.xref, +code.xref, +a tt { + background-color: #fbfbfb; + border-bottom: 1px solid #fff; +} + +a.reference { + text-decoration: none; + border-bottom: 1px dotted #004b6b; +} + +/* Don't put an underline on images */ +a.image-reference, +a.image-reference:hover { + border-bottom: none; +} + +a.reference:hover { + border-bottom: 1px solid #6d4100; +} + +a.footnote-reference { + text-decoration: none; + font-size: 0.7em; + vertical-align: top; + border-bottom: 1px dotted #004b6b; +} + +a.footnote-reference:hover { + border-bottom: 1px solid #6d4100; +} + +a:hover tt, +a:hover code { + background: #eee; +} + +@media screen and (max-width: 870px) { + div.sphinxsidebar { + display: none; + } + + div.document { + width: 100%; + } + + div.documentwrapper { + margin-left: 0; + margin-top: 0; + margin-right: 0; + margin-bottom: 0; + } + + div.bodywrapper { + margin-top: 0; + margin-right: 0; + margin-bottom: 0; + margin-left: 0; + } + + ul { + margin-left: 0; + } + + li > ul { + /* Matches the 30px from the "ul, ol" selector above */ + margin-left: 30px; + } + + .document { + width: auto; + } + + .footer { + width: auto; + } + + .bodywrapper { + margin: 0; + } + + .footer { + width: auto; + } + + .github { + display: none; + } +} + +@media screen and (max-width: 875px) { + body { + margin: 0; + padding: 20px 30px; + } + + div.documentwrapper { + float: none; + background: #fff; + } + + div.sphinxsidebar { + display: block; + float: none; + width: 102.5%; + margin: 50px -30px -20px -30px; + padding: 10px 20px; + background: #333; + color: #fff; + } + + div.sphinxsidebar h3, + div.sphinxsidebar h4, + div.sphinxsidebar p, + div.sphinxsidebar h3 a { + color: #fff; + } + + div.sphinxsidebar a { + color: #aaa; + } + + div.sphinxsidebar p.logo { + display: none; + } + + div.document { + width: 100%; + margin: 0; + } + + div.footer { + display: none; + } + + div.bodywrapper { + margin: 0; + } + + div.body { + min-height: 0; + padding: 0; + } + + .rtd_doc_footer { + display: none; + } + + .document { + width: auto; + } + + .footer { + width: auto; + } + + .footer { + width: auto; + } + + .github { + display: none; + } +} + +/* misc. */ + +.revsys-inline { + display: none !important; +} + +/* Hide ugly table cell borders in ..bibliography:: directive output */ +table.docutils.citation, +table.docutils.citation td, +table.docutils.citation th { + border: none; + /* Below needed in some edge cases; if not applied, bottom shadows appear */ + -moz-box-shadow: none; + -webkit-box-shadow: none; + box-shadow: none; +} + +/* relbar */ + +.related { + line-height: 30px; + width: 100%; + font-size: 0.9rem; +} + +.related.top { + border-bottom: 1px solid #eee; + margin-bottom: 20px; +} + +.related.bottom { + border-top: 1px solid #eee; +} + +.related ul { + padding: 0; + margin: 0; + list-style: none; +} + +.related li { + display: inline; +} + +nav#rellinks { + float: right; +} + +nav#rellinks li + li:before { + content: "|"; +} + +nav#breadcrumbs li + li:before { + content: "\00BB"; +} + +/* Hide certain items when printing */ +@media print { + div.related { + display: none; + } +} diff --git a/docs/_build/html/_static/basic.css b/docs/_build/html/_static/basic.css new file mode 100644 index 00000000..16098eaf --- /dev/null +++ b/docs/_build/html/_static/basic.css @@ -0,0 +1,962 @@ +/* + * basic.css + * ~~~~~~~~~ + * + * Sphinx stylesheet -- basic theme. + * + * :copyright: Copyright 2007-2023 by the Sphinx team, see AUTHORS. + * :license: BSD, see LICENSE for details. + * + */ + +/* -- main layout ----------------------------------------------------------- */ + +div.clearer { + clear: both; +} + +div.section::after { + display: block; + content: ""; + clear: left; +} + +/* -- relbar ---------------------------------------------------------------- */ + +div.related { + width: 100%; + font-size: 90%; +} + +div.related h3 { + display: none; +} + +div.related ul { + margin: 0; + padding: 0 0 0 10px; + list-style: none; +} + +div.related li { + display: inline; +} + +div.related li.right { + float: right; + margin-right: 5px; +} + +/* -- sidebar --------------------------------------------------------------- */ + +div.sphinxsidebarwrapper { + padding: 10px 5px 0 10px; +} + +div.sphinxsidebar { + float: left; + width: 230px; + margin-left: -100%; + font-size: 90%; + word-wrap: break-word; + overflow-wrap: break-word; +} + +div.sphinxsidebar ul { + list-style: none; +} + +div.sphinxsidebar ul ul, +div.sphinxsidebar ul.want-points { + margin-left: 20px; + list-style: square; +} + +div.sphinxsidebar ul ul { + margin-top: 0; + margin-bottom: 0; +} + +div.sphinxsidebar form { + margin-top: 10px; +} + +div.sphinxsidebar input { + border: 1px solid #98dbcc; + font-family: sans-serif; + font-size: 1em; +} + +div.sphinxsidebar #searchbox form.search { + overflow: hidden; +} + +div.sphinxsidebar #searchbox input[type="text"] { + float: left; + width: 80%; + padding: 0.25em; + box-sizing: border-box; +} + +div.sphinxsidebar #searchbox input[type="submit"] { + float: left; + width: 20%; + border-left: none; + padding: 0.25em; + box-sizing: border-box; +} + +img { + border: 0; + max-width: 100%; +} + +/* -- search page ----------------------------------------------------------- */ + +ul.search { + margin: 10px 0 0 20px; + padding: 0; +} + +ul.search li { + padding: 5px 0 5px 20px; + background-image: url(file.png); + background-repeat: no-repeat; + background-position: 0 7px; +} + +ul.search li a { + font-weight: bold; +} + +ul.search li p.context { + color: #888; + margin: 2px 0 0 30px; + text-align: left; +} + +ul.keywordmatches li.goodmatch a { + font-weight: bold; +} + +/* -- index page ------------------------------------------------------------ */ + +table.contentstable { + width: 90%; + margin-left: auto; + margin-right: auto; +} + +table.contentstable p.biglink { + line-height: 150%; +} + +a.biglink { + font-size: 1.3em; +} + +span.linkdescr { + font-style: italic; + padding-top: 5px; + font-size: 90%; +} + +/* -- general index --------------------------------------------------------- */ + +table.indextable { + width: 100%; +} + +table.indextable td { + text-align: left; + vertical-align: top; +} + +table.indextable ul { + margin-top: 0; + margin-bottom: 0; + list-style-type: none; +} + +table.indextable > tbody > tr > td > ul { + padding-left: 0em; +} + +table.indextable tr.pcap { + height: 10px; +} + +table.indextable tr.cap { + margin-top: 10px; + background-color: #f2f2f2; +} + +img.toggler { + margin-right: 3px; + margin-top: 3px; + cursor: pointer; +} + +div.modindex-jumpbox { + border-top: 1px solid #ddd; + border-bottom: 1px solid #ddd; + margin: 1em 0 1em 0; + padding: 0.4em; +} + +div.genindex-jumpbox { + border-top: 1px solid #ddd; + border-bottom: 1px solid #ddd; + margin: 1em 0 1em 0; + padding: 0.4em; +} + +/* -- domain module index --------------------------------------------------- */ + +table.modindextable td { + padding: 2px; + border-collapse: collapse; +} + +/* -- general body styles --------------------------------------------------- */ + +div.body { + min-width: inherit; + max-width: 800px; +} + +div.body p, +div.body dd, +div.body li, +div.body blockquote { + -moz-hyphens: auto; + -ms-hyphens: auto; + -webkit-hyphens: auto; + hyphens: auto; +} + +a.headerlink { + visibility: hidden; +} + +a:visited { + color: #551a8b; +} + +h1:hover > a.headerlink, +h2:hover > a.headerlink, +h3:hover > a.headerlink, +h4:hover > a.headerlink, +h5:hover > a.headerlink, +h6:hover > a.headerlink, +dt:hover > a.headerlink, +caption:hover > a.headerlink, +p.caption:hover > a.headerlink, +div.code-block-caption:hover > a.headerlink { + visibility: visible; +} + +div.body p.caption { + text-align: inherit; +} + +div.body td { + text-align: left; +} + +.first { + margin-top: 0 !important; +} + +p.rubric { + margin-top: 30px; + font-weight: bold; +} + +img.align-left, +figure.align-left, +.figure.align-left, +object.align-left { + clear: left; + float: left; + margin-right: 1em; +} + +img.align-right, +figure.align-right, +.figure.align-right, +object.align-right { + clear: right; + float: right; + margin-left: 1em; +} + +img.align-center, +figure.align-center, +.figure.align-center, +object.align-center { + display: block; + margin-left: auto; + margin-right: auto; +} + +img.align-default, +figure.align-default, +.figure.align-default { + display: block; + margin-left: auto; + margin-right: auto; +} + +.align-left { + text-align: left; +} + +.align-center { + text-align: center; +} + +.align-default { + text-align: center; +} + +.align-right { + text-align: right; +} + +/* -- sidebars -------------------------------------------------------------- */ + +div.sidebar, +aside.sidebar { + margin: 0 0 0.5em 1em; + border: 1px solid #ddb; + padding: 7px; + background-color: #ffe; + width: 40%; + float: right; + clear: right; + overflow-x: auto; +} + +p.sidebar-title { + font-weight: bold; +} + +nav.contents, +aside.topic, +div.admonition, +div.topic, +blockquote { + clear: left; +} + +/* -- topics ---------------------------------------------------------------- */ + +nav.contents, +aside.topic, +div.topic { + border: 1px solid #ccc; + padding: 7px; + margin: 10px 0 10px 0; +} + +p.topic-title { + font-size: 1.1em; + font-weight: bold; + margin-top: 10px; +} + +/* -- admonitions ----------------------------------------------------------- */ + +div.admonition { + margin-top: 10px; + margin-bottom: 10px; + padding: 7px; +} + +div.admonition dt { + font-weight: bold; +} + +p.admonition-title { + margin: 0px 10px 5px 0px; + font-weight: bold; +} + +div.body p.centered { + text-align: center; + margin-top: 25px; +} + +/* -- content of sidebars/topics/admonitions -------------------------------- */ + +div.sidebar > :last-child, +aside.sidebar > :last-child, +nav.contents > :last-child, +aside.topic > :last-child, +div.topic > :last-child, +div.admonition > :last-child { + margin-bottom: 0; +} + +div.sidebar::after, +aside.sidebar::after, +nav.contents::after, +aside.topic::after, +div.topic::after, +div.admonition::after, +blockquote::after { + display: block; + content: ""; + clear: both; +} + +/* -- tables ---------------------------------------------------------------- */ + +table.docutils { + margin-top: 10px; + margin-bottom: 10px; + border: 0; + border-collapse: collapse; +} + +table.align-center { + margin-left: auto; + margin-right: auto; +} + +table.align-default { + margin-left: auto; + margin-right: auto; +} + +table caption span.caption-number { + font-style: italic; +} + +table caption span.caption-text { +} + +table.docutils td, +table.docutils th { + padding: 1px 8px 1px 5px; + border-top: 0; + border-left: 0; + border-right: 0; + border-bottom: 1px solid #aaa; +} + +th { + text-align: left; + padding-right: 5px; +} + +table.citation { + border-left: solid 1px gray; + margin-left: 1px; +} + +table.citation td { + border-bottom: none; +} + +th > :first-child, +td > :first-child { + margin-top: 0px; +} + +th > :last-child, +td > :last-child { + margin-bottom: 0px; +} + +/* -- figures --------------------------------------------------------------- */ + +div.figure, +figure { + margin: 0.5em; + padding: 0.5em; +} + +div.figure p.caption, +figcaption { + padding: 0.3em; +} + +div.figure p.caption span.caption-number, +figcaption span.caption-number { + font-style: italic; +} + +div.figure p.caption span.caption-text, +figcaption span.caption-text { +} + +/* -- field list styles ----------------------------------------------------- */ + +table.field-list td, +table.field-list th { + border: 0 !important; +} + +.field-list ul { + margin: 0; + padding-left: 1em; +} + +.field-list p { + margin: 0; +} + +.field-name { + -moz-hyphens: manual; + -ms-hyphens: manual; + -webkit-hyphens: manual; + hyphens: manual; +} + +/* -- hlist styles ---------------------------------------------------------- */ + +table.hlist { + margin: 1em 0; +} + +table.hlist td { + vertical-align: top; +} + +/* -- object description styles --------------------------------------------- */ + +.sig { + font-family: "Consolas", "Menlo", "DejaVu Sans Mono", + "Bitstream Vera Sans Mono", monospace; +} + +.sig-name, +code.descname { + background-color: transparent; + font-weight: bold; +} + +.sig-name { + font-size: 1.1em; +} + +code.descname { + font-size: 1.2em; +} + +.sig-prename, +code.descclassname { + background-color: transparent; +} + +.optional { + font-size: 1.3em; +} + +.sig-paren { + font-size: larger; +} + +.sig-param.n { + font-style: italic; +} + +/* C++ specific styling */ + +.sig-inline.c-texpr, +.sig-inline.cpp-texpr { + font-family: unset; +} + +.sig.c .k, +.sig.c .kt, +.sig.cpp .k, +.sig.cpp .kt { + color: #0033b3; +} + +.sig.c .m, +.sig.cpp .m { + color: #1750eb; +} + +.sig.c .s, +.sig.c .sc, +.sig.cpp .s, +.sig.cpp .sc { + color: #067d17; +} + +/* -- other body styles ----------------------------------------------------- */ + +ol.arabic { + list-style: decimal; +} + +ol.loweralpha { + list-style: lower-alpha; +} + +ol.upperalpha { + list-style: upper-alpha; +} + +ol.lowerroman { + list-style: lower-roman; +} + +ol.upperroman { + list-style: upper-roman; +} + +:not(li) > ol > li:first-child > :first-child, +:not(li) > ul > li:first-child > :first-child { + margin-top: 0px; +} + +:not(li) > ol > li:last-child > :last-child, +:not(li) > ul > li:last-child > :last-child { + margin-bottom: 0px; +} + +ol.simple ol p, +ol.simple ul p, +ul.simple ol p, +ul.simple ul p { + margin-top: 0; +} + +ol.simple > li:not(:first-child) > p, +ul.simple > li:not(:first-child) > p { + margin-top: 0; +} + +ol.simple p, +ul.simple p { + margin-bottom: 0; +} + +aside.footnote > span, +div.citation > span { + float: left; +} +aside.footnote > span:last-of-type, +div.citation > span:last-of-type { + padding-right: 0.5em; +} +aside.footnote > p { + margin-left: 2em; +} +div.citation > p { + margin-left: 4em; +} +aside.footnote > p:last-of-type, +div.citation > p:last-of-type { + margin-bottom: 0em; +} +aside.footnote > p:last-of-type:after, +div.citation > p:last-of-type:after { + content: ""; + clear: both; +} + +dl.field-list { + display: grid; + grid-template-columns: fit-content(30%) auto; +} + +dl.field-list > dt { + font-weight: bold; + word-break: break-word; + padding-left: 0.5em; + padding-right: 5px; +} + +dl.field-list > dd { + padding-left: 0.5em; + margin-top: 0em; + margin-left: 0em; + margin-bottom: 0em; +} + +dl { + margin-bottom: 15px; +} + +dd > :first-child { + margin-top: 0px; +} + +dd ul, +dd table { + margin-bottom: 10px; +} + +dd { + margin-top: 3px; + margin-bottom: 10px; + margin-left: 30px; +} + +.sig dd { + margin-top: 0px; + margin-bottom: 0px; +} + +.sig dl { + margin-top: 0px; + margin-bottom: 0px; +} + +dl > dd:last-child, +dl > dd:last-child > :last-child { + margin-bottom: 0; +} + +dt:target, +span.highlighted { + background-color: #fbe54e; +} + +rect.highlighted { + fill: #fbe54e; +} + +dl.glossary dt { + font-weight: bold; + font-size: 1.1em; +} + +.versionmodified { + font-style: italic; +} + +.system-message { + background-color: #fda; + padding: 5px; + border: 3px solid red; +} + +.footnote:target { + background-color: #ffa; +} + +.line-block { + display: block; + margin-top: 1em; + margin-bottom: 1em; +} + +.line-block .line-block { + margin-top: 0; + margin-bottom: 0; + margin-left: 1.5em; +} + +.guilabel, +.menuselection { + font-family: sans-serif; +} + +.accelerator { + text-decoration: underline; +} + +.classifier { + font-style: oblique; +} + +.classifier:before { + font-style: normal; + margin: 0 0.5em; + content: ":"; + display: inline-block; +} + +abbr, +acronym { + border-bottom: dotted 1px; + cursor: help; +} + +.translated { + background-color: rgba(207, 255, 207, 0.2); +} + +.untranslated { + background-color: rgba(255, 207, 207, 0.2); +} + +/* -- code displays --------------------------------------------------------- */ + +pre { + overflow: auto; + overflow-y: hidden; /* fixes display issues on Chrome browsers */ +} + +pre, +div[class*="highlight-"] { + clear: both; +} + +span.pre { + -moz-hyphens: none; + -ms-hyphens: none; + -webkit-hyphens: none; + hyphens: none; + white-space: nowrap; +} + +div[class*="highlight-"] { + margin: 1em 0; +} + +td.linenos pre { + border: 0; + background-color: transparent; + color: #aaa; +} + +table.highlighttable { + display: block; +} + +table.highlighttable tbody { + display: block; +} + +table.highlighttable tr { + display: flex; +} + +table.highlighttable td { + margin: 0; + padding: 0; +} + +table.highlighttable td.linenos { + padding-right: 0.5em; +} + +table.highlighttable td.code { + flex: 1; + overflow: hidden; +} + +.highlight .hll { + display: block; +} + +div.highlight pre, +table.highlighttable pre { + margin: 0; +} + +div.code-block-caption + div { + margin-top: 0; +} + +div.code-block-caption { + margin-top: 1em; + padding: 2px 5px; + font-size: small; +} + +div.code-block-caption code { + background-color: transparent; +} + +table.highlighttable td.linenos, +span.linenos, +div.highlight span.gp { + /* gp: Generic.Prompt */ + user-select: none; + -webkit-user-select: text; /* Safari fallback only */ + -webkit-user-select: none; /* Chrome/Safari */ + -moz-user-select: none; /* Firefox */ + -ms-user-select: none; /* IE10+ */ +} + +div.code-block-caption span.caption-number { + padding: 0.1em 0.3em; + font-style: italic; +} + +div.code-block-caption span.caption-text { +} + +div.literal-block-wrapper { + margin: 1em 0; +} + +code.xref, +a code { + background-color: transparent; + font-weight: bold; +} + +h1 code, +h2 code, +h3 code, +h4 code, +h5 code, +h6 code { + background-color: transparent; +} + +.viewcode-link { + float: right; +} + +.viewcode-back { + float: right; + font-family: sans-serif; +} + +div.viewcode-block:target { + margin: -1px -10px; + padding: 0 10px; +} + +/* -- math display ---------------------------------------------------------- */ + +img.math { + vertical-align: middle; +} + +div.body div.math p { + text-align: center; +} + +span.eqno { + float: right; +} + +span.eqno a.headerlink { + position: absolute; + z-index: 1; +} + +div.math:hover a.headerlink { + visibility: visible; +} + +/* -- printout stylesheet --------------------------------------------------- */ + +@media print { + div.document, + div.documentwrapper, + div.bodywrapper { + margin: 0 !important; + width: 100%; + } + + div.sphinxsidebar, + div.related, + div.footer, + #top-link { + display: none; + } +} diff --git a/docs/_build/html/_static/custom.css b/docs/_build/html/_static/custom.css new file mode 100644 index 00000000..2a924f1d --- /dev/null +++ b/docs/_build/html/_static/custom.css @@ -0,0 +1 @@ +/* This file intentionally left blank. */ diff --git a/docs/_build/html/_static/doctools.js b/docs/_build/html/_static/doctools.js new file mode 100644 index 00000000..c9109ceb --- /dev/null +++ b/docs/_build/html/_static/doctools.js @@ -0,0 +1,157 @@ +/* + * doctools.js + * ~~~~~~~~~~~ + * + * Base JavaScript utilities for all Sphinx HTML documentation. + * + * :copyright: Copyright 2007-2023 by the Sphinx team, see AUTHORS. + * :license: BSD, see LICENSE for details. + * + */ +"use strict"; + +const BLACKLISTED_KEY_CONTROL_ELEMENTS = new Set([ + "TEXTAREA", + "INPUT", + "SELECT", + "BUTTON", +]); + +const _ready = (callback) => { + if (document.readyState !== "loading") { + callback(); + } else { + document.addEventListener("DOMContentLoaded", callback); + } +}; + +/** + * Small JavaScript module for the documentation. + */ +const Documentation = { + init: () => { + Documentation.initDomainIndexTable(); + Documentation.initOnKeyListeners(); + }, + + /** + * i18n support + */ + TRANSLATIONS: {}, + PLURAL_EXPR: (n) => (n === 1 ? 0 : 1), + LOCALE: "unknown", + + // gettext and ngettext don't access this so that the functions + // can safely bound to a different name (_ = Documentation.gettext) + gettext: (string) => { + const translated = Documentation.TRANSLATIONS[string]; + switch (typeof translated) { + case "undefined": + return string; // no translation + case "string": + return translated; // translation exists + default: + return translated[0]; // (singular, plural) translation tuple exists + } + }, + + ngettext: (singular, plural, n) => { + const translated = Documentation.TRANSLATIONS[singular]; + if (typeof translated !== "undefined") + return translated[Documentation.PLURAL_EXPR(n)]; + return n === 1 ? singular : plural; + }, + + addTranslations: (catalog) => { + Object.assign(Documentation.TRANSLATIONS, catalog.messages); + Documentation.PLURAL_EXPR = new Function( + "n", + `return (${catalog.plural_expr})`, + ); + Documentation.LOCALE = catalog.locale; + }, + + /** + * helper function to focus on search bar + */ + focusSearchBar: () => { + document.querySelectorAll("input[name=q]")[0]?.focus(); + }, + + /** + * Initialise the domain index toggle buttons + */ + initDomainIndexTable: () => { + const toggler = (el) => { + const idNumber = el.id.substr(7); + const toggledRows = document.querySelectorAll(`tr.cg-${idNumber}`); + if (el.src.substr(-9) === "minus.png") { + el.src = `${el.src.substr(0, el.src.length - 9)}plus.png`; + toggledRows.forEach((el) => (el.style.display = "none")); + } else { + el.src = `${el.src.substr(0, el.src.length - 8)}minus.png`; + toggledRows.forEach((el) => (el.style.display = "")); + } + }; + + const togglerElements = document.querySelectorAll("img.toggler"); + togglerElements.forEach((el) => + el.addEventListener("click", (event) => toggler(event.currentTarget)), + ); + togglerElements.forEach((el) => (el.style.display = "")); + if (DOCUMENTATION_OPTIONS.COLLAPSE_INDEX) togglerElements.forEach(toggler); + }, + + initOnKeyListeners: () => { + // only install a listener if it is really needed + if ( + !DOCUMENTATION_OPTIONS.NAVIGATION_WITH_KEYS && + !DOCUMENTATION_OPTIONS.ENABLE_SEARCH_SHORTCUTS + ) + return; + + document.addEventListener("keydown", (event) => { + // bail for input elements + if (BLACKLISTED_KEY_CONTROL_ELEMENTS.has(document.activeElement.tagName)) + return; + // bail with special keys + if (event.altKey || event.ctrlKey || event.metaKey) return; + + if (!event.shiftKey) { + switch (event.key) { + case "ArrowLeft": + if (!DOCUMENTATION_OPTIONS.NAVIGATION_WITH_KEYS) break; + + const prevLink = document.querySelector('link[rel="prev"]'); + if (prevLink && prevLink.href) { + window.location.href = prevLink.href; + event.preventDefault(); + } + break; + case "ArrowRight": + if (!DOCUMENTATION_OPTIONS.NAVIGATION_WITH_KEYS) break; + + const nextLink = document.querySelector('link[rel="next"]'); + if (nextLink && nextLink.href) { + window.location.href = nextLink.href; + event.preventDefault(); + } + break; + } + } + + // some keyboard layouts may need Shift to get / + switch (event.key) { + case "/": + if (!DOCUMENTATION_OPTIONS.ENABLE_SEARCH_SHORTCUTS) break; + Documentation.focusSearchBar(); + event.preventDefault(); + } + }); + }, +}; + +// quick alias for translations +const _ = Documentation.gettext; + +_ready(Documentation.init); diff --git a/docs/_build/html/_static/documentation_options.js b/docs/_build/html/_static/documentation_options.js new file mode 100644 index 00000000..8ad5f1d9 --- /dev/null +++ b/docs/_build/html/_static/documentation_options.js @@ -0,0 +1,13 @@ +const DOCUMENTATION_OPTIONS = { + VERSION: "v2.3.1b1", + LANGUAGE: "en", + COLLAPSE_INDEX: false, + BUILDER: "html", + FILE_SUFFIX: ".html", + LINK_SUFFIX: ".html", + HAS_SOURCE: true, + SOURCELINK_SUFFIX: ".txt", + NAVIGATION_WITH_KEYS: false, + SHOW_SEARCH_SUMMARY: true, + ENABLE_SEARCH_SHORTCUTS: true, +}; diff --git a/docs/_build/html/_static/file.png b/docs/_build/html/_static/file.png new file mode 100644 index 00000000..a858a410 Binary files /dev/null and b/docs/_build/html/_static/file.png differ diff --git a/docs/_build/html/_static/language_data.js b/docs/_build/html/_static/language_data.js new file mode 100644 index 00000000..7c1c9ffc --- /dev/null +++ b/docs/_build/html/_static/language_data.js @@ -0,0 +1,216 @@ +/* + * language_data.js + * ~~~~~~~~~~~~~~~~ + * + * This script contains the language-specific data used by searchtools.js, + * namely the list of stopwords, stemmer, scorer and splitter. + * + * :copyright: Copyright 2007-2023 by the Sphinx team, see AUTHORS. + * :license: BSD, see LICENSE for details. + * + */ + +var stopwords = [ + "a", + "and", + "are", + "as", + "at", + "be", + "but", + "by", + "for", + "if", + "in", + "into", + "is", + "it", + "near", + "no", + "not", + "of", + "on", + "or", + "such", + "that", + "the", + "their", + "then", + "there", + "these", + "they", + "this", + "to", + "was", + "will", + "with", +]; + +/* Non-minified version is copied as a separate JS file, is available */ + +/** + * Porter Stemmer + */ +var Stemmer = function () { + var step2list = { + ational: "ate", + tional: "tion", + enci: "ence", + anci: "ance", + izer: "ize", + bli: "ble", + alli: "al", + entli: "ent", + eli: "e", + ousli: "ous", + ization: "ize", + ation: "ate", + ator: "ate", + alism: "al", + iveness: "ive", + fulness: "ful", + ousness: "ous", + aliti: "al", + iviti: "ive", + biliti: "ble", + logi: "log", + }; + + var step3list = { + icate: "ic", + ative: "", + alize: "al", + iciti: "ic", + ical: "ic", + ful: "", + ness: "", + }; + + var c = "[^aeiou]"; // consonant + var v = "[aeiouy]"; // vowel + var C = c + "[^aeiouy]*"; // consonant sequence + var V = v + "[aeiou]*"; // vowel sequence + + var mgr0 = "^(" + C + ")?" + V + C; // [C]VC... is m>0 + var meq1 = "^(" + C + ")?" + V + C + "(" + V + ")?$"; // [C]VC[V] is m=1 + var mgr1 = "^(" + C + ")?" + V + C + V + C; // [C]VCVC... is m>1 + var s_v = "^(" + C + ")?" + v; // vowel in stem + + this.stemWord = function (w) { + var stem; + var suffix; + var firstch; + var origword = w; + + if (w.length < 3) return w; + + var re; + var re2; + var re3; + var re4; + + firstch = w.substr(0, 1); + if (firstch == "y") w = firstch.toUpperCase() + w.substr(1); + + // Step 1a + re = /^(.+?)(ss|i)es$/; + re2 = /^(.+?)([^s])s$/; + + if (re.test(w)) w = w.replace(re, "$1$2"); + else if (re2.test(w)) w = w.replace(re2, "$1$2"); + + // Step 1b + re = /^(.+?)eed$/; + re2 = /^(.+?)(ed|ing)$/; + if (re.test(w)) { + var fp = re.exec(w); + re = new RegExp(mgr0); + if (re.test(fp[1])) { + re = /.$/; + w = w.replace(re, ""); + } + } else if (re2.test(w)) { + var fp = re2.exec(w); + stem = fp[1]; + re2 = new RegExp(s_v); + if (re2.test(stem)) { + w = stem; + re2 = /(at|bl|iz)$/; + re3 = new RegExp("([^aeiouylsz])\\1$"); + re4 = new RegExp("^" + C + v + "[^aeiouwxy]$"); + if (re2.test(w)) w = w + "e"; + else if (re3.test(w)) { + re = /.$/; + w = w.replace(re, ""); + } else if (re4.test(w)) w = w + "e"; + } + } + + // Step 1c + re = /^(.+?)y$/; + if (re.test(w)) { + var fp = re.exec(w); + stem = fp[1]; + re = new RegExp(s_v); + if (re.test(stem)) w = stem + "i"; + } + + // Step 2 + re = + /^(.+?)(ational|tional|enci|anci|izer|bli|alli|entli|eli|ousli|ization|ation|ator|alism|iveness|fulness|ousness|aliti|iviti|biliti|logi)$/; + if (re.test(w)) { + var fp = re.exec(w); + stem = fp[1]; + suffix = fp[2]; + re = new RegExp(mgr0); + if (re.test(stem)) w = stem + step2list[suffix]; + } + + // Step 3 + re = /^(.+?)(icate|ative|alize|iciti|ical|ful|ness)$/; + if (re.test(w)) { + var fp = re.exec(w); + stem = fp[1]; + suffix = fp[2]; + re = new RegExp(mgr0); + if (re.test(stem)) w = stem + step3list[suffix]; + } + + // Step 4 + re = + /^(.+?)(al|ance|ence|er|ic|able|ible|ant|ement|ment|ent|ou|ism|ate|iti|ous|ive|ize)$/; + re2 = /^(.+?)(s|t)(ion)$/; + if (re.test(w)) { + var fp = re.exec(w); + stem = fp[1]; + re = new RegExp(mgr1); + if (re.test(stem)) w = stem; + } else if (re2.test(w)) { + var fp = re2.exec(w); + stem = fp[1] + fp[2]; + re2 = new RegExp(mgr1); + if (re2.test(stem)) w = stem; + } + + // Step 5 + re = /^(.+?)e$/; + if (re.test(w)) { + var fp = re.exec(w); + stem = fp[1]; + re = new RegExp(mgr1); + re2 = new RegExp(meq1); + re3 = new RegExp("^" + C + v + "[^aeiouwxy]$"); + if (re.test(stem) || (re2.test(stem) && !re3.test(stem))) w = stem; + } + re = /ll$/; + re2 = new RegExp(mgr1); + if (re.test(w) && re2.test(w)) { + re = /.$/; + w = w.replace(re, ""); + } + + // and turn initial Y back to y + if (firstch == "y") w = firstch.toLowerCase() + w.substr(1); + return w; + }; +}; diff --git a/docs/_build/html/_static/minus.png b/docs/_build/html/_static/minus.png new file mode 100644 index 00000000..d96755fd Binary files /dev/null and b/docs/_build/html/_static/minus.png differ diff --git a/docs/_build/html/_static/plus.png b/docs/_build/html/_static/plus.png new file mode 100644 index 00000000..7107cec9 Binary files /dev/null and b/docs/_build/html/_static/plus.png differ diff --git a/docs/_build/html/_static/pygments.css b/docs/_build/html/_static/pygments.css new file mode 100644 index 00000000..e21c1e6e --- /dev/null +++ b/docs/_build/html/_static/pygments.css @@ -0,0 +1,289 @@ +pre { + line-height: 125%; +} +td.linenos .normal { + color: inherit; + background-color: transparent; + padding-left: 5px; + padding-right: 5px; +} +span.linenos { + color: inherit; + background-color: transparent; + padding-left: 5px; + padding-right: 5px; +} +td.linenos .special { + color: #000000; + background-color: #ffffc0; + padding-left: 5px; + padding-right: 5px; +} +span.linenos.special { + color: #000000; + background-color: #ffffc0; + padding-left: 5px; + padding-right: 5px; +} +.highlight .hll { + background-color: #ffffcc; +} +.highlight { + background: #f8f8f8; +} +.highlight .c { + color: #8f5902; + font-style: italic; +} /* Comment */ +.highlight .err { + color: #a40000; + border: 1px solid #ef2929; +} /* Error */ +.highlight .g { + color: #000000; +} /* Generic */ +.highlight .k { + color: #004461; + font-weight: bold; +} /* Keyword */ +.highlight .l { + color: #000000; +} /* Literal */ +.highlight .n { + color: #000000; +} /* Name */ +.highlight .o { + color: #582800; +} /* Operator */ +.highlight .x { + color: #000000; +} /* Other */ +.highlight .p { + color: #000000; + font-weight: bold; +} /* Punctuation */ +.highlight .ch { + color: #8f5902; + font-style: italic; +} /* Comment.Hashbang */ +.highlight .cm { + color: #8f5902; + font-style: italic; +} /* Comment.Multiline */ +.highlight .cp { + color: #8f5902; +} /* Comment.Preproc */ +.highlight .cpf { + color: #8f5902; + font-style: italic; +} /* Comment.PreprocFile */ +.highlight .c1 { + color: #8f5902; + font-style: italic; +} /* Comment.Single */ +.highlight .cs { + color: #8f5902; + font-style: italic; +} /* Comment.Special */ +.highlight .gd { + color: #a40000; +} /* Generic.Deleted */ +.highlight .ge { + color: #000000; + font-style: italic; +} /* Generic.Emph */ +.highlight .ges { + color: #000000; +} /* Generic.EmphStrong */ +.highlight .gr { + color: #ef2929; +} /* Generic.Error */ +.highlight .gh { + color: #000080; + font-weight: bold; +} /* Generic.Heading */ +.highlight .gi { + color: #00a000; +} /* Generic.Inserted */ +.highlight .go { + color: #888888; +} /* Generic.Output */ +.highlight .gp { + color: #745334; +} /* Generic.Prompt */ +.highlight .gs { + color: #000000; + font-weight: bold; +} /* Generic.Strong */ +.highlight .gu { + color: #800080; + font-weight: bold; +} /* Generic.Subheading */ +.highlight .gt { + color: #a40000; + font-weight: bold; +} /* Generic.Traceback */ +.highlight .kc { + color: #004461; + font-weight: bold; +} /* Keyword.Constant */ +.highlight .kd { + color: #004461; + font-weight: bold; +} /* Keyword.Declaration */ +.highlight .kn { + color: #004461; + font-weight: bold; +} /* Keyword.Namespace */ +.highlight .kp { + color: #004461; + font-weight: bold; +} /* Keyword.Pseudo */ +.highlight .kr { + color: #004461; + font-weight: bold; +} /* Keyword.Reserved */ +.highlight .kt { + color: #004461; + font-weight: bold; +} /* Keyword.Type */ +.highlight .ld { + color: #000000; +} /* Literal.Date */ +.highlight .m { + color: #990000; +} /* Literal.Number */ +.highlight .s { + color: #4e9a06; +} /* Literal.String */ +.highlight .na { + color: #c4a000; +} /* Name.Attribute */ +.highlight .nb { + color: #004461; +} /* Name.Builtin */ +.highlight .nc { + color: #000000; +} /* Name.Class */ +.highlight .no { + color: #000000; +} /* Name.Constant */ +.highlight .nd { + color: #888888; +} /* Name.Decorator */ +.highlight .ni { + color: #ce5c00; +} /* Name.Entity */ +.highlight .ne { + color: #cc0000; + font-weight: bold; +} /* Name.Exception */ +.highlight .nf { + color: #000000; +} /* Name.Function */ +.highlight .nl { + color: #f57900; +} /* Name.Label */ +.highlight .nn { + color: #000000; +} /* Name.Namespace */ +.highlight .nx { + color: #000000; +} /* Name.Other */ +.highlight .py { + color: #000000; +} /* Name.Property */ +.highlight .nt { + color: #004461; + font-weight: bold; +} /* Name.Tag */ +.highlight .nv { + color: #000000; +} /* Name.Variable */ +.highlight .ow { + color: #004461; + font-weight: bold; +} /* Operator.Word */ +.highlight .pm { + color: #000000; + font-weight: bold; +} /* Punctuation.Marker */ +.highlight .w { + color: #f8f8f8; +} /* Text.Whitespace */ +.highlight .mb { + color: #990000; +} /* Literal.Number.Bin */ +.highlight .mf { + color: #990000; +} /* Literal.Number.Float */ +.highlight .mh { + color: #990000; +} /* Literal.Number.Hex */ +.highlight .mi { + color: #990000; +} /* Literal.Number.Integer */ +.highlight .mo { + color: #990000; +} /* Literal.Number.Oct */ +.highlight .sa { + color: #4e9a06; +} /* Literal.String.Affix */ +.highlight .sb { + color: #4e9a06; +} /* Literal.String.Backtick */ +.highlight .sc { + color: #4e9a06; +} /* Literal.String.Char */ +.highlight .dl { + color: #4e9a06; +} /* Literal.String.Delimiter */ +.highlight .sd { + color: #8f5902; + font-style: italic; +} /* Literal.String.Doc */ +.highlight .s2 { + color: #4e9a06; +} /* Literal.String.Double */ +.highlight .se { + color: #4e9a06; +} /* Literal.String.Escape */ +.highlight .sh { + color: #4e9a06; +} /* Literal.String.Heredoc */ +.highlight .si { + color: #4e9a06; +} /* Literal.String.Interpol */ +.highlight .sx { + color: #4e9a06; +} /* Literal.String.Other */ +.highlight .sr { + color: #4e9a06; +} /* Literal.String.Regex */ +.highlight .s1 { + color: #4e9a06; +} /* Literal.String.Single */ +.highlight .ss { + color: #4e9a06; +} /* Literal.String.Symbol */ +.highlight .bp { + color: #3465a4; +} /* Name.Builtin.Pseudo */ +.highlight .fm { + color: #000000; +} /* Name.Function.Magic */ +.highlight .vc { + color: #000000; +} /* Name.Variable.Class */ +.highlight .vg { + color: #000000; +} /* Name.Variable.Global */ +.highlight .vi { + color: #000000; +} /* Name.Variable.Instance */ +.highlight .vm { + color: #000000; +} /* Name.Variable.Magic */ +.highlight .il { + color: #990000; +} /* Literal.Number.Integer.Long */ diff --git a/docs/_build/html/_static/searchtools.js b/docs/_build/html/_static/searchtools.js new file mode 100644 index 00000000..c6058086 --- /dev/null +++ b/docs/_build/html/_static/searchtools.js @@ -0,0 +1,590 @@ +/* + * searchtools.js + * ~~~~~~~~~~~~~~~~ + * + * Sphinx JavaScript utilities for the full-text search. + * + * :copyright: Copyright 2007-2023 by the Sphinx team, see AUTHORS. + * :license: BSD, see LICENSE for details. + * + */ +"use strict"; + +/** + * Simple result scoring code. + */ +if (typeof Scorer === "undefined") { + var Scorer = { + // Implement the following function to further tweak the score for each result + // The function takes a result array [docname, title, anchor, descr, score, filename] + // and returns the new score. + /* + score: result => { + const [docname, title, anchor, descr, score, filename] = result + return score + }, + */ + + // query matches the full name of an object + objNameMatch: 11, + // or matches in the last dotted part of the object name + objPartialMatch: 6, + // Additive scores depending on the priority of the object + objPrio: { + 0: 15, // used to be importantResults + 1: 5, // used to be objectResults + 2: -5, // used to be unimportantResults + }, + // Used when the priority is not in the mapping. + objPrioDefault: 0, + + // query found in title + title: 15, + partialTitle: 7, + // query found in terms + term: 5, + partialTerm: 2, + }; +} + +const _removeChildren = (element) => { + while (element && element.lastChild) element.removeChild(element.lastChild); +}; + +/** + * See https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Regular_Expressions#escaping + */ +const _escapeRegExp = (string) => + string.replace(/[.*+\-?^${}()|[\]\\]/g, "\\$&"); // $& means the whole matched string + +const _displayItem = (item, searchTerms, highlightTerms) => { + const docBuilder = DOCUMENTATION_OPTIONS.BUILDER; + const docFileSuffix = DOCUMENTATION_OPTIONS.FILE_SUFFIX; + const docLinkSuffix = DOCUMENTATION_OPTIONS.LINK_SUFFIX; + const showSearchSummary = DOCUMENTATION_OPTIONS.SHOW_SEARCH_SUMMARY; + const contentRoot = document.documentElement.dataset.content_root; + + const [docName, title, anchor, descr, score, _filename] = item; + + let listItem = document.createElement("li"); + let requestUrl; + let linkUrl; + if (docBuilder === "dirhtml") { + // dirhtml builder + let dirname = docName + "/"; + if (dirname.match(/\/index\/$/)) + dirname = dirname.substring(0, dirname.length - 6); + else if (dirname === "index/") dirname = ""; + requestUrl = contentRoot + dirname; + linkUrl = requestUrl; + } else { + // normal html builders + requestUrl = contentRoot + docName + docFileSuffix; + linkUrl = docName + docLinkSuffix; + } + let linkEl = listItem.appendChild(document.createElement("a")); + linkEl.href = linkUrl + anchor; + linkEl.dataset.score = score; + linkEl.innerHTML = title; + if (descr) { + listItem.appendChild(document.createElement("span")).innerHTML = + " (" + descr + ")"; + // highlight search terms in the description + if (SPHINX_HIGHLIGHT_ENABLED) + // set in sphinx_highlight.js + highlightTerms.forEach((term) => + _highlightText(listItem, term, "highlighted"), + ); + } else if (showSearchSummary) + fetch(requestUrl) + .then((responseData) => responseData.text()) + .then((data) => { + if (data) + listItem.appendChild(Search.makeSearchSummary(data, searchTerms)); + // highlight search terms in the summary + if (SPHINX_HIGHLIGHT_ENABLED) + // set in sphinx_highlight.js + highlightTerms.forEach((term) => + _highlightText(listItem, term, "highlighted"), + ); + }); + Search.output.appendChild(listItem); +}; +const _finishSearch = (resultCount) => { + Search.stopPulse(); + Search.title.innerText = _("Search Results"); + if (!resultCount) + Search.status.innerText = Documentation.gettext( + "Your search did not match any documents. Please make sure that all words are spelled correctly and that you've selected enough categories.", + ); + else + Search.status.innerText = _( + `Search finished, found ${resultCount} page(s) matching the search query.`, + ); +}; +const _displayNextItem = ( + results, + resultCount, + searchTerms, + highlightTerms, +) => { + // results left, load the summary and display it + // this is intended to be dynamic (don't sub resultsCount) + if (results.length) { + _displayItem(results.pop(), searchTerms, highlightTerms); + setTimeout( + () => _displayNextItem(results, resultCount, searchTerms, highlightTerms), + 5, + ); + } + // search finished, update title and status message + else _finishSearch(resultCount); +}; + +/** + * Default splitQuery function. Can be overridden in ``sphinx.search`` with a + * custom function per language. + * + * The regular expression works by splitting the string on consecutive characters + * that are not Unicode letters, numbers, underscores, or emoji characters. + * This is the same as ``\W+`` in Python, preserving the surrogate pair area. + */ +if (typeof splitQuery === "undefined") { + var splitQuery = (query) => + query + .split(/[^\p{Letter}\p{Number}_\p{Emoji_Presentation}]+/gu) + .filter((term) => term); // remove remaining empty strings +} + +/** + * Search Module + */ +const Search = { + _index: null, + _queued_query: null, + _pulse_status: -1, + + htmlToText: (htmlString) => { + const htmlElement = new DOMParser().parseFromString( + htmlString, + "text/html", + ); + htmlElement.querySelectorAll(".headerlink").forEach((el) => { + el.remove(); + }); + const docContent = htmlElement.querySelector('[role="main"]'); + if (docContent !== undefined) return docContent.textContent; + console.warn( + "Content block not found. Sphinx search tries to obtain it via '[role=main]'. Could you check your theme or template.", + ); + return ""; + }, + + init: () => { + const query = new URLSearchParams(window.location.search).get("q"); + document + .querySelectorAll('input[name="q"]') + .forEach((el) => (el.value = query)); + if (query) Search.performSearch(query); + }, + + loadIndex: (url) => + (document.body.appendChild(document.createElement("script")).src = url), + + setIndex: (index) => { + Search._index = index; + if (Search._queued_query !== null) { + const query = Search._queued_query; + Search._queued_query = null; + Search.query(query); + } + }, + + hasIndex: () => Search._index !== null, + + deferQuery: (query) => (Search._queued_query = query), + + stopPulse: () => (Search._pulse_status = -1), + + startPulse: () => { + if (Search._pulse_status >= 0) return; + + const pulse = () => { + Search._pulse_status = (Search._pulse_status + 1) % 4; + Search.dots.innerText = ".".repeat(Search._pulse_status); + if (Search._pulse_status >= 0) window.setTimeout(pulse, 500); + }; + pulse(); + }, + + /** + * perform a search for something (or wait until index is loaded) + */ + performSearch: (query) => { + // create the required interface elements + const searchText = document.createElement("h2"); + searchText.textContent = _("Searching"); + const searchSummary = document.createElement("p"); + searchSummary.classList.add("search-summary"); + searchSummary.innerText = ""; + const searchList = document.createElement("ul"); + searchList.classList.add("search"); + + const out = document.getElementById("search-results"); + Search.title = out.appendChild(searchText); + Search.dots = Search.title.appendChild(document.createElement("span")); + Search.status = out.appendChild(searchSummary); + Search.output = out.appendChild(searchList); + + const searchProgress = document.getElementById("search-progress"); + // Some themes don't use the search progress node + if (searchProgress) { + searchProgress.innerText = _("Preparing search..."); + } + Search.startPulse(); + + // index already loaded, the browser was quick! + if (Search.hasIndex()) Search.query(query); + else Search.deferQuery(query); + }, + + /** + * execute search (requires search index to be loaded) + */ + query: (query) => { + const filenames = Search._index.filenames; + const docNames = Search._index.docnames; + const titles = Search._index.titles; + const allTitles = Search._index.alltitles; + const indexEntries = Search._index.indexentries; + + // stem the search terms and add them to the correct list + const stemmer = new Stemmer(); + const searchTerms = new Set(); + const excludedTerms = new Set(); + const highlightTerms = new Set(); + const objectTerms = new Set(splitQuery(query.toLowerCase().trim())); + splitQuery(query.trim()).forEach((queryTerm) => { + const queryTermLower = queryTerm.toLowerCase(); + + // maybe skip this "word" + // stopwords array is from language_data.js + if (stopwords.indexOf(queryTermLower) !== -1 || queryTerm.match(/^\d+$/)) + return; + + // stem the word + let word = stemmer.stemWord(queryTermLower); + // select the correct list + if (word[0] === "-") excludedTerms.add(word.substr(1)); + else { + searchTerms.add(word); + highlightTerms.add(queryTermLower); + } + }); + + if (SPHINX_HIGHLIGHT_ENABLED) { + // set in sphinx_highlight.js + localStorage.setItem( + "sphinx_highlight_terms", + [...highlightTerms].join(" "), + ); + } + + // console.debug("SEARCH: searching for:"); + // console.info("required: ", [...searchTerms]); + // console.info("excluded: ", [...excludedTerms]); + + // array of [docname, title, anchor, descr, score, filename] + let results = []; + _removeChildren(document.getElementById("search-progress")); + + const queryLower = query.toLowerCase(); + for (const [title, foundTitles] of Object.entries(allTitles)) { + if ( + title.toLowerCase().includes(queryLower) && + queryLower.length >= title.length / 2 + ) { + for (const [file, id] of foundTitles) { + let score = Math.round((100 * queryLower.length) / title.length); + results.push([ + docNames[file], + titles[file] !== title ? `${titles[file]} > ${title}` : title, + id !== null ? "#" + id : "", + null, + score, + filenames[file], + ]); + } + } + } + + // search for explicit entries in index directives + for (const [entry, foundEntries] of Object.entries(indexEntries)) { + if (entry.includes(queryLower) && queryLower.length >= entry.length / 2) { + for (const [file, id] of foundEntries) { + let score = Math.round((100 * queryLower.length) / entry.length); + results.push([ + docNames[file], + titles[file], + id ? "#" + id : "", + null, + score, + filenames[file], + ]); + } + } + } + + // lookup as object + objectTerms.forEach((term) => + results.push(...Search.performObjectSearch(term, objectTerms)), + ); + + // lookup as search terms in fulltext + results.push(...Search.performTermsSearch(searchTerms, excludedTerms)); + + // let the scorer override scores with a custom scoring function + if (Scorer.score) results.forEach((item) => (item[4] = Scorer.score(item))); + + // now sort the results by score (in opposite order of appearance, since the + // display function below uses pop() to retrieve items) and then + // alphabetically + results.sort((a, b) => { + const leftScore = a[4]; + const rightScore = b[4]; + if (leftScore === rightScore) { + // same score: sort alphabetically + const leftTitle = a[1].toLowerCase(); + const rightTitle = b[1].toLowerCase(); + if (leftTitle === rightTitle) return 0; + return leftTitle > rightTitle ? -1 : 1; // inverted is intentional + } + return leftScore > rightScore ? 1 : -1; + }); + + // remove duplicate search results + // note the reversing of results, so that in the case of duplicates, the highest-scoring entry is kept + let seen = new Set(); + results = results.reverse().reduce((acc, result) => { + let resultStr = result + .slice(0, 4) + .concat([result[5]]) + .map((v) => String(v)) + .join(","); + if (!seen.has(resultStr)) { + acc.push(result); + seen.add(resultStr); + } + return acc; + }, []); + + results = results.reverse(); + + // for debugging + //Search.lastresults = results.slice(); // a copy + // console.info("search results:", Search.lastresults); + + // print the results + _displayNextItem(results, results.length, searchTerms, highlightTerms); + }, + + /** + * search for object names + */ + performObjectSearch: (object, objectTerms) => { + const filenames = Search._index.filenames; + const docNames = Search._index.docnames; + const objects = Search._index.objects; + const objNames = Search._index.objnames; + const titles = Search._index.titles; + + const results = []; + + const objectSearchCallback = (prefix, match) => { + const name = match[4]; + const fullname = (prefix ? prefix + "." : "") + name; + const fullnameLower = fullname.toLowerCase(); + if (fullnameLower.indexOf(object) < 0) return; + + let score = 0; + const parts = fullnameLower.split("."); + + // check for different match types: exact matches of full name or + // "last name" (i.e. last dotted part) + if (fullnameLower === object || parts.slice(-1)[0] === object) + score += Scorer.objNameMatch; + else if (parts.slice(-1)[0].indexOf(object) > -1) + score += Scorer.objPartialMatch; // matches in last name + + const objName = objNames[match[1]][2]; + const title = titles[match[0]]; + + // If more than one term searched for, we require other words to be + // found in the name/title/description + const otherTerms = new Set(objectTerms); + otherTerms.delete(object); + if (otherTerms.size > 0) { + const haystack = `${prefix} ${name} ${objName} ${title}`.toLowerCase(); + if ( + [...otherTerms].some((otherTerm) => haystack.indexOf(otherTerm) < 0) + ) + return; + } + + let anchor = match[3]; + if (anchor === "") anchor = fullname; + else if (anchor === "-") anchor = objNames[match[1]][1] + "-" + fullname; + + const descr = objName + _(", in ") + title; + + // add custom score for some objects according to scorer + if (Scorer.objPrio.hasOwnProperty(match[2])) + score += Scorer.objPrio[match[2]]; + else score += Scorer.objPrioDefault; + + results.push([ + docNames[match[0]], + fullname, + "#" + anchor, + descr, + score, + filenames[match[0]], + ]); + }; + Object.keys(objects).forEach((prefix) => + objects[prefix].forEach((array) => objectSearchCallback(prefix, array)), + ); + return results; + }, + + /** + * search for full-text terms in the index + */ + performTermsSearch: (searchTerms, excludedTerms) => { + // prepare search + const terms = Search._index.terms; + const titleTerms = Search._index.titleterms; + const filenames = Search._index.filenames; + const docNames = Search._index.docnames; + const titles = Search._index.titles; + + const scoreMap = new Map(); + const fileMap = new Map(); + + // perform the search on the required terms + searchTerms.forEach((word) => { + const files = []; + const arr = [ + { files: terms[word], score: Scorer.term }, + { files: titleTerms[word], score: Scorer.title }, + ]; + // add support for partial matches + if (word.length > 2) { + const escapedWord = _escapeRegExp(word); + Object.keys(terms).forEach((term) => { + if (term.match(escapedWord) && !terms[word]) + arr.push({ files: terms[term], score: Scorer.partialTerm }); + }); + Object.keys(titleTerms).forEach((term) => { + if (term.match(escapedWord) && !titleTerms[word]) + arr.push({ files: titleTerms[word], score: Scorer.partialTitle }); + }); + } + + // no match but word was a required one + if (arr.every((record) => record.files === undefined)) return; + + // found search word in contents + arr.forEach((record) => { + if (record.files === undefined) return; + + let recordFiles = record.files; + if (recordFiles.length === undefined) recordFiles = [recordFiles]; + files.push(...recordFiles); + + // set score for the word in each file + recordFiles.forEach((file) => { + if (!scoreMap.has(file)) scoreMap.set(file, {}); + scoreMap.get(file)[word] = record.score; + }); + }); + + // create the mapping + files.forEach((file) => { + if (fileMap.has(file) && fileMap.get(file).indexOf(word) === -1) + fileMap.get(file).push(word); + else fileMap.set(file, [word]); + }); + }); + + // now check if the files don't contain excluded terms + const results = []; + for (const [file, wordList] of fileMap) { + // check if all requirements are matched + + // as search terms with length < 3 are discarded + const filteredTermCount = [...searchTerms].filter( + (term) => term.length > 2, + ).length; + if ( + wordList.length !== searchTerms.size && + wordList.length !== filteredTermCount + ) + continue; + + // ensure that none of the excluded terms is in the search result + if ( + [...excludedTerms].some( + (term) => + terms[term] === file || + titleTerms[term] === file || + (terms[term] || []).includes(file) || + (titleTerms[term] || []).includes(file), + ) + ) + break; + + // select one (max) score for the file. + const score = Math.max(...wordList.map((w) => scoreMap.get(file)[w])); + // add result to the result list + results.push([ + docNames[file], + titles[file], + "", + null, + score, + filenames[file], + ]); + } + return results; + }, + + /** + * helper function to return a node containing the + * search summary for a given text. keywords is a list + * of stemmed words. + */ + makeSearchSummary: (htmlText, keywords) => { + const text = Search.htmlToText(htmlText); + if (text === "") return null; + + const textLower = text.toLowerCase(); + const actualStartPosition = [...keywords] + .map((k) => textLower.indexOf(k.toLowerCase())) + .filter((i) => i > -1) + .slice(-1)[0]; + const startWithContext = Math.max(actualStartPosition - 120, 0); + + const top = startWithContext === 0 ? "" : "..."; + const tail = startWithContext + 240 < text.length ? "..." : ""; + + let summary = document.createElement("p"); + summary.classList.add("context"); + summary.textContent = + top + text.substr(startWithContext, 240).trim() + tail; + + return summary; + }, +}; + +_ready(Search.init); diff --git a/docs/_build/html/_static/sphinx_highlight.js b/docs/_build/html/_static/sphinx_highlight.js new file mode 100644 index 00000000..1aa8a45d --- /dev/null +++ b/docs/_build/html/_static/sphinx_highlight.js @@ -0,0 +1,155 @@ +/* Highlighting utilities for Sphinx HTML documentation. */ +"use strict"; + +const SPHINX_HIGHLIGHT_ENABLED = true; + +/** + * highlight a given string on a node by wrapping it in + * span elements with the given class name. + */ +const _highlight = (node, addItems, text, className) => { + if (node.nodeType === Node.TEXT_NODE) { + const val = node.nodeValue; + const parent = node.parentNode; + const pos = val.toLowerCase().indexOf(text); + if ( + pos >= 0 && + !parent.classList.contains(className) && + !parent.classList.contains("nohighlight") + ) { + let span; + + const closestNode = parent.closest("body, svg, foreignObject"); + const isInSVG = closestNode && closestNode.matches("svg"); + if (isInSVG) { + span = document.createElementNS("http://www.w3.org/2000/svg", "tspan"); + } else { + span = document.createElement("span"); + span.classList.add(className); + } + + span.appendChild(document.createTextNode(val.substr(pos, text.length))); + const rest = document.createTextNode(val.substr(pos + text.length)); + parent.insertBefore(span, parent.insertBefore(rest, node.nextSibling)); + node.nodeValue = val.substr(0, pos); + /* There may be more occurrences of search term in this node. So call this + * function recursively on the remaining fragment. + */ + _highlight(rest, addItems, text, className); + + if (isInSVG) { + const rect = document.createElementNS( + "http://www.w3.org/2000/svg", + "rect", + ); + const bbox = parent.getBBox(); + rect.x.baseVal.value = bbox.x; + rect.y.baseVal.value = bbox.y; + rect.width.baseVal.value = bbox.width; + rect.height.baseVal.value = bbox.height; + rect.setAttribute("class", className); + addItems.push({ parent: parent, target: rect }); + } + } + } else if (node.matches && !node.matches("button, select, textarea")) { + node.childNodes.forEach((el) => _highlight(el, addItems, text, className)); + } +}; +const _highlightText = (thisNode, text, className) => { + let addItems = []; + _highlight(thisNode, addItems, text, className); + addItems.forEach((obj) => + obj.parent.insertAdjacentElement("beforebegin", obj.target), + ); +}; + +/** + * Small JavaScript module for the documentation. + */ +const SphinxHighlight = { + /** + * highlight the search words provided in localstorage in the text + */ + highlightSearchWords: () => { + if (!SPHINX_HIGHLIGHT_ENABLED) return; // bail if no highlight + + // get and clear terms from localstorage + const url = new URL(window.location); + const highlight = + localStorage.getItem("sphinx_highlight_terms") || + url.searchParams.get("highlight") || + ""; + localStorage.removeItem("sphinx_highlight_terms"); + url.searchParams.delete("highlight"); + window.history.replaceState({}, "", url); + + // get individual terms from highlight string + const terms = highlight + .toLowerCase() + .split(/\s+/) + .filter((x) => x); + if (terms.length === 0) return; // nothing to do + + // There should never be more than one element matching "div.body" + const divBody = document.querySelectorAll("div.body"); + const body = divBody.length ? divBody[0] : document.querySelector("body"); + window.setTimeout(() => { + terms.forEach((term) => _highlightText(body, term, "highlighted")); + }, 10); + + const searchBox = document.getElementById("searchbox"); + if (searchBox === null) return; + searchBox.appendChild( + document + .createRange() + .createContextualFragment( + '", + ), + ); + }, + + /** + * helper function to hide the search marks again + */ + hideSearchWords: () => { + document + .querySelectorAll("#searchbox .highlight-link") + .forEach((el) => el.remove()); + document + .querySelectorAll("span.highlighted") + .forEach((el) => el.classList.remove("highlighted")); + localStorage.removeItem("sphinx_highlight_terms"); + }, + + initEscapeListener: () => { + // only install a listener if it is really needed + if (!DOCUMENTATION_OPTIONS.ENABLE_SEARCH_SHORTCUTS) return; + + document.addEventListener("keydown", (event) => { + // bail for input elements + if (BLACKLISTED_KEY_CONTROL_ELEMENTS.has(document.activeElement.tagName)) + return; + // bail with special keys + if (event.shiftKey || event.altKey || event.ctrlKey || event.metaKey) + return; + if ( + DOCUMENTATION_OPTIONS.ENABLE_SEARCH_SHORTCUTS && + event.key === "Escape" + ) { + SphinxHighlight.hideSearchWords(); + event.preventDefault(); + } + }); + }, +}; + +_ready(() => { + /* Do not call highlightSearchWords() when we are on the search page. + * It will highlight words from the *previous* search query. + */ + if (typeof Search === "undefined") SphinxHighlight.highlightSearchWords(); + SphinxHighlight.initEscapeListener(); +}); diff --git a/docs/_build/html/genindex.html b/docs/_build/html/genindex.html new file mode 100644 index 00000000..6eaa9e5a --- /dev/null +++ b/docs/_build/html/genindex.html @@ -0,0 +1,82 @@ + + + + + + + Index — DataFog v2.3.1b1 documentation + + + + + + + + + + + +
+
+
+
+

Index

+ +
+
+
+
+ +
+
+ + + diff --git a/docs/_build/html/index.html b/docs/_build/html/index.html new file mode 100644 index 00000000..08ab21db --- /dev/null +++ b/docs/_build/html/index.html @@ -0,0 +1,130 @@ + + + + + + + + + + Welcome to DataFog’s documentation! — DataFog v2.3.1b1 documentation + + + + + + + + + + + + +
+
+
+
+
+

+ Welcome to DataFog’s documentation! +

+
+
+
+

+ Indices and tables +

+ +
+
+
+
+ +
+
+ + + diff --git a/docs/_build/html/objects.inv b/docs/_build/html/objects.inv new file mode 100644 index 00000000..c8a7b89f Binary files /dev/null and b/docs/_build/html/objects.inv differ diff --git a/docs/_build/html/search.html b/docs/_build/html/search.html new file mode 100644 index 00000000..3339787c --- /dev/null +++ b/docs/_build/html/search.html @@ -0,0 +1,94 @@ + + + + + + + Search — DataFog v2.3.1b1 documentation + + + + + + + + + + + + + + + +
+
+
+
+

Search

+ + + +

+ Searching for multiple words only shows matches that contain all + words. +

+ +
+ + + +
+ +
+
+
+
+ +
+
+ + + diff --git a/docs/_build/html/searchindex.js b/docs/_build/html/searchindex.js new file mode 100644 index 00000000..ba441ddc --- /dev/null +++ b/docs/_build/html/searchindex.js @@ -0,0 +1,30 @@ +Search.setIndex({ + docnames: ["index"], + filenames: ["index.rst"], + titles: ["Welcome to DataFog\u2019s documentation!"], + terms: { index: 0, modul: 0, search: 0, page: 0 }, + objects: {}, + objtypes: {}, + objnames: {}, + titleterms: { welcom: 0, datafog: 0, "": 0, document: 0, indic: 0, tabl: 0 }, + envversion: { + "sphinx.domains.c": 3, + "sphinx.domains.changeset": 1, + "sphinx.domains.citation": 1, + "sphinx.domains.cpp": 9, + "sphinx.domains.index": 1, + "sphinx.domains.javascript": 3, + "sphinx.domains.math": 2, + "sphinx.domains.python": 4, + "sphinx.domains.rst": 2, + "sphinx.domains.std": 2, + sphinx: 60, + }, + alltitles: { + "Welcome to DataFog\u2019s documentation!": [ + [0, "welcome-to-datafog-s-documentation"], + ], + "Indices and tables": [[0, "indices-and-tables"]], + }, + indexentries: {}, +}); diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 00000000..73bea360 --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,27 @@ +# Configuration file for the Sphinx documentation builder. +# +# For the full list of built-in configuration values, see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Project information ----------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information + +project = "DataFog" +copyright = "2024, Sid Mohan" +author = "Sid Mohan" +release = "v2.3.1b1" + +# -- General configuration --------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration + +extensions = ["sphinx.ext.autodoc"] + +templates_path = ["_templates"] +exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] + + +# -- Options for HTML output ------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output + +html_theme = "alabaster" +html_static_path = ["_static"] diff --git a/docs/index.rst b/docs/index.rst new file mode 100644 index 00000000..9971b10c --- /dev/null +++ b/docs/index.rst @@ -0,0 +1,20 @@ +.. DataFog documentation master file, created by + sphinx-quickstart on Tue Mar 12 16:44:44 2024. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Welcome to DataFog's documentation! +=================================== + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + + + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/docs/make.bat b/docs/make.bat new file mode 100644 index 00000000..32bb2452 --- /dev/null +++ b/docs/make.bat @@ -0,0 +1,35 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=. +set BUILDDIR=_build + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.https://www.sphinx-doc.org/ + exit /b 1 +) + +if "%1" == "" goto help + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% + +:end +popd diff --git a/setup.py b/setup.py index 4291b2a7..2cb3c865 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ setup( name="datafog", - version="2.2.0", + version="2.3.1", author="Sid Mohan", author_email="sid@datafog.ai", description="Scan, redact, and manage PII in your documents before they get uploaded to a Retrieval Augmented Generation (RAG) system.", diff --git a/sotu_2023.txt b/sotu_2023.txt deleted file mode 100644 index 0ccef318..00000000 --- a/sotu_2023.txt +++ /dev/null @@ -1,667 +0,0 @@ -Mr. Speaker, Madam Vice President, our First Lady and Second Gentleman — good to see you guys up there — members of Congress — - -And, by the way, Chief Justice, I may need a court order. She gets to go to the game tomorr- — next week. I have to stay home. We got to work something out here. - -Members of the Cabinet, leaders of our military, Chief Justice, Associate Justices, and retired Justices of the Supreme Court, and to you, my fellow Americans: - -You know, I start tonight by congratulating the 118th Congress and the new Speaker of the House, Kevin McCarthy. - -Speaker, I don’t want to ruin your reputation, but I look forward to working with you. - -And I want to congratulate the new Leader of the House Democrats, the first African American Minority Leader in history, Hakeem Jeffries. - -He won despite the fact I campaigned for him. - -Congratulations to the longest-serving Leader in the history of the United States Senate, Mitch McConnell. Where are you, Mitch? - -And congratulations to Chuck Schumer, another — you know, another term as Senate Minority [Majority] Leader. You know, I think you — only this time you have a slightly bigger majority, Mr. Leader. And you’re the Majority Leader. About that much bigger? Yeah. - -Well, I tell you what — I want to give specolec- — special recognition to someone who I think is going to be considered the greatest Speaker in the history of the House of Representatives: Nancy Pelosi. - -Folks, the story of America is a story of progress and resilience, of always moving forward, of never, ever giving up. It’s a story unique among all nations. - -We’re the only country that has emerged from every crisis we’ve ever entered stronger than we got into it. - -Look, folks, that’s what we’re doing again. - -Two years ago, the economy was reeling. I stand here tonight, after we’ve created, with the help of many people in this room, 12 million new jobs — more jobs created in two years than any President has created in four years — because of you all, because of the American people. - -Two years ago — and two years ago, COVID had shut down — our businesses were closed, our schools were robbed of so much. And today, COVID no longer controls our lives. - -And two years ago, our democracy faced its greatest threat since the Civil War. And today, though bruised, our democracy remains unbowed and unbroken. - -As we gather here tonight, we’re writing the next chapter in the great American story — a story of progress and resilience. - -When world leaders ask me to define America — and they do, believe it or not — I say I can define it in one word, and I mean this: possibilities. We don’t think anything is beyond our capacity. Everything is a possibility. - -You know, we’re often told that Democrats and Republicans can’t work together. But over the past two years, we proved the cynics and naysayers wrong. - -Yes, we disagreed plenty. And yes, there were times when Democrats went alone. - -But time and again, Democrats and Republicans came together. Came together to defend a stronger and safer Europe. You came together to pass one in a gen- — one-in-a-generation — once-in-a-generation infrastructure law building bridges connecting our nation and our people. We came together to pass one the most significant law ever helping victims exposed to toxic burn pits. And, in fact — it’s important. - -And, in fact, I signed over 300 bipartisan pieces of legislation since becoming President, from reauthorizing the Violence Against Women Act to the Electoral Count Reform Act, the Respect for Marriage Act that protects the right to marry the person you love. - -And to my Republican friends, if we could work together in the last Congress, there’s no reason we can’t work together and find consensus on important things in this Congress as well. - -I think — folks, you all are just as informed as I am, but I think the people sent us a clear message: Fighting for the sake of fighting, power for the sake of power, conflict for the sake of conflict gets us nowhere. - -That’s always been my vision of our country, and I know it’s many of yours: to restore the soul of this nation; to rebuild the backbone of America, America’s middle class; and to unite the country. - -That’s always been my vision for the country. To restore the soul of the nation. To rebuild the backbone of America - the middle class. To unite the country. - -We’ve been sent here to finish the job, in my view. - -For decades, the middle class has been hollowed out in more than — and not in one administration, but for a long time. Too many good-paying manufacturing jobs moved overseas. Factories closed down. Once-thriving cities and towns that many of you represent became shadows of what they used to be. And along the way, something else we lost: pride, our sense of self-worth. - -I ran for President to fundamentally change things. To make sure the economy works for everyone so we can all feel that pride in what we do. To build an economy from the bottom up and the middle out, not from the top down. Because when the middle class does well, the poor have a ladder up and the wealthy still do very well. We all do well. - -I know a lot of you always kid me for always quoting my dad. But my dad used to say, “Joey, a job is about a lot more than a paycheck.” He really would say this. “It’s about a lot more than a paycheck. It’s about your dignity. It’s about respect. It’s about being able to look your kid in the eye and say, ‘Honey, it’s going to be okay’ and mean it.” - -Well, folks, so let’s look at the results. We’re not finished yet, by any stretch of the imagination. But unemployment rate is at 3.4 percent –- a 50-year low. And near record — and near record unemployment — near record unemployment for Black and Hispanic workers. - -We’ve already created, with your help, 800,000 good-paying manufacturing jobs — the fastest growth in 40 years. - -And where is it written — where is it written that America can’t lead the world in manufacturing? And I don’t know where that’s written. - -For too many decades, we imported projects and exported jobs. Now, thanks to what you’ve all done, we’re exporting American products and creating American jobs. - -Folks, inflation — inflation has been a global problem because the pandemic dirup- — disrupted our supply chains, and Putin’s unfair and brutal war in Ukraine disrupted ener- — energy supplied as well as food supplies, blocking all that grain in Ukraine. - -But we’re better positioned than any country on Earth right now. But we have more to do. - -But here at home, inflation is coming down. Here at home, gas prices are down $1.50 from their peak. - -Food inflation is coming down — not fast enough, but coming down. - -Inflation has fallen every month for the last six months, while take-home pay has gone up. - -Additionally, over the last two years, a record 10 million Americans applied to start new businesses. Ten million. - -And, by the way, every time — every time someone starts a small business, it’s an act of hope. - -And, Madam Vice President, I want to thank you for leading that effort to ensure that small businesses have access to capital and the historic laws we enacted that are going to just come into being. - -Standing here last year, I shared with you a story of American genius and possibilities. - -Semiconductors — small computer chips the size of a fingerprint that power everything from cellphones to automobiles and so much more. These chips were invented in America. Let’s get that straight: They were invented in America. - -And we used to make 40 percent of the world’s chips. In the last several decades, we lost our edge. We’re down to only producing 10 percent. - -We all saw what happened during the pandemic when chip factories shut down overseas. - -Today’s automobiles need 3,000 chips — each of those automobiles — but American automobiles [automakers] couldn’t make enough cars because there weren’t enough chips. - -Car prices went up. People got laid off. So did everything from refrigerators to cellphones. - -We can never let that happen again. - -That’s why — that’s why we came together to pass the bipartisan CHIPS and Science Act. - -Folks, I know I’ve been criticized for saying this, but I’m not changing my view. We’re going to make sure the supply chain for America begins in America — the supply chain begins in America. - -And we’ve already created — we’ve already created 800,000 new manufacturing jobs without this law, before the law kicks in. - -With this new law, we’re going to create hundreds of thousands of new jobs across the country. And I mean all across the country, throughout — not just the coast, but through the middle of the country as well. - -That’s going to come from companies that have announced more than $300 billion in investments in American manufacturing over the next few years. - -Outside of Columbus, Ohio, Intel is building semiconductor factories on a thousand acres — literally a field of dreams. - -It’s going to create 10,000 jobs, that one investment; 7,000 construction jobs; 3,000 jobs in those factories once they’re finished. They call them factors. Jobs paying an average of $130,000 a year, and many do not require a college degree. - -Jobs — because we worked together, these jobs where people don’t have to leave home to search for opportunity. - -And it’s just getting started. - -Think about the new homes, the small businesses, the big — the medium-sized businesses. So much more that’s going to be needed to support those three thou- — those 3,000 permanent jobs and the factories that are going to be built. - -Talk to mayors and governors, Democrats and Republicans, and they’ll tell you what this means for their communities. - -We’re seeing these fields of dreams transform the Heartland. But to maintain the strongest economy in the world, we need the best infrastructure in the world. - -And, folks, as you all know, we used to be number one in the world in infrastructure. We’ve sunk to 13th in the world. The United States of America — 13th in the world in infrastructure, modern infrastructure. - -But now we’re coming back because we came together and passed the Bipartisan Infrastructure Law — the largest investment in infrastructure since President Eisenhower’s Interstate Highway System. - -Folks, already we’ve funded over 20,000 projects, including major airports from Boston to Atlanta to Portland — projects that are going to put thousands of people to work rebuilding our highways, our bridges, our railroads, our tunnels, ports, airports, clean water, high-speed Internet all across America — urban, rural, Tribal. - -And, folks, we’re just getting started. We’re just getting started. - -And I mean this sincerely: I want to thank my Republican friends who voted for the law. And my Republican friends who voted against it as well — but I’m still — I still get asked to fund the projects in those districts as well, but don’t worry. I promised I’d be a President for all Americans. We’ll fund these projects. And I’ll see you at the groundbreaking. - -Look, this law — this law will further unite all of America. - -Projects like the Brent Spence Bridge in Kentucky over the Ohio River. Built 60 years ago. Badly in need of repairs. One of the nation’s most congested freight routes, carrying $2 billion worth of freight every single day across the Ohio River. - -And, folks, we’ve been talking about fixing it for decades, but we’re really finally going to get it done. - -I went there last month with Democrats and Republicans in — from both states — to deliver a commitment of $1.6 billion for this project. - -And while I was there, I met a young woman named Saria, who’s here tonight. I don’t know where Saria is. Is she up in the box? I don’t know. Saria, how are you? - -Well, Saria — for 30 years — for 30 years — I learned — she told me she’d been a proud member of the Iron workers Local 44, known as — — known as the “Cowboys in the Sky” — — the folks who built — who built Cincinnati’s skyline. - -Saria said she can’t wait to be 10 stories above the Ohio River building that new bridge. God bless her. That’s pride. - -And that’s what we’re also building — we’re building back pride. - -Look, we’re also replacing poisonous lead pipes that go into 10 million homes in America, 400,000 schools and childcare centers so every child in America — every child in American can drink the water, instead of having permanent damage to their brain. - -Look, we’re making sure — — we’re making sure that every community — every community in America has access to affordable, high-speed Internet. - -No parent should have to drive by a McDonald’s parking lot to help their — do their homework online with their kids, which many — thousands were doing across the country. - -And when we do these projects — and, again, I get criticized about this, but I make no excuses for it — we’re going to buy American. We’re going to buy American. - -Folks — — and it’s totally — it’s totally consistent with international trade rules. Buy American has been the law since 1933. But for too long, past administrations — Democrat and Republican — have fought to get around it. Not anymore. - -Tonight, I’m also announcing new standards to require all construction materials used in federal infra- — infrastructure projects to be made in America. Made in America. I mean it. Lumber, glass, drywall, fiber-optic cable. - -And on my watch, American roads, bridges, and American highways are going to be made with American products as well. - -Folks, my economic plan is about investing in places and people that have been forgotten. So many of you listening tonight, I know you feel it. So many of you felt like you’ve just simply been forgotten. Amid the economic upheaval of the past four decades, too many people have been left behind and treated like they’re invisible. - -Maybe that’s you, watching from home. You remember the jobs that went away. You remember them, don’t you? - -The folks at home remember them. You wonder whether the path even exists anymore for your children to get ahead without having to move away. - -Well, that’s why — I get that. That’s why we’re building an economy where no one is left behind. - -Jobs are coming back, pride is coming back because of choices we made in the last several years. - -You know, this is, in my view, a blue-collar blueprint to rebuild America and make a real difference in your lives at home. - -For example, too many of you lay in bed at night, like my dad did, staring at the ceiling, wondering what in God’s name happens if yo- — if your spouse gets cancer or your child gets deadly ill or if something happens to you. What are you going — are you going to have the money to pay for those medical bills? Are you going to have to sell the house or try to get a second mortgage on it? - -I get it. I get it. - -With the Inflation Reduction Act that I signed into law, we’re taking on powerful interests to bring healthcare costs down so you can sleep better at night with more security. - -You know, we pay more for prescription drugs than any nation in the world. Let me say it again: We pay more for prescription drugs than any major nation on Earth. - -For example, 1 in 10 Americans has diabetes. Many of you in this chamber do and in the audience. But every day, millions need insulin to control their diabetes so they can literally stay alive. Insulin has been around for over 100 years. The guy who invented it didn’t even patent it because he wanted it to be available for everyone. - -It costs the drug companies roughly $10 a vial to make that insulin. Package it and all, you may get up to $13. But Big Pharma has been unfairly charging people hundreds of dollars — $4- to $500 a month — making rec- — record profits. Not anymore. Not anymore. - -So — so many things that we did are only now coming to fruition. We said we were doing this and we said we’d pass the law to do it, but people didn’t know because the law didn’t take effect until January 1 of this year. - -We capped the cost of insulin at $35 a month for seniors on Medicare. But people are just finding out. I’m sure you’re getting the same calls I’m getting. - -We capped insulin for seniors at $35 per month. It’s time to do it for everyone. - -Look, there are millions of other Americans who do not — are not on Medicare, including 200,000 young people with Type 1 diabetes who need these insulin — need this insulin to stay alive. - -Let’s finish the job this time. Let’s cap the cost of insulin for everybody at $35. - -Folks — and Big Pharma is still going to do very well, I promise you all. I promise you they’re going to do very well. - -This law also — this law also caps — and it won’t even go into effect until 2025. It costs [caps] out-of-pocket drug costs for seniors on Medicare at a maximum of $2,000 a year. You don’t have to pay more than $2,000 a year, no matter how much your drug costs are. Because you know why? You all know it. - -Many of you, like many of my family, have cancer. You know the drugs can range from $10-, $11-, $14-, $15,000 for the cancer drugs. - -And if drug prices rise faster than inflation, drug companies are going to have to pay Medicare back the difference. - -And we’re finally — we’re finally giving Medicare the power to negotiate drug prices. - -Bringing down — bringing down prescription drug costs doesn’t just save seniors money, it cuts the federal deficit by billions of dollars — — by hundreds of billions of dollars because these prescription drugs are drugs purchased by Medicare to make — keep their commitment to the seniors. - -Well, guess what? Instead of paying 4- or 500 bucks a month, you’re paying 15. That’s a lot of savings for the federal government. - -And, by the way, why wouldn’t we want that? - -Now, some members here are threatening — and I know it’s not an official party position, so I’m not going to exaggerate — but threatening to repeal the Inflation Reduction Act. - -As my coach — that’s okay. That’s fair. As my football coach used to say, “Lots of luck in your senior year.” - -Make no mistake, if you try anything to raise the cost of prescription drugs, I will veto it. - -And, look, I’m pleased to say that more Americans health — have health insurance now than ever in history. A record 16 million people are enrolled in the Affordable Care Act. - -And thanks — thanks to the law I signed last year, saving — millions are saving $800 a year on their premiums. - -And, by the way, that law was written — and the benefit expires in 2025. So, my plea to some of you, at least in this audience: Let’s finish the job and make those savings permanent. Expand coverage on Medicaid. - -Look, the Inflation Reduction Act is also the most significant investment ever in climate change — ever. Lowering utility bills, creating American jobs, leading the world to a clean energy future. - -I visited the devastating aftermath of record floods, droughts, storms, and wildfires from Arizona to New Mexico to all the way up to the Canadian border. - -More timber has been burned that I’ve observed from helicopters than the entire state of Missouri. And we don’t have global warming? Not a problem. - -In addition to emergency recovery from Puerto Rico to Florida to Idaho, we’re rebuilding for the long term. - -New electric grids that are able to weather major storms and not — prevent those fire — forest fires. Roads and water systems to withstand the next big flood. Clean energy to cut pollution and create jobs in communities often left behind. - -We’re going to build 500,000 electric vehicle charging stations, installed across the country by tens of thousands of IBEW workers. - -And we’re helping families save more than $1,000 a year with tax credits to purchase of electric vehicles and efficient — and efficient appliances — energy-efficient appliances. - -Historic conservation efforts to be responsible stewards of our land. - -Let’s face reality. The climate crisis doesn’t care if you’re in a red or a blue state. It’s an existential threat. - -We have an obligation not to ourselves, but to our children and grandchildren to confront it. - -I’m proud of how the — how America, at last, is stepping up to the challenge. We’re still going to need oil and gas for a while, but guess what — — no, we do — but there’s so much more to do. We got to finish the job. - -And we pay for these investments in our future by finally making the wealthiest and biggest corporations begin to pay their fair share. Just begin. - -Look, I’m a capitalist. I’m a capitalist. But pay your fair share. - -I think a lot of you at home — a lot of you at home agree with me and many people that you know: The tax system is not fair. It is not fair. - -Look, the idea that in 2020, 55 of the largest corporations in America, the Fortune 500, made $40 billion in profits and paid zero in federal taxes? Zero. - -Folks, it’s simply not fair. - -But now, because of the law I signed, billion-dollar companies have to pay a minimum of 15 percent. God love them. Fifteen percent. That’s less than a nurse pays. - -Let me be crystal clear. I said at the very beginning: Under my plans, as long as I’m President, nobody earning less than $400,000 will pay an additional penny in taxes. Nobody. Not one penny. - -But let’s finish the job. There’s more to do. - -We have to reward work, not just wealth. Pass my proposal for the billionaire minimum tax. You know, there’s a thousand billionaires in America — it’s up from about 600 at the beginning of my term — but no billionaire should be paying a lower tax rate than a school teacher or a firefighter. No, I mean it. Think about it. - -We made every wealthy corporation pay a minimum tax. It’s time to do the same for billionaires. - -I mean, look, I know you all aren’t enthusiastic about that, but think about it. Think about it. - -Have you noticed — Big Oil just reported its profits. Record profits. Last year, they made $200 billion in the midst of a global energy crisis. I think it’s outrageous. - -Why? They invested too little of that profit to increase domestic production. And when I talked to a couple of them, they say, “We were afraid you were going to shut down all the oil wells and all the oil refineries anyway, so why should we invest in them?” I said, “We’re going to need oil for at least another decade, and that’s going to exceed…” — and beyond that. We’re going to need it. Production. - -If they had, in fact, invested in the production to keep gas prices down — instead they used the record profits to buy back their own stock, rewarding their CEOs and shareholders. - -Corporations ought to do the right thing. - -That’s why I propose we quadruple the tax on corporate stock buybacks and encourage long- — — long-term investments. They’ll still make considerable profit. - -Let’s finish the job and close the loopholes that allow the very wealthy to avoid paying their taxes. - -Instead of cutting the number of audits for wealthy taxpayers, I just signed a law to reduce the deficit by $114 billion by cracking down on wealthy tax cheats. That’s being fiscally responsible. - -In the last two years, my administration has cut the deficit by more than $1.7 trillion –- the largest deficit reduction in American history. - -Under the previous administration, the American deficit went up four years in a row. - -Because of those record deficits, no President added more to the national debt in any four years than my predecessor. - -Nearly 25 percent of the entire national debt that took over 200 years to accumulate was added by just one administration alone — the last one. They’re the facts. Check it out. Check it out. - -How did Congress respond to that debt? They did the right thing. They lifted the debt ceiling three times without preconditions or crisis. They paid the American bill to prevent an economic disaster of the country. - -So, tonight I’m asking the Congress to follow suit. Let us commit here tonight that the full faith and credit of the United States of America will never, ever be questioned. - -So my — many of — some of my Republican friends want to take the economy hostage — I get it — unless I agree to their economic plans. All of you at home should know what those plans are. - -Instead of making the wealthy pay their fair share, some Republicans — some Republicans want Medicare and Social Security to sunset. I’m not saying it’s a majority — - -Let me give you — - -Anybody who doubts it, contact my office. I’ll give you a copy. I’ll give you a copy of the proposal. - -That means Congress doesn’t vote — - -Well, I’m glad to see — no, I tell you, I enjoy conversion. - -You know, it means if Congress doesn’t keep the programs the way they are, they’d go away. - -Other Republicans say — I’m not saying it’s a majority of you. I don’t even think it’s a significant — - -— but it’s being proposed by individuals. - -I’m not — politely not naming them, but it’s being proposed by some of you. - -Look, folks, the idea is that we’re not going to be — we’re not going to be moved into being threatened to default on the debt if we don’t respond. - -Folks — so, folks, as we all apparently agree, Social Security and Medicare is off the — off the books now, right? They’re not to be touched? - -All right. All right. We got unanimity! Social Security and Medicare are a lifeline for millions of seniors. Americans have to pay into them from the very first paycheck they’ve started. - -So, tonight, let’s all agree — and we apparently are — let’s stand up for seniors. Stand up and show them we will not cut Social Security. We will not cut Medicare. - -President Biden wants to strengthen social security and medicare. House Republicans are threatening to cut them. - -Those benefits belong to the American people. They earned it. And if anyone tries to cut Social Security — which apparently no one is going to do — and if anyone tries to cut Medicare, I’ll stop them. I’ll veto it. - -And, look, I’m not going to allow them to take away — be taken away. Not today. Not tomorrow. Not ever. - -But apparently, it’s not going to be a problem. - -Next month, when I offer my fiscal plan, I ask my Republican friends to lay down their plan as well. I really mean it. Let’s sit down together and discuss our mutual plans together. Let’s do that. - -I can tell you, the plan I’m going to show you is going to cut the deficit by another $2 trillion. And it won’t cut a single bit of Medicare or Social Security. - -In fact, we’re going to extend the Medicare Trust Fund at least two decades, because that’s going to be the next argument: how do we make — keep it solvent. Right? - -Well, I will not raise taxes on anyone making under 400 grand. But we’ll pay for it the way we talked about tonight: by making sure that the wealthy and big corporations pay their fair share. - -Look — look, look, here’s — here’s the deal. They aren’t just taking advantage of the tax code, they’re taking advantage of you, the American consumer. - -Here’s my message to all of you out there: I have your back. We’re already preventing Americans who are [from] receiving surprise medical bills, stopping 1 billion dollar [1 million] surprise bills per month so far. - -We’re protecting seniors’ life savings by cracking down on nursing homes that commit fraud, endanger patient safety, or prescribe drugs that are not needed. - -Millions of Americans can now save thousands of dollars because they can finally get a hearing aid over the counter without a prescription. - -Look, capitalism without competition is not capitalism. It’s extortion. It’s exploitation. - -Last year, I cracked down, with the help of many of you, on foreign shipping companies that were making you pay higher prices for every good coming into the country. - -I signed a bipartisan bill that cut shipping costs by 90 percent, helping American farmers, businessmen, and consumers. - -Let’s finish the job. Pass the bipartisan legislation to strengthen and — to strengthen antitrust enforcement and forbeg — and prevent big online platforms from giving their own products an unfair advantage. - -My administration is also taking on junk fees, those hidden surcharges too many companies use to make you pay more. - -For example, we’re making airlines show you the full ticket price upfront, refund your money if your flight is cancelled or delayed. We’ve reduced exorbitant bank overdrafts by saving consumers more than $1 billion a year. - -We’re cutting credit card late fees by 75 percent, from $30 to $8. - -Look, junk fees may not matter to the very wealthy, but they matter to most other folks in homes like the one I grew up in, like many of you did. They add up to hundreds of dollars a month. They make it harder for you to pay your bills or afford that family trip. - -I know how unfair it feels when a company overcharges you and gets away with it. Not anymore. - -We’ve written a bill to stop it all. It’s called the Junk Fee Prevention Act. We’re going to ban surprise resort fees that hotels charge on your bill. Those fees can cost you up to $90 a night at hotels that aren’t even resorts. - -It’s time to end excessive serve fees for concert tickets. Pass the Junk Fee Prevention Act. - -We — the idea that cable, Internet, and cellphone companies can charge you $200 or more if you decide to switch to another provider. Give me a break. - -We can stop service fees on tickets to concerts and sporting events and make companies disclose all the fees upfront. - -And we’ll prohibit airlines from charging $50 roundtrip for a family just to be able to sit together. Baggage fees are bad enough. Airlines can’t treat your child like a piece of baggage. - -Americans are tired of being — we’re tired of being played for suckers. - -So pass — pass the Junk Fee Prevention Act so companies stop ripping us off. - -For too long, workers have been getting stiffed, but not anymore. We’re going to be — we’re beginning to restore the dignity of work. - -For example, I — I should have known this, but I didn’t until two years ago: Thirty million workers have to sign non-compete agreements for the jobs they take. Thirty million. So a cashier at a burger place can’t walk across town and take the same job at another burger place and make a few bucks more. - -It just changed. Well, they just changed it because we exposed it. That was part of the deal, guys. Look it up. But not anymore. - -We’re banning those agreements so companies have to compete for workers and pay them what they’re worth. - -And I must tell you, this is bound to get a response from my friends on my left, with the right. - -I’m so sick and tired of companies breaking the law by preventing workers from organizing. Pass the PRO Act! Because businesses have a right — workers have a right to form a union. And let’s guarantee all workers have a living wage. - -Let’s make sure working parents can afford to raise a family with sick days, paid family and medical leave, affordable childcare. That’s going to enable millions of more people to go and stay at work. - -And let’s restore the full Child Tax Credit — — which gave tens of millions of parents some breathing room and cut child poverty in half to the lowest level in history. - -And, by the way, when we do all of these things, we increase productivity, we increase economic growth. - -So let’s finish the job and get more families access to affordable, quality housing. - -Let’s get seniors who want to stay in their homes the care they need to do so. Let’s give more breathing room to millions of family caregivers looking after their loved ones. - -Pass my plan so we get seniors and people with disabilities the home care services they need — — and support the workers who are doing God’s work. - -These plans are fully paid for, and we can afford to do them. - -Restoring the dignity of work means making education an affordable ticket to the middle class. - -You know, when we made public education — 12 years of it — universal in the last century, we made the best-educated, best-paid — we became the best-education, best-paid nation in the world. - -But the rest of the world has caught up. It has caught up. - -Jill, my wife, who teaches full-time, has an expression. I hope I get it right, kid. “Any nation that out-educates is going to out-compete us.” Any nation that out-educates is going to out-compete us. - -Folks, we all know 12 years of education is not enough to win the economic competition of the 21st century. If we want to have the best-educated workforce, let’s finish the job by providing access to preschool for three and four years old. Studies show that children who go to preschool are nearly 50 percent more likely to finish high school and go on to earn a two- or four-year degree, no matter their background they came from. - -Let’s give public school teachers a raise. - -We’re making progress by reducing student debt, increasing Pell Grants for working and middle-class families. - -Let’s finish the job and connect students to career opportunities starting in high school, provide access to two years of community college — the best career training in America, in addition to being a pathway to a four-year degree. - -Let’s offer every American a path to a good career, whether they go to college or not. - -And, folks — folks, in the midst of the COVID crisis, when schools were closed and we were shutting down everything, let’s recognize how far we came in the fight against the pandemic itself. - -While the virus is not gone, thanks to the resilience of the American people and the ingenuity of medicine, we’ve broken the COVID grip on us. - -COVID deaths are down by 90 percent. We’ve saved millions of lives and opened up our country — we opened our country back up. And soon, we’ll end the public health emergency. - -But — that’s called a public health emergency. - -But we’ll remember the toll and pain that’s never going to go away. More than a million Americans lost their lives to COVID. A million. Families grieving. Children orphaned. Empty chairs at the dining room table constantly reminding you that she used to sit there. Remembering them, we remain vigilant. - -We still need to monitor dozens of variants and support new vaccines and treatments. So Congress needs to fund these efforts and keep America safe. - -And as we emerge from this crisis stronger, we’re also — got to double down prosecuting criminals who stole relief money meant to keep workers and small businesses afloat. - -Before I came to office, you remember, during that campaign, the big issue was about inspector generals who would protect taxpayers’ dollars, who were sidelined. They were fired. Many people said, “We don’t need them.” And fraud became rampant. - -Last year, I told you the watchdogs are back. Since then — since then, we’ve recovered billions of taxpayers’ dollars. - -Now let’s triple the anti-fraud strike force going after these criminals, double the statute of limitations on these crimes, and crack down on identity fraud by criminal syndicates stealing billions of dollars — billions of dollars from the American people. - -And the data shows that for every dollar we put into fighting fraud, the taxpayer will get back at least 10 times as much. It matters. It matters. - -Look, COVID left its scars, like the spike in violent crime in 2020 — the first year of the pandemic. We have an obligation to make sure all people are safe. - -Public safety depends on public trust, as all of us know. But too often, that trust is violated. - -Joining us tonight are the parents of Tyre Nichols — welcome — who had to bury Tyre last week. - -As many of you personally know, there’s no words to describe the heartache or grief of losing a child. But imagine — imagine if you lost that child at the hands of the law. Imagine having to worry whether your son or daughter came home from walking down the street or playing in the park or just driving a car. - -Most of us in here have never had to have “the talk” — “the talk” — that brown and Black parents have had to have with their children. - -Beau, Hunter, Ashley — my children — I never had to have the talk with them. I never had to tell them, “If a police officer pulls you over, turn your interior lights on right away. Don’t reach for your license. Keep your hands on the steering wheel.” - -Imagine having to worry like that every single time your kid got in a car. - -Here’s what Tyre’s mother shared with me when I spoke to her, when I asked her how she finds the courage to carry on and speak out. With the faith of God, she said her son was, quote, “a beautiful soul” and “something good will come of this.” - -Imagine how much courage and character that takes. - -It’s up to us, to all of us. We all want the same thing: neighborhoods free of violence, law enfircement [sic] — law enforcement who earns the community’s trust. Just as every cop, when they pin on that badge in the morning, has a right to be able to go home at night, so does everybody else out there. Our children have a right to come home safely. - -Equal protection under the law is a covenant we have with each other in America. - -We know police officers put their lives on the line every single night and day. And we know we ask them, in many cases, to do too much — to be counselors, social workers, psychologists — responding to drug overdoses, mental health crises, and so much more. In one sense, we ask much too much of them. - -I know most cops and their families are good, decent, honorable people — the vast majority. And they risk — and they risk their lives every time they put that shield on. - -But what happened to Tyre in Memphis happens too often. We have to do better. Give law enforcement the real training they need. Hold them to higher standards. Help them to succeed in keeping them safe. - -We also need more first responders and professionals to address the growing mental health, substance abuse challenges. More resources to reduce violent crime and gun crime. More community intervention programs. More investments in housing, education, and job training. All this can help prevent violence in the first place. - -And when police officers or police departments violate the public trust, they must be held accountable. - -With the support — with the support of families of victims, civil rights groups, and law enforcement, I signed an executive order for all federal officers, banning chokeholds, restricting no-knock warrants, and other key elements of the George Floyd Act. - -Let’s commit ourselves to make the words of Tyler’s [Tyre’s] mom true: Something good must come from this. Something good. - -And all of us — all of us — folks, it’s difficult, but it’s simple: All of us in the cha- — in this chamber, we need to rise to this moment. We can’t turn away. Let’s do what we know in our hearts that we need to do. Let’s come together to finish the job on police reform. Do something. Do something. - -Ban assault weapons. - -That was the plea of parents who lost their children in Uvalde — I met with every one of them — “Do something about gun violence.” Thank God — thank God we did, passing the most sweeping gun safety law in three decades. - -That includes things like — that the majority of responsible gun owners already support: enhanced background checks for 18- to 21 years old, red-flag laws keeping guns out of the hands of people who are a danger to themselves and others. - -But we know our work is not done. Joining us tonight is Brandon Tsay, a 26-year-old hero. - -Brandon put his college dreams on hold — to be at his mom’s side — his mom’s side when she was dying from cancer. And Brandon — Brandon now works at the dance studio started by his grandparents. - -And two weeks ago, during the Lunar New Year celebrations, he heard the studio door close, and he saw a man standing there pointing a semi-automatic pistol at him. He thought he was going to die, but he thought about the people inside. - -In that instant, he found the courage to act and wrestled the semi-automatic pistol away from the gunman who had already killed 11 people in another dance studio. Eleven. - -He saved lives. It’s time we do the same. - -Ban assault weapons now! Ban them now! Once and for all. - -I led the fight to do that in 1994. And in 10 years that ban was law, mass shootings went down. After we let it expire in a Republican administration, mass shootings tripled. - -Let’s finish the job and ban these assault weapons. - -And let’s also come together on immigration. Make it a bipartisan issue once again. - -We know — we now have a record number of personnel working to secure the border, arresting 8,000 human smugglers, seizing over 23,000 pounds of fentanyl in just the last several months. - -We’ve launched a new border plan last month. Unlawful migration from Cuba, Haiti, Nicaragua, and Venezuela has come down 97 percent as a consequence of that. - -But American border problems won’t be fixed until Congress acts. If we don’t pass my comprehensive immigration reform, at least pass my plan to provide the equipment and officers to secure the border — and a pathway to citizenship for DREAMers, those on temporary status, farmworkers, essential workers. - -Here in the People’s House, it’s our duty to protect all the people’s rights and freedoms. Congress must restore the right and — - -Congress must restore the right that was taken away in Roe v. Wade — and protect Roe v. Wade. Give every woman the constitutional right. - -The Vice President and I are doing everything to protect access to reproductive healthcare and safeguard patient safety. But already, more than a dozen states are enforcing extreme abortion bans. - -Make no mistake about it: If Congress passes a national ban, I will veto it. - -It’s time to pass the Equality Act. - -But let’s also pass — let’s also pass the bipartisan Equality Act to ensure LBG- — LGBTQ Americans, especially transgender young people, can live with safety and dignity. - -Our strength — our strength is not just the example of our power, but the power of our example. Let’s remember, the world is watching. - -I spoke from this chamber one year ago, just days after Vladimir Putin unleashed his brutal attack against Ukraine, a murderous assault, evoking images of death and destruction Europe suffered in World War Two. - -Putin’s invasion has been a test for the ages — a test for America, a test for the world. Would we stand for the most basic of principles? Would we stand for sovereignty? Would we stand for the right of people to live free of tyranny? Would we stand for the defense of democracy? For such defense matters to us because it keeps peace and prevents open season on would-be aggressors that threatens our prosperity. - -One year later, we know the answer. Yes, we would. And we did. We did. - -And together, we did what America always does at our best. We led. We united NATO. We built a global coalition. We stood against Putin’s aggression. We stood with the Ukrainian people. - -Tonight, we’re once again joined by Ukrainians’ Ambassador to the United States. She represents not her — just her nation but the courage of her people. Ambassador is — our Ambassador is here, united in our — we’re united in our support of your country. - -Will you stand so we can all take a look at you? Thank you. Because we’re going to stand with you as long as it takes. - -Our nation is working for more freedom, more dignity, and more — more peace, not just in Europe, but everywhere. - -Before I came to office, the story was about how the People’s Republic of China was increasing its power and America was failing in the world. Not anymore. - -We made clear and I made clear in my personal conversations, which have been many, with President Xi that we seek competition, not conflict. But I will make no apologies that we’re investing and — to make America stronger. - -Investing in American innovation and industries that will define the future that China intends to be dominating. - -Investing in our alliances and working with our allies to protect advanced technologies so they will not be used against us. - -Modernizing our military to safeguard stability and determine — deter aggression. - -Today, we’re in the strongest position in decades to compete with China or anyone else in the world. Anyone else in the world. - -And I’m committed — I’m committed to work with China where we can advance American interests and benefit the world. But make no mistake about it: As we made clear last week, if China threatens our sovereignty, we will act to protect our country. And we did. - -Look, let’s be clear: Winning the competition should unite all of us. - -We face serious challenges across the world. But in the past two years, democracies have become stronger, not weaker. Autocracies have grown weaker, not stronger. - -Name me a world leader who’d change places with Xi Jinping. Name me one. Name me one. - -America is rallying the world to meet those challenges — from climate to global health to food insecurity to terrorism to territorial aggression. - -Allies are stepping up, spending more, and doing more. Look, the bridges we’re forming between partners in the Pacific and those in the Atlantic. And those who bet against America are learning how wrong they are. It’s never, ever been a good bet to bet against America. Never. - -Well — - -When I came to office, most assured that bipartisanship — assumed — was impossible. But I never believed it. That’s why a year ago, I offered a Unity Agenda to the nation as I stood here. - -We made real progress together. - -We passed the law making it easier for doctors to prescribe effective treatments for opioid addiction. - -We passed the gun safety law, making historic investments in mental health. - -We launched the ARPA-H drive for breakthroughs in the fight against cancer, Alzheimer’s, and diabetes, and so much more. - -We passed the Heath Robinson PACT Act, named after the late Iraq War veteran whose story about exposure to toxic burn pits I shared here last year. - -And I understand something about those burn pits. - -But there is so much more to do. And we can do it together. - -Joining us tonight is a father named Doug from Newton, New Hampshire. He wrote Jill, my wife, a letter — and me as well — about his courageous daughter, Courtney. A contagious laugh. His sister’s best friend — her sister’s best friend. - -He shared a story all too familiar to millions of Americans and many of you in the audience. Courtney discovered pills in high school. It spiraled into addiction and eventually death from a fentanyl overdose. She was just 20 years old. - -Describing the last eight years without her, Doug said, “There is no worse pain.” Yet, their family has turned pain into purpose, working to end the stigma and change laws. He told us he wants to “start a journey towards American recovery.” - -Doug, we’re with you. Fentanyl is killing more than 70,000 Americans a year. Big — - -Big — you got it. - -So let’s launch a major surge to stop fentanyl production and the sale and trafficking. With more drug detection machines, inspection cargo, stop pills and powder at the border. Working with couriers, like FedEx, to inspect more packages for drugs. Strong penalties to crack down on fentanyl trafficking. - -Second, let’s do more on mental health, especially for our children. When millions of young people are struggling with bullying, violence, trauma, we owe them greater access to mental health care at their schools. - -We must finally hold social media companies accountable for experimenting they’re doing — running [on] children for profit. - -And it’s time to pass bipartisan legislation to stop Big Tech from collecting personal data on kids and teenagers online, ban targeted advertising to children, and impose stricter limits on the personal data that companies collect on all of us. - -Third, let’s do more to keep this nation’s one fully sacred obligation: to equip those we send into harm’s way and care for them and their families when they come home. - -Job training, job placement for veterans and their spouses as they come to — return to civilian life. Helping veterans to afford their rent, because no one should be homeless in America, especially someone who served the country. - -Denis McDoungin [sic] — Denis McDonough is here, of the VA. We had our first real discussion when I asked him to take the job. I’m glad he did. We were losing up to 25 veterans a day on suicide. Now we’re losing 17 a day to the silent scourge of suicide. Seventeen veterans a day are committing suicide, more than all the people being killed in the wars. - -Folks, VA — VA is doing everything it can, including expanding mental health screening, proven programs that recruits veterans to help other veterans understand what they’re going through, get them the help they need. We got to do more. - -And fourth, last year, Jill and I reignited the Cancer Moonshot that I was able to start with, and President Obama asked me to lead our administration on this issue. - -Our goal is to cut the cancer death rates at least by 50 percent in the next 25 years, turn more cancers from death sentences to treatable diseases, provide more support for patients and their families. - -It’s personal to so many of us — so many of us in this audience. - -Joining us are Maurice and Kandice, an Irishman and a daughter of immigrants from Panama. They met and fell in love in New York City and got married in the same chapel as Jill and I got married in New York City. Kindred spirits. - -He wrote us a letter about his little daughter, Ava. And I saw her just before I came over. She was just a year old when she was diagnosed with a rare kidney disease — cancer. After 26 blood transfusions, 11 rounds of radiation, 8 rounds of cheno [sic] — chemo, 1 kidney removed, given a 5 percent survival rate. - -He wrote how, in the darkest moments, he thought, “If she goes, I can’t stay.” - -Many of you have been through that as well. Jill and I understand that, like so many of you. - -And he read Jill’s book describing our family’s cancer journey and how we tried to steal moments of joy where we could with Beau. - -For them, that glimmer of joy was the half-smile of their baby girl. It meant everything to them. They never gave up hope, and little Ava never gave up hope. She turns four next month. - -They just found out Ava is beating the odds and is on her way to being cured of cancer. And she’s watching from the White House tonight, if she’s not asleep already. - -For the lives we can save — for the lives we can save and the lives we have lost, let this be a truly American moment that rallies the country and the world together and prove that we can still do big things. - -Twenty years ago, under the leadership of President Bush and countless advocates and champions, he undertook a bipartisan effort through PEPFAR to transform the global fight against HIV/AIDS. It’s been a huge success. He thought big. He thought large. He moved! - -I believe we can do the same thing with cancer. Let’s end cancer as we know it and cure some cancers once and for all. - -Folks, there’s one reason why we’ve been able to do all of these things: our democracy itself. It’s the most fundamental thing of all. With democracy, everything is possible. Without it, nothing is. - -Over the last few years, our democracy has been threatened and attacked, put at risk — put to the test in this very room on January the 6th. - -And then, just a few months ago, an unhinged Big Lie assailant unleashed a political violence at the home of the then-Speaker of the House of Representatives, using the very same language the insurrectionists used as they stalked these halls and chanted on January 6th. - -Here tonight, in this chamber, is the man who bears the scars of that brutal attack but is as tough and as strong and as resilient as they get: my friend, Paul Pelosi. Paul, stand up. - -But such a heinous act should have never happened. We must all speak out. There is no place for political violence in America. - -We have to protect the right to vote, not suppress the — that fundamental right. Honor the results of our elections, not subvert the will of the people. We have to uphold the rule of the law and restore trust in our institutions of democracy. And we must give hate and extremism in any form no safe harbor. - -Democracy must not be a partisan issue. It’s an American issue. - -Every generation of Americans have faced a moment where they have been called to protect our democracy, defend it, stand up for it. And this is our moment. - -My fellow Americans, we meet tonight at an inflection point, one of those moments that only a few generations ever face, where the direction we now take is going to decide the course of this nation for decades to come. - -We’re not bystanders of history. We’re not powerless before the forces that confront us. It’s within our power of We the People. - -We’re facing the test of our time. We have to be the nation we’ve always been at our best: optimistic, hopeful, forward-looking. A nation that embraces light over dark, hope over fear, unity over division, stability over chaos. - -We have to see each other not as enemies, but as fellow Americans. We’re a good people. The only nation in the world built on an idea — the only one. Other nations are defined by geography, ethnicity, but we’re the only nation based on an idea that all of us, every one of us, is created equal in the image of God. A nation that stands as a beacon to the world. A nation in a new age of possibilities. - -So I have come to fulfil my constitutional obligation to report on the state of the Union. And here is my — my report: Because the soul of this nation is strong, because the backboken [sic] — backbone of this nation is strong, because the people of this nation are strong, the state of the Union is strong. - -Because the soul of this nation is strong. Because the backbone of this nation is strong. Because the people of this nation are strong. The State of the Union is Strong. - -I’m not new to this place. I stand here tonight having served as long as about any one of you who have ever served here. But I’ve never been more optimistic about our future — about the future of America. - -We just have to remember who we are. We’re the United States of America. And there’s nothing — nothing beyond our capacity if we do it together. - -God bless you all. And may God protect our troops. Thank you. \ No newline at end of file diff --git a/src/datafog/__about__.py b/src/datafog/__about__.py index 916f802f..d6c25731 100644 --- a/src/datafog/__about__.py +++ b/src/datafog/__about__.py @@ -1,2 +1,2 @@ # SSOT for the package version -__version__ = "2.2.0" +__version__ = "2.3.1" diff --git a/src/datafog/__init__.py b/src/datafog/__init__.py index 547e17af..d51e13f3 100644 --- a/src/datafog/__init__.py +++ b/src/datafog/__init__.py @@ -1,5 +1,6 @@ # datafog-python/src/datafog/__init__.py import json +import logging import pandas as pd import requests @@ -8,6 +9,8 @@ from .__about__ import __version__ from .pii_tools import PresidioEngine +logger = logging.getLogger(__name__).setLevel(logging.ERROR) + __all__ = [ "__version__", "PresidioEngine", @@ -15,14 +18,51 @@ class DataFog: + """ + DataFog class for performing privacy operations on input data. + + This class uses the Spacy library to process and analyze input data for + personally identifiable information (PII) and applies specified privacy + operations to protect sensitive data. + + Attributes: + nlp (spacy.lang): Spacy language model for PII detection. + """ + def __init__(self): + """ + Initialize the DataFog instance. + + Loads the Spacy language model for PII detection. + """ self.nlp = spacy.load("en_spacy_pii_fast") @staticmethod def client(): + """ + Create a new instance of the DataFog client. + + Returns: + DataFog: A new instance of the DataFog client. + """ return DataFog() def __call__(self, input_source, privacy_operation): + """ + Process the input data and apply the specified privacy operation. + + Args: + input_source (Union[str, pd.DataFrame]): The input data source. + Can be a URL, file path, or a string containing the data. + Supported file formats: CSV, TXT, JSON, Parquet. + privacy_operation (str): The privacy operation to apply. + Supported operations: 'redact', 'annotate'. + Returns: + str: The processed text with the applied privacy operation. + + Raises: + ValueError: If an unsupported input source type or privacy operation is provided. + """ if isinstance(input_source, str): if input_source.startswith(("http://", "https://")): print("Downloading file from URL") @@ -61,6 +101,5 @@ def __call__(self, input_source, privacy_operation): raise ValueError( f"Unsupported privacy operation: {privacy_operation}" ) - # Add more privacy operations as needed return text diff --git a/src/datafog/pii_tools/PresidioEngine/__init__.py b/src/datafog/pii_tools/PresidioEngine/__init__.py index 52d6b8df..ffb73aa3 100644 --- a/src/datafog/pii_tools/PresidioEngine/__init__.py +++ b/src/datafog/pii_tools/PresidioEngine/__init__.py @@ -1,17 +1,96 @@ -from presidio_analyzer import AnalyzerEngine, RecognizerRegistry +import logging +from typing import List, Optional + +from presidio_analyzer import ( + AnalyzerEngine, + Pattern, + PatternRecognizer, + RecognizerRegistry, +) from presidio_analyzer.nlp_engine import NlpEngineProvider from .analyzer import CustomSpacyRecognizer +logger = logging.getLogger("presidio-engine-init").setLevel(logging.ERROR) + + +def create_ad_hoc_deny_list_recognizer( + deny_list=Optional[List[str]], +) -> Optional[PatternRecognizer]: + """ + Create an ad-hoc recognizer based on a provided deny list. + + Args: + deny_list (Optional[List[str]]): List of terms to be denied. Default is None. + + Returns: + Optional[PatternRecognizer]: A PatternRecognizer based on the provided deny list, + or None if no deny list is provided. + """ + if not deny_list: + return None + + deny_list_recognizer = PatternRecognizer( + supported_entity="CUSTOM_PII", deny_list=deny_list + ) + return deny_list_recognizer + + +def create_ad_hoc_regex_recognizer( + regex: str, entity_type: str, score: float, context: Optional[List[str]] = None +) -> Optional[PatternRecognizer]: + """ + Create an ad-hoc recognizer based on a provided regular expression. + + Args: + regex (str): The regular expression pattern to match. + entity_type (str): The entity type associated with the pattern. + score (float): The confidence score for the pattern. + context (Optional[List[str]]): Additional context for the pattern. Default is None. + + Returns: + Optional[PatternRecognizer]: A PatternRecognizer based on the provided regex, + or None if no regex is provided. + """ + if not regex: + return None + pattern = Pattern(name="Regex Pattern", regex=regex, score=score) + regex_recognizer = PatternRecognizer( + supported_entity=entity_type, patterns=[pattern], context=context + ) + return regex_recognizer + -# Helper methods def analyzer_engine(): - """Return AnalyzerEngine.""" + """ + Create and configure an AnalyzerEngine. + + Returns: + AnalyzerEngine: The configured AnalyzerEngine instance. + """ spacy_recognizer = CustomSpacyRecognizer() configuration = { "nlp_engine_name": "spacy", "models": [{"lang_code": "en", "model_name": "en_spacy_pii_fast"}], + "ner_model_configuration": { + "model_to_presidio_entity_mapping": { + "PER": "PERSON", + "PERSON": "PERSON", + "NORP": "NRP", + "FAC": "FACILITY", + "LOC": "LOCATION", + "GPE": "LOCATION", + "LOCATION": "LOCATION", + "ORG": "ORGANIZATION", + "ORGANIZATION": "ORGANIZATION", + "DATE": "DATE_TIME", + "TIME": "DATE_TIME", + }, + "low_confidence_score_multiplier": 0.4, + "low_score_entity_names": ["ORG", "ORGANIZATION"], + "labels_to_ignore": ["DATE_TIME", "NRP"], + }, } # Create NLP engine based on configuration @@ -35,6 +114,17 @@ def analyzer_engine(): def annotate(text, analysis_results): + """ + Annotate the input text with the analysis results. + + Args: + text (str): The input text to annotate. + analysis_results (List[AnalysisResult]): The list of analysis results. + + Returns: + List[Union[str, Tuple[str, str]]]: The annotated text as a list of tokens, + where each token is either a string or a tuple of (text, entity_type). + """ tokens = [] # sort by start index results = sorted(analysis_results, key=lambda x: x.start) @@ -55,10 +145,42 @@ def annotate(text, analysis_results): def scan(text, **kwargs): - # Set default values for any parameters not provided + """ + Analyze the input text using the Analyzer engine and provided arguments. + + Args: + text (str): The input text to analyze. + **kwargs: Additional keyword arguments for the analysis. + - language (str): The language of the input text. Default is "en". + - score_threshold (float): The minimum score threshold for entity recognition. Default is 0.35. + - nlp_artifacts (Optional[NlpArtifacts]): NLP artifacts to use for analysis. Default is None. + - entities (List[str]): List of entities to analyze. Default is an empty list. + - allow_list (List[str]): List of terms to allow. Default is an empty list. + - deny_list (List[str]): List of terms to deny. Default is an empty list. + + Returns: + List[AnalysisResult]: The list of analysis results. + """ kwargs.setdefault("language", "en") kwargs.setdefault("score_threshold", 0.35) kwargs.setdefault("nlp_artifacts", None) + kwargs.setdefault("entities", []) + kwargs.setdefault("allow_list", []) + kwargs.setdefault("deny_list", []) + + """Analyze input using Analyzer engine and input arguments (kwargs).""" + if "entities" not in kwargs or "All" in kwargs["entities"]: + kwargs["entities"] = None + + if "deny_list" in kwargs and kwargs["deny_list"] is not None: + ad_hoc_recognizer = create_ad_hoc_deny_list_recognizer(kwargs["deny_list"]) + kwargs["ad_hoc_recognizers"] = [ad_hoc_recognizer] if ad_hoc_recognizer else [] + del kwargs["deny_list"] + + if "regex_params" in kwargs and len(kwargs["regex_params"]) > 0: + ad_hoc_recognizer = create_ad_hoc_regex_recognizer(*kwargs["regex_params"]) + kwargs["ad_hoc_recognizers"] = [ad_hoc_recognizer] if ad_hoc_recognizer else [] + del kwargs["regex_params"] # init analyzer instance analyzer = analyzer_engine() diff --git a/src/datafog/pii_tools/PresidioEngine/analyzer.py b/src/datafog/pii_tools/PresidioEngine/analyzer.py index 3b78db23..36a94c1f 100644 --- a/src/datafog/pii_tools/PresidioEngine/analyzer.py +++ b/src/datafog/pii_tools/PresidioEngine/analyzer.py @@ -3,10 +3,23 @@ from presidio_analyzer import AnalysisExplanation, LocalRecognizer, RecognizerResult -logger = logging.getLogger("presidio-module") +logger = logging.getLogger("custom-spacy-recognizer").setLevel(logging.ERROR) class CustomSpacyRecognizer(LocalRecognizer): + """ + Custom Spacy Recognizer for PII detection. + + This recognizer uses a pre-trained Spacy model to identify PII entities in text. + + Attributes: + ENTITIES (List[str]): List of supported entities. + DEFAULT_EXPLANATION (str): Default explanation format for recognized entities. + CHECK_LABEL_GROUPS (List[Tuple[Set, Set]]): Groups of labels to check for entity matching. + MODEL_LANGUAGES (Dict[str, str]): Mapping of language codes to pre-trained model names. + PRESIDIO_EQUIVALENCES (Dict[str, str]): Mapping of Spacy entity labels to Presidio entity types. + """ + ENTITIES = [ "LOCATION", "PERSON", @@ -45,6 +58,17 @@ def __init__( context: Optional[List[str]] = None, ner_strength: float = 0.85, ): + """ + Initialize the CustomSpacyRecognizer. + + Args: + supported_language (str): The language supported by the recognizer. Default is "en". + supported_entities (Optional[List[str]]): List of supported entities. If None, defaults to ENTITIES. + check_label_groups (Optional[Tuple[Set, Set]]): Groups of labels to check for entity matching. + If None, defaults to CHECK_LABEL_GROUPS. + context (Optional[List[str]]): Additional context for the recognizer. Default is None. + ner_strength (float): The strength of the named entity recognition. Default is 0.85. + """ self.ner_strength = ner_strength self.check_label_groups = ( check_label_groups if check_label_groups else self.CHECK_LABEL_GROUPS @@ -61,8 +85,10 @@ def load(self) -> None: def get_supported_entities(self) -> List[str]: """ - Return supported entities by this model. - :return: List of the supported entities. + Get the list of supported entities by the recognizer. + + Returns: + List[str]: List of supported entities. """ return self.supported_entities @@ -83,6 +109,17 @@ def build_spacy_explanation( return explanation def analyze(self, text, entities, nlp_artifacts=None): # noqa D102 + """ + Analyze the given text for specified entities using the provided NLP artifacts. + + Args: + text (str): The text to analyze. + entities (List[str]): The list of entities to look for in the text. + nlp_artifacts (Optional[NlpArtifacts]): The NLP artifacts to use for analysis. If None, an empty list is returned. + + Returns: + List[RecognizerResult]: The list of recognized entities in the text. + """ results = [] if not nlp_artifacts: logger.warning("No NLP artifacts provided for analysis") @@ -120,6 +157,17 @@ def analyze(self, text, entities, nlp_artifacts=None): # noqa D102 def __check_label( entity: str, label: str, check_label_groups: Tuple[Set, Set] ) -> bool: + """ + Check if the given entity and label belong to any of the specified label groups. + + Args: + entity (str): The entity to check. + label (str): The label to check. + check_label_groups (Tuple[Set, Set]): The groups of labels to check against. + + Returns: + bool: True if the entity and label belong to any of the label groups, False otherwise. + """ return any( [entity in egrp and label in lgrp for egrp, lgrp in check_label_groups] ) diff --git a/tests/files/output_files/output.md b/tests/files/output_files/output.md index 41bc9a4f..3f0fe66a 100644 --- a/tests/files/output_files/output.md +++ b/tests/files/output_files/output.md @@ -2,12 +2,7 @@ PERSON LOCATION PERSON PERSON -DATE_TIME -NRP -DATE_TIME PERSON -NRP -NRP LOCATION PERSON PERSON @@ -16,64 +11,27 @@ PERSON LOCATION PERSON LOCATION -DATE_TIME -DATE_TIME -DATE_TIME -DATE_TIME -NRP -DATE_TIME -DATE_TIME -DATE_TIME -DATE_TIME -DATE_TIME -DATE_TIME -NRP LOCATION -NRP -NRP -DATE_TIME -NRP -NRP -NRP LOCATION -NRP LOCATION LOCATION LOCATION -DATE_TIME PERSON PERSON -DATE_TIME PERSON -NRP -DATE_TIME LOCATION -NRP -NRP PERSON LOCATION LOCATION LOCATION -DATE_TIME -DATE_TIME -NRP -DATE_TIME -NRP LOCATION LOCATION -DATE_TIME -DATE_TIME -NRP LOCATION LOCATION LOCATION -NRP -DATE_TIME LOCATION LOCATION LOCATION -NRP -NRP LOCATION LOCATION PERSON @@ -82,128 +40,46 @@ LOCATION LOCATION LOCATION LOCATION -NRP -NRP -NRP LOCATION LOCATION LOCATION -DATE_TIME LOCATION -DATE_TIME -DATE_TIME -NRP -NRP PERSON -DATE_TIME PERSON PERSON PERSON -DATE_TIME -DATE_TIME LOCATION PERSON LOCATION LOCATION LOCATION -NRP LOCATION -NRP -NRP PERSON -DATE_TIME -NRP -NRP -DATE_TIME LOCATION LOCATION PERSON -NRP -NRP -NRP -DATE_TIME -DATE_TIME -DATE_TIME LOCATION -DATE_TIME -DATE_TIME LOCATION -NRP -DATE_TIME -DATE_TIME -NRP PERSON -NRP -DATE_TIME LOCATION -NRP LOCATION LOCATION -NRP LOCATION LOCATION LOCATION LOCATION PERSON -NRP -DATE_TIME LOCATION PERSON LOCATION -DATE_TIME -DATE_TIME -NRP -NRP -DATE_TIME -DATE_TIME -DATE_TIME -NRP -DATE_TIME -DATE_TIME LOCATION -NRP -NRP -NRP -NRP -NRP -DATE_TIME PERSON -NRP -NRP -DATE_TIME -DATE_TIME -DATE_TIME -NRP -DATE_TIME -DATE_TIME -NRP -NRP -NRP -DATE_TIME -NRP LOCATION -NRP -DATE_TIME -DATE_TIME -DATE_TIME -DATE_TIME -DATE_TIME -DATE_TIME PERSON -DATE_TIME LOCATION -DATE_TIME -NRP -NRP -NRP PERSON LOCATION -DATE_TIME -NRP -DATE_TIME -DATE_TIME PERSON -DATE_TIME LOCATION PERSON PERSON @@ -217,120 +93,69 @@ PERSON PERSON PERSON LOCATION -DATE_TIME -DATE_TIME -DATE_TIME PERSON -DATE_TIME PERSON PERSON PERSON -DATE_TIME -DATE_TIME -DATE_TIME -NRP -DATE_TIME LOCATION LOCATION LOCATION LOCATION -NRP PERSON PERSON PERSON PERSON -NRP -DATE_TIME -DATE_TIME PERSON LOCATION LOCATION PERSON LOCATION -DATE_TIME LOCATION PERSON -NRP -DATE_TIME -NRP LOCATION LOCATION LOCATION PERSON LOCATION -NRP LOCATION -DATE_TIME -DATE_TIME LOCATION LOCATION -NRP -DATE_TIME LOCATION -DATE_TIME PERSON LOCATION LOCATION LOCATION LOCATION -DATE_TIME LOCATION -DATE_TIME -DATE_TIME PERSON LOCATION LOCATION PERSON PERSON -NRP -DATE_TIME -DATE_TIME PERSON -NRP PERSON -NRP LOCATION LOCATION PERSON PERSON LOCATION -DATE_TIME LOCATION -DATE_TIME PERSON PERSON -DATE_TIME LOCATION LOCATION -NRP LOCATION LOCATION PERSON LOCATION -DATE_TIME PERSON PERSON PERSON -DATE_TIME PERSON LOCATION -DATE_TIME -NRP PERSON -DATE_TIME -DATE_TIME -DATE_TIME -DATE_TIME -DATE_TIME PERSON LOCATION -NRP -NRP -NRP -DATE_TIME -DATE_TIME -NRP LOCATION -DATE_TIME LOCATION LOCATION diff --git a/tests/files/output_files/output.txt b/tests/files/output_files/output.txt index 41bc9a4f..3f0fe66a 100644 --- a/tests/files/output_files/output.txt +++ b/tests/files/output_files/output.txt @@ -2,12 +2,7 @@ PERSON LOCATION PERSON PERSON -DATE_TIME -NRP -DATE_TIME PERSON -NRP -NRP LOCATION PERSON PERSON @@ -16,64 +11,27 @@ PERSON LOCATION PERSON LOCATION -DATE_TIME -DATE_TIME -DATE_TIME -DATE_TIME -NRP -DATE_TIME -DATE_TIME -DATE_TIME -DATE_TIME -DATE_TIME -DATE_TIME -NRP LOCATION -NRP -NRP -DATE_TIME -NRP -NRP -NRP LOCATION -NRP LOCATION LOCATION LOCATION -DATE_TIME PERSON PERSON -DATE_TIME PERSON -NRP -DATE_TIME LOCATION -NRP -NRP PERSON LOCATION LOCATION LOCATION -DATE_TIME -DATE_TIME -NRP -DATE_TIME -NRP LOCATION LOCATION -DATE_TIME -DATE_TIME -NRP LOCATION LOCATION LOCATION -NRP -DATE_TIME LOCATION LOCATION LOCATION -NRP -NRP LOCATION LOCATION PERSON @@ -82,128 +40,46 @@ LOCATION LOCATION LOCATION LOCATION -NRP -NRP -NRP LOCATION LOCATION LOCATION -DATE_TIME LOCATION -DATE_TIME -DATE_TIME -NRP -NRP PERSON -DATE_TIME PERSON PERSON PERSON -DATE_TIME -DATE_TIME LOCATION PERSON LOCATION LOCATION LOCATION -NRP LOCATION -NRP -NRP PERSON -DATE_TIME -NRP -NRP -DATE_TIME LOCATION LOCATION PERSON -NRP -NRP -NRP -DATE_TIME -DATE_TIME -DATE_TIME LOCATION -DATE_TIME -DATE_TIME LOCATION -NRP -DATE_TIME -DATE_TIME -NRP PERSON -NRP -DATE_TIME LOCATION -NRP LOCATION LOCATION -NRP LOCATION LOCATION LOCATION LOCATION PERSON -NRP -DATE_TIME LOCATION PERSON LOCATION -DATE_TIME -DATE_TIME -NRP -NRP -DATE_TIME -DATE_TIME -DATE_TIME -NRP -DATE_TIME -DATE_TIME LOCATION -NRP -NRP -NRP -NRP -NRP -DATE_TIME PERSON -NRP -NRP -DATE_TIME -DATE_TIME -DATE_TIME -NRP -DATE_TIME -DATE_TIME -NRP -NRP -NRP -DATE_TIME -NRP LOCATION -NRP -DATE_TIME -DATE_TIME -DATE_TIME -DATE_TIME -DATE_TIME -DATE_TIME PERSON -DATE_TIME LOCATION -DATE_TIME -NRP -NRP -NRP PERSON LOCATION -DATE_TIME -NRP -DATE_TIME -DATE_TIME PERSON -DATE_TIME LOCATION PERSON PERSON @@ -217,120 +93,69 @@ PERSON PERSON PERSON LOCATION -DATE_TIME -DATE_TIME -DATE_TIME PERSON -DATE_TIME PERSON PERSON PERSON -DATE_TIME -DATE_TIME -DATE_TIME -NRP -DATE_TIME LOCATION LOCATION LOCATION LOCATION -NRP PERSON PERSON PERSON PERSON -NRP -DATE_TIME -DATE_TIME PERSON LOCATION LOCATION PERSON LOCATION -DATE_TIME LOCATION PERSON -NRP -DATE_TIME -NRP LOCATION LOCATION LOCATION PERSON LOCATION -NRP LOCATION -DATE_TIME -DATE_TIME LOCATION LOCATION -NRP -DATE_TIME LOCATION -DATE_TIME PERSON LOCATION LOCATION LOCATION LOCATION -DATE_TIME LOCATION -DATE_TIME -DATE_TIME PERSON LOCATION LOCATION PERSON PERSON -NRP -DATE_TIME -DATE_TIME PERSON -NRP PERSON -NRP LOCATION LOCATION PERSON PERSON LOCATION -DATE_TIME LOCATION -DATE_TIME PERSON PERSON -DATE_TIME LOCATION LOCATION -NRP LOCATION LOCATION PERSON LOCATION -DATE_TIME PERSON PERSON PERSON -DATE_TIME PERSON LOCATION -DATE_TIME -NRP PERSON -DATE_TIME -DATE_TIME -DATE_TIME -DATE_TIME -DATE_TIME PERSON LOCATION -NRP -NRP -NRP -DATE_TIME -DATE_TIME -NRP LOCATION -DATE_TIME LOCATION LOCATION