For October 22 (#64)

* clean up dataset page, and move half-baked to analysis

* move test_app assertion

* app_test uses fixture, not generated file. Fix #76

* check branch coverage. Fix #74

* label log slider

* move to separate file

* id parameter

* lower and upper parameter

* rename and reorg

* remove commented-out block

* columns checkbox group

* columns_ui

* stub per-column controls

* fix warning on startup, because there is a default contrib

* add a test that exercises conversion error handling

* execise Template param checks

* playwrite test of dynamic form

* test the log slider: click on the bar and it moves half-way

* comment linting

* Move stray files to utils directory

* move tests dir up and out

* Remove top-level coverage skip

* remove redundant css

* collect info from dynamic form elements

* numeric values

* pull columns into module

* rename

* add pragma to fix coverage

.coveragerc

@@ -3,11 +3,10 @@
 source = .
 
 omit =
-  # TODO
-  dp_creator_ii/app/*
+  tests/fixtures/*
 
 # More strict: Check transitions between lines, not just individual lines.
-# TODO: branch = True
+branch = True
 
 [report]
 show_missing = True

.pytest.ini

@@ -4,7 +4,7 @@
 filterwarnings =
 	error
 
-addopts = --doctest-glob '*.md' --doctest-modules --ignore dp_creator_ii/templates/ --ignore dp_creator_ii/tests/fixtures/
+addopts = --doctest-glob '*.md' --doctest-modules --ignore dp_creator_ii/utils/templates/ --ignore dp_creator_ii/tests/fixtures/
 
 # If an xfail starts passing unexpectedly, that should count as a failure:
 xfail_strict=true

dp_creator_ii/__init__.py

@@ -1,7 +1,7 @@
 """DP Creator II makes it easier to get started with Differential Privacy."""
 
 import shiny
-from dp_creator_ii.argparse_helpers import get_csv_contrib
+from dp_creator_ii.utils.argparse_helpers import get_csv_contrib
 
 
 __version__ = "0.0.1"

dp_creator_ii/app/__init__.py

@@ -17,11 +17,11 @@
 )
 
 
-def ctrl_c_reminder():
+def ctrl_c_reminder():  # pragma: no cover
     print("Session ended (Press CTRL+C to quit)")
 
 
-def server(input, output, session):
+def server(input, output, session):  # pragma: no cover
     dataset_panel.dataset_server(input, output, session)
     analysis_panel.analysis_server(input, output, session)
     results_panel.results_server(input, output, session)

dp_creator_ii/app/analysis_panel.py

@@ -2,29 +2,31 @@
 
 from shiny import ui, reactive, render
 
-from dp_creator_ii.mock_data import mock_data, ColumnDef
-from dp_creator_ii.app.plots import plot_error_bars_with_cutoff
+from dp_creator_ii.utils.mock_data import mock_data, ColumnDef
+from dp_creator_ii.app.components.plots import plot_error_bars_with_cutoff
+from dp_creator_ii.app.components.inputs import log_slider
+from dp_creator_ii.app.components.column_module import column_ui, column_server
+from dp_creator_ii.utils.csv_helper import read_field_names
+from dp_creator_ii.utils.argparse_helpers import get_csv_contrib
 
 
 def analysis_ui():
     return ui.nav_panel(
         "Define Analysis",
         ui.markdown(
-            "Select numeric columns of interest in *TODO*, "
+            "Select numeric columns of interest, "
             "and for each numeric column indicate the expected range, "
             "the number of bins for the histogram, "
             "and its relative share of the privacy budget."
         ),
-        ui.markdown(
-            "[TODO: Column selection]"
-            "(https://github.com/opendp/dp-creator-ii/issues/33)"
-        ),
+        ui.input_checkbox_group("columns_checkbox_group", None, []),
+        ui.output_ui("columns_ui"),
         ui.markdown(
             "What is your privacy budget for this release? "
             "Values above 1 will add less noise to the data, "
-            "but have greater risk of revealing individual data."
+            "but have a greater risk of revealing individual data."
         ),
-        ui.input_slider("log_epsilon_slider", None, -1, 1, 0, step=0.1),
+        log_slider("log_epsilon_slider", 0.1, 10.0),
         ui.output_text("epsilon"),
         ui.markdown(
             "## Preview\n"
@@ -38,8 +40,54 @@ def analysis_ui():
     )
 
 
-def analysis_server(input, output, session):
+def analysis_server(input, output, session):  # pragma: no cover
+    (csv_path, _contributions) = get_csv_contrib()
+
+    csv_path_from_cli_value = reactive.value(csv_path)
+
+    @reactive.effect
+    def _():
+        ui.update_checkbox_group(
+            "columns_checkbox_group",
+            label=None,
+            choices=csv_fields_calc(),
+        )
+
+    @render.ui
+    def columns_ui():
+        column_ids = input.columns_checkbox_group()
+        for column_id in column_ids:
+            column_server(column_id)
+        return [
+            [
+                ui.h3(column_id),
+                column_ui(column_id),
+            ]
+            for column_id in column_ids
+        ]
+
     @reactive.calc
+    def csv_path_calc():
+        csv_path_from_ui = input.csv_path_from_ui()
+        if csv_path_from_ui is not None:
+            return csv_path_from_ui[0]["datapath"]
+        return csv_path_from_cli_value.get()
+
+    @render.text
+    def csv_path():
+        return csv_path_calc()
+
+    @reactive.calc
+    def csv_fields_calc():
+        path = csv_path_calc()
+        if path is None:
+            return None
+        return read_field_names(path)
+
+    @render.text
+    def csv_fields():
+        return csv_fields_calc()
+
     def epsilon_calc():
         return pow(10, input.log_epsilon_slider())
 

dp_creator_ii/app/components/column_module.py

@@ -0,0 +1,34 @@
+from shiny import ui, render, module
+
+
+@module.ui
+def column_ui():  # pragma: no cover
+    return [
+        ui.input_numeric("min", "Min", 0),
+        ui.input_numeric("max", "Max", 10),
+        ui.input_numeric("bins", "Bins", 10),
+        ui.input_select(
+            "weight",
+            "Weight",
+            choices={
+                1: "Least accurate",
+                2: "Less accurate",
+                4: "More accurate",
+                8: "Most accurate",
+            },
+        ),
+        ui.output_code("col_config"),
+    ]
+
+
+@module.server
+def column_server(input, output, session):  # pragma: no cover
+    @output
+    @render.code
+    def col_config():
+        return {
+            "min": input.min(),
+            "max": input.max(),
+            "bins": input.bins(),
+            "weight": input.weight(),
+        }

dp_creator_ii/app/components/inputs.py

@@ -0,0 +1,29 @@
+from math import log10
+from shiny import ui
+
+
+def log_slider(id, lower, upper):
+    # Rather than engineer a new widget, hide the numbers we don't want.
+    # The rendered widget doesn't have a unique ID, but the following
+    # element does, so we can use some fancy CSS to get the preceding element.
+    # Long term solution is just to make our own widget.
+    return (
+        ui.tags.table(
+            ui.HTML(
+                f"""
+<style>
+.irs:has(+ #{id}) .irs-min, .irs-max, .irs-single {{
+    display: none;
+}}
+</style>
+"""
+            ),
+            ui.tags.tr(
+                ui.tags.td(lower),
+                ui.tags.td(
+                    ui.input_slider(id, None, log10(lower), log10(upper), 0, step=0.1),
+                ),
+                ui.tags.td(upper),
+            ),
+        ),
+    )

dp_creator_ii/app/plots.py → dp_creator_ii/app/components/plots.py

@@ -4,7 +4,7 @@
 
 def plot_error_bars_with_cutoff(
     y_values, x_min_label="min", x_max_label="max", y_cutoff=0, y_error=0
-):
+):  # pragma: no cover
     x_values = 0.5 + np.arange(len(y_values))
     x_values_above = []
     x_values_below = []

dp_creator_ii/app/css/styles.css

@@ -5,8 +5,3 @@ body {
 #top_level_nav {
     margin-bottom: 1em;
 }
-
-/* Rather than engineer a new widget, hide the numbers we don't want. */
-.irs-min, .irs-max, .irs-single {
-    display: none;
-}

dp_creator_ii/app/dataset_panel.py

@@ -1,60 +1,28 @@
 from shiny import ui, reactive, render
 
-from dp_creator_ii.argparse_helpers import get_csv_contrib
-from dp_creator_ii.csv_helper import read_field_names
-from dp_creator_ii.app.ui_helpers import output_code_sample
-from dp_creator_ii.template import make_privacy_unit_block
+from dp_creator_ii.utils.argparse_helpers import get_csv_contrib
+from dp_creator_ii.app.components.outputs import output_code_sample
+from dp_creator_ii.utils.template import make_privacy_unit_block
 
 
 def dataset_ui():
     (_csv_path, contributions) = get_csv_contrib()
 
     return ui.nav_panel(
         "Select Dataset",
-        "TODO: Pick dataset",
-        ui.input_file("csv_path_from_ui", "Choose CSV file", accept=[".csv"]),
-        "CSV path from either CLI or UI:",
-        ui.output_text("csv_path"),
-        "CSV fields:",
+        ui.input_file("csv_path_from_ui", "Choose CSV file:", accept=[".csv"]),
         ui.markdown(
             "How many rows of the CSV can one individual contribute to? "
             'This is the "unit of privacy" which will be protected.'
         ),
-        ui.output_text("csv_fields"),
         ui.input_numeric("contributions", "Contributions", contributions),
         output_code_sample("unit_of_privacy_python"),
         ui.input_action_button("go_to_analysis", "Define analysis"),
         value="dataset_panel",
     )
 
 
-def dataset_server(input, output, session):
-    (csv_path, _contributions) = get_csv_contrib()
-
-    csv_path_from_cli_value = reactive.value(csv_path)
-
-    @reactive.calc
-    def csv_path_calc():
-        csv_path_from_ui = input.csv_path_from_ui()
-        if csv_path_from_ui is not None:
-            return csv_path_from_ui[0]["datapath"]
-        return csv_path_from_cli_value.get()
-
-    @render.text
-    def csv_path():
-        return csv_path_calc()
-
-    @reactive.calc
-    def csv_fields_calc():
-        path = csv_path_calc()
-        if path is None:
-            return None
-        return read_field_names(path)
-
-    @render.text
-    def csv_fields():
-        return csv_fields_calc()
-
+def dataset_server(input, output, session):  # pragma: no cover
     @render.code
     def unit_of_privacy_python():
         contributions = input.contributions()

dp_creator_ii/app/results_panel.py

@@ -1,7 +1,7 @@
 from shiny import ui, render
 
-from dp_creator_ii.template import make_notebook_py, make_script_py
-from dp_creator_ii.converters import convert_py_to_nb
+from dp_creator_ii.utils.template import make_notebook_py, make_script_py
+from dp_creator_ii.utils.converters import convert_py_to_nb
 
 
 def results_ui():
@@ -29,7 +29,7 @@ def results_ui():
     )
 
 
-def results_server(input, output, session):
+def results_server(input, output, session):  # pragma: no cover
     @render.download(
         filename="dp-creator-script.py",
         media_type="text/x-python",

dp_creator_ii/argparse_helpers.py → dp_creator_ii/utils/argparse_helpers.py

@@ -107,7 +107,7 @@ def _get_demo_csv_contrib():
 def get_csv_contrib():  # pragma: no cover
     args = _get_args()
     if args.demo:
-        if args.csv_path is not None or args.contributions is not None:
+        if args.csv_path is not None:
             warn('"--demo" overrides "--csv" and "--contrib"')
         return _get_demo_csv_contrib()
     return (args.csv_path, args.contributions)

dp_creator_ii/converters.py → dp_creator_ii/utils/converters.py

@@ -28,9 +28,11 @@ def convert_py_to_nb(python_str, execute=False):
         )
         try:
             result = subprocess.run(argv, check=True, text=True, capture_output=True)
-        except subprocess.CalledProcessError:  # pragma: no cover
+        except subprocess.CalledProcessError:
             if not execute:
-                raise
+                # Might reach here if jupytext is not installed.
+                # Error quickly instead of trying to recover.
+                raise  # pragma: no cover
             # Install kernel if missing
             # TODO: Is there a better way to do this?
             subprocess.run(

dp_creator_ii/template.py → dp_creator_ii/utils/template.py

@@ -9,7 +9,7 @@ def __init__(self, path, template=None):
             template_path = Path(__file__).parent / "templates" / path
             self._template = template_path.read_text()
         if template is not None:
-            if path is not None:  # pragma: no cover
+            if path is not None:
                 raise Exception('"path" and "template" are mutually exclusive')
             self._path = "template-instead-of-path"
             self._template = template

dp_creator_ii/templates/README.md → dp_creator_ii/utils/templates/README.md

@@ -1 +1 @@
-Strings of ALL CAPS are replaced in these templates. Keeping them in a format which can actually be parsed as python makes some things easier, but it is also reinventing the wheel. We may revisit this.
+Strings of ALL CAPS are replaced in these templates. Keeping them in a format which can actually be parsed as python makes some things easier, but it is also reinventing the wheel. We may revisit this.

tests/fixtures/fake.csv

@@ -0,0 +1,7 @@
+student_id,class_year,hw_number,grade
+1234,1,1,90
+1234,1,2,95
+1234,1,3,85
+6789,2,1,70
+6789,2,2,100
+6789,2,3,90