Add an option to set a destination directory for the simulation outpu…

…t (local) (#34) * Add an option to set a destination directory for the simulation output * code cleanup * doc and version * test naming and test doc * fixes to CHANGELOG --------- Co-authored-by: Andrey Abramov
pylhc · Nov 13, 2023 · 507905c · 507905c
1 parent b5fb84c
commit 507905c
Show file tree

Hide file tree

Showing 30 changed files with 1,260 additions and 554 deletions.
diff --git a/.gitignore b/.gitignore
@@ -243,9 +243,14 @@ Temporary Items
 # Neovim
 .nvimlog
 
+# Intellij
 /.idea/codeStyles/codeStyleConfig.xml
 /.idea/misc.xml
 /.idea/modules.xml
 /.idea/inspectionProfiles/profiles_settings.xml
 /.idea/vcs.xml
 /.idea/PyhDToolkit.iml
+
+# Other
+tst_*
+
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,18 @@
 # `pylhc-submitter` Changelog
 
+## Version 2.0.0
+
+- General code cleanup/refactoring/documentation:
+  - Partly breaks backward compatibility, if individual methods of the `job_submitter`-functionality have been used.
+  - Does not affect any setups simply calling the `main()` function of `job_submitter.py` or calling the `job_submitter` as a module.
+  - Apart from some fixed imports, following the new structure, the `autosix` module has been untouched. 
+
+
+- New Feature of `job_submitter`:
+  - `output_destination` input parameter, which sets an output directory in which the folder-stucture 
+    for the jobs will be replicated and the job's `job_output_dir` will be copied into "manually" at the end of the job, 
+    instead of having the directory transferred back to the `working directory` by htcondor.
+
 ## Version 1.1.1
 
 - Uses `concat` instead of `append` to stack the DataFrames.

diff --git a/doc/Makefile b/doc/Makefile
@@ -48,9 +48,9 @@ html:
 	@echo "Build finished. The HTML pages are in $(BUILDDIR)/html."
 
 josch:
-	$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) /home/jdilly/Software/Documentation/submitter-doc
+	$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) ../../Documentation/submitter-doc
 	@echo
-	@echo "Build finished. The HTML pages are in /home/jdilly/Software/Documentation/submitter-doc."
+	@echo "Build finished. The HTML pages are in ../../Documentation/submitter-doc."
 
 dirhtml:
 	$(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml

diff --git a/doc/_static/css/custom.css b/doc/_static/css/custom.css
@@ -1,7 +1,41 @@
+:root {
+  --nav-side-width: 300px;  /* default is 300px */
+  /* for 100% width */
+  /*--nav-content-width: 100%;*/
+  /*--local-toc-width: 300px;*/
+  /*--nav-content-width-wide: calc(100% - var(--local-toc-width)); /* 100% here is fullscreen */
+  /*--local-toc-left: calc(100% - var(--local-toc-width)); /* 100% here is w/o sidebar */
+
+  /* for fixed widths */
+  --nav-content-width: 800px;  /* default is 800px */
+  --nav-content-width-wide: var(--nav-content-width);
+  --local-toc-width: calc(100% - var(--nav-content-width-wide));
+  --local-toc-left: calc(var(--nav-content-width-wide) + var(--nav-side-width));
+}
+
+/* main content width */
+.wy-nav-content {
+  max-width: var(--nav-content-width);
+}
+
+/* Sidebar width */
+.wy-nav-side {
+  width: var(--nav-side-width);
+}
+
 .wy-side-nav-search {
   background: rgb(243,244,247);
 }
 
+.wy-side-nav-search > a {
+  color: black;
+}
+
+.wy-side-nav-search> a img.logo {
+  width: 50%;
+}
+
+
 .wy-side-nav-search > div.version {
   color: black;
 }
@@ -182,3 +216,60 @@ em.sig-param span.default_value {
 .rst-content table.field-list th {
   padding: 16px;
 }
+
+
+/* Create local table of contents
+   ------------------------------
+  inspired by https://github.com/readthedocs/sphinx_rtd_theme/pull/919
+  and https://github.com/readthedocs/sphinx_rtd_theme/issues/764
+  see also _templates/layout.html
+ */
+
+#local-table-of-contents {
+  padding-bottom: 20px;
+  /* display: none; */
+}
+
+/* Mask entry of main header (chapter) */
+#local-table-of-contents a[href="#"]{
+  /*display: none;*/
+}
+
+/* indent subsections */
+#local-table-of-contents ul > ul {
+  padding-left: 0px;
+  margin-left: 20px;
+  padding-right: 0;
+  padding-bottom: 5px;
+}
+
+
+#local-table-of-contents-title {
+  margin-bottom: 10px;
+}
+
+/* Show in Sidebar if window width is larger than nav-side + nav-content + toc-width */
+@media screen and (min-width: 1200px) {
+  .wy-nav-content {
+    max-width: var(--nav-content-width-wide);
+  }
+
+  #local-table-of-contents {
+    display: block;
+    position: fixed;
+    margin-left: 15px;
+    overflow-y: auto;
+    height: 95%;
+    top: 45px;
+    left: var(--local-toc-left);
+    width: var(--local-toc-width);
+  }
+
+  #local-table-of-contents-title {
+    display: block;
+    font-size: 16px;
+    width: 100%;
+    padding-top: 10px;
+    padding-bottom: 5px;
+  }
+}
diff --git a/doc/_templates/layout.html b/doc/_templates/layout.html
@@ -0,0 +1,12 @@
+{% extends "!layout.html" %}
+{% block document %} 
+  {%- if toc|length > title|length + 75 %}
+  <nav id="local-table-of-contents" role="navigation" aria-labelledby="local-table-of-contents-title">
+    <h4 id="local-table-of-contents-title">On This Page</h4>
+    {{ toc }}
+  </nav>
+  {%- endif %}
+
+  {{ super() }}
+{% endblock %}
+
diff --git a/doc/conf.py b/doc/conf.py
@@ -66,9 +66,11 @@ def about_package(init_posixpath: pathlib.Path) -> dict:
     "sphinx.ext.githubpages",
     "sphinx.ext.napoleon",
 ]
+autosectionlabel_prefix_document = True
+autosectionlabel_maxdepth = 2
 
 # Add any paths that contain templates here, relative to this directory.
-# templates_path = ['_templates']
+templates_path = ['_templates']
 
 # The suffix(es) of source filenames.
 # You can specify multiple suffix as a list of string:
@@ -84,6 +86,11 @@ def about_package(init_posixpath: pathlib.Path) -> dict:
 copyright_ = "2019, pyLHC/OMC-TEAM"
 author = ABOUT_PYLHC_SUBMITTER["__author__"]
 
+# Override link in 'Edit on Github'
+rst_prolog = f"""
+:github_url: {ABOUT_PYLHC_SUBMITTER['__url__']}
+"""
+
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the
 # built documents.
@@ -98,7 +105,7 @@ def about_package(init_posixpath: pathlib.Path) -> dict:
 #
 # This is also used if you do content translation via gettext catalogs.
 # Usually you set "language" from the command line for these cases.
-language = None
+language = 'en'
 
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.

diff --git a/doc/entrypoints/autosix.rst b/doc/entrypoints/autosix.rst
@@ -1,2 +1,3 @@
 .. automodule:: pylhc_submitter.autosix
     :members:
+    :noindex:
diff --git a/doc/entrypoints/job_submitter.rst b/doc/entrypoints/job_submitter.rst
@@ -1,2 +1,3 @@
 .. automodule:: pylhc_submitter.job_submitter
     :members:
+    :noindex:
diff --git a/doc/modules/constants.rst b/doc/modules/constants.rst
@@ -1,15 +1,25 @@
 Constants Definitions
-**************************
+*********************
 
 
 .. automodule:: pylhc_submitter.constants.general
     :members:
+    :noindex:
 
 
 .. automodule:: pylhc_submitter.constants.external_paths
     :members:
+    :noindex:
 
+.. automodule:: pylhc_submitter.constants.job_submitter
+    :members:
+    :noindex:
+
+.. automodule:: pylhc_submitter.constants.htcondor
+    :members:
+    :noindex:
 
 .. automodule:: pylhc_submitter.constants.autosix
     :members:
+    :noindex:
 
diff --git a/doc/modules/htc.rst b/doc/modules/htc.rst
diff --git a/doc/modules/sixdesk_tools.rst b/doc/modules/sixdesk_tools.rst
@@ -3,21 +3,28 @@ Sixdesk Tools
 
 .. automodule:: pylhc_submitter.sixdesk_tools.stages
     :members:
+    :noindex:
 
 .. automodule:: pylhc_submitter.sixdesk_tools.create_workspace
     :members:
+    :noindex:
 
 .. automodule:: pylhc_submitter.sixdesk_tools.submit
     :members:
+    :noindex:
 
 .. automodule:: pylhc_submitter.sixdesk_tools.post_process_da
     :members:
+    :noindex:
 
 .. automodule:: pylhc_submitter.sixdesk_tools.extract_data_from_db
     :members:
+    :noindex:
 
 .. automodule:: pylhc_submitter.sixdesk_tools.utils
     :members:
+    :noindex:
 
 .. automodule:: pylhc_submitter.sixdesk_tools.troubleshooting
     :members:
+    :noindex:
diff --git a/doc/modules/submitter.rst b/doc/modules/submitter.rst
@@ -0,0 +1,19 @@
+Submitter
+*********
+
+.. automodule:: pylhc_submitter.submitter.htc_utils
+    :members:
+    :noindex:
+
+
+.. automodule:: pylhc_submitter.submitter.iotools
+    :members:
+    :noindex:
+
+.. automodule:: pylhc_submitter.submitter.mask
+    :members:
+    :noindex:
+
+.. automodule:: pylhc_submitter.submitter.runners
+    :members:
+    :noindex:
diff --git a/doc/modules/utils.rst b/doc/modules/utils.rst
@@ -3,7 +3,9 @@ Utilities
 
 .. automodule:: pylhc_submitter.utils.iotools
     :members:
+    :noindex:
 
 
 .. automodule:: pylhc_submitter.utils.logging_tools
     :members:
+    :noindex:
diff --git a/pylhc_submitter/__init__.py b/pylhc_submitter/__init__.py
@@ -10,7 +10,7 @@
 __title__ = "pylhc_submitter"
 __description__ = "pylhc-submitter contains scripts to simplify the creation and submission of jobs to HTCondor at CERN"
 __url__ = "https://github.com/pylhc/submitter"
-__version__ = "1.1.1"
+__version__ = "2.0.0"
 __author__ = "pylhc"
 __author_email__ = "pylhc@github.com"
 __license__ = "MIT"

diff --git a/pylhc_submitter/autosix.py b/pylhc_submitter/autosix.py
@@ -193,34 +193,17 @@
 
 import numpy as np
 import tfs
-from generic_parser import EntryPointParameters, entrypoint, DotDict
+from generic_parser import EntryPointParameters, entrypoint
 from generic_parser.entry_datatypes import DictAsString
 
-from pylhc_submitter.constants.autosix import (
-    HEADER_BASEDIR,
-    SIXENV_REQUIRED,
-    SIXENV_OPTIONAL,
-    AutoSixEnvironment,
-)
-from pylhc_submitter.htc.mask import generate_jobdf_index
-from pylhc_submitter.job_submitter import (
-    JOBSUMMARY_FILE,
-    COLUMN_JOBID,
-)
-from pylhc_submitter.sixdesk_tools.create_workspace import (
-    set_max_materialize
-)
-from pylhc_submitter.sixdesk_tools.stages import Stage, STAGE_ORDER
-from pylhc_submitter.sixdesk_tools.utils import (
-    is_locked,
-    check_mask,
-)
-from pylhc_submitter.utils.iotools import (
-    PathOrStr,
-    save_config,
-    make_replace_entries_iterable,
-    keys_to_path
-)
+from pylhc_submitter.constants.autosix import (HEADER_BASEDIR, SIXENV_OPTIONAL, SIXENV_REQUIRED,
+                                               AutoSixEnvironment)
+from pylhc_submitter.constants.job_submitter import COLUMN_JOBID, JOBSUMMARY_FILE
+from pylhc_submitter.submitter.mask import generate_jobdf_index
+from pylhc_submitter.sixdesk_tools.stages import STAGE_ORDER, Stage
+from pylhc_submitter.sixdesk_tools.utils import check_mask, is_locked
+from pylhc_submitter.utils.iotools import (PathOrStr, keys_to_path, make_replace_entries_iterable,
+                                           save_config)
 from pylhc_submitter.utils.logging_tools import log_setup
 
 LOG = logging.getLogger(__name__)

diff --git a/pylhc_submitter/constants/autosix.py b/pylhc_submitter/constants/autosix.py
@@ -1,6 +1,6 @@
 """
-Constants: Autosix
-----------------------------------
+Autosix
+-------
 
 Collections of constants and paths used in autosix.
 

diff --git a/pylhc_submitter/constants/external_paths.py b/pylhc_submitter/constants/external_paths.py
@@ -1,6 +1,6 @@
 """
-Constants: External Paths
--------------------------
+External Paths
+--------------
 
 Specific constants relating to external paths to be used,
 to help with consistency.

diff --git a/pylhc_submitter/constants/general.py b/pylhc_submitter/constants/general.py
@@ -1,6 +1,6 @@
 """
-Constants: General
-------------------
+General
+-------
 
 General constants to help with consistency.
 """