diff --git a/doc/Makefile b/doc/Makefile new file mode 100644 index 000000000..ac5036632 --- /dev/null +++ b/doc/Makefile @@ -0,0 +1,194 @@ +# Makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +PAPER = +BUILDDIR = build +BASENAME = $(subst .,,$(subst $() $(),,regolith)) + +# User-friendly check for sphinx-build +ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) +$(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/) +endif + +# Internal variables. +PAPEROPT_a4 = -D latex_paper_size=a4 +PAPEROPT_letter = -D latex_paper_size=letter +ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source +# the i18n builder cannot share the environment and doctrees with the others +I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source + +.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext + +help: + @echo "Please use \`make ' where is one of" + @echo " html to make standalone HTML files" + @echo " dirhtml to make HTML files named index.html in directories" + @echo " singlehtml to make a single large HTML file" + @echo " pickle to make pickle files" + @echo " json to make JSON files" + @echo " htmlhelp to make HTML files and a HTML help project" + @echo " qthelp to make HTML files and a qthelp project" + @echo " devhelp to make HTML files and a Devhelp project" + @echo " epub to make an epub" + @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" + @echo " latexpdf to make LaTeX files and run them through pdflatex" + @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" + @echo " text to make text files" + @echo " man to make manual pages" + @echo " texinfo to make Texinfo files" + @echo " info to make Texinfo files and run them through makeinfo" + @echo " gettext to make PO message catalogs" + @echo " changes to make an overview of all changed/added/deprecated items" + @echo " xml to make Docutils-native XML files" + @echo " pseudoxml to make pseudoxml-XML files for display purposes" + @echo " linkcheck to check all external links for integrity" + @echo " doctest to run all doctests embedded in the documentation (if enabled)" + +clean: + rm -rf $(BUILDDIR)/* + +html: + $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." + +dirhtml: + $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." + +singlehtml: + $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml + @echo + @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." + +pickle: + $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle + @echo + @echo "Build finished; now you can process the pickle files." + +json: + $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json + @echo + @echo "Build finished; now you can process the JSON files." + +htmlhelp: + $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp + @echo + @echo "Build finished; now you can run HTML Help Workshop with the" \ + ".hhp project file in $(BUILDDIR)/htmlhelp." + +qthelp: + $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp + @echo + @echo "Build finished; now you can run "qcollectiongenerator" with the" \ + ".qhcp project file in $(BUILDDIR)/qthelp, like this:" + @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/$(BASENAME).qhcp" + @echo "To view the help file:" + @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/$(BASENAME).qhc" + +devhelp: + $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp + @echo + @echo "Build finished." + @echo "To view the help file:" + @echo "# mkdir -p $$HOME/.local/share/devhelp/$(BASENAME)" + @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/$(BASENAME)" + @echo "# devhelp" + +epub: + $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub + @echo + @echo "Build finished. The epub file is in $(BUILDDIR)/epub." + +latex: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo + @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." + @echo "Run \`make' in that directory to run these through (pdf)latex" \ + "(use \`make latexpdf' here to do that automatically)." + +latexpdf: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo "Running LaTeX files through pdflatex..." + $(MAKE) -C $(BUILDDIR)/latex all-pdf + @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." + +latexpdfja: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo "Running LaTeX files through platex and dvipdfmx..." + $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja + @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." + +text: + $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text + @echo + @echo "Build finished. The text files are in $(BUILDDIR)/text." + +man: + $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man + @echo + @echo "Build finished. The manual pages are in $(BUILDDIR)/man." + +texinfo: + $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo + @echo + @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." + @echo "Run \`make' in that directory to run these through makeinfo" \ + "(use \`make info' here to do that automatically)." + +info: + $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo + @echo "Running Texinfo files through makeinfo..." + make -C $(BUILDDIR)/texinfo info + @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." + +gettext: + $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale + @echo + @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." + +changes: + $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes + @echo + @echo "The overview file is in $(BUILDDIR)/changes." + +linkcheck: + $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck + @echo + @echo "Link check complete; look for any errors in the above output " \ + "or in $(BUILDDIR)/linkcheck/output.txt." + +doctest: + $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest + @echo "Testing of doctests in the sources finished, look at the " \ + "results in $(BUILDDIR)/doctest/output.txt." + +# Manual publishing to the gh-pages branch + +GITREPOPATH = $(shell cd $(CURDIR) && git rev-parse --git-dir) +GITREMOTE = origin +GITREMOTEURL = $(shell git config --get remote.$(GITREMOTE).url) +GITLASTCOMMIT = $(shell git rev-parse --short HEAD) + +publish: + @test -d build/html || \ + ( echo >&2 "Run 'make html' first!"; false ) + git show-ref --verify --quiet refs/heads/gh-pages || \ + git branch --track gh-pages $(GITREMOTE)/gh-pages + test -d build/gh-pages || \ + git clone -s -b gh-pages $(GITREPOPATH) build/gh-pages + cd build/gh-pages && \ + git pull $(GITREMOTEURL) gh-pages + rsync -acv --delete --exclude=.git --exclude=.rsync-exclude \ + --exclude-from=build/gh-pages/.rsync-exclude \ + --link-dest=$(CURDIR)/build/html build/html/ build/gh-pages/ + cd build/gh-pages && \ + git add --all . && \ + git diff --cached --quiet || \ + git commit -m "Sync with the source at $(GITLASTCOMMIT)." + cd build/gh-pages && \ + git push origin gh-pages diff --git a/doc/make.bat b/doc/make.bat new file mode 100644 index 000000000..2be830693 --- /dev/null +++ b/doc/make.bat @@ -0,0 +1,36 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=source +set BUILDDIR=build +set SPHINXPROJ=PackagingScientificPython + +if "%1" == "" goto help + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.http://sphinx-doc.org/ + exit /b 1 +) + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% + +:end +popd diff --git a/docs/Makefile b/doc/source/Makefile similarity index 100% rename from docs/Makefile rename to doc/source/Makefile diff --git a/doc/source/_static/.placeholder b/doc/source/_static/.placeholder new file mode 100644 index 000000000..e69de29bb diff --git a/docs/_static/regolith-logo-white.png b/doc/source/_static/regolith-logo-white.png similarity index 100% rename from docs/_static/regolith-logo-white.png rename to doc/source/_static/regolith-logo-white.png diff --git a/docs/_static/regolith-logo-white.svg b/doc/source/_static/regolith-logo-white.svg similarity index 100% rename from docs/_static/regolith-logo-white.svg rename to doc/source/_static/regolith-logo-white.svg diff --git a/docs/_static/regolith-logo.ico b/doc/source/_static/regolith-logo.ico similarity index 100% rename from docs/_static/regolith-logo.ico rename to doc/source/_static/regolith-logo.ico diff --git a/docs/_static/regolith-logo.png b/doc/source/_static/regolith-logo.png similarity index 100% rename from docs/_static/regolith-logo.png rename to doc/source/_static/regolith-logo.png diff --git a/docs/_static/regolith-logo.svg b/doc/source/_static/regolith-logo.svg similarity index 100% rename from docs/_static/regolith-logo.svg rename to doc/source/_static/regolith-logo.svg diff --git a/docs/_static/rg-dungeon.js b/doc/source/_static/rg-dungeon.js similarity index 100% rename from docs/_static/rg-dungeon.js rename to doc/source/_static/rg-dungeon.js diff --git a/docs/_static/unicodetiles.css b/doc/source/_static/unicodetiles.css similarity index 100% rename from docs/_static/unicodetiles.css rename to doc/source/_static/unicodetiles.css diff --git a/docs/_static/unicodetiles.min.js b/doc/source/_static/unicodetiles.min.js similarity index 100% rename from docs/_static/unicodetiles.min.js rename to doc/source/_static/unicodetiles.min.js diff --git a/docs/_themes/blackcloud/static/blackcloud.css_t b/doc/source/_themes/blackcloud/static/blackcloud.css_t similarity index 100% rename from docs/_themes/blackcloud/static/blackcloud.css_t rename to doc/source/_themes/blackcloud/static/blackcloud.css_t diff --git a/docs/_themes/blackcloud/theme.conf b/doc/source/_themes/blackcloud/theme.conf similarity index 100% rename from docs/_themes/blackcloud/theme.conf rename to doc/source/_themes/blackcloud/theme.conf diff --git a/docs/api/app.rst b/doc/source/api/app.rst similarity index 100% rename from docs/api/app.rst rename to doc/source/api/app.rst diff --git a/docs/api/basebuilder.rst b/doc/source/api/basebuilder.rst similarity index 100% rename from docs/api/basebuilder.rst rename to doc/source/api/basebuilder.rst diff --git a/docs/api/broker.rst b/doc/source/api/broker.rst similarity index 100% rename from docs/api/broker.rst rename to doc/source/api/broker.rst diff --git a/docs/api/builder.rst b/doc/source/api/builder.rst similarity index 100% rename from docs/api/builder.rst rename to doc/source/api/builder.rst diff --git a/docs/api/chained_db.rst b/doc/source/api/chained_db.rst similarity index 100% rename from docs/api/chained_db.rst rename to doc/source/api/chained_db.rst diff --git a/docs/api/commands.rst b/doc/source/api/commands.rst similarity index 100% rename from docs/api/commands.rst rename to doc/source/api/commands.rst diff --git a/docs/api/cvbuilder.rst b/doc/source/api/cvbuilder.rst similarity index 100% rename from docs/api/cvbuilder.rst rename to doc/source/api/cvbuilder.rst diff --git a/docs/api/database.rst b/doc/source/api/database.rst similarity index 100% rename from docs/api/database.rst rename to doc/source/api/database.rst diff --git a/docs/api/dates.rst b/doc/source/api/dates.rst similarity index 100% rename from docs/api/dates.rst rename to doc/source/api/dates.rst diff --git a/docs/api/deploy.rst b/doc/source/api/deploy.rst similarity index 100% rename from docs/api/deploy.rst rename to doc/source/api/deploy.rst diff --git a/docs/api/htmlbuilder.rst b/doc/source/api/htmlbuilder.rst similarity index 100% rename from docs/api/htmlbuilder.rst rename to doc/source/api/htmlbuilder.rst diff --git a/docs/api/index.rst b/doc/source/api/index.rst similarity index 100% rename from docs/api/index.rst rename to doc/source/api/index.rst diff --git a/docs/api/interactive.rst b/doc/source/api/interactive.rst similarity index 100% rename from docs/api/interactive.rst rename to doc/source/api/interactive.rst diff --git a/docs/api/main.rst b/doc/source/api/main.rst similarity index 100% rename from docs/api/main.rst rename to doc/source/api/main.rst diff --git a/doc/source/api/regolith.example_package.rst b/doc/source/api/regolith.example_package.rst new file mode 100644 index 000000000..e8fdb885f --- /dev/null +++ b/doc/source/api/regolith.example_package.rst @@ -0,0 +1,31 @@ +.. _example_package documentation: + +|title| +======= + +.. |title| replace:: regolith.example_package package + +.. automodule:: regolith.example_package + :members: + :undoc-members: + :show-inheritance: + +|foo| +----- + +.. |foo| replace:: regolith.example_package.foo module + +.. automodule:: regolith.example_package.foo + :members: + :undoc-members: + :show-inheritance: + +|bar| +----- + +.. |bar| replace:: regolith.example_package.bar module + +.. automodule:: regolith.example_package.foo + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/regolith.rst b/doc/source/api/regolith.rst new file mode 100644 index 000000000..44aceb0e3 --- /dev/null +++ b/doc/source/api/regolith.rst @@ -0,0 +1,30 @@ +:tocdepth: -1 + +|title| +======= + +.. |title| replace:: regolith package + +.. automodule:: regolith + :members: + :undoc-members: + :show-inheritance: + +Subpackages +----------- + +.. toctree:: + regolith.example_package + +Submodules +---------- + +|module| +-------- + +.. |module| replace:: regolith.example_submodule module + +.. automodule:: regolith.example_submodule + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/api/runcontrol.rst b/doc/source/api/runcontrol.rst similarity index 100% rename from docs/api/runcontrol.rst rename to doc/source/api/runcontrol.rst diff --git a/docs/api/tools.rst b/doc/source/api/tools.rst similarity index 100% rename from docs/api/tools.rst rename to doc/source/api/tools.rst diff --git a/docs/api/validators.rst b/doc/source/api/validators.rst similarity index 100% rename from docs/api/validators.rst rename to doc/source/api/validators.rst diff --git a/docs/collections/citations.rst b/doc/source/collections/citations.rst similarity index 100% rename from docs/collections/citations.rst rename to doc/source/collections/citations.rst diff --git a/docs/collections/courses.rst b/doc/source/collections/courses.rst similarity index 100% rename from docs/collections/courses.rst rename to doc/source/collections/courses.rst diff --git a/docs/collections/index.rst b/doc/source/collections/index.rst similarity index 100% rename from docs/collections/index.rst rename to doc/source/collections/index.rst diff --git a/docs/collections/jobs.rst b/doc/source/collections/jobs.rst similarity index 100% rename from docs/collections/jobs.rst rename to doc/source/collections/jobs.rst diff --git a/docs/collections/news.rst b/doc/source/collections/news.rst similarity index 100% rename from docs/collections/news.rst rename to doc/source/collections/news.rst diff --git a/docs/commands/fs-to-mongo.rst b/doc/source/commands/fs-to-mongo.rst similarity index 100% rename from docs/commands/fs-to-mongo.rst rename to doc/source/commands/fs-to-mongo.rst diff --git a/docs/commands/index.rst b/doc/source/commands/index.rst similarity index 100% rename from docs/commands/index.rst rename to doc/source/commands/index.rst diff --git a/docs/commands/validate.rst b/doc/source/commands/validate.rst similarity index 100% rename from docs/commands/validate.rst rename to doc/source/commands/validate.rst diff --git a/doc/source/conf.py b/doc/source/conf.py new file mode 100644 index 000000000..5f7abb709 --- /dev/null +++ b/doc/source/conf.py @@ -0,0 +1,424 @@ +# -*- coding: utf-8 -*- +# +# xo documentation build configuration file, created by +# sphinx-quickstart on Sun Jan 27 00:12:33 2013. +# +# This file is execfile()d with the current directory set to its containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +import json +import tempfile +from collections.abc import MutableMapping +from subprocess import check_output +from textwrap import indent + +import cloud_sptheme as csp + +from regolith import __version__ as REGOLITH_VERSION +from regolith.fsclient import _id_key, dump_json, json_to_yaml +from regolith.main import CONNECTED_COMMANDS, DISCONNECTED_COMMANDS +from regolith.schemas import EXEMPLARS, SCHEMAS + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# sys.path.insert(0, os.path.abspath('.')) + +# -- General configuration ----------------------------------------------------- + +# If your documentation needs a minimal Sphinx version, state it here. +# needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be extensions +# coming with Sphinx (named 'sphinx.ext.*') or your custom ones. +extensions = [ + "sphinx.ext.autodoc", + "sphinx.ext.mathjax", + "sphinx.ext.napoleon", + "sphinx.ext.viewcode", +] + +napoleon_google_docstring = False +napoleon_use_param = False +napoleon_use_ivar = True + +# Add any paths that contain templates here, relative to this directory. +templates_path = ["_templates"] + +# The suffix of source filenames. +source_suffix = ".rst" + +# The encoding of source files. +# source_encoding = 'utf-8-sig' + +# The master toctree document. +master_doc = "index" + +# General information about the project. +project = "regolith" +copyright = "2015, Anthony Scopatz" + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +version = REGOLITH_VERSION.rsplit(".", 1)[0] + +# The full version, including alpha/beta/rc tags. +release = REGOLITH_VERSION + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# language = None + +# There are two options for replacing |today|: either, you set today to some +# non-false value, then it is used: +# today = '' +# Else, today_fmt is used as the format for a strftime call. +# today_fmt = '%B %d, %Y' + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +exclude_patterns = ["_build"] + +# The reST default role (used for this markup: `text`) to use for all documents. +# default_role = None + +# If true, '()' will be appended to :func: etc. cross-reference text. +# add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +# add_module_names = True + +# If true, sectionauthor and moduleauthor directives will be shown in the +# output. They are ignored by default. +# show_authors = False + +# The name of the Pygments (syntax highlighting) style to use. +# pygments_style = 'sphinx' +pygments_style = "pastie" + +# A list of ignored prefixes for module index sorting. +# modindex_common_prefix = [] + + +# -- Options for HTML output --------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +html_theme = "redcloud" +# html_theme = 'blackcloud' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +# html_theme_options = {} +html_theme_options = {"roottarget": "index"} + +# Add any paths that contain custom themes here, relative to this directory. +# html_theme_path = [] +# html_theme_path = [csp.get_theme_dir()] +html_theme_path = ["_themes", csp.get_theme_dir()] + +# The name for this set of Sphinx documents. If None, it defaults to +# " v documentation". +html_title = "regolith - software group content managment system" + +# A shorter title for the navigation bar. Default is the same as html_title. +html_short_title = "regolith" + +# The name of an image file (relative to this directory) to place at the top +# of the sidebar. +html_logo = "_static/regolith-logo.svg" + +# The name of an image file (within the static path) to use as favicon of the +# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 +# pixels large. +html_favicon = "_static/regolith-logo.ico" + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ["_static"] + +# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, +# using the given strftime format. +# html_last_updated_fmt = '%b %d, %Y' + +# If true, SmartyPants will be used to convert quotes and dashes to +# typographically correct entities. +# html_use_smartypants = True + +# Custom sidebar templates, maps document names to template names. +# html_sidebars = {} + +# Additional templates that should be rendered to pages, maps page names to +# template names. +# html_additional_pages = {} + +# If false, no module index is generated. +html_domain_indices = False + +# If false, no index is generated. +html_use_index = False + +# If true, the index is split into individual pages for each letter. +html_split_index = False + +# If true, links to the reST sources are added to the pages. +html_show_sourcelink = False + +# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. +html_show_sphinx = False + +# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. +# html_show_copyright = True + +# If true, an OpenSearch description file will be output, and all pages will +# contain a tag referring to it. The value of this option must be the +# base URL from which the finished HTML is served. +# html_use_opensearch = '' + +# This is the file name suffix for HTML files (e.g. ".xhtml"). +# html_file_suffix = None + +# Output file base name for HTML help builder. +htmlhelp_basename = "regolithdoc" + +# -- Options for LaTeX output -------------------------------------------------- + +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + # 'papersize': 'letterpaper', + # The font size ('10pt', '11pt' or '12pt'). + # 'pointsize': '10pt', + # Additional stuff for the LaTeX preamble. + # 'preamble': '', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, author, documentclass [howto/manual]). +latex_documents = [ + ( + "index", + "regolith.tex", + "Regolith Documentation", + "Anthony Scopatz", + "manual", + ) +] + +# The name of an image file (relative to this directory) to place at the top of +# the title page. +# latex_logo = None + +# For "manual" documents, if this is true, then toplevel headings are parts, +# not chapters. +# latex_use_parts = False + +# If true, show page references after internal links. +# latex_show_pagerefs = False + +# If true, show URL addresses after external links. +# latex_show_urls = False + +# Documents to append as an appendix to all manuals. +# latex_appendices = [] + +# If false, no module index is generated. +# latex_domain_indices = True + + +# -- Options for manual page output -------------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [("index", "regolith", "regolith docs", ["Anthony Scopatz"], 1)] + +# If true, show URL addresses after external links. +# man_show_urls = False + + +# -- Options for Texinfo output ------------------------------------------------ + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + ( + "index", + "regolith", + "regolith documentation", + "Anthony Scopatz", + "regolith", + "Research group managment software.", + "Miscellaneous", + ) +] + + +# Documents to append as an appendix to all manuals. +# texinfo_appendices = [] + +# If false, no module index is generated. +# texinfo_domain_indices = True + +# How to display URL addresses: 'footnote', 'no', or 'inline'. +# texinfo_show_urls = 'footnote' + + +def format_key(schema, key, indent_str=""): + s = "" + if key == "schema" or key == "required": + return s + line_format = ":{key}: {type}, {description}, required\n" + line_format_o = ":{key}: {type}, {description}, optional\n" + if not schema.get("required", False): + lf = line_format_o + else: + lf = line_format + if "type" in schema.get(key, {}) or "anyof_type" in schema.get(key, {}): + schema = schema[key] + if "type" in schema or "anyof_type" in schema: + s += indent( + lf.format( + key=key, + description=schema.get("description", ""), + type=schema.get("type", schema.get("anyof_type", "")), + ), + indent_str, + ) + allowed = schema.get("allowed", "") or schema.get("eallowed", "") + if allowed: + s += indent("\nAllowed values: \n", indent_str + "\t") + for allow in allowed: + if allow: + s += indent("* {}\n".format(allow), indent_str + "\t" * 2) + else: + s += indent("* ``''``\n", indent_str + "\t" * 2) + s = s.replace(", , ", ", ") + if schema.get("schema", False): + s += "\n" + for inner_key in schema.get("schema", ()): + s += format_key(schema["schema"], inner_key, indent_str=indent_str + "\t") + + return s + + +def build_schema_doc(key): + fn = "collections/" + key + ".rst" + with open(fn, "w") as f: + s = "" + s += key.title() + "\n" + s += "=" * len(key) + "\n" + s += SCHEMAS[key]["_description"]["description"] + "\n\n" + s += "Schema\n------\nThe following lists key names mapped to its " "type and meaning for each entry.\n\n" + schema = SCHEMAS[key] + schema_list = list(schema.keys()) + schema_list.sort() + for k in schema_list: + if k not in ["_description"]: + s += format_key(schema[k], key=k) + s += "\n\n" + s += "YAML Example\n------------\n\n" + s += ".. code-block:: yaml\n\n" + temp = tempfile.NamedTemporaryFile() + temp2 = tempfile.NamedTemporaryFile() + documents = EXEMPLARS[key] + if isinstance(documents, MutableMapping): + documents = [documents] + documents = {doc["_id"]: doc for doc in documents} + dump_json(temp.name, documents) + docs = sorted(documents.values(), key=_id_key) + lines = [json.dumps(doc, sort_keys=True, indent=4, separators=(",", ": ")) for doc in docs] + jd = "\n".join(lines) + json_to_yaml(temp.name, temp2.name) + with open(temp2.name, "r") as ff: + s += indent(ff.read(), "\t") + s += "\n\n" + s += "JSON/Mongo Example\n------------------\n\n" + s += ".. code-block:: json\n\n" + s += indent(jd, "\t") + s += "\n" + f.write(s) + + +for k in SCHEMAS: + build_schema_doc(k) + + +docs_not_in_schemas = ["citations", "courses", "jobs", "news"] +all_collection_docs = sorted(list(SCHEMAS.keys()) + docs_not_in_schemas) + +collections_header = """.. _regolith_collections: + +================= +Collections +================= +The following contain the regolith schemas and examples in both YAML and JSON/Mongo. + +.. toctree:: + :maxdepth: 1 + +""" + +collections_header += indent("\n".join(all_collection_docs), " ") + +with open("collections/index.rst", "w") as f: + f.write(collections_header) + + +def build_cli_doc(cli): + fn = "commands/" + cli + ".rst" + out = check_output(["regolith", cli, "-h"]).decode("utf-8") + s = "{}\n".format(cli) + "=" * len(cli) + "\n\n" + s += ".. code-block:: bash\n\n" + s += indent(out, "\t") + "\n" + if cli == "validate": + s += """Misc +---- + +This can also be added as a git hook by adding the following to +``.git/hooks/pre-commit`` + +.. code-block:: sh + + #!/bin/sh + regolith validate + +This can be enabled with ``chmod +x .git/hooks/pre-commit``""" + with open(fn, "w") as f: + f.write(s) + + +# build CLI docs +clis = sorted(set(CONNECTED_COMMANDS.keys()) | set(DISCONNECTED_COMMANDS.keys())) +for cli in clis: + build_cli_doc(cli) + +cli_index = """.. _commands: + +================= +Regolith Commands +================= +Shell commands for regolith + +{} + +.. toctree:: + :maxdepth: 1 + +""" +cli_index += indent("\n".join(clis), " ") +with open("commands/index.rst", "w") as f: + out = check_output(["regolith", "-h"]).decode("utf-8") + s = ".. code-block:: bash\n\n" + s += indent(out, "\t") + "\n" + f.write(cli_index.format(s)) diff --git a/docs/database_maintenance/index.rst b/doc/source/database_maintenance/index.rst similarity index 100% rename from docs/database_maintenance/index.rst rename to doc/source/database_maintenance/index.rst diff --git a/docs/database_maintenance/mongo_maintenance.rst b/doc/source/database_maintenance/mongo_maintenance.rst similarity index 100% rename from docs/database_maintenance/mongo_maintenance.rst rename to doc/source/database_maintenance/mongo_maintenance.rst diff --git a/docs/developer_notes/dp_regolith_prums_to_mongodb.md b/doc/source/developer_notes/dp_regolith_prums_to_mongodb.md similarity index 100% rename from docs/developer_notes/dp_regolith_prums_to_mongodb.md rename to doc/source/developer_notes/dp_regolith_prums_to_mongodb.md diff --git a/docs/helper_gui.rst b/doc/source/helper_gui.rst similarity index 100% rename from docs/helper_gui.rst rename to doc/source/helper_gui.rst diff --git a/docs/helper_gui_images/a_projectum.png b/doc/source/helper_gui_images/a_projectum.png similarity index 100% rename from docs/helper_gui_images/a_projectum.png rename to doc/source/helper_gui_images/a_projectum.png diff --git a/docs/helper_gui_images/l_members.png b/doc/source/helper_gui_images/l_members.png similarity index 100% rename from docs/helper_gui_images/l_members.png rename to doc/source/helper_gui_images/l_members.png diff --git a/docs/helper_gui_images/layout.png b/doc/source/helper_gui_images/layout.png similarity index 100% rename from docs/helper_gui_images/layout.png rename to doc/source/helper_gui_images/layout.png diff --git a/docs/helper_gui_images/u_logurl.png b/doc/source/helper_gui_images/u_logurl.png similarity index 100% rename from docs/helper_gui_images/u_logurl.png rename to doc/source/helper_gui_images/u_logurl.png diff --git a/docs/helper_gui_images/workflow1.png b/doc/source/helper_gui_images/workflow1.png similarity index 100% rename from docs/helper_gui_images/workflow1.png rename to doc/source/helper_gui_images/workflow1.png diff --git a/docs/helper_gui_images/workflow10.png b/doc/source/helper_gui_images/workflow10.png similarity index 100% rename from docs/helper_gui_images/workflow10.png rename to doc/source/helper_gui_images/workflow10.png diff --git a/docs/helper_gui_images/workflow12.png b/doc/source/helper_gui_images/workflow12.png similarity index 100% rename from docs/helper_gui_images/workflow12.png rename to doc/source/helper_gui_images/workflow12.png diff --git a/docs/helper_gui_images/workflow3.png b/doc/source/helper_gui_images/workflow3.png similarity index 100% rename from docs/helper_gui_images/workflow3.png rename to doc/source/helper_gui_images/workflow3.png diff --git a/docs/helper_gui_images/workflow6.png b/doc/source/helper_gui_images/workflow6.png similarity index 100% rename from docs/helper_gui_images/workflow6.png rename to doc/source/helper_gui_images/workflow6.png diff --git a/docs/helper_gui_images/workflow8.png b/doc/source/helper_gui_images/workflow8.png similarity index 100% rename from docs/helper_gui_images/workflow8.png rename to doc/source/helper_gui_images/workflow8.png diff --git a/doc/source/index.rst b/doc/source/index.rst new file mode 100644 index 000000000..8dd6b62dd --- /dev/null +++ b/doc/source/index.rst @@ -0,0 +1,721 @@ +.. role:: bash(code) + :language: bash +.. role:: python(code) + :language: python + +.. raw:: html + + + + +
+

welcome to the regolith docs

+
+ + +Regolith +======== +Regolith is a content management system for software & research groups. +Regolith creates and manages a database of people, publications, projects, +proposals & grants, courses, and more! From this database, regolith is then +able to: + +* Generate a group website, +* Generate CVs and publication lists for the group members, +* Act as a grade book for your courses, and more! + +Databases may be file-based (JSON and YAML) or MongoDB-based. + +Regolith is developed as a `regro project `_ + +Example Sites +============= +The following are some sample websites that are powered by regolith, even though +building +websites is just one of the many facets of this tool: + +1. `ERGS Home Page `_ +2. `Technical WorkShop on Fuel Cycle Simulation `_ + +Installation +============ +1. Make your first database +---------------------------- +The quickest way to get started is to set up your first minimal database using a +handy cookie cutter. These instructions use the command line and assume you know +how to use the terminal/cmd prompt, and that you know how to install software from +either Pypi using :bash:`pip` or Anaconda/Miniconda using :bash:`conda`. The +instructions use the linux shell commands which should work on Mac and linux +computers, and on windows if you are running from at Git Bash terminal (recommended) +but will be slightly different (but still work) on a windows cmd terminal. + +First install the cookiecutter package if you don't already have it + +.. code-block:: sh + + $ conda install cookiecutter + +or + +.. code-block:: sh + + $ pip install cookiecutter + +Next, clone the GitHub repository with the handy beginning database template + +.. code-block:: sh + + $ git clone git@github.com/sbillinge/regolithdb-cookiecutter.git + +to get it using SSH or + +.. code-block:: sh + + $ git clone https://github.com/sbillinge/regolithdb-cookiecutter.git + +to get it using the HTTPS protocol (just use whichever works for you) + +Make a note of the path to the resulting :bash:`regolithdb-cookiecutter` directory, +(e.g., :bash:`/c/Users/me/scratch/regolithdb-cookiecutter` but yours will be different). +This is not your database, this is just the template and will be removed shortly. + +Next, in a new terminal, or in the same terminal, move to the directory where +you want to install your own permanent database. For example, we like to +create a directory off our home directory called :bash:`dbs` where we will keep +all of our databases (believe me, once you start using Regolith you will want +to make more and more) + +.. code-block:: bash + + $ cd ~ # takes you to your home directory + $ mkdir dbs # creates the dbs directory if it is not already there + $ cd dbs # change dir to the new dbs directory + +Now by running cookiecutter your starting db will be built from the template + +.. code-block:: bash + + $ cookiecutter //regolithdb-cookiecutter + +The program will ask a series of questions and you can type responses. Take your +time and answer the questions as accurately as possible, because you are already +entering data into your database! + +Here is an example, and the questions look like + +.. code-block:: bash + + $ cookiecutter ~/scratch/regolithdb-cookiecutter/ + database_name [my-cv-db]: + my_first_name [Albert]: Simon + my_last_name [Einstein]: Billinge + id_for_me [aeinstein]: sbillinge + my_group_name [Einstein Group]: Billinge Group + +and so on. If you just hit enter the cookie-cutter will use the default values +and you will build a database for Einstein, but type the values you want in +answer to each question to make your own. + +If you make a mistake just type CTL^C +and do it again. You may have to remove the directory if it has already been +created, for example, :bash:`$ rm -r my-cv-db`. Watch what you type here and +be careful not to remove something you care about by mistake! + +When you are happy with your database setup, type + +.. code-block:: bash + + $ ls + +which lists all the files in your current directory, +and you should see a directory called :bash:`my-cv-db` or whatever you chose to +call you database. OK, let's go and look at our database. change directory into it and do a directory +listing, + +.. code-block:: bash + + $ cd my-cv-db + $ ls + +or open a file +browser such as windows explorer and check out what is in there. + +You will see a direcotry called :bash:`db` and a file called :bash:`regolithrc.json`. +All of the collections in your database are in the :bash:`db` directory. The +:bash:`regolithrc.json` contains a bunch of information that Regolith needs to run and do its business. + +You can use the Regolith program to do many things with, and to, your +database. But you must always run Regolith from a directory that contains a +:bash:`regolithrc.json` file. Since you are in a directory that contains one, +you can run Regolith from here, but first you have to install it.... + +2. install Regolith +--------------------- +Regolith packages are available from conda-forge and PyPI: + +**conda:** + +.. code-block:: sh + + $ conda install -c conda-forge regolith + +**pip:** + +.. code-block:: sh + + $ pip install regolith + +The Regolith code is migrating quickly these days. If you prefer you can +install from the GitHub repository mode and get the latest changes. +In that case, clone the `GitHub repository `_, +change directory to the top level directory in that cloned repository where the +:bash:`setup.py` file is. From inside your virtual environment, type + +.. code-block:: sh + + $ pip install regolith -e + +which installs regolith in this environment in develop mode. In this mode, the +version of Regolith you run will change each time you update from the repo +leading to instability so be careful. + +To check that your installation is working, let's have Regolith make us a +todo list from our database. + +Make sure you are in a directory that +contains a :bash:`regolithrc.json` file (which you should be, i.e., the +top level directory of :bash:`~/dbs/my-cv-db`, if you have been +following these instructions) and type + +.. code-block:: sh + + $ regolith helper l_todos + +and you should see something like + +.. code-block:: sh + + loading .\./db\todos.yml... + dumping todos... + usage: regolith helper [-h] [-s STATI [STATI ...]] [--short [SHORT]] + [-t TAGS [TAGS ...]] [-a ASSIGNED_TO] + [-b [ASSIGNED_BY]] [--date DATE] + [-f FILTER [FILTER ...]] + helper_target + + positional arguments: + helper_target helper target to run. Currently valid targets are: + ['a_expense', 'a_grppub_readlist', 'a_manurev', + 'a_presentation', 'a_projectum', 'a_proposal', + 'a_proprev', 'a_todo', 'f_prum', 'f_todo', + 'l_abstract', 'l_contacts', 'l_grants', 'l_members', + 'l_milestones', 'l_progress', 'l_projecta', 'l_todo', + 'u_contact', 'u_institution', 'u_logurl', + 'u_milestone', 'u_todo', 'v_meetings', 'lister', + 'makeappointments'] + + optional arguments: + -h, --help show this help message and exit + -s STATI [STATI ...], --stati STATI [STATI ...] + Filter tasks with specific status from ['started', + 'finished', 'cancelled', 'paused']. Default is + started. + --short [SHORT] Filter tasks with estimated duration <= 30 mins, but + if a number is specified, the duration of the filtered + tasks will be less than that number of minutes. + -t TAGS [TAGS ...], --tags TAGS [TAGS ...] + Filter tasks by tags. Items are returned if they + contain any of the tags listed + -a ASSIGNED_TO, --assigned-to ASSIGNED_TO + Filter tasks that are assigned to this user id. + Default id is saved in user.json. + -b [ASSIGNED_BY], --assigned-by [ASSIGNED_BY] + Filter tasks that are assigned to other members by + this user id. Default id is saved in user.json. + --date DATE Enter a date such that the helper can calculate how + many days are left from that date to the due-date. + Default is today. + -f FILTER [FILTER ...], --filter FILTER [FILTER ...] + Search this collection by giving key element pairs. + '-f description paper' will return tasks with + description containing 'paper' + If the indices are far from being in numerical order, please renumber them by running regolith helper u_todo -r + (index) action (days to due date|importance|expected duration (mins)|tags|assigned by) + -------------------------------------------------------------------------------- + started: + (1) Do all the things to set up todos in regolith (59|3|60.0||None) + ------------------------------ + Tasks (decreasing priority going up) + ------------------------------ + 2021-07-29(59 days): (1) Do all the things to set up todos in regolith (59|3|60.0||None) + ------------------------------ + Deadlines: + ------------------------------ + +After all the help messages is your list of Todo items. There is just one item, +:bash:`Do all the things to set up todos in regolith`. + +OK, your Regolith is working. If it isn't working, consider joining, browsing +and posting questions to the `regolith-users `_ +Google group. + +Quick(ish) Start +================ +OK, let's use Regolith to build our cv. Why not. again, in a terminal navigate +to the top level directory of your database (where the :bash:`regolithrc.json` +file is). and type: + +.. code-block:: sh + + $ regolith build cv + +Regolith will take information from the various collections in your database and +build them into your academic cv according to a pre-determined template. The +current template builds the cv using latex. If your computer has latex installed +and Regolith can find it, your cv should appear as a pdf document in the directory +:bash:`my-cv-db/_build` (or more generally :bash:`///_build`). +All your built documents will appear in the :bash:`_build` +directory. + +If you don't have latex installed we can have Regolith build the latex +source file for the cv but without trying to +render it to PDF, + +.. code-block:: sh + + $ regolith build cv --no-pdf + +The latex source is a text file that you will find in the :bash:`_build` directory +and you can open it in a text editor. Even without latex installed you can +render it by opening a free account at http://overleaf.com starting a new blank +project, uploading the :bash:`.tex` and :bash:`.bib` files to +that project and hitting the :bash:`recompile` button. + +Whether it builds on your computer or on overleaf, it should look something like + +.. image:: ../_static/cv.pdf + +If, for some reason, the publication list doesn't render +correctly, try running the latex command again. If you are going to +do much building with regolith it is definitely recommended to install latex on +your computer, such as MikTeX for windows (latex comes installed with many linux +systems and is easily installed on IOS). + +What Next? +=========== + +You have not spent too much time entering data into your database yet, but you +can already build a number of different things. Try building your +resume (:bash:`$ regolith build resume`), your publication list +(:bash:`$ regolith build publist`) and your presentation list +(:bash:`$ regolith build preslist`). You can even build a web-page +for your group (:bash:`$ regolith build html`). It will look pretty +ugly until we set it up properly with a nice template, but all the content +will be dynamically built from the latest info in your databases. + +To see everything you can build, type :bash:`$ regolith build --help`. +To build some of those things you will need more collections that are not +in the cookie cutter template, for example, +:bash:`proposals` and :bash:`grants` collections, but you get the idea. + +So next we might want to work on those collections and start adding more data. +This can be done in a couple of ways. Probably the simplest to begin +with is just use a text editor or IDE like PyCharm. The :bash:`yml` files are +yaml files, which is a human readable way of storing information that can be +read and understood by python. Please read about it `here `_ +if you are not familiar with it. However, briefly to get you started, it encodes +whether information is part of a list or a dictionary by indentation and semantics. +For example, + +.. code-block:: sh + + key: + - list item + - another list item + +would be read by python as :python:`{"key": ["list item", "another list item"]}`, +and a collection consisting of a list of dictionaries would look like this in yaml: + +.. code-block:: sh + + id: + - name: Arthur + quest: To find the Holy Grail + favorite_color: Blue + - name: Sir Lancelot + quest: To find the Holy Grail + favorite_color: Green, no pink + +Long story short, you can update your database by directly editing the file, +and this is quick and easy when you get comfortable with the YAML syntax, +but can be frustrating as you are learning it. + +If you want to check what +fields are allowed or required in a collection look at the Collections part of +the docs, :ref:`regolith_collections`, which are built from the Regolith schema +(or directly look at the +schema in :bash:`schema.py`). You can automatically check if your database +edits are valid by running :bash:`$ regolith validate`. + +Getting Help from Helpers +========================== +Regolith builders build documents, but there are a small but growing number of +tools that either will run popular queries on the database and print the results +to the terminal ("lister helpers" with :bash:`l_` prefixes -- you +already used one, +it was the lister helper that builds your todo list). + +There are also helpers +that help you to add documents to your database collections. These are +"adder helpers" with :bash:`a_` prefixes. An important adder helper is +:bash:`a_todo` helper that will add a todo item to your list. + +"Updater +helpers" will update existing entries in your databases and have prefix +:bash:`u_`. + +An important special kind of updater helper is a "finish helper" that will mark +something as finished (and give it a finish date). So when you do that +pesky 15th todo item on your todo list, run +:bash:`regolith helper f_todo -i 15` to finish it. + +That is a lot of typing to finish a todo, so consider setting up an alias in +the config file for your terminal program (my terminals run bash so I put the +alias in the :bash:`.bashrc` file in my home directory (:bash:`$ cd ~` to get there). +With this alias I just type :bash:`rhlt 15` to finish that 15th todo item. + +To explore what helpers are there so you can play with them, type + +.. code-block:: sh + + $ regolith helper + +and hit return. It will return a list of available helpers, e.g., + +.. code-block:: sh + + $ regolith helper + usage: regolith helper [-h] helper_target + regolith helper: error: the following arguments are required: helper_target + usage: regolith helper [-h] helper_target + + positional arguments: + helper_target helper target to run. Currently valid targets are: + ['a_expense', 'a_grppub_readlist', 'a_manurev', + 'a_presentation', 'a_projectum', 'a_proposal', 'a_proprev', + 'a_todo', 'f_prum', 'f_todo', 'l_abstract', 'l_contacts', + 'l_grants', 'l_members', 'l_milestones', 'l_progress', + 'l_projecta', 'l_todo', 'u_contact', 'u_institution', + 'u_logurl', 'u_milestone', 'u_todo', 'v_meetings', 'lister', + 'makeappointments'] + +then if you want to know how to use any of the helpers type + +.. code-block:: sh + + $ regolith helper + +and hit return, e.g., + +.. code-block:: sh + + $ regolith helper l_contacts + usage: regolith helper [-h] [-v] [-n NAME] [-i INST] [-d DATE] [-r RANGE] + [-o NOTES] [-f FILTER [FILTER ...]] + [-k KEYS [KEYS ...]] + helper_target run + regolith helper: error: the following arguments are required: run + usage: regolith helper [-h] [-v] [-n NAME] [-i INST] [-d DATE] [-r RANGE] + [-o NOTES] [-f FILTER [FILTER ...]] + [-k KEYS [KEYS ...]] + helper_target run + + positional arguments: + helper_target helper target to run. Currently valid targets are: + ['a_expense', 'a_grppub_readlist', 'a_manurev', + 'a_presentation', 'a_projectum', 'a_proposal', + 'a_proprev', 'a_todo', 'f_prum', 'f_todo', + 'l_abstract', 'l_contacts', 'l_grants', 'l_members', + 'l_milestones', 'l_progress', 'l_projecta', 'l_todo', + 'u_contact', 'u_institution', 'u_logurl', + 'u_milestone', 'u_todo', 'v_meetings', 'lister', + 'makeappointments'] + run run the lister. To see allowed optional arguments, + type "regolith helper l_contacts". + + optional arguments: + -h, --help show this help message and exit + -v, --verbose Increases the verbosity of the output. + -n NAME, --name NAME name or name fragment (single argument only) to use to + find contacts. + -i INST, --inst INST institution or an institution fragment (single + argument only) to use to find contacts. + -d DATE, --date DATE approximate date in ISO format (YYYY-MM-DD) + corresponding to when the contact was entered in the + database. Comes with a default range of 4 months + centered around the date; change range using --range + argument. + -r RANGE, --range RANGE + range (in months) centered around date d specified by + --date, i.e. (d +/- r/2). + -o NOTES, --notes NOTES + fragment (single argument only) to be found in the + notes section of a contact. + -f FILTER [FILTER ...], --filter FILTER [FILTER ...] + Search this collection by giving key element pairs. + -k KEYS [KEYS ...], --keys KEYS [KEYS ...] + Specify what keys to return values from when running + --filter. If no argument is given the default is just + the id. + +you then would rerun the command giving all required, and any +optional, command +line arguments. e.g., + +.. code-block:: sh + + $ regolith helper l_contacts run --name frank -v + +will return all contacts in the contacts collection where :bash:`frank` appears anywhere +in the name, such as :bash:`Frankie Valli`, :bash:`Baron von Frankenstein` +and :bash:`Anne Frank` (if they are in your contacts). The :bash:`-v` +command stands for :bash:`verbose` which means more information is +returned than if you don't type :bash:`-v`. You can try it now: + +.. code-block:: sh + + $ regolith helper l_contacts run -n auth -v + +Setting up Gitlab repository information for API requests +========================================================= + +Some helpers have features that make API requests to GitLab (or GitHub). For example, the a_presentation helper has a functionality that +creates a repository in a designated GitLab group. In order to use these features, the target repository +information needs to be defined in your configuration files (:code:`regolithrc.json`, :code:`user.json`). + +Setting up Destination Repo Information +--------------------------------------- + +The designated repository information should be defined in :code:`regolithrc.json` in the directory in which you are +running the helper. Create a collection of repository targets designated as :code:`repos` (see below for an example). +according to the following pattern. We will use as an example an entry that will +allow :code:`a_presentation` to successfully create a repository in a group called `talks` +on a GitLab instance. + +:code:`a_presentation` looks for a rep with the entry :code:`_id` with value ``"talk_repo"``. + +.. code-block:: json + +"repos":[ + {"_id": "talk_repo", # a_presentation looks for the entry with this ID + "params": {"namespace_id": "35", # These params are handed to the API post request. + "initialize_with_readme": "True" # "name" is also needed but a_presentation generates that automatially + }, + "url": "https://gitlab.example.com", # The URL of the main GitLab/GitHub instance + "api_route": "/api/v4/projects/", # This is the route to the REST-API. The value + # shown here is correct for GitLab at the time of writing + "namespace_name": "talks" # the name of group/org which corresponds to the namespace_id above. + }, + { + "_id": "another_example_repo", + [...] + } + ] + +The namespace ID is the repository's group ID which can be found on the target repository's main page. +The :code:`url` and :code:`api_route` should be in the format above, including the dashes. + +For more information on the required request info, or to see a list of additional attributes +that can also be defined in the request (e.g. ``initialize_with_readme``, ``description``, etc.), +see GitHub or GitLab API documentation, e.g., for GitLab the `GitLab docs `_. +(Note that additional attributes can be defined under ``params``, where needed.) + +Setting up your Private Access Token +------------------------------------ + +Your personal/private API request token should be defined in :code:`user.json`, which can be found in your +~/.config directory. Similarly, define a distinct ID for each private token. For example, to create a repo +in GitLab, you should define your authentication token with the ID, ``"gitlab_private_token"``: + +.. code-block:: json + + [ + { + "_id": "gitlab_private_token", + "token": "" + }, + { + "_id": "example_token", + [...] + } + ] + +To learn more about creating a personal access token, refer to the +`Gitlab docs `_. +Note that your personal access token should have the ``api`` scope enabled in order to make a successful request. + +To change the target directory, you can change the parameters (or IDs) in the function +:code:`create_repo(destination_id, token_info_id, rc)` in `a_presentationhelper.py` to +the IDs of your desired repo info and corresponding token. + +Setting up GitHub repository information for API requests +========================================================= + +Using the filter capabilities in the helpers +============================================ + +Most helpers have a filter field. This allows you to filter the relevant collection +before running the helper functionality. + +The logic of filter is the following. A document will be valid if the value of key contains value for all keys and values using AND logic. + +As an example, if we consider filtering in the :bash:`l_milestones` helper we will get the following behavior. :bash:`l_milestones` operates on the :bash:`projecta` collection, so the filter will be applied to this collection. +If you specify :bash:`--filter lead voe` it will return all documents where :bash:`voe` appears in the value for the :bash:`lead` field (e.g., if there is someone with an id of :bash:`carvoe` and another person with and id of :bash:`voedemort` the filter will return all the documents where either of these people are :bash:`lead`). + +If you then select :bash:`current` and :bash:`verbose` the helper will do the normal thing of returning in verbose form the current milestones, but it will do it on the filtered collection. + +A slight gotcha is that since filter uses "in" in its logic, if the type of the key value is a string it will find all strings that contain that fragment, as above, but if the type of the key value is a list it will return documents where the specified value is in the list, so :bash:`--filter group_member voe` will return all the documents where :bash:`voe` is listed as a group member, but it won't return any documents where :bash:`carvoe` or :bash:`voedemort` are listed as a group member. + +The filter uses AND logic and operates such that :bash:`--filter lead voe grants mygrant21 status finished` will return all prums that are led by :bash:`carvoe` or :bash:`voedemort` that acknowledge the mygrant21 grant and are finished. Actually, similar behavior can be obtained also by selecting :bash:`--lead voe --stati finished --filter grants mygrant21` + +unfortunately the filter function does not currently recurse, so it will only operate on top level key-value pairs where the type of the value is a string or a list or a tuple. + +Backing up and protecting your work +=================================== +Now you have started saving your precious life's work in your regolith database +you better start protecting it and backing it up. One low overhead approach +for this is simply to set up your database directory to be backed up remotely as +a Google drive or Dropbox synced directory, for example. + +However, Regolith is set up to +work with git and GitHub and this is a powerful option if you are +comfortable with it. This gets more useful when you want to start +sharing databases with group members, for example, using GitHub access rights. +It is also possible to make sure people's edits to the database won't +break things by setting up continuous integration (CI) that runs some +validation and builders and makes sure they don't crash before the +edits are accepted. This is much more advanced usage which you +should save for later. + +To get started with the GitHub option, the next thing to do is to turn your database directory +on your filesystem into a git repository and link it to a repository on +your personal space on GitHub (you will need a GitHub account). You can make +that repo private so the world +cannot see your todo list, or public so that the world can see the web-page +you build from it. We will get back to this later, but Regolith will build +collections from across databases, so you can have parts of your :bash:`people` +collection private and other parts public. Depending which :bash:`regolithrc.json` +file you use to build with, you can pull from the public, or private, or both +parts. Again, this is a peep to the future. +For now, let's assume you just want to back up and keep versions of +you private database. You will make a repository on your personal GitHub account +and synchronize your local database with this repo. +`Instructions for doing this are here `_ + +Once you get everything set up you will want to periodically (meaning +frequently) type + + +.. code-block:: sh + + $ git commit -a -m "my commit message" + $ git push + +This will add, commit and push all files that git is tracking that have been +updated locally. If you add a new file to the repository and want it in the +GitHub backup, you will have to explicitly add it before committing, + +.. code-block:: sh + + $ git add my_new_file.py + $ git commit -m "an even more informative commit message" + +:bash:`$ git commit` commits (i.e., checks in) to the git database (yes, +git, like you now, is using a database backend) everything that has been added, or staged, for commit. +:bash:`$ git commit -a` automatically adds all files that git is tracking +(have been previously committed in the past) that have been edited and then +commits them. + +They are now safely captured in the git database and you can retrieve them +later if you accidentally delete your personal database or mess it up some other way. +But this version of the git database is still stored on your local computer, so +if you spill coffee on your computer, you may lose everything. :bash:`$ git push` pushes all +these updates to a remote computer on the internet at the GitHub headquarters. +Git and GitHub form a wonderful but complicated infrastructure, it is well worth +getting to know how to use them well. For now, we have used it to +secure your precious database. Remember to make frequent pushes. + +OK, you are started with your Regolith database. Go play. Regolith +can do many more complicated things to help with administering your +research group, or whatever you are working on. We will continue to +add tutorials below explaining some of these things, so check back +from time to time. And remember join and to ask questions at the `regolith-users `_ +Google group. They will get answered. + + + +Tutorials +========= +.. toctree:: + :maxdepth: 1 + + tutorials/index + +Run Control +=========== + +.. toctree:: + :maxdepth: 1 + + rc + + +Database Collections +==================== +**Collections** are the regolith (and mongo) abstraction for *tables*. +**Entries** (or *rows*) in +a collection must follow the schema defined below. In general, the following notions +hold: + +* An entry is a dictionary with string keys. +* Each entry must contain a unique identifier. This is called ``"_id"`` in JSON + and Mongo, and is simply the top-level key in YAML. +* A collection is a list of entries that follow the same schema. + +Not all regolith actions will use every collection type. It is common for regolith +projects to just use some of the collections below. For example, building a group +website will use different collections than managing students and grades in a course! +With these points in mind, feel free to dive into the databases below! + +.. toctree:: + :maxdepth: 2 + + collections/index + + +Regolith API +============ +For those who want to dive deeper into the library itself. + +.. toctree:: + :maxdepth: 1 + + api/index + + +Regolith Commands +================= +Shell commmands for regolith + +.. toctree:: + :maxdepth: 2 + + commands/index + + +Database Maintenance +========= +.. toctree:: + :maxdepth: 1 + + database_maintenance/index diff --git a/doc/source/license.rst b/doc/source/license.rst new file mode 100644 index 000000000..3ba00c3d5 --- /dev/null +++ b/doc/source/license.rst @@ -0,0 +1,34 @@ +:tocdepth: -1 + +.. index:: license + +License +####### + +OPEN SOURCE LICENSE AGREEMENT +============================= + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/docs/make.bat b/doc/source/make.bat similarity index 100% rename from docs/make.bat rename to doc/source/make.bat diff --git a/docs/rc.rst b/doc/source/rc.rst similarity index 100% rename from docs/rc.rst rename to doc/source/rc.rst diff --git a/docs/regolithrc.json b/doc/source/regolithrc.json similarity index 100% rename from docs/regolithrc.json rename to doc/source/regolithrc.json diff --git a/doc/source/release.rst b/doc/source/release.rst new file mode 100644 index 000000000..27cd0cc94 --- /dev/null +++ b/doc/source/release.rst @@ -0,0 +1,5 @@ +:tocdepth: -1 + +.. index:: release notes + +.. include:: ../../CHANGELOG.rst diff --git a/docs/tutorials/broker.rst b/doc/source/tutorials/broker.rst similarity index 100% rename from docs/tutorials/broker.rst rename to doc/source/tutorials/broker.rst diff --git a/docs/tutorials/index.rst b/doc/source/tutorials/index.rst similarity index 100% rename from docs/tutorials/index.rst rename to doc/source/tutorials/index.rst