diff --git a/CHANGELOG.md b/CHANGELOG.md index 125403380..f4a67b905 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,128 @@ # VisiData version history +# v2.1 (2020-12-06) + + - [add] add bulk rows and cols leave cursor on first added (like add singles) + - [add] add colname input to `addcol-new` + - [aggregators] add mode and stdev to aggregator options (thanks @jsvine for PR #754) + - [api] add options.unset() + - [columns] add hidden 'keycol' to **ColumnsSheet** (thanks @geekscrapy for feature request #768) + - [cli] support running as `python -m visidata` (thanks @abitrolly for PR #785) + - [cli] add `#!vd -p` as first line of `.vdj` for executable vd script + - [cli] allow `=` in `.vd` replay parameters + - [clipboard] clipboard commands now require some selected rows #681 + - [commands] add unset-option command bound to `d` on OptionsSheet #733 + - [config] `--config=''` now ignores visidatarc (thanks @rswgnu for feature request #777) + - [defer] commit changes, even if no deferred changes + - [deprecated] add traceback warnings for deprecated calls (thanks @ajkerrigan for PR #724) + - [display] add sort indication #582 + - [display] show ellipsis on left side with non-zero hoffset (thanks @frosencrantz for feature request #751) + - [expr] allow column attributes as variables (thanks @frosencrantz for feature request #659) + - [freq] change `numeric_binning` back to False by default + - [input] Shift+Arrow within `edit-cell` to move cursor and re-enter edit mode + - [loaders http] have automatic API pagination (thanks @geekscrapy for feature request #480) + - [loaders json] improve loading speedup 50% (thanks @lxcode for investigating and pointing this out #765) + - this makes JSON saving non-deterministic in Python 3.6, as the order of fields output is dependent on the order within the dict + - (this is the default behaviour for dicts in Python 3.7+) + - [loaders json] try loading as jsonl before json (inverted) + - jsonl is a streamable format, so this way it doesn't have to wait for the entire contents to be loaded before failing to parse as json and then trying to parse as jsonl + - fixes api loading with http so that contents of each response are added as they happen + - unfurl toplevel lists + - functionally now jsonl and json are identical + - [loaders json] try parsing `options.json_indent` as int (thanks @frosencrantz for the bug report #753) + this means json output can't be indented with a number. this seems like an uncommon use case + - [loaders json] skip lines starting with `#` + - [loaders pdf] `options.pdf_tables` to parse tables from pdf with tabular + - [loaders sqlite] use rowid to update and delete rows + - note that this will not work with WITHOUT ROWID sqlite tables + - [loaders xlsx] add active column (thanks @kbd for feature request #726) + - [loaders zip] add extract-file, extract-selected, extract-file-to, extract-selected-to commands + - [macros] add improved macro system (thanks @bob-u for feature request #755) + - `m` (`macro-record`) begins recording macro; `m` prompts for keystroke, and completes recording + - macro can then be executed everytime provided keystroke is used, will override existing keybinding + - `gm` opens an index of all existing macros, can be directly viewed with `Enter` and then modified with `Ctrl+S` + - macros will run command on current row, column sheet + - remove deprecated `z Ctrl+D` older iteration of macro system + - [regex] use capture names for column names, if available, in `capture-col` (thanks @tsibley for PR #808) + - allows for pre-determining friendlier column names, saving a renaming step later + - [save] `g Ctrl+S` is `save-sheets-selected` on **IndexSheet** + - new command allows some or all sheets on an **IndexSheet** to be saved (and not the sheets on the sheet stack) + - [saver] add fixed-width saver (uses col.width) + - [saver sqlite] ensureLoaded when saving sheets to sqlite db + - [search] `search-next` and `searchr-next` are now bound to n and N (was `next-search` and `search-prev`) + - [select] differentiate select-equal- and select-exact- (thanks @geekscrapy for feature request #734) + - previous select-equal- matched type value + - now select-equal- matches display value + - add `z,` and `gz,` bindings for select-exact-cell/-row + - [sheets] sorting on **SheetsSheet** now does not sort **SheetsSheet** itself. (thanks @klartext and @geekscrapy for bug reports #761 #518) + - [status] use `color_working` for progress indicator (thanks @geekscrapy for feature request #804) + - [types] add floatsi parser (sponsored feature by @anjakefala #661) + - floatsi type now parses SI strings (like 2.3M) + - use `z%` to set column type to floatsi + +## Bugfixes + + - [api] expose visidata.view (thanks @alekibango for bug report #732) + - [color] use `color_column_sep` for sep chars (thanks @geekscrapy for bug report) + - [defer] frozen columns should not be deferred (thanks @frosencrantz for bug report #786) + - [dir] fix commit-sheet and delete-row on DirSheet + - [draw] fix display for off-screen cursor with multiline rows + - [expr] remove duplicate tabbing suggestions (thanks @geekscrapy for bug report #747) + - [expr] never include computing column (thanks @geekscrapy for bug report #756) + - only checks for self-reference; 2+ cycles still raises RecursionException + - caches are now for each cell, instead of for each row + - [freeze] freeze-sheet with errors should replace with null + - [loaders frictionless] assume JSON if no format (thanks scls19fr for bug report #803) + - from https://specs.frictionlessdata.io/data-resource/#data-location): + - a consumer of resource object MAY assume if no format or mediatype property is provided that the data is JSON and attempt to process it as such. + + - [loaders hdf5] misc bugfixes to hdf5 dataset loading (thanks @amotl for PR #728) + - [loaders jsonl] fix copy-rows + - [loaders pandas] support loading Python objects directly (thanks @ajkerrigan for PR #816 and scls19fr for bug report #798) + - [loaders pandas] ensure all column names are strings (thanks @ajkerrigan for PR #816 and scls19fr for bug report #800) + - [loaders pandas] build frequency table using a copy of the source (thanks @ajkerrigan for PR #816 and scls19fr for bug report #802) + - [loaders sqlite] fix commit-sheet + - [loaders sqlite] fix commit deletes + - [loaders xlsx] only reload Workbook sheets to avoid error (thanks @aborruso for bug report #797) + - [loaders vdj] fix add-row + - [man] fix warnings with manpage (thanks @jsvine for the bug report #718) + - [movement] fix scroll-cells (thanks @jsvine for bug report #762) + - [numeric binning] perform degenerate binning when number of bins greater than number of values + - (instead of when greater than width of bins) + - [numeric binning] if width of bins is 1, fallback to degenerate binning + - [numeric binning] degenerate binning should resemble non-numeric binning (thanks @setop for bug report #791) + - [options] fix `confirm_overwrite` in batch mode + - fix `-y` to set `confirm_overwrite` to False (means, no confirmation necessary for overwrite) + - make `confirm()` always fail in batch mode + - make `confirm_overwrite` a sheet-specific option + - [plugins] only reload **Plugins Sheet** if not already loaded + - [replay] move to replay context after getting sheet (thanks @rswgnu for bug report #796) + - [replay] do not push replaying .vd on sheet stack (thanks @rswgnu for bug report #795) + - [scroll] zj/zk do nothing in single-line mode (thanks @jsvine for suggestion) + - [shell] empty stdin to avoid hanging process (thanks @frosencrantz for bug report #752) + - [status] handle missing attributes in `disp_rstatus_fmt` (thanks @geekscrapy for bug report #764) + - [tabulate] fix savers to save in their own format (thanks @frosencrantz for bug report #723) + - [typing] fix indefinite hang for typing (thanks @lxcode for issue #794) + - [windows] add Ctrl+M as alias for Ctrl+J #741 (thanks @bob-u for bug report #741) + - [windows man] package man/vd.txt as a fallback for when man is not available on os (thanks @bob-u for bug report #745) + +## Plugins +- add conll loader to **PluginsSheet** (thanks @polm) +- remove livesearch +- add clickhouse loader + +## Commands +- if `options.some_selected_rows` is True, `setcol-expr`, `setcol-iter`, `setcol-subst`, `setcol-subst`, `setcol-subst-all` will return all rows, if none selected + +## API +- [columns] add Column.visibleWidth +- [open] additionally search for `open_filetype` within the vd scope +- [select] rename `someSelectedRows` to `onlySelectedRows` +- [select] add new `someSelectedRows` and `options.some_selected_rows` (thanks maufdez for feature request #767) + - if options is True, and no rows are selected, `someSelectedRows` will return all rows +- [status] allow non-hashable status msgs by deduping based on stringified contents +- [isNumeric] isNumeric is part of vdobj + # v2.0.1 (2020-10-13) ## Bugfixes diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 000000000..7c5bd336b --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1 @@ +don't be a dick diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 515e3784e..57c206281 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -60,6 +60,7 @@ Some examples of great bug reports: ## Submitting Source Code +Check out the [Plugin Authors Guide](https://visidata.org/docs/api) for an overview of the API. Code in `plugins/` or `visidata/loaders/` is welcome, as long as it is useful to someone and safe for everyone. Updates or additions to the core code should be proposed via an [Github Issue](https://github.com/saulpw/visidata/issues/new/choose) before submitting a PR. diff --git a/MANIFEST.in b/MANIFEST.in index 731f3a3a0..ce2d37747 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,3 +1,4 @@ include README.md include LICENSE.gpl3 include visidata/man/vd.1 +include visidata/man/vd.txt diff --git a/README.md b/README.md index 547b30b98..1f42e892a 100644 --- a/README.md +++ b/README.md @@ -1,82 +1,55 @@ -[![Gitpod ready-to-code](https://img.shields.io/badge/Gitpod-ready--to--code-blue?logo=gitpod)](https://gitpod.io/#https://github.com/saulpw/visidata) -# VisiData v2.0.1 [![CircleCI](https://circleci.com/gh/saulpw/visidata/tree/stable.svg?style=svg)](https://circleci.com/gh/saulpw/visidata/tree/stable) +# VisiData v2.1 [![twitter @VisiData][1.1]][1] [![CircleCI](https://circleci.com/gh/saulpw/visidata/tree/develop.svg?style=svg)](https://circleci.com/gh/saulpw/visidata/tree/develop) [![Gitpod ready-to-code](https://img.shields.io/badge/Gitpod-ready--to--code-blue?logo=gitpod)](https://gitpod.io/#https://github.com/saulpw/visidata) A terminal interface for exploring and arranging tabular data. ![Frequency table](http://visidata.org/freq-move-row.gif) -## Dependencies +VisiData supports tsv, csv, sqlite, json, xlsx (Excel), hdf5, and [many other formats](https://visidata.org/formats). -- Linux, OS/X or Windows -- Python 3.6+ -- python-dateutil -- other modules may be required for opening particular data sources - - see [requirements.txt](https://github.com/saulpw/visidata/blob/stable/requirements.txt) or the [supported sources](https://visidata.org/formats) in the vd manpage +## Platform requirements -## Getting started +- Linux, OS/X, or Windows (with WSL) +- Python 3.6+ +- additional Python modules are required for certain formats and sources -### Installation +## Install -Each package contains the full loader suite but differs in which loader dependencies will get installed by default. +To install the latest release from PyPi: -The base VisiData package concerns loaders whose dependencies are covered by the Python3 standard library. + pip3 install visidata -Base loaders: tsv, csv, json, sqlite, and fixed width text. +To install the cutting edge `develop` branch (no warranty expressed or implied): -|Platform |Package Manager | Command | Out-of-box Loaders | -|-------------------|----------------------------------------|----------------------------------------------|----------------------| -|all |[pip3](https://visidata.org/install#pip3) | `pip3 install visidata` | Base | -|all |[conda](https://visidata.org/install#conda) | `conda install --channel conda-forge visidata` | Base, http, html, xls(x) | -|MacOS |[Homebrew](https://visidata.org/install#brew) | `brew install saulpw/vd/visidata` | Base, http, html, xls(x) | -|Linux (Debian/Ubuntu) |[apt](https://visidata.org/install#apt) | [full instructions](https://visidata.org/install#apt) | Base, http, html, xls(x) | -|Linux (Debian/Ubuntu) |[dpkg](https://visidata.org/install#dpkg) | [full instructions](https://visidata.org/install#dpkg) | Base, http, html, xls(x) | -|Windows |[WSL](https://visidata.org/install#wsl) | Windows is not directly supported (use WSL) | N/A | -|all |[github](https://visidata.org/install#git) | `pip3 install git+https://github.com/saulpw/visidata.git@stable` | Base | -|Linux (NixOS)|[nix](https://visidata.org/install#nix)| `nix-env -i visidata`|Base, yaml, xls(x), hdf5, html, pandas, shp | + pip3 install git+https://github.com/saulpw/visidata.git@develop -Please see [/install](https://visidata.org/install) for detailed instructions, additional information, and troubleshooting. +See [visidata.org/install](https://visidata.org/install) for detailed instructions for all available platforms and package managers. ### Usage - $ vd [] ... - $ | vd [] - -VisiData supports tsv, csv, xlsx, hdf5, sqlite, json and more (see the [list of supported sources](https://visidata.org/formats)). + $ vd + $ | vd -Use `-f ` to force a particular filetype. +Press `Ctrl+Q` to quit at any time. +Hundreds of other commands and options are also available; see the documentation. ### Documentation +* [VisiData documentation](https://visidata.org/docs) +* [Plugin Author's Guide and API Reference](https://visidata.org/docs/api) +* [Quick reference](https://visidata.org/man) (available within `vd` with `Ctrl+H`), which has a list of commands and options. * [Intro to VisiData Tutorial](https://jsvine.github.io/intro-to-visidata/) by [Jeremy Singer-Vine](https://www.jsvine.com/) -* Quick reference: `Ctrl+H` within `vd` will open the [man page](https://visidata.org/man), which has a list of all commands and options. -* [/docs](https://visidata.org/docs) contains a collection of howto recipes. ### Help and Support If you have a question, issue, or suggestion regarding VisiData, please [create an issue on Github](https://github.com/saulpw/visidata/issues) or chat with us at #visidata on [freenode.net](https://webchat.freenode.net/). -Here are some concrete ways you can help make VisiData even more awesome: - -* Write a blogpost (or tweet or whatever) about a VisiData command or feature you use frequently, and share it with us! -* Expand VisiData to support .xyz proprietary data format. Creating a loader [is really straightforward](https://www.visidata.org/docs/api/loaders.html). -* Create and maintain [new installation packages](https://github.com/saulpw/visidata/labels/packaging). -* Acknowledge the realities of late-stage capitalism and [give regular old money](https://www.patreon.com/saulpw). - -## Other applications within the VisiData ecosystem - -The core interface paradigm--rows and columns--can be used to create efficient terminal workflows with a minimum of effort for almost any application. These have been prototyped as proof of this concept: - -- [vgit](https://github.com/saulpw/visidata/tree/stable/plugins/vgit): a git interface -- [vsh](https://github.com/saulpw/vsh): a collection of utilities like `vping` and `vtop`. -- [vdgalcon](https://github.com/saulpw/vdgalcon): a port of the classic game [Galactic Conquest](https://www.galcon.com) - -Other workflows can also be created as separate apps using the visidata module. These apps can be very small and provide a lot of functionality; for example, see the included [viewtsv](https://visidata.org/docs/viewtsv). +If you use VisiData regularly, please [support me on Patreon](https://www.patreon.com/saulpw)! ## License -VisiData, including the main `vd` application, addons, loaders, and other code in this repository, is available for use and redistribution under GPLv3. +Code in the `stable` branch of this repository, including the main `vd` application, loaders, and plugins, is available for use and redistribution under GPLv3. ## Credits @@ -85,3 +58,8 @@ VisiData is conceived and developed by Saul Pwanson ``. Anja Kefala `` maintains the documentation and packages for all platforms. Many thanks to numerous other [contributors](https://visidata.org/credits/), and to those wonderful users who provide feedback, for helping to make VisiData the awesome tool that it is. + +[1.1]: http://i.imgur.com/tXSoThF.png +[1]: http://www.twitter.com/VisiData + + diff --git a/dev/ROADMAP b/dev/ROADMAP index 33b7ef91e..c371e23c4 100644 --- a/dev/ROADMAP +++ b/dev/ROADMAP @@ -1,35 +1,60 @@ -# Roadmap to 2.0 - -## API - -The primary goal of 2.0 is a stable, documented Python API that can be used to create an broad ecosystem of loaders, plugins, and apps. - -The Book of VisiData will describe the api and all functions and objects available to plugins and command execstrs. - -## Split panes - -+ horizontal split -- one pane can be dependent on the other; e.g. move cursor in one, see preview update in other -+ use for choosing aggregators, command help, other choose() - -- DirSheet file preview #309 - -## add jsonl as more robust system format -+ add .vdj format (visidata cmdlog as jsonl) -+ migrate plugins.tsv to be jsonl format - -## Minor features, refactors, cleanup - -- 'modified' indicator on status bar, also enables quitguard -- remove all asserts; replace with error() on case-by-case basis - -- [options] allow list or dict of enum values for default -- [options] for options that have been renamed, add aliases for compatibility - - e.g. `tsv_delimiter` and `tsv_row_delimiter` become 'proper' options with 'delimiter', 'delim', and 'd' becoming aliases - -# Features - -- Memory sheet; can give things names and use in expressions. - - 'show' commands are copied there by default. - -- [columns] add helpstr for all specialized-sheet columns +# Roadmap for 2.x + +1. holdovers from 2.0 + - [canvas] API docs + - [options] option enums + - [options] user-defined option aliases + - [splitpane] File preview in directory view + - [defermods] 'modified since last save' indicator on status bar + - [scroll cell] zh/zl Left hand side of a cell with content hidden doesn't show ellipsis #751 + +2. Persistence + - input history #736 #468 + - key indexes for better joining and lookups + +3. Interface discoverability for commands #247 #742 + - clickable menu canvas + - more clickable affordances all around + - clickable motd + - possibly add popup modals + +4. more expressive expressions + - Memory sheet; can give things names and use in expressions. #392 + - access column values from execstr #655 + - shortcut name for current column #659 + +5. better asynchronicity + - async thread pool + - streaming architecture #366 #656 + +6. Loaders/Savers + - frictionless saver #237 + - RSS reader #157 + - toml loader #735 + - HexSheet for unknown/binary files #548 + - .ods loader (LibreOffice/OpenOffice spreadsheet) #473 + + - jsonl load and save round-trip (minimal diff) #429 + +## other features + +a) generate non-terminal graphs (ggplot) +b) automatic reload into time series +c) intra-cell coloring (for search results, markup) + +# major plugin projects + +1. fully operational SQL viewer/editor + + - #282: Select starting table in postgres from command-line + - #579: [Postgres] Allow inserting / deleting rows + - #522: [postgres] parms in options + - #586: SQL query data + - #727: [postgres] Transaction error when viewing table + - #729: Integrate generic SQL loader + +2. web scraper + + - #480: HTTP API pagination loader + - #465: Ability to load from the contents of a cell. + - #505: [html] Provide way to access non-table elements diff --git a/dev/checklists/add-aggregator.md b/dev/checklists/add-aggregator.md new file mode 100644 index 000000000..9ae8e384a --- /dev/null +++ b/dev/checklists/add-aggregator.md @@ -0,0 +1,4 @@ +- API for adding aggregators can be found [here](https://www.visidata.org/docs/api/columns.html#aggregators). A description is necessary. +- [ ] Review whether new aggregator should be included in **DescribeSheet**. If so, add it to `describe_aggrs`. +- [ ] Check if aggregator replaces an existing **DescribeSheet** column, remove it from the **DescribeSheet**, if so. +- [ ] add the aggregator to visidata:docs/group.md diff --git a/dev/formats.jsonl b/dev/formats.jsonl index 31f027178..a143874a8 100644 --- a/dev/formats.jsonl +++ b/dev/formats.jsonl @@ -6,7 +6,7 @@ {"filetype": "hdf5", "aliases": "h5", "requirements": "h5py", "format": "Hierarchical Data Format", "VisiData loader": "yes", "VisiData saver": "", "version_added": "0.28", "created": "199x", "creator": "NCSA", "description": "", "open format": "yes", "nestable": "", "plottable": "", "format_url": "https://support.hdfgroup.org/HDF5/"} {"filetype": "sqlite", "filetype_url": "#sqlite", "aliases": "db", "requirements": "", "format": "sqlite", "VisiData loader": "yes", "VisiData saver": "", "version_added": "0.42", "created": "2000", "creator": "D. Richard Hipp", "description": "full SQL relational database; the most used database engine in the world", "open format": "public domain", "nestable": "", "plottable": "", "format_url": "https://sqlite.org/"} {"filetype": "xls", "aliases": "", "requirements": "xlrd", "format": "Excel spreadsheets", "VisiData loader": "yes", "VisiData saver": "", "version_added": "0.42", "created": "1987", "creator": "Microsoft", "description": "", "open format": "no", "nestable": "", "plottable": "", "format_url": "https://msdn.microsoft.com/en-us/library/office/cc313154(v=office.12).aspx"} -{"filetype": "fixed", "filetype_url": "#fixed", "aliases": "", "requirements": "", "format": "fixed width text", "VisiData loader": "yes", "VisiData saver": "", "version_added": "0.97", "created": "", "creator": "", "description": "not recommended for new data", "open format": "prehistory", "nestable": "", "plottable": "", "format_url": "https://stackoverflow.com/questions/7666780/why-are-fixed-width-file-formats-still-in-use"} +{"filetype": "fixed", "filetype_url": "#fixed", "aliases": "", "requirements": "", "format": "fixed width text", "VisiData loader": "yes", "VisiData saver": "displayed text (as of 2.1)", "version_added": "0.97", "created": "", "creator": "", "description": "not recommended for new data", "open format": "prehistory", "nestable": "", "plottable": "", "format_url": "https://stackoverflow.com/questions/7666780/why-are-fixed-width-file-formats-still-in-use"} {"filetype": "postgres", "filetype_url": "#postgres", "aliases": "", "requirements": "", "format": "PostgreSQL database", "VisiData loader": "yes", "VisiData saver": "", "version_added": "0.97", "created": "1996", "creator": "", "description": "", "open format": "", "nestable": "", "plottable": "", "format_url": ""} {"filetype": "vd", "filetype_url": "#vd", "aliases": "vdj", "requirements": "", "format": "VisiData command log", "VisiData loader": "yes", "VisiData saver": "", "version_added": "0.97", "created": "2017", "creator": "VisiData", "description": "replayable", "open format": "yes", "nestable": "", "plottable": "", "format_url": "http://visidata.org/docs/save-restore/"} {"filetype": "mbtiles", "filetype_url": "#mbtiles", "aliases": "", "requirements": "mapbox-vector-tile", "format": "MapBox Tileset", "VisiData loader": "yes", "VisiData saver": "", "version_added": "0.98", "created": "2011", "creator": "MapBox", "description": "based on sqlite", "open format": "yes", "nestable": "", "plottable": "plottable", "format_url": "https://docs.mapbox.com/help/glossary/mbtiles/"} diff --git a/dev/history.jsonl b/dev/history.jsonl new file mode 100644 index 000000000..fe5080072 --- /dev/null +++ b/dev/history.jsonl @@ -0,0 +1,8 @@ +{"date": "2020-02-29", "event": "\"VisiData is wonderful...\" - @jeremyphoward", "url": "https://twitter.com/jeremyphoward/status/1233956135226314753"} +{"date": "2018-03-04", "event": "[HN] VisiData Lightning Demo at PyCascades 2018", "url": "https://news.ycombinator.com/item?id=16515299"} +{"date": "2018-01-31", "event": "r/commandline post by @anjakefala", "url": "https://www.reddit.com/r/commandline/comments/7ua1hj/shell_show_and_tell_terminal_spreadsheet_tool/"} +{"date": "2017-06-29", "event": "[HN] Show HN: VisiData - vi for data", "url": "https://news.ycombinator.com/item?id=14662860"} +{"date": "2020-03-27", "event": "anja wins year-old bet, >140 users every weekday", "url": ""} +{"date": "2020-05-13", "event": "changelog podcast released", "url": "https://changelog.com/podcast/394"} +{"date": "2020-12-05", "event": "anja's tweet about visidata being a tool everyone should gets >100 likes", "url": "https://twitter.com/nevoitbien/status/1335398803604586498"} +{"date": "2020-12-06", "event": "[HN] VisiData in 60 seconds", "url": "https://news.ycombinator.com/item?id=25322091"} diff --git a/dev/mkman.sh b/dev/mkman.sh index 558572494..f6cbf5ea5 100755 --- a/dev/mkman.sh +++ b/dev/mkman.sh @@ -21,3 +21,4 @@ $MAN/parse_options.py $BUILD/vd-cli.inc $BUILD/vd-opts.inc soelim -rt -I $BUILD $BUILD/vd.inc > $BUILD/vd-pre.1 preconv -r -e utf8 $BUILD/vd-pre.1 > $MAN/vd.1 +MANWIDTH=80 man $MAN/vd.1 > $MAN/vd.txt diff --git a/dev/test.sh b/dev/test.sh index d24cdf79d..80079f691 100755 --- a/dev/test.sh +++ b/dev/test.sh @@ -20,10 +20,10 @@ for i in $TESTS ; do outbase=${i##tests/} if [ "${i%-nosave.vd}-nosave" == "${i%.vd}" ]; then - PYTHONPATH=. bin/vd --play "$i" --batch + PYTHONPATH=. bin/vd --play "$i" --batch --config tests/.visidatarc --visidata-dir tests/.visidata else for goldfn in tests/golden/${outbase%.vd}.*; do - PYTHONPATH=. bin/vd --confirm-overwrite=False --play "$i" --batch --output "$goldfn" + PYTHONPATH=. bin/vd --confirm-overwrite=False --play "$i" --batch --output "$goldfn" --config tests/.visidatarc --visidata-dir tests/.visidata echo "save: $goldfn" done fi diff --git a/dev/types.jsonl b/dev/types.jsonl new file mode 100644 index 000000000..3dfd78b30 --- /dev/null +++ b/dev/types.jsonl @@ -0,0 +1,7 @@ +{"type": "anytype", "description": "pass-through", "numeric": "", "command": "type-anytype", "keystrokes": "z~"} +{"type": "str", "description": "string", "numeric": "", "command": "type-str", "keystrokes": "~"} +{"type": "date", "description": "date/time", "numeric": "Y", "command": "type-date", "keystrokes": "@"} +{"type": "int", "description": "integer", "numeric": "Y", "command": "type-int", "keystrokes": "#"} +{"type": "float", "description": "decimal", "numeric": "Y", "command": "type-float", "keystrokes": "%"} +{"type": "currency", "description": "decimal with units", "numeric": "Y", "command": "type-currency", "keystrokes": "$"} +{"type": "vlen", "description": "container size", "numeric": "Y", "command": "type-vlen", "keystrokes": "z#"} diff --git a/docs/api/columns.rst b/docs/api/columns.rst index 0e55970d6..af934ad66 100644 --- a/docs/api/columns.rst +++ b/docs/api/columns.rst @@ -17,6 +17,9 @@ Instead, apps and plugins should call ``getValue`` and ``setValue``, which provi .. autoattribute:: visidata.Column.name .. autoattribute:: visidata.Column.type .. autoattribute:: visidata.Column.width +.. autoattribute:: visidata.Column.visibleWidth +.. versionadded:: 2.1 + .. autoattribute:: visidata.Column.fmtstr .. autoattribute:: visidata.Column.hidden @@ -156,11 +159,16 @@ Types API ~~~~~~~~~~~ .. autoclass:: visidata.vd.addType +.. autofunction:: visidata.vd.isNumeric +.. versionadded:: 2.1 .. autofunction:: visidata.isNumeric +.. deprecated:: 2.1 + Use ``vd.isNumeric`` instead. Examples ~~~~~~~~~ +:: # Add an ip_address type. vd.addType(ipaddress.ip_address, icon=':', formatter=lambda fmt,ip: str(ip)) diff --git a/docs/api/data.rst b/docs/api/data.rst index 40fd3de67..9d7f945ab 100644 --- a/docs/api/data.rst +++ b/docs/api/data.rst @@ -5,6 +5,9 @@ Each TableSheet has a set of *selected rows*, which is a strict subset of the ro .. autoattribute:: visidata.TableSheet.selectedRows .. autoattribute:: visidata.TableSheet.someSelectedRows +.. versionchanged:: 2.1 +.. autoattribute:: visidata.TableSheet.onlySelectedRows +.. versionchanged:: 2.1 .. autoattribute:: visidata.TableSheet.nSelectedRows .. autofunction:: visidata.TableSheet.selectRow diff --git a/docs/api/index.rst b/docs/api/index.rst index 4add3d32b..d2c67b5ef 100644 --- a/docs/api/index.rst +++ b/docs/api/index.rst @@ -51,6 +51,6 @@ This extends VisiData so that when :kbd:`1` is pressed on any sheet, the command Toplevel functions =================== -.. autofunction visidata.vd.view -.. autofunction visidata.vd.view_pandas -.. autofunction visidata.vd.run +.. autofunction:: visidata.vd.view +.. autofunction:: visidata.vd.view_pandas +.. autofunction:: visidata.vd.run diff --git a/docs/api/interface.rst b/docs/api/interface.rst index 0d81fcdf4..2337ec314 100644 --- a/docs/api/interface.rst +++ b/docs/api/interface.rst @@ -91,7 +91,7 @@ Examples passwd = vd.input("password: ", display=False) # initial value is the formatted value under the cursor - vd.status(vd.input("text to show: ", value=cursorDisplayValue)) + vd.status(vd.input("text to show: ", value=cursorDisplay)) .. _colors: diff --git a/docs/api/loaders.rst b/docs/api/loaders.rst index 99b1f9478..c0c66935b 100644 --- a/docs/api/loaders.rst +++ b/docs/api/loaders.rst @@ -34,6 +34,16 @@ The *p* argument is a :ref:`visidata.Path`. The actual loading happens in the Sheet. An existing :ref:`sheet type` can be used, or a new sheet type can be created. +If the loader is within a plugin, the ``open_`` should be decorated with a ``@VisiData,api`` in order to make them available through the ``vd`` object's scope. + +:: + + @VisiData.api + def open_readme(vd, p): + return ReadmeSheet(p.name, source=p) + +Note, the change in the ``open_`` function signature, when decorated. + Step 2. Create a Sheet subclass ------------------------------- diff --git a/docs/api/options.rst b/docs/api/options.rst index 4288ca0c3..50d165927 100644 --- a/docs/api/options.rst +++ b/docs/api/options.rst @@ -66,6 +66,8 @@ Options API .. autofunction:: visidata.vd.options.get .. autofunction:: visidata.vd.options.set +.. autofunction:: visidata.vd.options.unset +.. versionadded:: 2.1 .. autofunction:: visidata.vd.options.getall The dict returned by ``options.getall('foo_')`` is designed to be used as kwargs to other loaders, so that their options can be passed through VisiData transparently. diff --git a/docs/api/plugins.rst b/docs/api/plugins.rst index b1ff06ad3..257251ef7 100644 --- a/docs/api/plugins.rst +++ b/docs/api/plugins.rst @@ -41,6 +41,9 @@ Multiple plugins can be defined in the same manifest; one line per plugin in the Complete "Hello world" plugin example ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +hello.py +^^^^^^^^^^^^^^^^ + :: '''This plugin adds the ``hello-world`` command to all sheets. diff --git a/docs/async.md b/docs/async.md deleted file mode 100644 index 1ad704cfb..000000000 --- a/docs/async.md +++ /dev/null @@ -1,83 +0,0 @@ -- Date: 2017-12-27 -- VisiData v1.0 - -# Maintaining a responsive interface - -## `@asyncthread` - -Use the `@asyncthread` decorator on a function to make it execute in a thread. -The return value is the spawned thread (which can often be ignored by the caller); the return value of the original function is effectively lost. - -Cells which are being computed in a separate thread should have that thread as their value until their result is available. -This will show the `options.disp_pending` notation and allow the user to interact with the specific thread (via e.g. `z^Y` and others). - -Each thread is added to `Sheet.currentThreads` for the current sheet. -Note that a thread spawned by calling a function on a different sheet will add the thread to the currentThreads for the topmost/current sheet instead. - -### Canceling threads - -The user can cancel all `Sheet.currentThreads` with `^C`. - -Internally, `cancelThread(*threads)` will send each thread an `EscapeException`, which percolates up the stack to be caught by the thread entry point. -EscapeException inherits from BaseException instead of Exception, so that threads can still have catch-all try blocks with `except Exception:`. -An unqualified `except:` clause is bad practice (as always); when used in an async function, it will make the thread uncancelable. - -### Wait for threads to finish - -`sync(expectedThreads)` will wait for all but some number of `expectedThreads` to finish. - -This will only rarely be useful. - -# Threads Sheet (`^T`) - -All threads (active, aborted, and completed) are added to `VisiData.threads`, which can be viewed as the ThreadsSheet via `^T`. -Threads which take less than `min_thread_time_s` (hardcoded in `asyncthread.py` to 10ms) are removed, to reduce clutter. - -- Press `ENTER` (on the Threads Sheet) on a thread to view its performance profile (if `options.profile_threads` was True when the thread started). -- Press `^_` (anywhere) to toggle profiling of the main thread. - -## Profiling - -The view of a performance profile in VisiData is the output from `pstats.Stats.print_stats()`. - -- `z^S` on the performance profile will call `dump_stats()` and save the profile data to the given filename, for analysis with e.g. [pyprof2calltree]() and [kcachegrind](). -- (`z^S` because the raw text can be saved with `^S` as usual. Ideally, `^S` to a file with a `.pyprof` extension on a profile sheet would do this instead.) - -# Progress counters - -In all `@asyncthread` functions, a `Progress` counter should be used to provide a progress percentage, which appears in the right-hand status. - -## Progress as iterable - -When iterating over a potentially large sequence: - - for item in Progress(iterable): - -This is just like `for item in iterable`, but it also keeps track of progress, to display on the right status line. - -- This only displays if used in another thread (but is harmless if not). -- Use Progress around the innermost iterations for maximum granularity and apparent responsiveness. -- But this incurs a small amount of overhead, so if a tight loop needs every last optimization, use it with an outer iterator instead (if there is one). -- Multiple Progress objects used in parallel will stack properly. -- Multiple Progress objects used serially will make the progress indicator reset (which is better than having no indicator at all). - -If `iterable` does not know its own length, it (or an approximation) should be passed as the `total` keyword argument: - - for item in Progress(iterable, total=approx_size): - -The `Progress` object contributes 1 towards the total for each iteration. -To contribute a different amount, use `Progress.addProgress(n)` (n-1 if being used as an iterable, as 1 will be added automatically). - -## Progress as context manager - -To manage `Progress` without wrapping an iterable, use it as a context manager with only a `total` keyword argument, and call `addProgress` as progress is made: - - with Progress(total=amt) as prog: - while amt > 0: - some_amount = some_work() - prog.addProgress(some_amount) - amt -= some_amount - -- Using `Progress()` other than as an iterable or a context manager will have no effect. - ---- diff --git a/docs/columns.md b/docs/columns.md index f8379eddc..ae343d48e 100644 --- a/docs/columns.md +++ b/docs/columns.md @@ -1,5 +1,5 @@ -- Update: 2020-06-17 -- Version: VisiData 2.0 +- Update: 2020-10-01 +- Version: VisiData 2.0.1 # Columns @@ -9,7 +9,7 @@ Commands(s) Operation ------------ ----------- `!` pins the current column on the left as a key column `H` `L` slides the current column **one position** to the left/right -`gH` `gL` slides the current column **all the way** to the left/right of the sheet +`gH` `gL` slides the current column **all the way** to the left/right of its section --- @@ -29,6 +29,10 @@ Commands(s) Operation ###### How to unhide columns +1. Press `gv` to unhide all columns on current sheet. + +**or** + 1. Press `Shift+C` on the source sheet to open its **Columns sheet**. 2. Move the cursor right to the **width** column. 3. Move the cursor down to the row which represents the column you wish to unhide. Currently, that cell should contain the value **0**. @@ -132,9 +136,24 @@ uses the commands for column splitting and transformation with [xd/puzzles.tsv]( ### - `:` adds new columns derived from splitting the current column at positions defined by a *regex pattern*. The current row will be used to infer the number of columns that will be created. -- `;` adds new columns derived from pulling the contents of the current column which match the *regex within capture groups*. This command also requires an example row. +- `;` adds new columns derived from pulling the contents of the current column which match the *regex within capture groups*. The new columns are named using the capture group index, or if named capture groups are used, the capture group names. This command also requires an example row. - `*` followed by *regex*`/`*substring* replaces the text which matches the capture groups in *regex* with the contents of *substring*. *substring* may include backreferences (*\1* etc). +## [How do I substitute text in my column] + +The `*` command can be used to do content transformations of cells. The `g*` variant transforms in-place, instead of creating a new column. + +The following example uses [benchmarks.csv](https://raw.githubusercontent.com/saulpw/visidata/stable/sample_data/benchmarks.csv). + +**Question** Transform the **SKU** values of *food* to *nutri*. + +1. Move cursor to **SKU** column. +2. Press `gs` to select all rows. +3. Type `g*` folowed by *food/nutri*. + +- tests/transform-cols.vd + + --- ## [How to expand columns that contain nested data](#expand) {#expand} @@ -170,7 +189,35 @@ Note that by default the expansion logic will look for nested columns in **up to ## [How to create derivative columns](#derived) {#derived} -The `=` command takes a Python expression as input, evaluates the expression, and creates a new column from the result. Column names can be supplied as variables, in order to have the expression performed on the column cell-by-cell. VisiData supports `Tab` autocompletion of column names. +The `=` command takes a Python expression as input and creates a new column, where each cell evaluates the expression in the context of its row. + +These variables and functions are available in the scope of an expression: + +- **Column names** evaluate to the typed value of the cell in the named column for the same row. +- **`vd`** attributes and methods; use `Ctrl+X vd` to view the vd object, or [see the API](). +- **`Sheet`** attributes and methods; use `g Ctrl+Y` to view the sheet object (or see the API). +- **Global** functions and variables (add your own in your .visidatarc). +- **modules** that have been `import`ed in Python + - if you need a module that hasn't already been imported at runtime, use `g Ctrl+X import `. + +- **`sheet`**: the current sheet (a TableSheet object) +- **`col`**: the current column (as a Column object; use for Column metadata) +- **`row`**: the current row (a Python object of the internal rowtype) + +Additional attributes can be added to sheets and columns. + +`col` deliberately returns a Column object, but any other Column object is interpreted as the value within that column for the same row. + +For example, this customizes addcol-expr to set the `curcol` attribute on the new ExprColumn to a snapshot of the current cursor column (at the time the expression column is added): + +``` +Sheet.addCommand('=', 'addcol-expr', 'addColumnAtCursor(ExprColumn(inputExpr("new column expr="), curcol=cursorCol))', 'create + new column from Python expression, with column names as variables') +``` + +Then, an expression can use `curcol` as though it referred to the value in the saved column. + +`Tab` autocompletion when inputting an expression will cycle through valid column names only. The following examples use the file [sample.tsv](https://raw.githubusercontent.com/saulpw/visidata/stable/sample_data/sample.tsv). diff --git a/docs/group.md b/docs/group.md index 82cbccb65..4af59201b 100644 --- a/docs/group.md +++ b/docs/group.md @@ -19,6 +19,7 @@ Aggregator Description `min` smallest value in the group `max` largest value in the group `avg`/`mean` average value of the group +`mode` most frequently appearing value in group `median` median value in the group `q3/q4/q5/q10` add quantile aggregators to group (e.g. q4 adds p25, p50, p75) `sum` total summation of all numbers in the group @@ -26,6 +27,7 @@ Aggregator Description `count` number of values in the group `keymax` key of the row with the largest value in the group `list` gathers values in column into a list +`stdev` standard deviation of values The follow howtos will have examples of workflows involving grouping of data and statistical aggregation. diff --git a/docs/index.md b/docs/index.md index 0e2273b17..b3db83cb8 100644 --- a/docs/index.md +++ b/docs/index.md @@ -13,9 +13,11 @@ ## References -* [quick reference guide](/man) +* [quick reference guide for 2.0](/man) * all available commands and options * also available as a manpage via `man vd` and from inside VisiData with `Ctrl+H` +* [quick reference guide for 1.5.2](/docs/v1.5.2/man) +* [VisiData Cheat Sheet](https://jsvine.github.io/visidata-cheat-sheet/en/) * [keyboard layout of commands](/docs/kblayout) ## 'How to' recipes @@ -79,8 +81,6 @@ # For developers VisiData can interact with data from any source or in any format. - +* [VisiData API Guide](/docs/api) * [guide to contributing](/contributing) * [viewtsv annotated](/docs/viewtsv) -* [async](/docs/async) -* [graphics reference](/docs/graphics) diff --git a/docs/join.md b/docs/join.md index f6dd3bfc6..9c0b829f9 100644 --- a/docs/join.md +++ b/docs/join.md @@ -1,25 +1,22 @@ -- Update: 2018-08-19 -- Version: VisiData 1.3.1 +- Update: 2020-10-27 +- Version: VisiData 2.0.1 # Combining datasets ## How to join two datasets -1. Load the datasets into VisiData. +1. Open the datasets in VisiData. a. `vd d1.tsv d2.tsv` **or** b. Press `o` and enter a filepath for each file. - -2. Press `Shift+S` to open the **Sheets sheet**. -3. Use `s` or `t` to select the sheets to merge. -4. Press `g Shift+C` to open a **Columns sheet** with all of the columns from selected sheets. -5. Press `g!` on the rows that reference the indices on which the join will be performed. At least one key column, per sheet, should be set. -6. Press `Ctrl+^` or `S` to return to the **Sheets sheet**. -7. Optional: If performing a left outer join, use `Shift+J` or `Shift+K` to reorder the sheets. The first sheet will be the one for whom all rows will be retained. -8. Type `&` followed by the *jointype* to execute the join +2. Press `S` to open up the **Sheets Sheet**. Through here, you can navigate to every sheet by pressing `Enter` on the row it is referenced in. +3. Navigate to the sheets you want the join, and set their shared columns as key columns with `!`. +4. Press `S` to return to the **Sheets sheet**. Select the sheets you want to merge with `s`. +5. Optional: If performing a left outer join, use `Shift+J` or `Shift+K` to reorder the sheets. The first sheet will be the one for whom all rows will be retained. +6. Type `&` to open the join-chooser, and select your desired jointype with `Enter`. jointype description --------- ------------- @@ -28,10 +25,11 @@ jointype description `full` keeps all rows from all sheets (union) `diff` keeps only rows NOT in all sheets `extend` keeps all rows and retain **SheetType** from first selected sheet +`merge` Merges differences from other sheets into first sheet ## How to append two datasets -1. Load the datasets into VisiData. +1. Open the datasets with VisiData. 2. Press `Shift+S` to open the **Sheets sheet**. 3. Use `s` or `t` to select the sheets to merge. -4. Type `&` followed by `append` to concatenate the selected datasets. +4. Type `&` and press `Enter` on `append` to concatenate the selected datasets. diff --git a/plugins/clickhouse.py b/plugins/clickhouse.py new file mode 100644 index 000000000..33bb6de0a --- /dev/null +++ b/plugins/clickhouse.py @@ -0,0 +1,72 @@ + +from visidata import vd, addGlobals, asyncthread, Sheet, ItemColumn +from visidata import * + +__all__ = ['openurl_clickhouse'] + +vd.option('clickhouse_host', '', '') +vd.option('clickhouse_port', 9000, '') + +def openurl_clickhouse(p, filetype=None): + url = urlparse(p.given) + options.clickhouse_host = url.hostname + options.clickhouse_port = url.port + return ClickhouseIndexSheet(p.name, source=p) + +BaseSheet.addCommand(ALT+'c', 'open-clickhouse', 'vd.push(vd.clickhouse_queries)') + +class ClickhouseQuerySheet(Sheet): + columns = [ + ColumnItem('name', 0), + ColumnItem('query', 1), + ColumnItem('nrows', 2), + ColumnItem('notes', 3), + ] + _rowtype=lambda: ['', '', 0, ''] + + def openRow(self, row): + return ClickhouseSheet(row[0], query=row[1]) + +@VisiData.lazy_property +def clickhouse_client(vd): + from clickhouse_driver import Client + return Client(**options.getall('clickhouse_')) + +@VisiData.lazy_property +def clickhouse_queries(self): + return ClickhouseQuerySheet("queries") + + +class ClickhouseSheet(Sheet): + @asyncthread + def reload(self): + if isinstance(self.source, ClickhouseSheet): + vd.clickhouse_client.execute('USE %s' % self.source.dbname) + + self.rows = [] + self.columns = [] + self.result = vd.clickhouse_client.execute(self.query, with_column_types=True) + result = self.result[0] + + for i, r in enumerate(self.result[1]): + self.addColumn(ItemColumn(r[0], i)) + + for r in result: + self.addRow(r) + +class ClickhouseIndexSheet(IndexSheet): + @asyncthread + def reload(self): + self.rows = [] + for r in vd.clickhouse_client.execute('SHOW DATABASES'): + self.addRow(ClickhouseDbSheet(r[0], source=self)) + +class ClickhouseDbSheet(IndexSheet): + @asyncthread + def reload(self): + vd.clickhouse_client.execute('USE %s' % self.name) + self.rows = [] + for r in vd.clickhouse_client.execute('SHOW TABLES'): + self.addRow(ClickhouseSheet(r[0], source=self, query='SELECT * FROM %s LIMIT 100' % r[0])) + +addGlobals({'openurl_clickhouse': openurl_clickhouse}) diff --git a/plugins/plugins.jsonl b/plugins/plugins.jsonl index a9f7389b8..80cc6ab46 100644 --- a/plugins/plugins.jsonl +++ b/plugins/plugins.jsonl @@ -1,8 +1,7 @@ {"name": "vfake", "description": "replace column with fake values", "maintainer": "Saul Pwanson @saulpw", "latest_release": "2020-10-06", "url": "https://raw.githubusercontent.com/saulpw/visidata/0d8af5a226818aa2f99a2f74e08543152b22a36d/plugins/vfake.py", "latest_ver": "1.1", "visidata_ver": "2.0", "pydeps": "Faker", "vdplugindeps": "", "sha256": "8153b04a364ac3db2ee310503f762924c4dc472445477af9e027c4d0e921af7f"} -{"name": "vddedupe", "description": "adds commands for selection and removal of rows which are duplicates of prior rows", "maintainer": "Jeremy Singer-Vine @jsvine", "latest_release": "2019-01-01", "url": "https://raw.githubusercontent.com/jsvine/visidata-plugins/fc070edde97c543ed345667cba58110d08b4895d/plugins/vddedupe.py", "latest_ver": "0.0.1", "visidata_ver": "1.5.2", "pydeps": "", "vdplugindeps": "", "sha256": "c9adf5323e7fbdc537ceedb58de30af2c145141eba1d53288a1e187cfd28452d"} -{"name": "vdnormcol", "description": "normalises column names in any given sheet so that the names are unique, valid Python identifiers, and only composed of lowercase letters, numbers, and underscores", "maintainer": "Jeremy Singer-Vine @jsvine", "latest_release": "2018-12-31", "url": "https://raw.githubusercontent.com/jsvine/visidata-plugins/fc070edde97c543ed345667cba58110d08b4895d/plugins/vdnormcol.py", "latest_ver": "0.0.0", "visidata_ver": "2.0", "pydeps": "", "vdplugindeps": "", "sha256": "77d1d40c31cb7d66c47decffa85c5db120e81969ab10c6dd41d436992995e72e"} -{"name": "vdfec", "description": "loader for .fec files from the Federal Election Commission", "maintainer": "Jeremy Singer-Vine @jsvine", "latest_release": "2019-04-21", "url": "https://raw.githubusercontent.com/jsvine/visidata-plugins/fc070edde97c543ed345667cba58110d08b4895d/plugins/vdfec.py", "latest_ver": "0.0.0", "visidata_ver": "1.5.2", "pydeps": "fecfile", "vdplugindeps": "", "sha256": "e917170e5bb74ac6242f81113aa48ba7e36763ad32cb190420acb93c3f62765c"} -{"name": "livesearch", "description": "filter rows as you search", "maintainer": "Saul Pwanson @saulpw", "latest_release": "2019-06-09", "url": "https://raw.githubusercontent.com/saulpw/visidata/a185cb5f734a58d14478cf13477246c7773245f8/plugins/livesearch.py", "latest_ver": "0.9", "visidata_ver": "2.-1", "pydeps": "", "vdplugindeps": "", "sha256": "57ba2685252f0a4659d2582e05e7d9bab9e680314883328a202ed220dbd0bb6f"} +{"name": "dedupe", "description": "adds commands for selection and removal of rows which are duplicates of prior rows", "maintainer": "Jeremy Singer-Vine @jsvine", "latest_release": "2020-10-11", "url": "https://raw.githubusercontent.com/jsvine/visidata-plugins/37cba82ed2cc49aedc1a377af984acfe1ef3d5cd/plugins/dedupe.py", "latest_ver": "0.1.0", "visidata_ver": "2.0", "pydeps": "", "vdplugindeps": "", "sha256": "2e077b8d62bc8ec2235d22d7d9711d99e7366c8c0ff20405f70647044cca67a1"} +{"name": "normcol", "description": "normalizes column names in any given sheet so that the names are unique, valid Python identifiers, and only composed of lowercase letters, numbers, and underscores", "maintainer": "Jeremy Singer-Vine @jsvine", "latest_release": "2020-10-11", "url": "https://raw.githubusercontent.com/jsvine/visidata-plugins/37cba82ed2cc49aedc1a377af984acfe1ef3d5cd/plugins/normcol.py", "latest_ver": "0.1.0", "visidata_ver": "2.0", "pydeps": "", "vdplugindeps": "", "sha256": "ede6b9e508e1ce7842e00d308c372837c9b4d4906b88e0ba62efd5b481f816f6"} +{"name": "fec", "description": "loader for .fec files from the Federal Election Commission", "maintainer": "Jeremy Singer-Vine @jsvine", "latest_release": "2019-04-21", "url": "https://raw.githubusercontent.com/jsvine/visidata-plugins/37cba82ed2cc49aedc1a377af984acfe1ef3d5cd/plugins/fec.py", "latest_ver": "0.0.0", "visidata_ver": "1.5.2", "pydeps": "fecfile", "vdplugindeps": "", "sha256": "e917170e5bb74ac6242f81113aa48ba7e36763ad32cb190420acb93c3f62765c"} {"name": "sparkline", "description": "add a sparkline column to visualise trends of numeric cells in a row", "maintainer": "Lucas Messenger @layertwo", "latest_release": "2020-09-13", "url": "https://raw.githubusercontent.com/saulpw/visidata/e4a373cd74b49b2b91876b17d7a1cd0cd0303848/plugins/sparkline.py", "latest_ver": "0.1", "visidata_ver": "2.0", "pydeps": "", "vdplugindeps": "", "sha256": "4256bea5535405a2478cb6827df4be86f28c69b3cb7f30cc08e2cd655e48b592"} {"name": "rownum", "description": "add column of original row ordering", "maintainer": "Saul Pwanson @saulpw", "latest_release": "2019-11-07", "url": "https://raw.githubusercontent.com/saulpw/visidata/a185cb5f734a58d14478cf13477246c7773245f8/plugins/rownum.py", "latest_ver": "0.9", "visidata_ver": "2.0", "pydeps": "", "vdplugindeps": "", "sha256": "caf896841639ecdef25c2f227f3370a4c4f0b0b31bdd72ef422dfe94ec5664f0"} {"name": "vmailcap", "description": "add mailcap-view(-selected) commands to DirSheet", "maintainer": "Saul Pwanson @saulpw", "latest_release": "2020-10-06", "url": "https://raw.githubusercontent.com/saulpw/visidata/0d8af5a226818aa2f99a2f74e08543152b22a36d/plugins/vmailcap.py", "latest_ver": "0.9", "visidata_ver": "2.0", "pydeps": "", "vdplugindeps": "", "sha256": "b8aa66821dfa38107ee45fe49a5262e766cc23a0ff362ce5aea5aa6e9b8e6fb0"} @@ -11,3 +10,4 @@ {"name": "genericSQL", "description": "add loaders for SQL databases (Oracle, MySQL)", "maintainer": "Andrew Swanson @aswan89", "latest_release": "2020-04-15", "url": "https://raw.githubusercontent.com/aswan89/visidata_plugin_genericSQL/1.0.0/generic_sql.py", "latest_ver": "1.0.0", "visidata_ver": "2.-3", "pydeps": "sqlalchemy cx_oracle mysqlclient pyodbc", "vdplugindeps": "", "sha256": "71bbaabaa4ffb973ef7745dc3eadca108b34a7e0f0bf6cf6631eca3fa4532513"} {"name": "diff", "description": "adds command to create diff sheets", "maintainer": "Anja Kefala @anjakefala", "latest_release": "2020-10-09", "url": "https://raw.githubusercontent.com/saulpw/visidata/492d558ff4aba55a192965a27ae6499f8d219072/plugins/diff.py", "latest_ver": "0.9", "visidata_ver": "2.0", "sha256": "0085bf8afdc9abf74b7dd52c251f7b7f7befc507aae262bbb656c2ab7379ddd1"} {"name": "marks", "description": "adds commands for marking selected rows, and selecting + viewing marked rows", "maintainer": "Saul Pwanson @saulpw", "latest_release": "2020-10-09", "url": "https://raw.githubusercontent.com/saulpw/visidata/54e49feee97a607cce5355bf0e3a79b2466c3d8a/plugins/marks.py", "latest_ver": "0.1", "visidata_ver": "2.0", "sha256": "f15977f327ccc387be208922b66afe4b65666912e8066d02352c53b1aaff941f"} +{"name": "conll", "description": "CoNLL data loader", "maintainer": "Paul McCann ", "latest_release": "2020-11-09", "url": "https://raw.githubusercontent.com/polm/visidata-conll/83579a939813b3a3bca638bbb15bb9e8cf4e08ac/conll.py", "latest_ver": "0.1.0", "visidata_ver": "2.0", "pydeps": "pyconll", "sha256": "ff0df4c817121e57780ddd3e16ae67703fa69129c9e33e83fe40dd5220ad93e2"} diff --git a/plugins/rownum.py b/plugins/rownum.py index 69857ba8c..f673932dc 100644 --- a/plugins/rownum.py +++ b/plugins/rownum.py @@ -72,4 +72,4 @@ def addcol_rownum(sheet): return newcol Sheet.addCommand(None, 'addcol-rownum', 'addcol_rownum()', helpstr='add column with original row ordering') -Sheet.addCommand(None, 'addcol-delta', 'addcol_delta()', helpstr='add column with delta of current column') +Sheet.addCommand(None, 'addcol-delta', 'addcol_delta(cursorVisibleColIndex)', helpstr='add column with delta of current column') diff --git a/requirements.txt b/requirements.txt index cc0cad566..4e00f83d8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ python-dateutil # date type -# optional requirements +# optional dependencies pandas==1.0.3 # dta (Stata) requests # http lxml # html/xml @@ -21,6 +21,7 @@ dnslib # pcap namestand # graphviz datapackage # frictionless .json pdfminer.six # pdf +tabula # pdf tables vobject # vcf tabulate # tabulate saver wcwidth # tabulate saver with unicode diff --git a/sample_data/hello.mnu b/sample_data/hello.mnu new file mode 100644 index 000000000..57655b2a3 --- /dev/null +++ b/sample_data/hello.mnu @@ -0,0 +1,2 @@ +x y text color command input cond status +3 4 hello world 217 underline show-status Hello World! diff --git a/setup.py b/setup.py index 9833bf4de..dfdbe23c6 100755 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ from setuptools import setup # tox can't actually run python3 setup.py: https://github.com/tox-dev/tox/issues/96 #from visidata import __version__ -__version__ = '2.0.1' +__version__ = '2.1' setup(name='visidata', version=__version__, @@ -25,7 +25,7 @@ packages=['visidata', 'visidata.loaders', 'visidata.tests'], include_package_data=True, data_files = [('share/man/man1', ['visidata/man/vd.1'])], - package_data={'visidata': ['man/vd.1']}, + package_data={'visidata': ['man/vd.1', 'man/vd.txt']}, license='GPLv3', classifiers=[ 'Development Status :: 5 - Production/Stable', diff --git a/tests/.visidata/.gitignore b/tests/.visidata/.gitignore new file mode 100644 index 000000000..0a2101fab --- /dev/null +++ b/tests/.visidata/.gitignore @@ -0,0 +1 @@ +/cache/ diff --git a/tests/.visidatarc b/tests/.visidatarc new file mode 100644 index 000000000..e69de29bb diff --git a/tests/aggregators-errors.vd b/tests/aggregators-errors.vd index ca6330c58..350210aee 100644 --- a/tests/aggregators-errors.vd +++ b/tests/aggregators-errors.vd @@ -1,5 +1,6 @@ sheet col row longname input keystrokes comment open-file sample_data/test.jsonl o + numeric_binning set-option True test key1 type-date @ test qty type-int # test amt type-float % diff --git a/tests/bulk-rename-cols.vd b/tests/bulk-rename-cols.vd new file mode 100644 index 000000000..ae326b906 --- /dev/null +++ b/tests/bulk-rename-cols.vd @@ -0,0 +1,5 @@ +sheet col row longname input keystrokes comment + open-file sample_data/sample.tsv o +sample columns-sheet C open Columns Sheet: edit column properties for current sheet +sample_columns select-rows gs +sample_columns name setcol-expr f"feature_{name}" g= set current column for selected rows to result of Python expression diff --git a/tests/capture-col-named.vd b/tests/capture-col-named.vd new file mode 100644 index 000000000..9a55c89fa --- /dev/null +++ b/tests/capture-col-named.vd @@ -0,0 +1,3 @@ +sheet col row longname input keystrokes comment + open-file sample_data/benchmark.csv o +benchmark Date capture-col (?P\d{1,2})/(?P\d{1,2})/(?P\d{4}) (?P\d{1,2}):(?P\d{2})(?P[ap]) ; add new column from capture groups of regex; requires example row diff --git a/tests/exp-digits.vd b/tests/exp-digits.vd index b035fb365..0399cbcb7 100644 --- a/tests/exp-digits.vd +++ b/tests/exp-digits.vd @@ -1,6 +1,6 @@ sheet col row longname input keystrokes comment incr_base set-option 0 -exp-digits_vd open-new 1 A open new blank sheet with number columns + open-new 1 A open new blank sheet with number columns unnamed 0 rename-col x ^ edit name of current column unnamed add-rows 100 ga add N blank rows unnamed select-rows gs diff --git a/tests/freq-error.vd b/tests/freq-error.vd index f6612b379..c14c3a2cb 100644 --- a/tests/freq-error.vd +++ b/tests/freq-error.vd @@ -1,6 +1,5 @@ sheet col row longname input keystrokes comment open-file sample_data/test.jsonl o -test numeric_binning set-option False test key1 type-date @ test qty type-int # test amt type-float % diff --git a/tests/freq-fmtstr.vd b/tests/freq-fmtstr.vd index c742631cd..986fd247d 100644 --- a/tests/freq-fmtstr.vd +++ b/tests/freq-fmtstr.vd @@ -1,9 +1,7 @@ sheet col row longname input keystrokes comment open-file sample_data/benchmark.csv o - numeric_binning set-option False benchmark Date type-date @ benchmark columns-sheet C benchmark_columns fmtstr キDate edit-cell %Y-%m e -benchmark_columns quit-sheet q benchmark Date freq-col F benchmark_Date_freq histogram hide-col - diff --git a/tests/golden/bulk-rename-cols.tsv b/tests/golden/bulk-rename-cols.tsv new file mode 100644 index 000000000..6fb25357e --- /dev/null +++ b/tests/golden/bulk-rename-cols.tsv @@ -0,0 +1,8 @@ +name width height type fmtstr value expr aggregators +feature_OrderDate 1 2016-01-06 0 +feature_Region 1 East 1 +feature_Rep 1 Jones 2 +feature_Item 1 Pencil 3 +feature_Units 1 95 4 +feature_Unit_Cost 1 1.99 5 +feature_Total 1 189.05 6 diff --git a/tests/golden/capture-col-named.tsv b/tests/golden/capture-col-named.tsv new file mode 100644 index 000000000..9f22b7dfd --- /dev/null +++ b/tests/golden/capture-col-named.tsv @@ -0,0 +1,50 @@ +Date Date_month Date_day Date_year Date_hour Date_min Date_meridiem Customer SKU Item Quantity Unit Paid +7/3/2018 1:47p 7 3 2018 1 47 p Robert Armstrong FOOD213 BFF Oh My Gravy! Beef & Salmon 2.8oz 4 $12.95 $51.8 +7/3/2018 3:32p 7 3 2018 3 32 p Kyle Kennedy FOOD121 Food, Adult Cat - 3.5 oz 1 $4.22 $4.22 +7/5/2018 4:15p 7 5 2018 4 15 p Douglas "Dougie" Powers FOOD121 Food, Adult Cat 3.5 oz 1 $4.22 $4.22 +7/6/2018 12:15p 7 6 2018 12 15 p 桜 高橋 (Sakura Takahashi) FOOD122 Food, Senior Wet Cat - 3 oz 12 $1.29 157¥ +7/10/2018 10:28a 7 10 2018 10 28 a David Attenborough NSCT201 Food, Salamander 30 $.05 $1.5 +7/10/2018 5:23p 7 10 2018 5 23 p Susan Ashworth CAT060 Cat, Korat (Felis catus) 1 $720.42 $720.42 +7/10/2018 5:23p 7 10 2018 5 23 p Susan Ashworth FOOD130 Food, Kitten 3kg 1 $14.94 $14.94 +7/13/2018 10:26a 7 13 2018 10 26 a Wil Wheaton NSCT523 Monster, Rust (Monstrus gygaxus) 1 $39.95 $39.95 +7/13/2018 3:49p 7 13 2018 3 49 p Robert Armstrong FOOD216 BFF Oh My Gravy! Chicken & Shrimp 2.8oz 4 $12.95 $51.8 +7/17/2018 9:01a 7 17 2018 9 01 a Robert Armstrong FOOD217 BFF Oh My Gravy! Duck & Tuna 2.8oz 4 $12.95 $51.8 +7/17/2018 11:30a 7 17 2018 11 30 a Helen Halestorm LAGO342 Rabbit (Oryctolagus cuniculus) 2 $32.94 $65.88 +7/18/2018 12:16p 7 18 2018 12 16 p 桜 高橋 (Sakura Takahashi) FOOD122 Food, Senior Wet Cat - 3 oz 6 $1.29 157¥ +7/19/2018 10:28a 7 19 2018 10 28 a Rubeus Hagrid FOOD170 Food, Dog - 5kg 5 $44.95 $224.75 +7/20/2018 2:13p 7 20 2018 2 13 p Jon Arbuckle FOOD167 Food, Premium Wet Cat - 3.5 oz 50 $3.95 $197.5 +7/23/2018 1:41p 7 23 2018 1 41 p Robert Armstrong FOOD215 BFF Oh My Gravy! Lamb & Tuna 2.8oz 4 $12.95 $51.8 +7/23/2018 4:23p 7 23 2018 4 23 p Douglas "Dougie" Powers TOY235 Laser Pointer 1 $16.12 $16.12 +7/24/2018 12:16p 7 24 2018 12 16 p 桜 高橋 (Sakura Takahashi) FOOD122 Food, Senior Wet Cat - 3 oz 3 $1.29 157¥ +7/26/2018 4:39p 7 26 2018 4 39 p Douglas "Dougie" Powers FOOD420 Food, Shark - 10 kg 1 $15.70 $15.7 +7/27/2018 12:16p 7 27 2018 12 16 p 桜 高橋 (Sakura Takahashi) FOOD122 Food, Senior Wet Cat - 3 oz 3 $1.29 157¥ +7/30/2018 12:17p 7 30 2018 12 17 p 桜 高橋 (Sakura Takahashi) RETURN Food, Senior Wet Cat - 3 oz 1 $1.29 157¥ +7/31/2018 5:42p 7 31 2018 5 42 p Rubeus Hagrid CAT060 Food, Dragon - 50kg 5 $720.42 $3602.1 +8/1/2018 2:44p 8 1 2018 2 44 p David Attenborough FOOD360 Food, Rhinocerous - 50kg 4 $5.72 $22.88 +8/2/2018 5:12p 8 2 2018 5 12 p Susan Ashworth CAT110 Cat, Maine Coon (Felix catus) 1 $1,309.68 $1309.68 +8/2/2018 5:12p 8 2 2018 5 12 p Susan Ashworth FOOD130 Food, Kitten 3kg 3 $14.94 $44.82 +8/6/2018 10:21a 8 6 2018 10 21 a Robert Armstrong FOOD212 BFF Oh My Gravy! Beef & Chicken 2.8oz 4 $12.95 $51.8 +8/7/2018 4:12p 8 7 2018 4 12 p Juan Johnson REPT082 Kingsnake, California (Lampropeltis getula) 1 $89.95 $89.95 +8/7/2018 4:12p 8 7 2018 4 12 p Juan Johnson RDNT443 Mouse, Pinky (Mus musculus) 1 $1.49 $1.49 +8/10/2018 4:31p 8 10 2018 4 31 p Robert Armstrong FOOD211 BFF Oh My Gravy! Chicken & Turkey 2.8oz 4 $12.95 $51.8 +8/13/2018 2:07p 8 13 2018 2 07 p Monica Johnson RDNT443 Mouse, Pinky (Mus musculus) 1 $1.49 $1.49 +8/13/2018 2:08p 8 13 2018 2 08 p María Fernández FOOD146 Forti Diet Prohealth Mouse/Rat 3lbs 2 $2.00 $4.0 +8/15/2018 11:57a 8 15 2018 11 57 a Mr. Praline RETURN Parrot, Norwegian Blue (Mopsitta tanta) 1 $2300.00 -$2300.0 +8/15/2018 3:48p 8 15 2018 3 48 p Kyle Kennedy FOOD121 Food, Adult Cat - 3.5 oz 2 $4.22 $8.44 +8/16/2018 11:50a 8 16 2018 11 50 a Helen Halestorm RETURN Rabbit (Oryctolagus cuniculus) 6 $0 $0.0 +8/16/2018 4:00p 8 16 2018 4 00 p Kyle Kennedy DOG010 Dog, Golden Retriever (Canis lupus familiaris) 1 $2,495.99 $2495.99 +8/16/2018 5:15p 8 16 2018 5 15 p Michael Smith BIRD160 Parakeet, Blue (Melopsittacus undulatus) 1 29.95 $31.85 +8/17/2018 9:26a 8 17 2018 9 26 a Rubeus Hagrid NSCT201 Food, Spider 5 $.05 $0.25 +8/20/2018 9:36a 8 20 2018 9 36 a Kyle Kennedy RETURN Dog, Golden Retriever (Canis lupus familiaris) 1 $1,247.99 -$1247.99 +8/20/2018 3:31p 8 20 2018 3 31 p Monica Johnson NSCT201 Crickets, Adult Live (Gryllus assimilis) 30 $.05 $1.5 +8/20/2018 5:12p 8 20 2018 5 12 p David Attenborough NSCT084 Food, Pangolin 30 $.17 $5.10 +8/21/2018 12:13p 8 21 2018 12 13 p Robert Armstrong FOOD214 BFF Oh My Gravy! Duck & Salmon 2.8oz 4 $12.95 $51.8 +8/22/2018 9:38a 8 22 2018 9 38 a David Attenborough BIRD160 Food, Quoll 1 29.95 $29.95 +8/22/2018 2:13p 8 22 2018 2 13 p Jon Arbuckle FOOD170 Food, Adult Dog - 5kg 1 $44.95 $44.95 +8/24/2018 11:42a 8 24 2018 11 42 a Robert Armstrong FOOD218 BFF Oh My Gravy! Chicken & Salmon 2.8oz 4 $12.95 $51.8 +8/27/2018 3:05p 8 27 2018 3 05 p Monica Johnson NSCT443 Mealworms, Large (Tenebrio molitor) 100ct 1 $1.99 $1.99 +8/28/2018 5:32p 8 28 2018 5 32 p Susan Ashworth CAT020 Cat, Scottish Fold (Felis catus) 1 $1,964.53 $1964.53 +8/28/2018 5:32p 8 28 2018 5 32 p Susan Ashworth FOOD130 Food, Kitten 3kg 2 $14.94 $29.88 +8/29/2018 10:07a 8 29 2018 10 07 a Robert Armstrong FOOD219 BFF Oh My Gravy! Chicken & Pumpkin 2.8oz 4 $12.95 $51.8 +8/31/2018 12:00a 8 31 2018 12 00 a Robert Armstrong FOOD219 BFF Oh My Gravy! Chicken & Pumpkin 2.8oz 144 $12.95 $1864.8 +8/31/2018 5:57p 8 31 2018 5 57 p Juan Johnson REPT217 Lizard, Spinytail (Uromastyx ornatus) 1 $99.95 $99.95 diff --git a/tests/golden/issue733.tsv b/tests/golden/issue733.tsv new file mode 100644 index 000000000..05a8fbd97 --- /dev/null +++ b/tests/golden/issue733.tsv @@ -0,0 +1,50 @@ +Date Customer SKU Item Quantity Unit Paid +2018-07-03 Robert Armstrong FOOD213 BFF Oh My Gravy! Beef & Salmon 2.8oz 4 $12.95 $51.8 +2018-07-03 Kyle Kennedy FOOD121 Food, Adult Cat - 3.5 oz 1 $4.22 $4.22 +2018-07-05 Douglas "Dougie" Powers FOOD121 Food, Adult Cat 3.5 oz 1 $4.22 $4.22 +2018-07-06 桜 高橋 (Sakura Takahashi) FOOD122 Food, Senior Wet Cat - 3 oz 12 $1.29 157¥ +2018-07-10 David Attenborough NSCT201 Food, Salamander 30 $.05 $1.5 +2018-07-10 Susan Ashworth CAT060 Cat, Korat (Felis catus) 1 $720.42 $720.42 +2018-07-10 Susan Ashworth FOOD130 Food, Kitten 3kg 1 $14.94 $14.94 +2018-07-13 Wil Wheaton NSCT523 Monster, Rust (Monstrus gygaxus) 1 $39.95 $39.95 +2018-07-13 Robert Armstrong FOOD216 BFF Oh My Gravy! Chicken & Shrimp 2.8oz 4 $12.95 $51.8 +2018-07-17 Robert Armstrong FOOD217 BFF Oh My Gravy! Duck & Tuna 2.8oz 4 $12.95 $51.8 +2018-07-17 Helen Halestorm LAGO342 Rabbit (Oryctolagus cuniculus) 2 $32.94 $65.88 +2018-07-18 桜 高橋 (Sakura Takahashi) FOOD122 Food, Senior Wet Cat - 3 oz 6 $1.29 157¥ +2018-07-19 Rubeus Hagrid FOOD170 Food, Dog - 5kg 5 $44.95 $224.75 +2018-07-20 Jon Arbuckle FOOD167 Food, Premium Wet Cat - 3.5 oz 50 $3.95 $197.5 +2018-07-23 Robert Armstrong FOOD215 BFF Oh My Gravy! Lamb & Tuna 2.8oz 4 $12.95 $51.8 +2018-07-23 Douglas "Dougie" Powers TOY235 Laser Pointer 1 $16.12 $16.12 +2018-07-24 桜 高橋 (Sakura Takahashi) FOOD122 Food, Senior Wet Cat - 3 oz 3 $1.29 157¥ +2018-07-26 Douglas "Dougie" Powers FOOD420 Food, Shark - 10 kg 1 $15.70 $15.7 +2018-07-27 桜 高橋 (Sakura Takahashi) FOOD122 Food, Senior Wet Cat - 3 oz 3 $1.29 157¥ +2018-07-30 桜 高橋 (Sakura Takahashi) RETURN Food, Senior Wet Cat - 3 oz 1 $1.29 157¥ +2018-07-31 Rubeus Hagrid CAT060 Food, Dragon - 50kg 5 $720.42 $3602.1 +2018-08-01 David Attenborough FOOD360 Food, Rhinocerous - 50kg 4 $5.72 $22.88 +2018-08-02 Susan Ashworth CAT110 Cat, Maine Coon (Felix catus) 1 $1,309.68 $1309.68 +2018-08-02 Susan Ashworth FOOD130 Food, Kitten 3kg 3 $14.94 $44.82 +2018-08-06 Robert Armstrong FOOD212 BFF Oh My Gravy! Beef & Chicken 2.8oz 4 $12.95 $51.8 +2018-08-07 Juan Johnson REPT082 Kingsnake, California (Lampropeltis getula) 1 $89.95 $89.95 +2018-08-07 Juan Johnson RDNT443 Mouse, Pinky (Mus musculus) 1 $1.49 $1.49 +2018-08-10 Robert Armstrong FOOD211 BFF Oh My Gravy! Chicken & Turkey 2.8oz 4 $12.95 $51.8 +2018-08-13 Monica Johnson RDNT443 Mouse, Pinky (Mus musculus) 1 $1.49 $1.49 +2018-08-13 María Fernández FOOD146 Forti Diet Prohealth Mouse/Rat 3lbs 2 $2.00 $4.0 +2018-08-15 Mr. Praline RETURN Parrot, Norwegian Blue (Mopsitta tanta) 1 $2300.00 -$2300.0 +2018-08-15 Kyle Kennedy FOOD121 Food, Adult Cat - 3.5 oz 2 $4.22 $8.44 +2018-08-16 Helen Halestorm RETURN Rabbit (Oryctolagus cuniculus) 6 $0 $0.0 +2018-08-16 Kyle Kennedy DOG010 Dog, Golden Retriever (Canis lupus familiaris) 1 $2,495.99 $2495.99 +2018-08-16 Michael Smith BIRD160 Parakeet, Blue (Melopsittacus undulatus) 1 29.95 $31.85 +2018-08-17 Rubeus Hagrid NSCT201 Food, Spider 5 $.05 $0.25 +2018-08-20 Kyle Kennedy RETURN Dog, Golden Retriever (Canis lupus familiaris) 1 $1,247.99 -$1247.99 +2018-08-20 Monica Johnson NSCT201 Crickets, Adult Live (Gryllus assimilis) 30 $.05 $1.5 +2018-08-20 David Attenborough NSCT084 Food, Pangolin 30 $.17 $5.10 +2018-08-21 Robert Armstrong FOOD214 BFF Oh My Gravy! Duck & Salmon 2.8oz 4 $12.95 $51.8 +2018-08-22 David Attenborough BIRD160 Food, Quoll 1 29.95 $29.95 +2018-08-22 Jon Arbuckle FOOD170 Food, Adult Dog - 5kg 1 $44.95 $44.95 +2018-08-24 Robert Armstrong FOOD218 BFF Oh My Gravy! Chicken & Salmon 2.8oz 4 $12.95 $51.8 +2018-08-27 Monica Johnson NSCT443 Mealworms, Large (Tenebrio molitor) 100ct 1 $1.99 $1.99 +2018-08-28 Susan Ashworth CAT020 Cat, Scottish Fold (Felis catus) 1 $1,964.53 $1964.53 +2018-08-28 Susan Ashworth FOOD130 Food, Kitten 3kg 2 $14.94 $29.88 +2018-08-29 Robert Armstrong FOOD219 BFF Oh My Gravy! Chicken & Pumpkin 2.8oz 4 $12.95 $51.8 +2018-08-31 Robert Armstrong FOOD219 BFF Oh My Gravy! Chicken & Pumpkin 2.8oz 144 $12.95 $1864.8 +2018-08-31 Juan Johnson REPT217 Lizard, Spinytail (Uromastyx ornatus) 1 $99.95 $99.95 diff --git a/tests/golden/save-benchmarks.rst b/tests/golden/save-benchmarks.rst index 6debff9d4..bcab632d5 100644 --- a/tests/golden/save-benchmarks.rst +++ b/tests/golden/save-benchmarks.rst @@ -1,50 +1,53 @@ -|| Date || Customer || SKU || Item || Quantity || Unit || Paid || -| 2018-07-03 | Robert Armstrong | FOOD213 | BFF Oh My Gravy! Beef & Salmon 2.8oz | 4 | 12.95 | $51.8 | -| 2018-07-03 | Kyle Kennedy | FOOD121 | Food, Adult Cat - 3.5 oz | 1 | 4.22 | $4.22 | -| 2018-07-05 | Douglas "Dougie" Powers | FOOD121 | Food, Adult Cat 3.5 oz | 1 | 4.22 | $4.22 | -| 2018-07-06 | 桜 高橋 (Sakura Takahashi) | FOOD122 | Food, Senior Wet Cat - 3 oz | 12 | 1.29 | 157¥ | -| 2018-07-10 | David Attenborough | NSCT201 | Food, Salamander | 30 | 0.05 | $1.5 | -| 2018-07-10 | Susan Ashworth | CAT060 | Cat, Korat (Felis catus) | 1 | 720.42 | $720.42 | -| 2018-07-10 | Susan Ashworth | FOOD130 | Food, Kitten 3kg | 1 | 14.94 | $14.94 | -| 2018-07-13 | Wil Wheaton | NSCT523 | Monster, Rust (Monstrus gygaxus) | 1 | 39.95 | $39.95 | -| 2018-07-13 | Robert Armstrong | FOOD216 | BFF Oh My Gravy! Chicken & Shrimp 2.8oz | 4 | 12.95 | $51.8 | -| 2018-07-17 | Robert Armstrong | FOOD217 | BFF Oh My Gravy! Duck & Tuna 2.8oz | 4 | 12.95 | $51.8 | -| 2018-07-17 | Helen Halestorm | LAGO342 | Rabbit (Oryctolagus cuniculus) | 2 | 32.94 | $65.88 | -| 2018-07-18 | 桜 高橋 (Sakura Takahashi) | FOOD122 | Food, Senior Wet Cat - 3 oz | 6 | 1.29 | 157¥ | -| 2018-07-19 | Rubeus Hagrid | FOOD170 | Food, Dog - 5kg | 5 | 44.95 | $224.75 | -| 2018-07-20 | Jon Arbuckle | FOOD167 | Food, Premium Wet Cat - 3.5 oz | 50 | 3.95 | $197.5 | -| 2018-07-23 | Robert Armstrong | FOOD215 | BFF Oh My Gravy! Lamb & Tuna 2.8oz | 4 | 12.95 | $51.8 | -| 2018-07-23 | Douglas "Dougie" Powers | TOY235 | Laser Pointer | 1 | 16.12 | $16.12 | -| 2018-07-24 | 桜 高橋 (Sakura Takahashi) | FOOD122 | Food, Senior Wet Cat - 3 oz | 3 | 1.29 | 157¥ | -| 2018-07-26 | Douglas "Dougie" Powers | FOOD420 | Food, Shark - 10 kg | 1 | 15.7 | $15.7 | -| 2018-07-27 | 桜 高橋 (Sakura Takahashi) | FOOD122 | Food, Senior Wet Cat - 3 oz | 3 | 1.29 | 157¥ | -| 2018-07-30 | 桜 高橋 (Sakura Takahashi) | RETURN | Food, Senior Wet Cat - 3 oz | 1 | 1.29 | 157¥ | -| 2018-07-31 | Rubeus Hagrid | CAT060 | Food, Dragon - 50kg | 5 | 720.42 | $3602.1 | -| 2018-08-01 | David Attenborough | FOOD360 | Food, Rhinocerous - 50kg | 4 | 5.72 | $22.88 | -| 2018-08-02 | Susan Ashworth | CAT110 | Cat, Maine Coon (Felix catus) | 1 | 1309.68 | $1309.68 | -| 2018-08-02 | Susan Ashworth | FOOD130 | Food, Kitten 3kg | 3 | 14.94 | $44.82 | -| 2018-08-06 | Robert Armstrong | FOOD212 | BFF Oh My Gravy! Beef & Chicken 2.8oz | 4 | 12.95 | $51.8 | -| 2018-08-07 | Juan Johnson | REPT082 | Kingsnake, California (Lampropeltis getula) | 1 | 89.95 | $89.95 | -| 2018-08-07 | Juan Johnson | RDNT443 | Mouse, Pinky (Mus musculus) | 1 | 1.49 | $1.49 | -| 2018-08-10 | Robert Armstrong | FOOD211 | BFF Oh My Gravy! Chicken & Turkey 2.8oz | 4 | 12.95 | $51.8 | -| 2018-08-13 | Monica Johnson | RDNT443 | Mouse, Pinky (Mus musculus) | 1 | 1.49 | $1.49 | -| 2018-08-13 | María Fernández | FOOD146 | Forti Diet Prohealth Mouse/Rat 3lbs | 2 | 2 | $4.0 | -| 2018-08-15 | Mr. Praline | RETURN | Parrot, Norwegian Blue (Mopsitta tanta) | 1 | 2300 | -$2300.0 | -| 2018-08-15 | Kyle Kennedy | FOOD121 | Food, Adult Cat - 3.5 oz | 2 | 4.22 | $8.44 | -| 2018-08-16 | Helen Halestorm | RETURN | Rabbit (Oryctolagus cuniculus) | 6 | 0 | $0.0 | -| 2018-08-16 | Kyle Kennedy | DOG010 | Dog, Golden Retriever (Canis lupus familiaris) | 1 | 2495.99 | $2495.99 | -| 2018-08-16 | Michael Smith | BIRD160 | Parakeet, Blue (Melopsittacus undulatus) | 1 | 29.95 | $31.85 | -| 2018-08-17 | Rubeus Hagrid | NSCT201 | Food, Spider | 5 | 0.05 | $0.25 | -| 2018-08-20 | Kyle Kennedy | RETURN | Dog, Golden Retriever (Canis lupus familiaris) | 1 | 1247.99 | -$1247.99 | -| 2018-08-20 | Monica Johnson | NSCT201 | Crickets, Adult Live (Gryllus assimilis) | 30 | 0.05 | $1.5 | -| 2018-08-20 | David Attenborough | NSCT084 | Food, Pangolin | 30 | 0.17 | $5.10 | -| 2018-08-21 | Robert Armstrong | FOOD214 | BFF Oh My Gravy! Duck & Salmon 2.8oz | 4 | 12.95 | $51.8 | -| 2018-08-22 | David Attenborough | BIRD160 | Food, Quoll | 1 | 29.95 | $29.95 | -| 2018-08-22 | Jon Arbuckle | FOOD170 | Food, Adult Dog - 5kg | 1 | 44.95 | $44.95 | -| 2018-08-24 | Robert Armstrong | FOOD218 | BFF Oh My Gravy! Chicken & Salmon 2.8oz | 4 | 12.95 | $51.8 | -| 2018-08-27 | Monica Johnson | NSCT443 | Mealworms, Large (Tenebrio molitor) 100ct | 1 | 1.99 | $1.99 | -| 2018-08-28 | Susan Ashworth | CAT020 | Cat, Scottish Fold (Felis catus) | 1 | 1964.53 | $1964.53 | -| 2018-08-28 | Susan Ashworth | FOOD130 | Food, Kitten 3kg | 2 | 14.94 | $29.88 | -| 2018-08-29 | Robert Armstrong | FOOD219 | BFF Oh My Gravy! Chicken & Pumpkin 2.8oz | 4 | 12.95 | $51.8 | -| 2018-08-31 | Robert Armstrong | FOOD219 | BFF Oh My Gravy! Chicken & Pumpkin 2.8oz | 144 | 12.95 | $1864.8 | -| 2018-08-31 | Juan Johnson | REPT217 | Lizard, Spinytail (Uromastyx ornatus) | 1 | 99.95 | $99.95 | \ No newline at end of file +========== ========================== ======= ============================================== ========== ======= ========= +Date Customer SKU Item Quantity Unit Paid +========== ========================== ======= ============================================== ========== ======= ========= +2018-07-03 Robert Armstrong FOOD213 BFF Oh My Gravy! Beef & Salmon 2.8oz 4 12.95 $51.8 +2018-07-03 Kyle Kennedy FOOD121 Food, Adult Cat - 3.5 oz 1 4.22 $4.22 +2018-07-05 Douglas "Dougie" Powers FOOD121 Food, Adult Cat 3.5 oz 1 4.22 $4.22 +2018-07-06 桜 高橋 (Sakura Takahashi) FOOD122 Food, Senior Wet Cat - 3 oz 12 1.29 157¥ +2018-07-10 David Attenborough NSCT201 Food, Salamander 30 0.05 $1.5 +2018-07-10 Susan Ashworth CAT060 Cat, Korat (Felis catus) 1 720.42 $720.42 +2018-07-10 Susan Ashworth FOOD130 Food, Kitten 3kg 1 14.94 $14.94 +2018-07-13 Wil Wheaton NSCT523 Monster, Rust (Monstrus gygaxus) 1 39.95 $39.95 +2018-07-13 Robert Armstrong FOOD216 BFF Oh My Gravy! Chicken & Shrimp 2.8oz 4 12.95 $51.8 +2018-07-17 Robert Armstrong FOOD217 BFF Oh My Gravy! Duck & Tuna 2.8oz 4 12.95 $51.8 +2018-07-17 Helen Halestorm LAGO342 Rabbit (Oryctolagus cuniculus) 2 32.94 $65.88 +2018-07-18 桜 高橋 (Sakura Takahashi) FOOD122 Food, Senior Wet Cat - 3 oz 6 1.29 157¥ +2018-07-19 Rubeus Hagrid FOOD170 Food, Dog - 5kg 5 44.95 $224.75 +2018-07-20 Jon Arbuckle FOOD167 Food, Premium Wet Cat - 3.5 oz 50 3.95 $197.5 +2018-07-23 Robert Armstrong FOOD215 BFF Oh My Gravy! Lamb & Tuna 2.8oz 4 12.95 $51.8 +2018-07-23 Douglas "Dougie" Powers TOY235 Laser Pointer 1 16.12 $16.12 +2018-07-24 桜 高橋 (Sakura Takahashi) FOOD122 Food, Senior Wet Cat - 3 oz 3 1.29 157¥ +2018-07-26 Douglas "Dougie" Powers FOOD420 Food, Shark - 10 kg 1 15.7 $15.7 +2018-07-27 桜 高橋 (Sakura Takahashi) FOOD122 Food, Senior Wet Cat - 3 oz 3 1.29 157¥ +2018-07-30 桜 高橋 (Sakura Takahashi) RETURN Food, Senior Wet Cat - 3 oz 1 1.29 157¥ +2018-07-31 Rubeus Hagrid CAT060 Food, Dragon - 50kg 5 720.42 $3602.1 +2018-08-01 David Attenborough FOOD360 Food, Rhinocerous - 50kg 4 5.72 $22.88 +2018-08-02 Susan Ashworth CAT110 Cat, Maine Coon (Felix catus) 1 1309.68 $1309.68 +2018-08-02 Susan Ashworth FOOD130 Food, Kitten 3kg 3 14.94 $44.82 +2018-08-06 Robert Armstrong FOOD212 BFF Oh My Gravy! Beef & Chicken 2.8oz 4 12.95 $51.8 +2018-08-07 Juan Johnson REPT082 Kingsnake, California (Lampropeltis getula) 1 89.95 $89.95 +2018-08-07 Juan Johnson RDNT443 Mouse, Pinky (Mus musculus) 1 1.49 $1.49 +2018-08-10 Robert Armstrong FOOD211 BFF Oh My Gravy! Chicken & Turkey 2.8oz 4 12.95 $51.8 +2018-08-13 Monica Johnson RDNT443 Mouse, Pinky (Mus musculus) 1 1.49 $1.49 +2018-08-13 María Fernández FOOD146 Forti Diet Prohealth Mouse/Rat 3lbs 2 2 $4.0 +2018-08-15 Mr. Praline RETURN Parrot, Norwegian Blue (Mopsitta tanta) 1 2300 -$2300.0 +2018-08-15 Kyle Kennedy FOOD121 Food, Adult Cat - 3.5 oz 2 4.22 $8.44 +2018-08-16 Helen Halestorm RETURN Rabbit (Oryctolagus cuniculus) 6 0 $0.0 +2018-08-16 Kyle Kennedy DOG010 Dog, Golden Retriever (Canis lupus familiaris) 1 2495.99 $2495.99 +2018-08-16 Michael Smith BIRD160 Parakeet, Blue (Melopsittacus undulatus) 1 29.95 $31.85 +2018-08-17 Rubeus Hagrid NSCT201 Food, Spider 5 0.05 $0.25 +2018-08-20 Kyle Kennedy RETURN Dog, Golden Retriever (Canis lupus familiaris) 1 1247.99 -$1247.99 +2018-08-20 Monica Johnson NSCT201 Crickets, Adult Live (Gryllus assimilis) 30 0.05 $1.5 +2018-08-20 David Attenborough NSCT084 Food, Pangolin 30 0.17 $5.10 +2018-08-21 Robert Armstrong FOOD214 BFF Oh My Gravy! Duck & Salmon 2.8oz 4 12.95 $51.8 +2018-08-22 David Attenborough BIRD160 Food, Quoll 1 29.95 $29.95 +2018-08-22 Jon Arbuckle FOOD170 Food, Adult Dog - 5kg 1 44.95 $44.95 +2018-08-24 Robert Armstrong FOOD218 BFF Oh My Gravy! Chicken & Salmon 2.8oz 4 12.95 $51.8 +2018-08-27 Monica Johnson NSCT443 Mealworms, Large (Tenebrio molitor) 100ct 1 1.99 $1.99 +2018-08-28 Susan Ashworth CAT020 Cat, Scottish Fold (Felis catus) 1 1964.53 $1964.53 +2018-08-28 Susan Ashworth FOOD130 Food, Kitten 3kg 2 14.94 $29.88 +2018-08-29 Robert Armstrong FOOD219 BFF Oh My Gravy! Chicken & Pumpkin 2.8oz 4 12.95 $51.8 +2018-08-31 Robert Armstrong FOOD219 BFF Oh My Gravy! Chicken & Pumpkin 2.8oz 144 12.95 $1864.8 +2018-08-31 Juan Johnson REPT217 Lizard, Spinytail (Uromastyx ornatus) 1 99.95 $99.95 +========== ========================== ======= ============================================== ========== ======= ========= \ No newline at end of file diff --git a/tests/golden/sort-levels.tsv b/tests/golden/sort-levels.tsv new file mode 100644 index 000000000..e4cbd54d6 --- /dev/null +++ b/tests/golden/sort-levels.tsv @@ -0,0 +1,50 @@ +primary secondary tertiary +1 1 1 +1 1 1 +1 1 1 +1 1 1 +1 1 1 +1 1 1 +1 1 1 +1 1 1 +1 1 1 +1 1 1 +1 1 1 +1 1 1 +1 1 1 +1 3 2 +1 3 2 +1 3 2 +1 3 2 +1 3 2 +1 3 2 +1 3 2 +1 3 2 +1 3 2 +1 3 2 +1 3 1 +1 3 1 +1 3 1 +2 1 3 +2 1 3 +2 1 3 +2 1 3 +2 1 3 +2 1 3 +2 2 4 +2 2 4 +2 2 4 +2 2 4 +2 2 4 +2 2 4 +2 2 4 +2 2 4 +2 2 4 +2 2 4 +2 2 3 +2 2 3 +2 2 3 +2 2 3 +2 2 3 +2 3 3 +2 3 2 diff --git a/tests/graph-cursor-nosave.vd b/tests/graph-cursor-nosave.vd new file mode 100644 index 000000000..fb076bfb9 --- /dev/null +++ b/tests/graph-cursor-nosave.vd @@ -0,0 +1,16 @@ +sheet col row longname input keystrokes comment + open-file http://visidata.org/usage.tsv o +usage daily_users type-int # +usage daily_users aggregate-col sum + +usage motd_fetches type-int # +usage motd_fetches aggregate-col sum + +usage date freq-col F +usage_date_freq date type-date @ +usage_date_freq exec-python import calendar g^X +usage_date_freq date addcol-expr date.isocalendar()[2] = +usage_date_freq date.isocalendar()[2] rename-col dotw ^ +usage_date_freq dotw addcol-expr 'Sat/Sun' if dotw >=6 else 'Mon-Fri' = +usage_date_freq 'Sat/Sun' if dotw >=6 else 'Mon-Fri' key-col ! +usage_date_freq date sort-desc ! +usage_date_freq daily_users_sum plot-column . +usage_date_freq_graph 2018-11-18 2019-03-26 51 140 delete-cursor d delete rows on source sheet contained within canvas cursor diff --git a/tests/graph-sincos-nosave.vd b/tests/graph-sincos-nosave.vd index 5c356e91f..d892fdafe 100644 --- a/tests/graph-sincos-nosave.vd +++ b/tests/graph-sincos-nosave.vd @@ -1,5 +1,5 @@ sheet col row longname input keystrokes comment -graph-sincos-nosave_vd add-sheet 1 A open new blank sheet with number columns + add-sheet 1 A open new blank sheet with number columns unnamed add-rows 360 ga add N blank rows unnamed select-rows gs unnamed 0 setcol-iter range(360) gz= set selected rows in this column to the values in the given Python sequence expression diff --git a/tests/histogram.vd b/tests/histogram.vd index 634281ff9..340767bc4 100644 --- a/tests/histogram.vd +++ b/tests/histogram.vd @@ -1,5 +1,6 @@ sheet col row longname input keystrokes comment open-file sample_data/benchmark.csv o + numeric_binning set-option True benchmark Quantity type-int # benchmark Quantity addcol-expr Quantity > 1 = benchmark Quantity > 1 select-col-regex True | diff --git a/tests/issue733.vd b/tests/issue733.vd new file mode 100644 index 000000000..575c93ff3 --- /dev/null +++ b/tests/issue733.vd @@ -0,0 +1,5 @@ +sheet col row longname input keystrokes comment + open-file sample_data/benchmark.csv o +benchmark Date type-date @ set type of current column to date +benchmark disp_date_fmt set-option %Y +benchmark disp_date_fmt unset-option diff --git a/tests/pivot-error.vd b/tests/pivot-error.vd index cf45387f0..a487f7e75 100644 --- a/tests/pivot-error.vd +++ b/tests/pivot-error.vd @@ -1,6 +1,5 @@ sheet col row longname input keystrokes comment open-file sample_data/test.jsonl o -test numeric_binning set-option False test key1 type-date @ test qty type-int # test amt type-float % diff --git a/tests/quantum-sum-nosave.vd b/tests/quantum-sum-nosave.vd index 127abef04..677acdeef 100644 --- a/tests/quantum-sum-nosave.vd +++ b/tests/quantum-sum-nosave.vd @@ -1,5 +1,5 @@ sheet col row longname input keystrokes comment -quantum-sum-nosave_vd add-sheet 1 A open new blank sheet with number columns + add-sheet 1 A open new blank sheet with number columns unnamed add-rows 100000 ga add N blank rows unnamed 0 addcol-expr random.random() = create new column from Python expression, with column names as variables unnamed random.random() type-float % set type of current column to float diff --git a/tests/sort-levels.tsv b/tests/sort-levels.tsv new file mode 100644 index 000000000..8e4853904 --- /dev/null +++ b/tests/sort-levels.tsv @@ -0,0 +1,50 @@ +primary secondary tertiary +1 1 1 +2 2 4 +1 3 2 +1 1 1 +1 3 2 +2 2 4 +1 1 1 +2 2 4 +2 2 4 +1 1 1 +1 3 2 +2 2 3 +2 2 4 +1 3 2 +2 2 4 +2 2 4 +1 1 1 +2 2 3 +1 1 1 +1 1 1 +1 3 2 +2 1 3 +2 1 3 +1 3 2 +1 3 2 +2 1 3 +2 2 4 +1 3 1 +2 1 3 +1 1 1 +2 3 3 +2 2 3 +2 1 3 +2 1 3 +1 1 1 +1 1 1 +1 3 2 +2 2 3 +2 2 4 +1 3 1 +1 1 1 +2 2 4 +1 3 2 +1 1 1 +1 3 1 +2 3 2 +1 3 2 +2 2 3 +1 1 1 diff --git a/tests/sort-levels.vd b/tests/sort-levels.vd new file mode 100644 index 000000000..9c41de817 --- /dev/null +++ b/tests/sort-levels.vd @@ -0,0 +1,5 @@ +sheet col row longname input keystrokes comment + open-file tests/sort-levels.tsv o +sort-levels primary sort-asc [ sort ascending by current column; replace any existing sort criteria +sort-levels secondary sort-asc-add z[ sort ascending by current column; add to existing sort criteria +sort-levels tertiary sort-desc-add z] sort descending by current column; add to existing sort criteria diff --git a/visidata/__init__.py b/visidata/__init__.py index 683ac419a..6d5bd7d00 100644 --- a/visidata/__init__.py +++ b/visidata/__init__.py @@ -1,6 +1,6 @@ 'VisiData: a curses interface for exploring and arranging tabular data' -__version__ = '2.0.1' +__version__ = '2.1' __version_info__ = 'VisiData v' + __version__ __author__ = 'Saul Pwanson ' __status__ = 'Production/Stable' @@ -95,6 +95,8 @@ def getGlobals(): import visidata.incr import visidata.customdate import visidata.misc +from .macros import * +from .menu import * from .loaders.csv import * from .loaders.archive import * diff --git a/visidata/__main__.py b/visidata/__main__.py new file mode 100644 index 000000000..fbba325bc --- /dev/null +++ b/visidata/__main__.py @@ -0,0 +1,3 @@ +from .main import vd_cli + +vd_cli() diff --git a/visidata/_input.py b/visidata/_input.py index 507b4327a..89f2aab1e 100644 --- a/visidata/_input.py +++ b/visidata/_input.py @@ -6,7 +6,7 @@ from visidata import EscapeException, ExpectedException, clipdraw, Sheet, VisiData from visidata import vd, options, theme, colors -from visidata import launchExternalEditor, suspend, ColumnItem, ENTER +from visidata import launchExternalEditor, suspend, ColumnItem __all__ = ['confirm', 'CompleteKey'] @@ -16,6 +16,22 @@ VisiData.init('lastInputs', lambda: collections.defaultdict(list)) # [input_type] -> list of prevInputs +class AcceptInput(Exception): + '*args[0]* is the input to be accepted' + +visidata.vd._nextCommands = [] + +@VisiData.api +def queueCommand(vd, longname): #, input=None, sheet=None, col=None, row=None): + vd._nextCommands.append(longname) + +def acceptThenFunc(*longnames): + def _acceptthen(v, i): + for longname in longnames: + vd.queueCommand(longname) + raise AcceptInput(v) + return _acceptthen + # editline helpers class EnableCursor: @@ -124,7 +140,6 @@ def editline(vd, scr, y, x, w, i=0, attr=curses.A_NORMAL, value='', fillchar=' ' 'A better curses line editing widget.' with EnableCursor(): ESC='^[' - ENTER='^J' TAB='^I' history_state = HistoryState(history) @@ -196,7 +211,7 @@ def find_nonword(s, a, b, incr): elif ch in ('^H', 'KEY_BACKSPACE', '^?'): i -= 1; v = delchar(v, i) elif ch == TAB: v, i = complete_state.complete(v, i, +1) elif ch == 'KEY_BTAB': v, i = complete_state.complete(v, i, -1) - elif ch == ENTER: break + elif ch in ['^J', '^M']: break # ENTER to accept value elif ch == '^K': v = v[:i] # ^Kill to end-of-line elif ch == '^O': v = launchExternalEditor(v) elif ch == '^R': v = str(value) # ^Reload initial value @@ -241,7 +256,10 @@ def editText(vd, y, x, w, record=True, display=True, **kwargs): v = vd.getLastArgs() if v is None: - v = vd.editline(vd.sheets[0]._scr, y, x, w, display=display, **kwargs) + try: + v = vd.editline(vd.sheets[0]._scr, y, x, w, display=display, **kwargs) + except AcceptInput as e: + v = e.args[0] # clear keyboard buffer to neutralize multi-line pastes (issue#585) curses.flushinp() @@ -299,7 +317,7 @@ def input(self, prompt, type=None, defaultLast=False, history=[], **kwargs): else: history = type - sheet = self.sheets[0] + sheet = self.activeSheet rstatuslen = self.drawRightStatus(sheet._scr, sheet) attr = 0 promptlen = clipdraw(sheet._scr, sheet.windowHeight-1, 0, prompt, attr, w=sheet.windowWidth-rstatuslen-1) @@ -312,6 +330,8 @@ def input(self, prompt, type=None, defaultLast=False, history=[], **kwargs): if ret: if isinstance(type, str): + if self.lastInputs[type] and self.lastInputs[type][-1] == ret: + return ret self.lastInputs[type].append(ret) elif defaultLast: history or vd.fail("no previous input") @@ -323,6 +343,9 @@ def input(self, prompt, type=None, defaultLast=False, history=[], **kwargs): @VisiData.global_api def confirm(vd, prompt, exc=EscapeException): 'Display *prompt* on status line and demand input that starts with "Y" or "y" to proceed. Raise *exc* otherwise. Return True.' + if options.batch: + return vd.fail('cannot confirm in batch mode: ' + prompt) + yn = vd.input(prompt, value='no', record=False)[:1] if not yn or yn not in 'Yy': msg = 'disconfirmed: ' + prompt @@ -366,11 +389,25 @@ def editCell(self, vcolidx=None, rowidx=None, value=None, **kwargs): y, h = self._rowLayout.get(rowidx, (0, 0)) value = value or col.getDisplayValue(self.rows[self.cursorRowIndex]) + bindings={ + 'kUP': acceptThenFunc('go-up', 'rename-col' if rowidx < 0 else 'edit-cell'), + 'KEY_SR': acceptThenFunc('go-up', 'rename-col' if rowidx < 0 else 'edit-cell'), + 'kDN': acceptThenFunc('go-down', 'rename-col' if rowidx < 0 else 'edit-cell'), + 'KEY_SF': acceptThenFunc('go-down', 'rename-col' if rowidx < 0 else 'edit-cell'), + 'KEY_SRIGHT': acceptThenFunc('go-right', 'rename-col' if rowidx < 0 else 'edit-cell'), + 'KEY_SLEFT': acceptThenFunc('go-left', 'rename-col' if rowidx < 0 else 'edit-cell'), + } + + bindings.update(kwargs.get('bindings', {})) + kwargs['bindings'] = bindings + editargs = dict(value=value, fillchar=options.disp_edit_fill, truncchar=options.disp_truncator) + editargs.update(kwargs) # update with user-specified args r = vd.editText(y, x, w, **editargs) + if rowidx >= 0: # if not header r = col.type(r) # convert input to column type, let exceptions be raised diff --git a/visidata/_open.py b/visidata/_open.py index a6c7bbbed..6cf029621 100644 --- a/visidata/_open.py +++ b/visidata/_open.py @@ -56,7 +56,7 @@ def openPath(vd, p, filetype=None): filetype = filetype.lower() - openfunc = vd.getGlobals().get('open_' + filetype) + openfunc = getattr(vd, 'open_' + filetype, vd.getGlobals().get('open_' + filetype)) if not openfunc: vd.warning('unknown "%s" filetype' % filetype) filetype = 'txt' diff --git a/visidata/_types.py b/visidata/_types.py index b826a9da0..990ca0420 100644 --- a/visidata/_types.py +++ b/visidata/_types.py @@ -47,17 +47,33 @@ def numericFormatter(fmtstr, typedval): return str(typedval) -vd.si_prefixes='p n u m . k M G T P Q'.split() +vd.si_prefixes='p n u m . kK M G T P Q'.split() + +def floatsi(*args): + if not args: + return 0.0 + if not isinstance(args[0], str): + return args[0] + + s=args[0].strip() + for i, p in enumerate(vd.si_prefixes): + if s[-1] in p: + return float(s[:-1]) * (1000 ** (i-4)) + + return float(s) + + def SIFormatter(fmtstr, val): level = 4 - while abs(val) > 1000: - val /= 1000 - level += 1 - while abs(val) < 0.001: - val *= 1000 - level -= 1 + if val != 0: + while abs(val) > 1000: + val /= 1000 + level += 1 + while abs(val) < 0.001: + val *= 1000 + level -= 1 - return numericFormatter(fmtstr, val) + (vd.si_prefixes[level] if level != 4 else '') + return numericFormatter(fmtstr, val) + (vd.si_prefixes[level][0] if level != 4 else '') class VisiDataType: @@ -101,14 +117,12 @@ def getType(vd, typetype): vdtype(dict, '') vdtype(list, '') -def isNumeric(col): +@VisiData.api +def isNumeric(vd, col): return col.type in (int,vlen,float,currency,date,floatsi) ## -def floatsi(*args): - return float(*args) - floatchars='+-0123456789.' def currency(*args): 'dirty float (strip non-numeric characters)' diff --git a/visidata/aggregators.py b/visidata/aggregators.py index 03e40aabb..116d51db9 100644 --- a/visidata/aggregators.py +++ b/visidata/aggregators.py @@ -1,6 +1,7 @@ import math import functools import collections +from statistics import mode, stdev from visidata import Progress, Column from visidata import * @@ -96,14 +97,16 @@ def quantiles(q, helpstr): return [percentile(round(100*i/q), helpstr) for i in range(1, q)] vd.aggregator('min', min, 'minimum value') -vd.aggregator('max', max, 'minimum value') +vd.aggregator('max', max, 'maximum value') vd.aggregator('avg', mean, 'arithmetic mean of values', type=float) vd.aggregator('mean', mean, 'arithmetic mean of values', type=float) vd.aggregator('median', median, 'median of values') +vd.aggregator('mode', mode, 'mode of values') vd.aggregator('sum', sum, 'sum of values') vd.aggregator('distinct', set, 'distinct values', type=vlen) vd.aggregator('count', lambda values: sum(1 for v in values), 'number of values', type=int) vd.aggregator('list', list, 'list of values') +vd.aggregator('stdev', stdev, 'standard deviation of values', type=float) vd.aggregators['q3'] = quantiles(3, 'tertiles (33/66th pctile)') vd.aggregators['q4'] = quantiles(4, 'quartiles (25/50/75th pctile)') diff --git a/visidata/basesheet.py b/visidata/basesheet.py index 1cd360389..b89cc4af0 100644 --- a/visidata/basesheet.py +++ b/visidata/basesheet.py @@ -207,8 +207,8 @@ def checkCursorNoExceptions(self): except Exception as e: vd.exceptionCaught(e) - def evalExpr(self, expr, row=None): - 'Evaluate Python expression *expr* in the context of *row*.' + def evalExpr(self, expr, **kwargs): + 'Evaluate Python expression *expr* in the context of *kwargs* (may vary by sheet type).' return eval(expr, vd.getGlobals(), None) diff --git a/visidata/canvas.py b/visidata/canvas.py index 8697e340e..f2204b8e2 100644 --- a/visidata/canvas.py +++ b/visidata/canvas.py @@ -417,8 +417,10 @@ def polygon(self, vertexes, attr=0, row=None): self.polylines.append((vertexes + [vertexes[0]], attr, row)) def qcurve(self, vertexes, attr=0, row=None): - 'quadratic curve from vertexes[0] to vertexes[2] with control point at vertexes[1]' - assert len(vertexes) == 3, len(vertexes) + 'Draw quadratic curve from vertexes[0] to vertexes[2] with control point at vertexes[1]' + if len(vertexes) != 3: + vd.fail('need exactly 3 points for qcurve (got %d)' % len(vertexes)) + x1, y1 = vertexes[0] x2, y2 = vertexes[1] x3, y3 = vertexes[2] diff --git a/visidata/choose.py b/visidata/choose.py index edeb890e1..715d6cc8a 100644 --- a/visidata/choose.py +++ b/visidata/choose.py @@ -5,13 +5,14 @@ option('fancy_chooser', True, 'a nicer selection interface for aggregators and jointype') @VisiData.api -def chooseOne(vd, L): - return vd.choose(L, 1) +def chooseOne(vd, choices): + 'Return one user-selected key from *choices*.' + return vd.choose(choices, 1) @VisiData.api def choose(vd, choices, n=None): - 'Return *n* (default 1) of *choices* elements (if list) or values (if dict).' + 'Return a list of 1 to *n* "key" from elements of *choices* (see chooseMany).' ret = vd.chooseMany(choices) or vd.fail('no choice made') if n and len(ret) > n: vd.fail('can only choose %s' % n) @@ -39,7 +40,7 @@ def chooseFancy(vd, choices): @VisiData.api def chooseMany(vd, choices): - '''*choices* is a list of dicts; each dict must have a unique "key" whose value has no spaces. Return a list of 1 or more keys, as chosen by the user. Handle replay correctly.''' + 'Return a list of 1 or more keys from *choices*, which is a list of dicts. Each element dict must have a unique "key", which must be typed directly by the user in non-fancy mode (therefore no spaces). All other items in the dicts are also shown in fancy chooser mode. Use previous choices from the replay input if available. Add chosen keys (space-separated) to the cmdlog as input for the current command.''' if vd.cmdlog: v = vd.getLastArgs() if v is not None: @@ -77,4 +78,4 @@ def throw_fancy(v, i): ChoiceSheet.addCommand(ENTER, 'choose-rows', 'makeChoice([cursorRow])') -ChoiceSheet.addCommand('g'+ENTER, 'choose-rows-selected', 'makeChoice(someSelectedRows)') +ChoiceSheet.addCommand('g'+ENTER, 'choose-rows-selected', 'makeChoice(onlySelectedRows)') diff --git a/visidata/clipboard.py b/visidata/clipboard.py index 6498aaea9..515c99f92 100644 --- a/visidata/clipboard.py +++ b/visidata/clipboard.py @@ -181,26 +181,26 @@ def saveToClipboard(sheet, rows, filetype=None): Sheet.addCommand('p', 'paste-after', 'paste_after(cursorRowIndex)', 'paste clipboard rows after current row') Sheet.addCommand('P', 'paste-before', 'paste_before(cursorRowIndex)', 'paste clipboard rows before current row') -Sheet.addCommand('gd', 'delete-selected', 'copyRows(selectedRows); deleteSelected()', 'delete (cut) selected rows and move them to clipboard') -Sheet.addCommand('gy', 'copy-selected', 'copyRows(selectedRows)', 'yank (copy) selected rows to clipboard') +Sheet.addCommand('gd', 'delete-selected', 'copyRows(onlySelectedRows); deleteSelected()', 'delete (cut) selected rows and move them to clipboard') +Sheet.addCommand('gy', 'copy-selected', 'copyRows(onlySelectedRows)', 'yank (copy) selected rows to clipboard') Sheet.addCommand('zy', 'copy-cell', 'copyCells(cursorCol, [cursorRow])', 'yank (copy) current cell to clipboard') Sheet.addCommand('zp', 'paste-cell', 'cursorCol.setValuesTyped([cursorRow], vd.clipcells[0]) if vd.clipcells else warning("no cells to paste")', 'set contents of current cell to last clipboard value') Sheet.addCommand('zd', 'delete-cell', 'vd.clipcells = [cursorDisplay]; cursorCol.setValues([cursorRow], None)', 'delete (cut) current cell and move it to clipboard') -Sheet.addCommand('gzd', 'delete-cells', 'vd.clipcells = list(vd.sheet.cursorCol.getDisplayValue(r) for r in selectedRows); cursorCol.setValues(selectedRows, None)', 'delete (cut) contents of current column for selected rows and move them to clipboard') +Sheet.addCommand('gzd', 'delete-cells', 'vd.clipcells = list(vd.sheet.cursorCol.getDisplayValue(r) for r in onlySelectedRows); cursorCol.setValues(onlySelectedRows, None)', 'delete (cut) contents of current column for selected rows and move them to clipboard') Sheet.bindkey('BUTTON2_PRESSED', 'go-mouse') Sheet.addCommand('BUTTON2_RELEASED', 'syspaste-cells', 'pasteFromClipboard(visibleCols[cursorVisibleColIndex:], rows[cursorRowIndex:])', 'paste into VisiData from system clipboard') Sheet.bindkey('BUTTON2_CLICKED', 'go-mouse') -Sheet.addCommand('gzy', 'copy-cells', 'copyCells(cursorCol, selectedRows)', 'yank (copy) contents of current column for selected rows to clipboard') -Sheet.addCommand('gzp', 'setcol-clipboard', 'for r, v in zip(selectedRows, itertools.cycle(vd.clipcells)): cursorCol.setValuesTyped([r], v)', 'set cells of current column for selected rows to last clipboard value') +Sheet.addCommand('gzy', 'copy-cells', 'copyCells(cursorCol, onlySelectedRows)', 'yank (copy) contents of current column for selected rows to clipboard') +Sheet.addCommand('gzp', 'setcol-clipboard', 'for r, v in zip(onlySelectedRows, itertools.cycle(vd.clipcells)): cursorCol.setValuesTyped([r], v)', 'set cells of current column for selected rows to last clipboard value') Sheet.addCommand('Y', 'syscopy-row', 'syscopyRows([cursorRow])', 'yank (copy) current row to system clipboard (using options.clipboard_copy_cmd)') -Sheet.addCommand('gY', 'syscopy-selected', 'syscopyRows(selectedRows)', 'yank (copy) selected rows to system clipboard (using options.clipboard_copy_cmd)') +Sheet.addCommand('gY', 'syscopy-selected', 'syscopyRows(onlySelectedRows)', 'yank (copy) selected rows to system clipboard (using options.clipboard_copy_cmd)') Sheet.addCommand('zY', 'syscopy-cell', 'syscopyCells(cursorCol, [cursorRow])', 'yank (copy) current cell to system clipboard (using options.clipboard_copy_cmd)') -Sheet.addCommand('gzY', 'syscopy-cells', 'syscopyCells(cursorCol, selectedRows)', 'yank (copy) contents of current column from selected rows to system clipboard (using options.clipboard_copy_cmd') +Sheet.addCommand('gzY', 'syscopy-cells', 'syscopyCells(cursorCol, onlySelectedRows)', 'yank (copy) contents of current column from selected rows to system clipboard (using options.clipboard_copy_cmd') Sheet.bindkey('KEY_DC', 'delete-cell'), Sheet.bindkey('gKEY_DC', 'delete-cells'), diff --git a/visidata/cmdlog.py b/visidata/cmdlog.py index bea4a565d..46e950bbc 100644 --- a/visidata/cmdlog.py +++ b/visidata/cmdlog.py @@ -13,7 +13,7 @@ # prefixes which should not be logged nonLogged = '''forget exec-longname undo redo quit show error errors statuses options threads jump -replay cancel save-cmdlog +replay cancel save-cmdlog macro go- search scroll prev next page start end zoom resize visibility mouse suspend redraw no-op help syscopy sysopen profile toggle'''.split() @@ -159,12 +159,10 @@ def beforeExecHook(self, sheet, cmd, args, keystrokes): colname = sheet.cursorCol.name or sheet.visibleCols.index(sheet.cursorCol) if contains(cmd.execstr, 'plotterCursorBox'): - assert not colname and not rowname bb = sheet.cursorBox colname = '%s %s' % (sheet.formatX(bb.xmin), sheet.formatX(bb.xmax)) rowname = '%s %s' % (sheet.formatY(bb.ymin), sheet.formatY(bb.ymax)) elif contains(cmd.execstr, 'plotterVisibleBox'): - assert not colname and not rowname bb = sheet.visibleBox colname = '%s %s' % (sheet.formatX(bb.xmin), sheet.formatX(bb.xmax)) rowname = '%s %s' % (sheet.formatY(bb.ymin), sheet.formatY(bb.ymax)) @@ -212,6 +210,10 @@ class CommandLog(_CommandLog, VisiDataMetaSheet): pass class CommandLogJsonl(_CommandLog, JsonLinesSheet): + + def newRow(self): + return JsonLinesSheet.newRow(self) + def iterload(self): for r in JsonLinesSheet.iterload(self): if isinstance(r, TypedWrapper): @@ -269,6 +271,11 @@ def delay(vd, factor=1): acquired = vd.semaphore.acquire(timeout=options.replay_wait*factor if not vd.paused else None) return acquired or not vd.paused +@VisiData.property +def activeSheet(vd): + 'Return top sheet on sheets stack, or cmdlog sheets stack empty.' + return vd.sheets[0] if vd.sheets else vd.cmdlog + @VisiData.api def replayOne(vd, r): @@ -280,23 +287,25 @@ def replayOne(vd, r): vs = None longname = getattr(r, 'longname', None) - if longname == 'set-option': + if longname in ['set-option', 'unset-option']: try: - if r.col: - options.set(r.row, r.input, r.col) + context = vs if r.sheet and vs else vd + if longname == 'set-option': + context.options.set(r.row, r.input, r.sheet or r.col or 'override') else: - options[r.row] = r.input + context.options.unset(r.row, r.sheet or r.col or 'override') escaped = False except Exception as e: vd.exceptionCaught(e) escaped = True else: - vd.moveToReplayContext(r, vs) if vs: vd.push(vs) else: - vs = vd.sheets[0] # use top sheet by default + vs = vd.activeSheet + + vd.moveToReplayContext(r, vs) if r.comment: vd.status(r.comment) @@ -337,7 +346,7 @@ def replay_sync(vd, cmdlog, live=False): cmdlog.cursorRowIndex += 1 prog.addProgress(1) - vd.sheets[0].ensureLoaded() + vd.activeSheet.ensureLoaded() vd.sync() while not vd.delay(): pass @@ -376,15 +385,6 @@ def replayStatus(vd): return ' │ %s %s/%s' % (x, vd.currentReplay.cursorRowIndex, len(vd.currentReplay.rows)) -@VisiData.api -def setOption(vd, optname, optval, obj=None): - if vd.cmdlog: - objname = options._opts.objname(obj) - vd.cmdlog.addRow(vd.cmdlog.newRow(col=objname, row=optname, - keystrokes='', input=str(optval), - longname='set-option')) - - @BaseSheet.property def cmdlog(sheet): rows = sheet.cmdlog_sheet.rows diff --git a/visidata/column.py b/visidata/column.py index 2beb4eb2e..4a8c20d2e 100644 --- a/visidata/column.py +++ b/visidata/column.py @@ -7,7 +7,6 @@ import time from visidata import option, options, anytype, stacktrace, vd -from visidata import isNumeric from visidata import asyncthread, dispwidth from visidata import wrapply, TypedWrapper, TypedExceptionWrapper from visidata import Extensible, AttrDict, undoAttrFunc @@ -52,7 +51,7 @@ def __init__(self, value=None, *, display=None, note=None, notecolor=None, error self.error = error # list of strings for stacktrace def __bool__(self): - return self.value + return bool(self.value) def __eq__(self, other): return self.value == other @@ -164,7 +163,7 @@ def type(self): @type.setter def type(self, t): if self._type != t: - vd.addUndo(setattr, self, 'type', self._type) + vd.addUndo(setattr, self, '_type', self.type) self._type = t @property @@ -496,7 +495,7 @@ def __init__(self, name, expr=None, **kwargs): def calcValue(self, row): t0 = time.perf_counter() - r = self.sheet.evalExpr(self.compiledExpr, row) + r = self.sheet.evalExpr(self.compiledExpr, row, col=self) t1 = time.perf_counter() self.ncalcs += 1 self.maxtime = max(self.maxtime, t1-t0) diff --git a/visidata/customdate.py b/visidata/customdate.py index cfc0f647f..c21856d58 100644 --- a/visidata/customdate.py +++ b/visidata/customdate.py @@ -17,4 +17,4 @@ def __new__(cls, *args, **kwargs): Sheet.addCommand('z@', 'type-customdate', 'cursorCol.type=cursorCol.type=customdate(input("date format: ", type="fmtstr"))', 'set type of current column to custom date format') -ColumnsSheet.addCommand('gz@', 'type-customdate-selected', 'someSelectedRows.type=customdate(input("date format: ", type="fmtstr"))', 'set type of selected columns to date') +ColumnsSheet.addCommand('gz@', 'type-customdate-selected', 'onlySelectedRows.type=customdate(input("date format: ", type="fmtstr"))', 'set type of selected columns to date') diff --git a/visidata/deprecated.py b/visidata/deprecated.py index cb383566f..5776bb37f 100644 --- a/visidata/deprecated.py +++ b/visidata/deprecated.py @@ -1,4 +1,4 @@ -from visidata import VisiData +from visidata import VisiData, vd import visidata alias = visidata.BaseSheet.bindkey @@ -6,11 +6,15 @@ def deprecated(ver, instead=''): def decorator(func): def wrapper(*args, **kwargs): - # ideally would include a stacktrace + import traceback + + for line in reversed(traceback.extract_stack(limit=6)[:-1]): + vd.warning(f' file {line.filename} at line {line.lineno} in {line.name}') + vd.warning(f'Deprecated call traceback (most recent last):') msg = f'{func.__name__} deprecated since v{ver}' if instead: msg += f'; use {instead}' - visidata.warning(msg) + vd.warning(msg) return func(*args, **kwargs) return wrapper return decorator @@ -30,7 +34,7 @@ def copyToClipboard(value): @deprecated('1.6') def replayableOption(optname, default, helpstr): - option(optname, default, helpstr, replay=True) + vd.option(optname, default, helpstr, replay=True) @deprecated('1.6') def SubrowColumn(*args, **kwargs): @@ -38,7 +42,7 @@ def SubrowColumn(*args, **kwargs): @deprecated('1.6') def DeferredSetColumn(*args, **kwargs): - return Column(*args, defer=True, **kwargs) + return visidata.Column(*args, defer=True, **kwargs) @deprecated('2.0') def bindkey_override(keystrokes, longname): @@ -58,7 +62,7 @@ def exec_keystrokes(self, keystrokes, vdglobals=None): @VisiData.api def filetype(vd, ext, constructor): 'Add constructor to handle the given file type/extension.' - globals().setdefault('open_'+ext, lambda p,ext=ext: constructor(p,name, source=p, filetype=ext)) + globals().setdefault('open_'+ext, lambda p,ext=ext: constructor(p.name, source=p, filetype=ext)) @deprecated('2.0', 'Sheet(namepart1, namepart2, ...)') @VisiData.global_api @@ -69,18 +73,22 @@ def joinSheetnames(vd, *sheetnames): @deprecated('2.0', 'PyobjSheet') @VisiData.global_api def load_pyobj(*names, **kwargs): - return PyobjSheet(*names, **kwargs) + return visidata.PyobjSheet(*names, **kwargs) @deprecated('2.0', 'PyobjSheet') @VisiData.global_api def push_pyobj(name, pyobj): - vs = PyobjSheet(name, source=pyobj) + vs = visidata.PyobjSheet(name, source=pyobj) if vs: return vd.push(vs) else: vd.error("cannot push '%s' as pyobj" % type(pyobj).__name__) -visidata.addGlobals({'load_pyobj': load_pyobj}) +@deprecated('2.1', 'vd.isNumeric instead') +def isNumeric(col): + return vd.isNumeric(col) + +visidata.addGlobals({'load_pyobj': load_pyobj, 'isNumeric': isNumeric}) # The longnames on the left are deprecated for 2.0 @@ -92,6 +100,7 @@ def push_pyobj(name, pyobj): alias('dup-row', 'dive-row') alias('next-search', 'search-next') alias('prev-search', 'search-prev') +alias('search-prev', 'searchr-next') alias('prev-sheet', 'jump-prev') alias('prev-value', 'go-prev-value') alias('next-value', 'go-next-value') diff --git a/visidata/describe.py b/visidata/describe.py index 3822adfa3..f1f98e205 100644 --- a/visidata/describe.py +++ b/visidata/describe.py @@ -87,7 +87,7 @@ def reloadColumn(self, srccol): d['errors'].append(sr) d['mode'] = self.calcStatistic(d, mode, vals) - if isNumeric(srccol): + if vd.isNumeric(srccol): for func in [min, max, sum, median]: # use type d[func.__name__] = self.calcStatistic(d, func, vals) for aggrname in options.describe_aggrs.split(): diff --git a/visidata/expr.py b/visidata/expr.py index 243478457..44a872d15 100644 --- a/visidata/expr.py +++ b/visidata/expr.py @@ -1,4 +1,4 @@ -from visidata import Progress, Sheet, Column, asyncthread, vd, ColumnExpr +from visidata import Progress, Sheet, Column, asyncthread, vd, ExprColumn class CompleteExpr: @@ -22,6 +22,10 @@ def __call__(self, val, state): varnames = [] varnames.extend(sorted((base+col.name) for col in self.sheet.columns if col.name.startswith(partial))) varnames.extend(sorted((base+x) for x in globals() if x.startswith(partial))) + + # Remove duplicate tabbing suggestions + varnames_dict = {var:None for var in varnames} + varnames = list(varnames_dict.keys()) return varnames[state%len(varnames)] @@ -32,6 +36,8 @@ def setValuesFromExpr(self, rows, expr): compiledExpr = compile(expr, '', 'eval') vd.addUndoSetValues([self], rows) for row in Progress(rows, 'setting'): + # Note: expressions that are only calculated once, do not need to pass column identity + # they can reference their "previous selves" once without causing a recursive problem self.setValueSafe(row, self.sheet.evalExpr(compiledExpr, row)) self.recalc() vd.status('set %d values = %s' % (len(rows), expr)) @@ -42,9 +48,9 @@ def inputExpr(self, prompt, *args, **kwargs): return vd.input(prompt, "expr", *args, completer=CompleteExpr(self), **kwargs) -Sheet.addCommand('=', 'addcol-expr', 'addColumnAtCursor(ColumnExpr(inputExpr("new column expr=")))', 'create new column from Python expression, with column names as variables') -Sheet.addCommand('g=', 'setcol-expr', 'cursorCol.setValuesFromExpr(selectedRows, inputExpr("set selected="))', 'set current column for selected rows to result of Python expression') -Sheet.addCommand('z=', 'setcell-expr', 'cursorCol.setValues([cursorRow], evalExpr(inputExpr("set expr="), cursorRow))', 'evaluate Python expression on current row and set current cell with result of Python expression') -Sheet.addCommand('gz=', 'setcol-iter', 'cursorCol.setValues(selectedRows, *list(itertools.islice(eval(input("set column= ", "expr", completer=CompleteExpr())), len(selectedRows))))', 'set current column for selected rows to the items in result of Python sequence expression') +Sheet.addCommand('=', 'addcol-expr', 'addColumnAtCursor(ExprColumn(inputExpr("new column expr="), curcol=cursorCol))', 'create new column from Python expression, with column names as variables') +Sheet.addCommand('g=', 'setcol-expr', 'cursorCol.setValuesFromExpr(someSelectedRows, inputExpr("set selected="))', 'set current column for selected rows to result of Python expression') +Sheet.addCommand('z=', 'setcell-expr', 'cursorCol.setValues([cursorRow], evalExpr(inputExpr("set expr="), cursorRow,))', 'evaluate Python expression on current row and set current cell with result of Python expression') +Sheet.addCommand('gz=', 'setcol-iter', 'cursorCol.setValues(someSelectedRows, *list(itertools.islice(eval(input("set column= ", "expr", completer=CompleteExpr())), len(someSelectedRows))))', 'set current column for selected rows to the items in result of Python sequence expression') Sheet.addCommand(None, 'show-expr', 'status(evalExpr(inputExpr("show expr="), cursorRow))', 'evaluate Python expression on current row and show result on status line') diff --git a/visidata/freeze.py b/visidata/freeze.py index 26d2d61a4..7fc794a4b 100644 --- a/visidata/freeze.py +++ b/visidata/freeze.py @@ -17,9 +17,9 @@ def calcRows_async(frozencol, rows, col): # no need to undo, addColumn undo is enough for r in Progress(rows, 'calculating'): try: - frozencol.setValue(r, col.getTypedValue(r)) + frozencol.putValue(r, col.getTypedValue(r)) except Exception as e: - frozencol.setValue(r, e) + frozencol.putValue(r, e) calcRows_async(frozencol, sheet.rows, col) return frozencol @@ -44,10 +44,11 @@ def reload(self): row = [] self.addRow(row) for col in self.source.visibleCols: - try: - row.append(col.getTypedValue(r)) - except Exception as e: + val = col.getTypedValue(r) + if isinstance(val, TypedExceptionWrapper): row.append(None) + else: + row.append(val) Sheet.addCommand("'", 'freeze-col', 'sheet.addColumnAtCursor(StaticColumn(cursorCol))', 'add a frozen copy of current column with all cells evaluated') diff --git a/visidata/freqtbl.py b/visidata/freqtbl.py index e5a57297f..d23a4911f 100644 --- a/visidata/freqtbl.py +++ b/visidata/freqtbl.py @@ -7,7 +7,7 @@ theme('disp_histogram', '*', 'histogram element character') option('disp_histolen', 50, 'width of histogram column') option('histogram_bins', 0, 'number of bins for histogram of numeric columns') -option('numeric_binning', True, 'bin numeric columns into ranges', replay=True) +option('numeric_binning', False, 'bin numeric columns into ranges', replay=True) def valueNames(discrete_vals, numeric_vals): @@ -59,7 +59,7 @@ def reload(self): self.column('percent').hide() self.column('histogram').hide() - if not [c for c in self.groupByCols if isNumeric(c)]: + if not [c for c in self.groupByCols if vd.isNumeric(c)]: self.orderBy(self.column('count'), reverse=True) def openRow(self, row): diff --git a/visidata/graph.py b/visidata/graph.py index 0197c6258..14e9a7024 100644 --- a/visidata/graph.py +++ b/visidata/graph.py @@ -5,8 +5,7 @@ def numericCols(cols): - # isNumeric from describe.py - return [c for c in cols if isNumeric(c)] + return [c for c in cols if vd.isNumeric(c)] class InvertedCanvas(Canvas): @@ -49,7 +48,7 @@ def reload(self): self.reset() vd.status('loading data points') - catcols = [c for c in self.xcols if not isNumeric(c)] + catcols = [c for c in self.xcols if not vd.isNumeric(c)] numcols = numericCols(self.xcols) for ycol in self.ycols: for rownum, row in enumerate(Progress(self.sourceRows, 'plotting')): # rows being plotted from source @@ -92,7 +91,7 @@ def moveToCol(self, colstr): return True def formatX(self, amt): - return ','.join(xcol.format(xcol.type(amt)) for xcol in self.xcols if isNumeric(xcol)) + return ','.join(xcol.format(xcol.type(amt)) for xcol in self.xcols if vd.isNumeric(xcol)) def formatY(self, amt): srccol = self.ycols[0] @@ -143,7 +142,7 @@ def createLabels(self): # TODO: if 0 line is within visible bounds, explicitly draw the axis # TODO: grid lines corresponding to axis labels - xname = ','.join(xcol.name for xcol in self.xcols if isNumeric(xcol)) or 'row#' + xname = ','.join(xcol.name for xcol in self.xcols if vd.isNumeric(xcol)) or 'row#' xname, _ = clipstr(xname, self.leftMarginPixels//2-2) self.plotlabel(0, self.plotviewBox.ymax+4, xname+'»', colors.color_graph_axis) diff --git a/visidata/help.py b/visidata/help.py index 536dc040e..fded3ac51 100644 --- a/visidata/help.py +++ b/visidata/help.py @@ -63,7 +63,8 @@ def openManPage(vd): from pkg_resources import resource_filename import os with SuspendCurses(): - os.system(' '.join(['man', resource_filename(__name__, 'man/vd.1')])) + if os.system(' '.join(['man', resource_filename(__name__, 'man/vd.1')])) != 0: + vd.push(TextSheet('man_vd', source=Path(resource_filename(__name__, 'man/vd.txt')))) # in VisiData, ^H refers to the man page diff --git a/visidata/join.py b/visidata/join.py index c3e33dc2b..52c473736 100644 --- a/visidata/join.py +++ b/visidata/join.py @@ -31,7 +31,7 @@ def createJoinedSheet(sheets, jointype=''): 'outer': 'all rows from first selected sheet', 'full': 'all rows from all sheets (union)', 'diff': 'only rows NOT in all sheets', - 'append': 'only columns from first sheet; extend with rows from all sheets', + 'append': 'columns all sheets; extend with rows from all sheets', 'extend': 'only rows from first sheet; extend with columns from all sheets', 'merge': 'merge differences from other sheets into first sheet', }.items()] diff --git a/visidata/loaders/_pandas.py b/visidata/loaders/_pandas.py index 5bfcd7d77..3ab4f5561 100644 --- a/visidata/loaders/_pandas.py +++ b/visidata/loaders/_pandas.py @@ -17,7 +17,9 @@ class DataFrameAdapter: def __init__(self, df): import pandas as pd - assert isinstance(df, pd.DataFrame) + if not isinstance(df, pd.DataFrame): + vd.fail('%s is not a dataframe' % type(df).__name__) + self.df = df def __len__(self): @@ -115,11 +117,20 @@ def reload(self): else: readfunc = getattr(pd, 'read_'+filetype) or vd.error('no pandas.read_'+filetype) df = readfunc(str(self.source), **options.getall('pandas_'+filetype+'_')) + else: + try: + df = pd.DataFrame(self.source) + except ValueError as err: + vd.fail('error building pandas DataFrame from source data: %s' % err) # reset the index here if type(df.index) is not pd.RangeIndex: df = df.reset_index() + # VisiData assumes string column names but pandas does not. Forcing string + # columns at load-time avoids various errors later. + df.columns = df.columns.astype(str) + self.columns = [] for col in (c for c in df.columns if not c.startswith("__vd_")): self.addColumn(Column( @@ -180,7 +191,7 @@ def unselectRow(self, row): return is_selected @property - def nSelected(self): + def nSelectedRows(self): self._checkSelectedIndex() return self._selectedMask.sum() diff --git a/visidata/loaders/archive.py b/visidata/loaders/archive.py index 3fac973a0..7ac9720ea 100644 --- a/visidata/loaders/archive.py +++ b/visidata/loaders/archive.py @@ -36,14 +36,21 @@ def openZipFile(self, fp, *args, **kwargs): vd.error(err) def openRow(self, fi): - zfp = zipfile.ZipFile(str(self.source), 'r') - decodedfp = codecs.iterdecode(self.openZipFile(zfp, fi), + decodedfp = codecs.iterdecode(self.openZipFile(self.zfp, fi), encoding=options.encoding, errors=options.encoding_errors) return vd.openSource(Path(fi.filename, fp=decodedfp, filesize=fi.file_size), filetype=options.filetype) + @asyncthread + def extract(self, *rows, path=None): + self.zfp.extractall(members=[r.filename for r in rows], path=path) + + @property + def zfp(self): + return zipfile.ZipFile(str(self.source), 'r') + def iterload(self): - with zipfile.ZipFile(str(self.source), 'r') as zf: + with self.zfp as zf: for zi in Progress(zf.infolist()): yield zi @@ -72,3 +79,9 @@ def iterload(self): with tarfile.open(name=str(self.source)) as tf: for ti in Progress(tf.getmembers()): yield ti + + +ZipSheet.addCommand('x', 'extract-file', 'extract(cursorRow)') +ZipSheet.addCommand('gx', 'extract-selected', 'extract(*onlySelectedRows)') +ZipSheet.addCommand('zx', 'extract-file-to', 'extract(cursorRow, path=inputPath("extract to: "))') +ZipSheet.addCommand('gzx', 'extract-selected-to', 'extract(*onlySelectedRows, path=inputPath("extract %d files to: " % nSelected))') diff --git a/visidata/loaders/fixed_width.py b/visidata/loaders/fixed_width.py index 648e20a68..2b5d4a648 100644 --- a/visidata/loaders/fixed_width.py +++ b/visidata/loaders/fixed_width.py @@ -70,3 +70,25 @@ def iterload(self): def setCols(self, headerlines): self.headerlines = headerlines + + +@VisiData.api +def save_fixed(vd, p, *vsheets): + with p.open_text(mode='w') as fp: + for sheet in vsheets: + if len(vsheets) > 1: + fp.write('%s\n\n' % vs.name) + + # headers + for col in sheet.visibleCols: + fp.write('{0:{width}}'.format(col.name, width=col.width)) + fp.write('\n') + + # rows + with Progress(gerund='saving'): + for dispvals in sheet.iterdispvals(format=True): + for col, val in dispvals.items(): + fp.write('{0:{align}{width}}'.format(val, width=col.width, align='>' if vd.isNumeric(col) else '<')) + fp.write('\n') + + vd.status('%s save finished' % p) diff --git a/visidata/loaders/frictionless.py b/visidata/loaders/frictionless.py index d38af9f9c..a1380a686 100644 --- a/visidata/loaders/frictionless.py +++ b/visidata/loaders/frictionless.py @@ -8,4 +8,4 @@ def iterload(self): import datapackage self.dp = datapackage.Package(self.source.open_text()) for r in Progress(self.dp.resources): - yield vd.openSource(self.source.with_name(r.descriptor['path']), filetype=r.descriptor['format']) + yield vd.openSource(self.source.with_name(r.descriptor['path']), filetype=r.descriptor.get('format', 'json')) diff --git a/visidata/loaders/graphviz.py b/visidata/loaders/graphviz.py index 517ed6e37..1b57238b9 100644 --- a/visidata/loaders/graphviz.py +++ b/visidata/loaders/graphviz.py @@ -1,5 +1,5 @@ from visidata import vd, options, option, TypedWrapper, asyncthread, Progress -from visidata import wrapply, clean_to_id, isNumeric, VisiData, SIFormatter +from visidata import wrapply, clean_to_id, VisiData, SIFormatter option('graphviz_edge_labels', True, 'whether to include edge labels on graphviz diagrams') @@ -29,7 +29,7 @@ def save_dot(vd, p, vs): downsrc = clean_to_id(str(src)) or src downdst = clean_to_id(str(dst)) or dst - edgenotes = [c.getTypedValue(row) for c in vs.nonKeyVisibleCols if not isNumeric(c)] + edgenotes = [c.getTypedValue(row) for c in vs.nonKeyVisibleCols if not vd.isNumeric(c)] edgetype = '-'.join(str(x) for x in edgenotes if is_valid(x)) color = assignedColors.get(edgetype, None) if not color: @@ -37,7 +37,7 @@ def save_dot(vd, p, vs): assignedColors[edgetype] = color if options.graphviz_edge_labels: - nodelabels = [wrapply(SIFormatter, '%0.1f', c.getTypedValue(row)) for c in vs.nonKeyVisibleCols if isNumeric(c)] + nodelabels = [wrapply(SIFormatter, '%0.1f', c.getTypedValue(row)) for c in vs.nonKeyVisibleCols if vd.isNumeric(c)] label = '/'.join(str(x) for x in nodelabels if is_valid(x)) else: label = '' diff --git a/visidata/loaders/hdf5.py b/visidata/loaders/hdf5.py index 8994eaa0f..873c2efb3 100644 --- a/visidata/loaders/hdf5.py +++ b/visidata/loaders/hdf5.py @@ -32,7 +32,7 @@ def iterload(self): elif len(source.shape) == 2: # matrix ncols = source.shape[1] for i in range(ncols): - self.addColumns(ColumnItem('', i, width=8), index=i) + self.addColumn(ColumnItem('', i, width=8), index=i) self.recalc() yield from source # copy else: @@ -42,10 +42,16 @@ def iterload(self): def openRow(self, row): + import h5py if isinstance(row, BaseSheet): return row - if isinstance(row, h5py.Object): - return H5ObjSheet(row) + if isinstance(row, h5py.HLObject): + return Hdf5ObjSheet(row) + + import numpy + from .npy import NpySheet + if isinstance(row, numpy.ndarray): + return NpySheet(None, npy=row) Hdf5ObjSheet.addCommand('A', 'dive-metadata', 'vd.push(SheetDict(cursorRow.name + "_attrs", source=cursorRow.attrs))', 'open metadata sheet for object referenced in current row') diff --git a/visidata/loaders/http.py b/visidata/loaders/http.py index 1a5a63682..abb81bfd5 100644 --- a/visidata/loaders/http.py +++ b/visidata/loaders/http.py @@ -30,8 +30,16 @@ def openurl_http(path, filetype=None): if not response.encoding: response.encoding = options.encoding + # Automatically paginate if a 'next' URL is given + def _iter_lines(path=path, response=response): + while response: + yield from response.iter_lines(decode_unicode=True) + + src = response.links.get('next', {}).get('url', None) + response = requests.get(src, stream=True) if src else None + # create resettable iterator over contents - fp = RepeatFile(iter_lines=response.iter_lines(decode_unicode=True)) + fp = RepeatFile(iter_lines=_iter_lines()) # call open_ with a usable Path return vd.openSource(Path(path.given, fp=fp), filetype=filetype) diff --git a/visidata/loaders/json.py b/visidata/loaders/json.py index 040efc9bd..6718b8ed6 100644 --- a/visidata/loaders/json.py +++ b/visidata/loaders/json.py @@ -9,13 +9,13 @@ option('default_colname', '', 'column name to use for non-dict rows') -def open_json(p): +def open_jsonobj(p): return JsonSheet(p.name, source=p) def open_jsonl(p): - return JsonLinesSheet(p.name, source=p) + return JsonSheet(p.name, source=p) -open_ndjson = open_ldjson = open_jsonl +open_ndjson = open_ldjson = open_json = open_jsonl class JsonSheet(PythonSheet): @@ -23,18 +23,30 @@ def iterload(self): self.colnames = {} # [colname] -> Column self.columns = [] - try: - with self.source.open_text() as fp: - ret = json.load(fp, object_pairs_hook=OrderedDict) - - if isinstance(ret, list): - yield from Progress(ret) - else: - yield ret + with self.source.open_text() as fp: + for L in fp: + try: + if L.startswith('#'): # skip commented lines + continue + ret = json.loads(L) + if isinstance(ret, list): + yield from Progress(ret) + else: + yield ret + + except ValueError as e: + if self.rows: # if any rows have been added already + e.stacktrace = stacktrace() + yield TypedExceptionWrapper(json.loads, L, exception=e) # an error on one line + else: + with self.source.open_text() as fp: + ret = json.load(fp) + if isinstance(ret, list): + yield from Progress(ret) + else: + yield ret + break - except ValueError as e: - vd.status('trying jsonl') - yield from JsonLinesSheet.iterload(self) def addRow(self, row, index=None): # Wrap non-dict rows in a dummy object with a predictable key name. @@ -56,19 +68,7 @@ def addRow(self, row, index=None): def newRow(self): return {} - -class JsonLinesSheet(JsonSheet): - def iterload(self): - self.colnames = {} # [colname] -> Column - self.columns = [] - with self.source.open_text() as fp: - for L in fp: - try: - yield json.loads(L, object_pairs_hook=OrderedDict) - except Exception as e: - e.stacktrace = stacktrace() - yield TypedExceptionWrapper(json.loads, L, exception=e) - +JsonLinesSheet=JsonSheet ## saving json and jsonl @@ -116,7 +116,12 @@ def save_json(vd, p, *vsheets): else: it = {vs.name: [_rowdict(vs.visibleCols, row) for row in vs.iterrows()] for vs in vsheets} - jsonenc = _vjsonEncoder(indent=options.json_indent) + try: + indent = int(options.json_indent) + except Exception: + indent = options.json_indent + + jsonenc = _vjsonEncoder(indent=indent) with Progress(gerund='saving'): for chunk in jsonenc.iterencode(it): fp.write(chunk) diff --git a/visidata/loaders/markdown.py b/visidata/loaders/markdown.py index 01d74262d..457756ed7 100644 --- a/visidata/loaders/markdown.py +++ b/visidata/loaders/markdown.py @@ -13,7 +13,7 @@ def markdown_escape(s, style='orgmode'): return ret def markdown_colhdr(col): - if isNumeric(col): + if vd.isNumeric(col): return ('-' * (col.width-1)) + ':' else: return '-' * (col.width or options.default_width) diff --git a/visidata/loaders/mbtiles.py b/visidata/loaders/mbtiles.py index d8d834375..e762e3bff 100644 --- a/visidata/loaders/mbtiles.py +++ b/visidata/loaders/mbtiles.py @@ -111,7 +111,7 @@ def iterpolylines(self, r): for poly in mpoly: yield poly+[poly[0]], self.plotColor(key), r else: - assert False, t + vd.warning('unknown geometry type %s' % t) @asyncthread def reload(self): diff --git a/visidata/loaders/pandas_freqtbl.py b/visidata/loaders/pandas_freqtbl.py index cf5e4f153..5c3874045 100644 --- a/visidata/loaders/pandas_freqtbl.py +++ b/visidata/loaders/pandas_freqtbl.py @@ -13,8 +13,13 @@ class DataFrameRowSliceAdapter: def __init__(self, df, mask): import pandas as pd import numpy as np - assert isinstance(df, pd.DataFrame) - assert isinstance(mask, pd.Series) and df.shape[0] == mask.shape[0] + if not isinstance(df, pd.DataFrame): + vd.fail('%s is not a dataframe' % type(df).__name__) + if not isinstance(mask, pd.Series): + vd.fail('mask %s is not a Series' % type(mask).__name__) + if df.shape[0] != mask.shape[0]: + vd.fail('dataframe and mask have different shapes (%s vs %s)' % (df.shape[0], mask.shape[0])) + self.df = df self.mask_bool = mask # boolean mask self.mask_iloc = np.where(mask.values)[0] # integer indexes corresponding to mask @@ -92,7 +97,7 @@ def reload(self): # that operates similarly to pd.cut. super().initCols() - df = self.source.rows.df + df = self.source.df.copy() # Implementation (special case): for one row, this degenerates # to .value_counts(); however this does not order in a stable manner. @@ -145,7 +150,9 @@ def reload(self): for element in Progress(value_counts.index): if len(self.groupByCols) == 1: element = (element,) - assert len(element) == len(self.groupByCols) + elif len(element) != len(self.groupByCols): + vd.fail('different number of index cols and groupby cols (%s vs %s)' % (len(element), len(self.groupByCols))) + mask = df[self.groupByCols[0].name] == element[0] for i in range(1, len(self.groupByCols)): mask = mask & (df[self.groupByCols[i].name] == element[i]) @@ -172,6 +179,6 @@ def expand_source_rows(source, vd, cursorRow): PandasFreqTableSheet.addCommand('t', 'stoggle-row', 'toggle([cursorRow]); cursorDown(1)', 'toggle selection of rows grouped in current row in source sheet') PandasFreqTableSheet.addCommand('s', 'select-row', 'select([cursorRow]); cursorDown(1)', 'select rows grouped in current row in source sheet') PandasFreqTableSheet.addCommand('u', 'unselect-row', 'unselect([cursorRow]); cursorDown(1)', 'unselect rows grouped in current row in source sheet') -PandasFreqTableSheet.addCommand(ENTER, 'dup-row', 'expand_source_rows(source, vd, cursorRow)', 'open copy of source sheet with rows that are grouped in current row') +PandasFreqTableSheet.addCommand(ENTER, 'open-row', 'expand_source_rows(source, vd, cursorRow)', 'open copy of source sheet with rows that are grouped in current row') PandasFreqTableSheet.class_options.numeric_binning = False diff --git a/visidata/loaders/pdf.py b/visidata/loaders/pdf.py index da39c19bf..4e3cafc55 100644 --- a/visidata/loaders/pdf.py +++ b/visidata/loaders/pdf.py @@ -2,10 +2,16 @@ from visidata import * +vd.option('pdf_tables', False, 'parse PDF for tables instead of pages of text', replay=True) + + def open_pdf(p): - return PdfSheet(p.name, source=p) + if vd.options.pdf_tables: + return TabulaSheet(p.name, source=p) + return PdfMinerSheet(p.name, source=p) + -class PdfSheet(TableSheet): +class PdfMinerSheet(TableSheet): rowtype='pages' # rowdef: [pdfminer.LTPage, pageid, text] columns=[ ColumnItem('pdfpage', 0, width=0), @@ -27,3 +33,10 @@ def iterload(self): interpreter = PDFPageInterpreter(newrsrcmgr, txtconv) interpreter.process_page(page) yield [page, page.pageid, output_string.getvalue()] + + +class TabulaSheet(IndexSheet): + def iterload(self): + import tabula + for i, t in enumerate(tabula.read_pdf(self.source, pages='all', multiple_tables=True)): + yield PandasSheet(self.source.name, i, source=t) diff --git a/visidata/loaders/rec.py b/visidata/loaders/rec.py index e90f5fcc9..73ad8c638 100644 --- a/visidata/loaders/rec.py +++ b/visidata/loaders/rec.py @@ -1,7 +1,7 @@ from visidata import * @VisiData.api -def open_rec(p): +def open_rec(vd, p): return RecIndexSheet(p.name, source=p) def decode_multiline(line, fp): diff --git a/visidata/loaders/sqlite.py b/visidata/loaders/sqlite.py index af430e8bb..eb9cd0b72 100644 --- a/visidata/loaders/sqlite.py +++ b/visidata/loaders/sqlite.py @@ -37,8 +37,9 @@ def iterload(self): tblname = self.tableName if not isinstance(self, SqliteIndexSheet): self.columns = [] + self.addColumn(ColumnItem('rowid', 0, type=int, width=0)) for i, r in enumerate(self.execute(conn, 'PRAGMA TABLE_INFO("%s")' % tblname)): - c = ColumnItem(r[1], i, type=sqltypes.get(r[2].upper(), anytype)) + c = ColumnItem(r[1], i+1, type=sqltypes.get(r[2].upper(), anytype)) self.addColumn(c) if r[-1]: @@ -46,7 +47,7 @@ def iterload(self): r = self.execute(conn, 'SELECT COUNT(*) FROM "%s"' % tblname).fetchall() rowcount = r[0][0] - for row in Progress(self.execute(conn, 'SELECT * FROM "%s"' % tblname), total=rowcount-1): + for row in Progress(self.execute(conn, 'SELECT rowid, * FROM "%s"' % tblname), total=rowcount-1): yield list(row) @asyncthread @@ -71,25 +72,34 @@ def values(row, cols): return vals with self.conn() as conn: - wherecols = self.keyCols or self.visibleCols + wherecols = [self.columns[0]] # self.column("rowid") for r in adds.values(): cols = self.visibleCols sql = 'INSERT INTO "%s" ' % self.tableName sql += '(%s)' % ','.join(c.name for c in cols) - sql += 'VALUES (%s)' % ','.join('?' for c in cols) - self.execute(conn, sql, parms=values(r, cols)) + sql += ' VALUES (%s)' % ','.join('?' for c in cols) + res = self.execute(conn, sql, parms=values(r, cols)) + if res.rowcount != res.arraysize: + vd.warning('not all rows inserted') # f'{res.rowcount}/{res.arraysize} rows inserted' for row, rowmods in mods.values(): sql = 'UPDATE "%s" SET ' % self.tableName sql += ', '.join('%s=?' % c.name for c, _ in rowmods.items()) sql += ' WHERE %s' % ' AND '.join('"%s"=?' % c.name for c in wherecols) - self.execute(conn, sql, - parms=values(row, [c for c, _ in rowmods.items()]) + list(c.getSavedValue(row) for c in wherecols)) - - for r in dels.values(): + newvals=values(row, [c for c, _ in rowmods.items()]) + # calcValue gets the 'previous' value (before update) + wherevals=list(Column.calcValue(c, row) or '' for c in wherecols) + res = self.execute(conn, sql, parms=newvals+wherevals) + if res.rowcount != res.arraysize: + vd.warning('not all rows updated') # f'{res.rowcount}/{res.arraysize} rows updated' + + for row in dels.values(): sql = 'DELETE FROM "%s" ' % self.tableName sql += ' WHERE %s' % ' AND '.join('"%s"=?' % c.name for c in wherecols) - self.execute(conn, sql, parms=list(c.getTypedValue(r) for c in wherecols)) + wherevals=list(Column.calcValue(c, row) for c in wherecols) + res = self.execute(conn, sql, parms=wherevals) + if res.rowcount != res.arraysize: + vd.warning('not all rows deleted') # f'{res.rowcount}/{res.arraysize} rows deleted' conn.commit() @@ -101,8 +111,8 @@ class SqliteIndexSheet(SqliteSheet, IndexSheet): tableName = 'sqlite_master' def iterload(self): for row in SqliteSheet.iterload(self): - if row[0] != 'index': - tblname = row[1] + if row[1] != 'index': + tblname = row[2] yield SqliteSheet(tblname, source=self, tableName=tblname, row=row) @@ -110,10 +120,11 @@ class SqliteQuerySheet(SqliteSheet): def iterload(self): with self.conn() as conn: self.columns = [] + self.addColumn(ColumnItem('rowid', 0, type=int)) self.result = self.execute(conn, self.query, parms=getattr(self, 'parms', [])) for i, desc in enumerate(self.result.description): - self.addColumn(ColumnItem(desc[0], i)) + self.addColumn(ColumnItem(desc[0], i+1)) for row in self.result: yield row @@ -132,6 +143,10 @@ def save_sqlite(vd, p, *vsheets): currency: 'REAL' } + for vs in vsheets: + vs.ensureLoaded() + vd.sync() + for vs in vsheets: tblname = clean_to_id(vs.name) sqlcols = [] diff --git a/visidata/loaders/texttables.py b/visidata/loaders/texttables.py index aaccd2112..39a086b60 100644 --- a/visidata/loaders/texttables.py +++ b/visidata/loaders/texttables.py @@ -4,7 +4,7 @@ try: import tabulate for fmt in tabulate.tabulate_formats: - def save_table(path, *sheets): + def save_table(path, *sheets, fmt=fmt): import tabulate with path.open_text(mode='w') as fp: diff --git a/visidata/loaders/xlsx.py b/visidata/loaders/xlsx.py index 8391288c5..b4218f947 100644 --- a/visidata/loaders/xlsx.py +++ b/visidata/loaders/xlsx.py @@ -1,11 +1,12 @@ from visidata import * +def open_xls(p): + return XlsIndexSheet(p.name, source=p) + def open_xlsx(p): return XlsxIndexSheet(p.name, source=p) -open_xls = open_xlsx - class XlsxIndexSheet(IndexSheet): 'Load XLSX file (in Excel Open XML format).' rowtype = 'sheets' # rowdef: xlsxSheet @@ -14,6 +15,7 @@ class XlsxIndexSheet(IndexSheet): ColumnAttr('name', width=0), # visidata Sheet name ColumnAttr('nRows', type=int), ColumnAttr('nCols', type=int), + Column('active', getter=lambda col,row: row.source is col.sheet.workbook.active), ] nKeys = 1 @@ -21,8 +23,10 @@ def iterload(self): import openpyxl self.workbook = openpyxl.load_workbook(str(self.source), data_only=True, read_only=True) for sheetname in self.workbook.sheetnames: - vs = XlsxSheet(self.name, sheetname, source=self.workbook[sheetname]) - vs.reload() + src = self.workbook[sheetname] + vs = XlsxSheet(self.name, sheetname, source=src) + if isinstance(src, openpyxl.Workbook): + vs.reload() yield vs @@ -85,7 +89,7 @@ def save_xlsx(vd, p, *sheets): for col, v in dispvals.items(): if col.type == date: v = datetime.datetime.fromtimestamp(int(v.timestamp())) - elif not isNumeric(col): + elif not vd.isNumeric(col): v = str(v) row.append(v) diff --git a/visidata/macros.py b/visidata/macros.py index 4c4ab3dea..819ff37c9 100644 --- a/visidata/macros.py +++ b/visidata/macros.py @@ -1,30 +1,65 @@ from visidata import * +from functools import wraps +vd.macroMode = None +vd.macrobindings = {} @VisiData.lazy_property def macrosheet(vd): macrospath = Path(os.path.join(options.visidata_dir, 'macros.tsv')) macrosheet = vd.loadInternalSheet(TsvSheet, macrospath, columns=(ColumnItem('command', 0), ColumnItem('filename', 1))) or vd.error('error loading macros') + real_macrosheet = IndexSheet('user_macros', rows=[], source=macrosheet) for ks, fn in macrosheet.rows: vs = vd.loadInternalSheet(CommandLog, Path(fn)) + vd.status(f"setting {ks}") setMacro(ks, vs) + real_macrosheet.addRow(vs) - return macrosheet + return real_macrosheet + +@VisiData.api +def runMacro(vd, macro): + vd.replay_sync(macro, live=True) def setMacro(ks, vs): - vd.bindkeys.set(ks, vs.name, 'override') - vd.commands.set(vs.name, vs, 'override') + vd.macrobindings[ks] = vs + BaseSheet.addCommand(ks, vs.name, 'runMacro(vd.macrobindings[keystrokes])') @CommandLog.api def saveMacro(self, rows, ks): vs = copy(self) - vs.rows = self.selectedRows + vs.rows = rows macropath = Path(fnSuffix(options.visidata_dir+"macro")) vd.save_vd(macropath, vs) setMacro(ks, vs) - append_tsv_row(vd.macrosheet, (ks, macropath)) + append_tsv_row(vd.macrosheet.source, (ks, macropath)) + +@CommandLog.api +@wraps(CommandLog.afterExecSheet) +def afterExecSheet(cmdlog, sheet, escaped, err): + if vd.macroMode and (vd.activeCommand is not None) and (vd.activeCommand is not UNLOADED): + cmd = copy(vd.activeCommand) + cmd.row = cmd.col = cmd.sheet = '' + vd.macroMode.addRow(cmd) + + # the following needs to happen at the end, bc + # once cmdlog.afterExecSheet.__wrapped__ runs, vd.activeCommand resets to None + cmdlog.afterExecSheet.__wrapped__(cmdlog, sheet, escaped, err) + +@CommandLog.api +def startMacro(cmdlog): + if vd.macroMode: + ks = vd.input('save macro for keystroke: ') + vd.cmdlog.saveMacro(vd.macroMode.rows, ks) + vd.macroMode = None + else: + vd.status("recording macro") + vd.macroMode = CommandLog('current_macro', rows=[]) + +vd.status(vd.macrosheet) -CommandLog.addCommand('z^D', 'save-macro', 'sheet.saveMacro(selectedRows or fail("no rows selected"), input("save macro for keystroke: "))', 'save selected rows to macro mapped to keystroke') +Sheet.addCommand('m', 'macro-record', 'vd.cmdlog.startMacro()') +Sheet.addCommand('gm', 'macro-sheet', 'vd.push(vd.macrosheet)') diff --git a/visidata/main.py b/visidata/main.py index fda66ce34..9b3f67aef 100755 --- a/visidata/main.py +++ b/visidata/main.py @@ -2,7 +2,7 @@ # Usage: $0 [] [ ...] # $0 [] --play [--batch] [-w ] [-o ] [field=value ...] -__version__ = '2.0.1' +__version__ = '2.1' __version_info__ = 'saul.pw/VisiData v' + __version__ from copy import copy @@ -33,7 +33,7 @@ def eval_vd(logpath, *args, **kwargs): src = Path(logpath.given, fp=io.StringIO(log), filesize=len(log)) vs = vd.openSource(src, filetype=src.ext) vs.name += '_vd' - vd.push(vs) + vs.reload() vs.vd = vd return vs @@ -59,15 +59,15 @@ def duptty(): return stdin, stdout option_aliases = {} -def optalias(abbr, name): - option_aliases[abbr] = name +def optalias(abbr, name, val=None): + option_aliases[abbr] = (name, val) optalias('f', 'filetype') optalias('p', 'play') optalias('b', 'batch') optalias('P', 'preplay') -optalias('y', 'confirm_overwrite') +optalias('y', 'confirm_overwrite', False) optalias('o', 'output') optalias('w', 'replay_wait') optalias('d', 'delimiter') @@ -130,7 +130,7 @@ def main_vd(): pass optname = optname.replace('-', '_') - optname = option_aliases.get(optname, optname) + optname, optval = option_aliases.get(optname, (optname, optval)) if optval is None: opt = options._get(optname) @@ -167,7 +167,7 @@ def main_vd(): elif current_args.get('play', None) and '=' in arg: # parse 'key=value' pairs for formatting cmdlog template in replay mode - k, v = arg.split('=') + k, v = arg.split('=', maxsplit=1) fmtkwargs[k] = v else: inputs.append((arg, copy(current_args))) @@ -188,7 +188,7 @@ def main_vd(): options.set(k, v, obj='override') # fetch motd and plugins *after* options parsing/setting - visidata.PluginsSheet().reload() + vd.pluginsSheet.ensureLoaded() domotd() if args.batch: @@ -275,6 +275,7 @@ def main_vd(): if vd.replay_sync(vs): # error return 1 else: + vd.currentReplay = vs vd.replay(vs) run() diff --git a/visidata/mainloop.py b/visidata/mainloop.py index 4846b3a8e..d0b13a6e4 100644 --- a/visidata/mainloop.py +++ b/visidata/mainloop.py @@ -74,6 +74,8 @@ def setWindows(vd, scr): @VisiData.api def draw_all(vd): 'Draw all sheets in all windows.' + if not vd.sheets: + return vd.draw_sheet(vd.win1, vd.sheets[0]) if vd.win2 and len(vd.sheets) > 1: vd.draw_sheet(vd.win2, vd.sheets[1]) @@ -104,11 +106,10 @@ def mainloop(self, scr): self.keystrokes = '' while True: - if not self.sheets: - # if no more sheets, exit + if not self.sheets and self.currentReplay is None: return - sheet = self.sheets[0] + sheet = self.activeSheet threading.current_thread().sheet = sheet vd.drawThread = threading.current_thread() @@ -117,6 +118,10 @@ def mainloop(self, scr): self.draw_all() + if vd._nextCommands: + sheet.execCommand(vd._nextCommands.pop(0), keystrokes=self.keystrokes) + continue + keystroke = self.getkeystroke(scr, sheet) if not keystroke and prefixWaiting and ESC in self.keystrokes: # timeout ESC @@ -197,7 +202,6 @@ def mainloop(self, scr): else: scr.timeout(curses_timeout) - def setupcolors(stdscr, f, *args): curses.raw() # get control keys instead of signals curses.meta(1) # allow "8-bit chars" diff --git a/visidata/man/vd.1 b/visidata/man/vd.1 index ccd6b026a..c12fbca65 100644 --- a/visidata/man/vd.1 +++ b/visidata/man/vd.1 @@ -1,4 +1,4 @@ -.Dd July 23, 2020 +.Dd Dec 5, 2020 .Dt vd \&1 "Quick Reference Guide" .Os Linux/MacOS . @@ -188,7 +188,7 @@ set name of current column to combined contents of current cell in selected rows set name of all visible columns to combined contents of current column for selected rows (or current row) .Pp .It Ic " =" Ar expr -.No create new column from Python Ar expr Ns , with column names as variables +.No create new column from Python Ar expr Ns , with column names, and attributes, as variables .It Ic " g=" Ar expr .No set current column for selected rows to result of Python Ar expr .It Ic "gz=" Ar expr @@ -253,9 +253,11 @@ select/toggle/unselect all rows from cursor to bottom .It Ic "z| z\e\ " Ns Ar expr .No select/unselect rows matching Python Ar expr No in any visible column .It Ic " \&," Ns " (comma)" -select rows matching current cell in current column +select rows matching typed value of current cell in current column .It Ic "g\&," -select rows matching current row in all visible columns +select rows matching typed value of current row in all visible columns +.It Ic "z\&, gz\&," +select rows matching display value of current cell/row in current column/all visible columns . .El . @@ -343,6 +345,8 @@ toggle insert mode set contents to previous/next in history .It Ic "Tab Shift+Tab" autocomplete input (when available) +.It Ic "Shift+Arrow" +.No move cursor in direction of Sy Arrow No and re-enter edit mode . .El . @@ -369,9 +373,9 @@ open .It Ic "z+" Ar aggregator .No display result of Ar aggregator No over values in selected rows for current column .It Ic " &" -.No concatenate top two sheets in Sy Sheets Stack No +.No concatenate top two sheets in Sy Sheets Stack .It Ic "g&" -.No concatenate all sheets in Sy Sheets Stack No +.No concatenate all sheets in Sy Sheets Stack .Pp .El .Ss Data Visualization @@ -466,6 +470,10 @@ show cursor position and bounds of current sheet on status line show version and copyright information on status line .It Ic " ^P" .No open Sy Status History +.It "m" Ar keystroke +.No first, begin recording macro; second, prompt for Ar keystroke No, and complete recording. Macro can then be executed everytime provided keystroke is used. Will override existing keybinding. Macros will run on current row, column, sheet. +.It "gm" +.No open an index of all existing macros. Can be directly viewed with Sy Enter Ns , and then modified with Sy ^S Ns . . .El .Pp @@ -591,8 +599,8 @@ toggle/unset selected columns as key columns on source sheet add Ar aggregator No to selected source columns .It Ic "g-" No (hyphen) hide selected columns on source sheet -.It Ic "g~ g# g% g$ g@ gz#" -set type of selected columns on source sheet to str/int/float/currency/date/len +.It Ic "g~ g# g% g$ g@ gz# z%" +set type of selected columns on source sheet to str/int/float/currency/date/len/floatsi .It Ic " Enter" .No open a Sy Frequency Table No sheet grouped by column referenced in current row .El @@ -624,6 +632,8 @@ add row to reference a new blank sheet .No reload all selected sheets .It Ic "z^C gz^C" abort async threads for current/selected sheets(s) +.It Ic "g^S" +save selected or all sheets .It Ic " &" Ar jointype .No merge selected sheets with visible columns from all, keeping rows according to Ar jointype Ns : .El @@ -654,7 +664,7 @@ abort async threads for current/selected sheets(s) .It Ic zO .No edit sheet options (apply to Sy current sheet No only) .It Ic gO -.No open Sy options.config No as Sy TextSheet No +.No open Sy options.config No as Sy TextSheet .El .Bl -inset -compact .It (sheet-specific commands) @@ -662,6 +672,8 @@ abort async threads for current/selected sheets(s) .Bl -tag -width XXXXXXXXXXXXXXX -compact -offset XXX .It Ic "Enter e" edit option at current row +.It Ic "d" +remove option override for this context .El . .Ss CommandLog (Shift+D) @@ -802,8 +814,6 @@ default column width default column height .It Sy --textwrap-cells Ns = Ns Ar "bool " No "True" wordwrap text for multiline rows -.It Sy --cmd-after-edit Ns = Ns Ar "str " No "go-down" -command longname to execute after successful edit .It Sy --quitguard No " False" confirm before quitting last sheet .It Sy --debug No " False" @@ -844,6 +854,8 @@ encoding passed to codecs.open encoding_errors passed to codecs.open .It Sy --bulk-select-clear No " False" clear selected rows before new bulk selections +.It Sy --some-selected-rows No " False" +if no rows selected, if True, someSelectedRows returns all rows; if False, fails .It Sy --delimiter Ns = Ns Ar "str " No " " field delimiter to use for tsv/usv filetype .It Sy --row-delimiter Ns = Ns Ar "str " No " @@ -879,7 +891,7 @@ a nicer selection interface for aggregators and jointype numeric aggregators to calculate on Describe sheet .It Sy --histogram-bins Ns = Ns Ar "int " No "0" number of bins for histogram of numeric columns -.It Sy --numeric-binning Ns = Ns Ar "bool " No "True" +.It Sy --numeric-binning No " False" bin numeric columns into ranges .It Sy --replay-wait Ns = Ns Ar "float " No "0.0" time to wait between replayed commands, in seconds @@ -950,6 +962,8 @@ table header when saving to html (y/s/n) if save_dot includes all internet hosts separately (y), combined (s), or does not include the internet (n) .It Sy --graphviz-edge-labels Ns = Ns Ar "bool " No "True" whether to include edge labels on graphviz diagrams +.It Sy --pdf-tables No " False" +parse PDF for tables instead of pages of text .It Sy --plugins-url Ns = Ns Ar "str " No "https://visidata.org/plugins/plugins.jsonl" source of plugins sheet .El @@ -1017,6 +1031,10 @@ separator between key columns and rest of columns .It Sy "disp_selected_note " No "\[u2022]" +.It Sy "disp_sort_asc " No "\[u2191]\[u219F]\[u21DE]\[u21E1]\[u21E7]\[u21D1]" +characters for ascending sort +.It Sy "disp_sort_desc " No "\[u2193]\[u21A1]\[u21DF]\[u21E3]\[u21E9]\[u21D3]" +characters for descending sort .It Sy "color_default " No "normal" the default color .It Sy "color_default_hdr " No "bold" @@ -1121,7 +1139,7 @@ color of active clue .No open a blank sheet named Ar newfile No if file does not exist .Pp .Dl Ic vd sample.xlsx +:sheet1:2:3 -.No launch with Sy sheet1 No at top-of-stack, and cursor at column Sy 2 No and row Sy 3 No +.No launch with Sy sheet1 No at top-of-stack, and cursor at column Sy 2 No and row Sy 3 .Pp .Dl Ic vd -P open-plugins .No preplay longname Sy open-plugins No before starting the session @@ -1144,7 +1162,7 @@ For example: Functions defined in .visidatarc are available in python expressions (e.g. in derived columns). . .Sh SUPPORTED SOURCES -These are the supported sources: +Core VisiData includes these sources: .Pp .Bl -inset -compact -offset xxx .It Sy tsv No (tab-separated value) @@ -1177,67 +1195,24 @@ These are the supported sources: .El .Pp .Bl -inset -compact -offset xxx -.It Sy yaml Ns / Ns Sy yml No (requires Sy PyYAML Ns ) -.Bl -inset -compact -offset xxx -.It Uses the safe YAML loader, which supports the most common uses of YAML. -.El -.Pp -.Bl -inset -compact -offset xxx -.It Sy pcap No ( requires Sy dpkt Ns , Sy dnslib Ns ) +.It Sy sqlite .Bl -inset -compact -offset xxx -.It View and investigate captured network traffic in a tabular format. +.It May include multiple tables. The initial sheet is the table directory; +.Sy Enter No loaders the entrie table into memory. Sy z^S No saves modifications to source. .El .El .Pp -.Bl -inset -compact -offset xxx -.It Sy png No (requires Sy pypng Ns ) -.Bl -inset -compact -offset xxx -.It Pixels can be edited and saved in data form. Images can be plotted with Ic "\&." No (dot). -.El -.El -. -.Pp -The following URL schemes are supported: +URL schemes are also supported: .Bl -inset -compact -offset xxx .It Sy http No (requires Sy requests Ns ); can be used as transport for with another filetype -.It Sy postgres No (requires Sy psycopg2 Ns ) -.El -. -.Pp -.Bl -inset -compact -.It The following sources may include multiple tables. The initial sheet is the table directory; -.Sy Enter No loads the entire table into memory. .El . .Pp -.Bl -inset -compact -offset xxx -.It Sy sqlite -.It Sy xlsx No (requires Sy openpyxl Ns ) -.It Sy xls No (requires Sy xlrd Ns ) -.It Sy hdf5 No (requires Sy h5py Ns ) -.It Sy ttf Ns / Ns Sy otf No (requires Sy fonttools Ns ) -.It Sy mbtiles No (requires Sy mapbox-vector-tile Ns ) -.It Sy htm Ns / Ns Sy html No (requires Sy lxml Ns ) -.It Sy xml No (requires Sy lxml Ns ) -.Bl -tag -width XXXX -compact -offset XXX -.It Sy " v" -show only columns in current row attributes -.It Sy za -add column for xml attribute -.El -.It Sy xpt No (SAS; requires Sy xport Ns ) -.It Sy sas7bdat No (SAS; requires Sy sas7bdat Ns ) -.It Sy sav No (SPSS; requires Sy savReaderWriter Ns ) -.It Sy dta No (Stata; requires Sy pandas Ns ) -.It Sy shp No (requires Sy pyshp Ns ) -.El +For a list of all remaining formats supported by VisiData, see https://visidata.org/formats. .Pp In addition, .Sy .zip Ns , Sy .gz Ns , Sy .bz2 Ns , and Sy .xz No files are decompressed on the fly. .Pp -.No VisiData has an adapter for Sy pandas Ns . To load a file format which is supported by Sy pandas Ns , pass Sy -f pandas data.foo Ns . This will call Sy pandas.read_foo() Ns . -.Pp -.No For example, Sy vd -f pandas data.parquet No loads a parquet file. Note that when using the Sy pandas No loader, the Sy .fileformat No file extension is mandatory . .Sh SUPPORTED OUTPUT FORMATS These are the supported savers: @@ -1247,18 +1222,13 @@ These are the supported savers: .It Sy csv No (comma-separated value) .It Sy json No (one object with all rows) .It Sy jsonl Ns / Ns Sy ndjson Ns / Ns Sy ldjson No (one object per line/row) -.It Sy sqlite No (multisave capable) .Bl -inset -compact -offset xxx .It All expanded subcolumns must be closed (with Sy "\&)" Ns ) to retain the same structure. -.It Sy .shp No files can be saved as Sy geoJSON Ns . .El -.It Sy md No (org-mode compatible markdown table) -.It Sy htm Ns / Ns Sy html No (requires Sy lxml Ns ) -.It Sy png No (requires Sy pypng Ns ) +.It Sy sqlite No (save to source with Sy z^S Ns ) +.It Sy md No (markdown table) .El .Pp -.No Multisave is supported by Sy html Ns , Sy md Ns , and Sy txt Ns ; Sy g^S No will save all sheets into a single output file. -.Pp . .Sh AUTHOR .Nm VisiData diff --git a/visidata/man/vd.inc b/visidata/man/vd.inc index a9dbbe98c..40ba6543a 100644 --- a/visidata/man/vd.inc +++ b/visidata/man/vd.inc @@ -1,4 +1,4 @@ -.Dd July 23, 2020 +.Dd Dec 5, 2020 .Dt vd \&1 "Quick Reference Guide" .Os Linux/MacOS . @@ -188,7 +188,7 @@ set name of current column to combined contents of current cell in selected rows set name of all visible columns to combined contents of current column for selected rows (or current row) .Pp .It Ic " =" Ar expr -.No create new column from Python Ar expr Ns , with column names as variables +.No create new column from Python Ar expr Ns , with column names, and attributes, as variables .It Ic " g=" Ar expr .No set current column for selected rows to result of Python Ar expr .It Ic "gz=" Ar expr @@ -253,9 +253,11 @@ select/toggle/unselect all rows from cursor to bottom .It Ic "z| z\e\ " Ns Ar expr .No select/unselect rows matching Python Ar expr No in any visible column .It Ic " \&," Ns " (comma)" -select rows matching current cell in current column +select rows matching typed value of current cell in current column .It Ic "g\&," -select rows matching current row in all visible columns +select rows matching typed value of current row in all visible columns +.It Ic "z\&, gz\&," +select rows matching display value of current cell/row in current column/all visible columns . .El . @@ -343,6 +345,8 @@ toggle insert mode set contents to previous/next in history .It Ic "Tab Shift+Tab" autocomplete input (when available) +.It Ic "Shift+Arrow" +.No move cursor in direction of Sy Arrow No and re-enter edit mode . .El . @@ -369,9 +373,9 @@ open .It Ic "z+" Ar aggregator .No display result of Ar aggregator No over values in selected rows for current column .It Ic " &" -.No concatenate top two sheets in Sy Sheets Stack No +.No concatenate top two sheets in Sy Sheets Stack .It Ic "g&" -.No concatenate all sheets in Sy Sheets Stack No +.No concatenate all sheets in Sy Sheets Stack .Pp .El .Ss Data Visualization @@ -466,6 +470,10 @@ show cursor position and bounds of current sheet on status line show version and copyright information on status line .It Ic " ^P" .No open Sy Status History +.It "m" Ar keystroke +.No first, begin recording macro; second, prompt for Ar keystroke No, and complete recording. Macro can then be executed everytime provided keystroke is used. Will override existing keybinding. Macros will run on current row, column, sheet. +.It "gm" +.No open an index of all existing macros. Can be directly viewed with Sy Enter Ns , and then modified with Sy ^S Ns . . .El .Pp @@ -591,8 +599,8 @@ toggle/unset selected columns as key columns on source sheet add Ar aggregator No to selected source columns .It Ic "g-" No (hyphen) hide selected columns on source sheet -.It Ic "g~ g# g% g$ g@ gz#" -set type of selected columns on source sheet to str/int/float/currency/date/len +.It Ic "g~ g# g% g$ g@ gz# z%" +set type of selected columns on source sheet to str/int/float/currency/date/len/floatsi .It Ic " Enter" .No open a Sy Frequency Table No sheet grouped by column referenced in current row .El @@ -624,6 +632,8 @@ add row to reference a new blank sheet .No reload all selected sheets .It Ic "z^C gz^C" abort async threads for current/selected sheets(s) +.It Ic "g^S" +save selected or all sheets .It Ic " &" Ar jointype .No merge selected sheets with visible columns from all, keeping rows according to Ar jointype Ns : .El @@ -654,7 +664,7 @@ abort async threads for current/selected sheets(s) .It Ic zO .No edit sheet options (apply to Sy current sheet No only) .It Ic gO -.No open Sy options.config No as Sy TextSheet No +.No open Sy options.config No as Sy TextSheet .El .Bl -inset -compact .It (sheet-specific commands) @@ -662,6 +672,8 @@ abort async threads for current/selected sheets(s) .Bl -tag -width XXXXXXXXXXXXXXX -compact -offset XXX .It Ic "Enter e" edit option at current row +.It Ic "d" +remove option override for this context .El . .Ss CommandLog (Shift+D) @@ -819,7 +831,7 @@ overwrite existing files without confirmation .No open a blank sheet named Ar newfile No if file does not exist .Pp .Dl Ic vd sample.xlsx +:sheet1:2:3 -.No launch with Sy sheet1 No at top-of-stack, and cursor at column Sy 2 No and row Sy 3 No +.No launch with Sy sheet1 No at top-of-stack, and cursor at column Sy 2 No and row Sy 3 .Pp .Dl Ic vd -P open-plugins .No preplay longname Sy open-plugins No before starting the session @@ -842,7 +854,7 @@ For example: Functions defined in .visidatarc are available in python expressions (e.g. in derived columns). . .Sh SUPPORTED SOURCES -These are the supported sources: +Core VisiData includes these sources: .Pp .Bl -inset -compact -offset xxx .It Sy tsv No (tab-separated value) @@ -875,67 +887,24 @@ These are the supported sources: .El .Pp .Bl -inset -compact -offset xxx -.It Sy yaml Ns / Ns Sy yml No (requires Sy PyYAML Ns ) -.Bl -inset -compact -offset xxx -.It Uses the safe YAML loader, which supports the most common uses of YAML. -.El -.Pp -.Bl -inset -compact -offset xxx -.It Sy pcap No ( requires Sy dpkt Ns , Sy dnslib Ns ) +.It Sy sqlite .Bl -inset -compact -offset xxx -.It View and investigate captured network traffic in a tabular format. +.It May include multiple tables. The initial sheet is the table directory; +.Sy Enter No loaders the entrie table into memory. Sy z^S No saves modifications to source. .El .El .Pp -.Bl -inset -compact -offset xxx -.It Sy png No (requires Sy pypng Ns ) -.Bl -inset -compact -offset xxx -.It Pixels can be edited and saved in data form. Images can be plotted with Ic "\&." No (dot). -.El -.El -. -.Pp -The following URL schemes are supported: +URL schemes are also supported: .Bl -inset -compact -offset xxx .It Sy http No (requires Sy requests Ns ); can be used as transport for with another filetype -.It Sy postgres No (requires Sy psycopg2 Ns ) -.El -. -.Pp -.Bl -inset -compact -.It The following sources may include multiple tables. The initial sheet is the table directory; -.Sy Enter No loads the entire table into memory. .El . .Pp -.Bl -inset -compact -offset xxx -.It Sy sqlite -.It Sy xlsx No (requires Sy openpyxl Ns ) -.It Sy xls No (requires Sy xlrd Ns ) -.It Sy hdf5 No (requires Sy h5py Ns ) -.It Sy ttf Ns / Ns Sy otf No (requires Sy fonttools Ns ) -.It Sy mbtiles No (requires Sy mapbox-vector-tile Ns ) -.It Sy htm Ns / Ns Sy html No (requires Sy lxml Ns ) -.It Sy xml No (requires Sy lxml Ns ) -.Bl -tag -width XXXX -compact -offset XXX -.It Sy " v" -show only columns in current row attributes -.It Sy za -add column for xml attribute -.El -.It Sy xpt No (SAS; requires Sy xport Ns ) -.It Sy sas7bdat No (SAS; requires Sy sas7bdat Ns ) -.It Sy sav No (SPSS; requires Sy savReaderWriter Ns ) -.It Sy dta No (Stata; requires Sy pandas Ns ) -.It Sy shp No (requires Sy pyshp Ns ) -.El +For a list of all remaining formats supported by VisiData, see https://visidata.org/formats. .Pp In addition, .Sy .zip Ns , Sy .gz Ns , Sy .bz2 Ns , and Sy .xz No files are decompressed on the fly. .Pp -.No VisiData has an adapter for Sy pandas Ns . To load a file format which is supported by Sy pandas Ns , pass Sy -f pandas data.foo Ns . This will call Sy pandas.read_foo() Ns . -.Pp -.No For example, Sy vd -f pandas data.parquet No loads a parquet file. Note that when using the Sy pandas No loader, the Sy .fileformat No file extension is mandatory . .Sh SUPPORTED OUTPUT FORMATS These are the supported savers: @@ -945,18 +914,13 @@ These are the supported savers: .It Sy csv No (comma-separated value) .It Sy json No (one object with all rows) .It Sy jsonl Ns / Ns Sy ndjson Ns / Ns Sy ldjson No (one object per line/row) -.It Sy sqlite No (multisave capable) .Bl -inset -compact -offset xxx .It All expanded subcolumns must be closed (with Sy "\&)" Ns ) to retain the same structure. -.It Sy .shp No files can be saved as Sy geoJSON Ns . .El -.It Sy md No (org-mode compatible markdown table) -.It Sy htm Ns / Ns Sy html No (requires Sy lxml Ns ) -.It Sy png No (requires Sy pypng Ns ) +.It Sy sqlite No (save to source with Sy z^S Ns ) +.It Sy md No (markdown table) .El .Pp -.No Multisave is supported by Sy html Ns , Sy md Ns , and Sy txt Ns ; Sy g^S No will save all sheets into a single output file. -.Pp . .Sh AUTHOR .Nm VisiData diff --git a/visidata/man/vd.txt b/visidata/man/vd.txt new file mode 100644 index 000000000..03c2ee0e2 --- /dev/null +++ b/visidata/man/vd.txt @@ -0,0 +1,898 @@ +vd(1) Quick Reference Guide vd(1) + +NAME + VisiData — a terminal utility for exploring and arranging tabular data + +SYNOPSIS + vd [options] [input ...] + vd [options] --play cmdlog [-w waitsecs] [--batch] [-o output] + [field=value] + vd [options] [input ...] +toplevel:subsheet:col:row + +DESCRIPTION + VisiData is an easy-to-use multipurpose tool to explore, clean, edit, and + restructure data. Rows can be selected, filtered, and grouped; columns + can be rearranged, transformed, and derived via regex or Python expres‐ + sions; and workflows can be saved, documented, and replayed. + + REPLAY MODE + -p, --play=cmdlog replay a saved cmdlog within the interface + -w, --replay-wait=seconds + wait seconds between commands + -b, --batch replay in batch mode (with no interface) + -o, --output=file save final visible sheet to file as .tsv + --replay-movement toggle --play to move cursor cell-by-cell + field=value replace "{field}" in cmdlog contents with value + + Commands During Replay + ^U pause/resume replay + ^N execute next row in replaying sheet + ^K cancel current replay + + GLOBAL COMMANDS + All keystrokes are case sensitive. The ^ prefix is shorthand for Ctrl. + + Keystrokes to start off with + ^Q abort program immediately + ^C cancel user input or abort all async threads on current + sheet + g^C abort all secondary threads + q quit current sheet + gq quit all sheets (clean exit) + + ^H view this man page + z^H view sheet of command longnames and keybindings + Space longname execute command by its longname + + U undo the most recent modification (requires enabled + options.undo) + R redo the most recent undo (requires enabled + options.undo) + + Cursor Movement + Arrow PgUp Home go as expected + h j k l go left/down/up/right + gh gj gk gl go all the way to the left/bottom/top/right of sheet + G gg go all the way to the bottom/top of sheet + ^B ^F scroll one page back/forward + zz scroll current row to center of screen + + ^^ (Ctrl+^) jump to previous sheet (swaps with current sheet) + + / ? regex search for regex forward/backward in current column + g/ g? regex search for regex forward/backward over all visible + columns + z/ z? expr search by Python expr forward/backward in current column + (with column names as variables) + n N go to next/previous match from last regex search + + < > go up/down current column to next value + z< z> go up/down current column to next null value + { } go up/down current column to next selected row + + c regex go to next column with name matching regex + r regex go to next row with key matching regex + zc zr number go to column/row number (0-based) + + H J K L slide current row/column left/down/up/right + gH gJ gK gL slide current row/column all the way to the left/bot‐ + tom/top/right of sheet + zH zJ zK zK number + slide current row/column number positions to the + left/down/up/right + + zh zj zk zl scroll one left/down/up/right + + Column Manipulation + _ (underscore) + toggle width of current column between full and default + width + g_ toggle widths of all visible columns between full and + default width + z_ number adjust width of current column to number + gz_ number adjust widths of all visible columns to Ar number + + - (hyphen) hide current column + z- reduce width of current column by half + gv unhide all columns + + ! z! toggle/unset current column as a key column + ~ # % $ @ z# + set type of current column to str/int/float/cur‐ + rency/date/len + ^ edit name of current column + g^ set names of all unnamed visible columns to contents of + selected rows (or current row) + z^ set name of current column to combined contents of cur‐ + rent cell in selected rows (or current row) + gz^ set name of all visible columns to combined contents of + current column for selected rows (or current row) + + = expr create new column from Python expr, with column names, + and attributes, as variables + g= expr set current column for selected rows to result of Python + expr + gz= expr set current column for selected rows to the items in + result of Python sequence expr + z= expr evaluate Python expression on current row and set + current cell with result of Python expr + + i (iota) add column with incremental values + gi set current column for selected rows to incremental + values + zi step add column with values at increment step + gzi step set current column for selected rows at increment step + + ' (tick) add a frozen copy of current column with all cells eval‐ + uated + g' open a frozen copy of current sheet with all visible + columns evaluated + z' gz' add/reset cache for current/all visible column(s) + + : regex add new columns from regex split; number of columns + determined by example row at cursor + ; regex add new columns from capture groups of regex (also + requires example row) + z; expr create new column from bash expr, with $columnNames as + variables + * regex/subst add column derived from current column, replacing regex + with subst (may include \1 backrefs) + g* gz* regex/subst + modify selected rows in current/all visible column(s), + replacing regex with subst (may include \1 backrefs) + + ( g( expand current/all visible column(s) of lists (e.g. [3]) + or dicts (e.g. {3}) fully + z( gz( depth expand current/all visible column(s) of lists (e.g. [3]) + or dicts (e.g. {3}) to given depth (0= fully) + ) unexpand current column; restore original column and re‐ + move other columns at this level + zM row-wise expand current column of lists (e.g. [3]) or + dicts (e.g. {3}) within that column + + Row Selection + s t u select/toggle/unselect current row + gs gt gu select/toggle/unselect all rows + zs zt zu select/toggle/unselect all rows from top to cursor + gzs gzt gzu select/toggle/unselect all rows from cursor to bottom + | \ regex select/unselect rows matching regex in current column + g| g\ regex select/unselect rows matching regex in any visible + column + z| z\ expr select/unselect rows matching Python expr in any visible + column + , (comma) select rows matching typed value of current cell in cur‐ + rent column + g, select rows matching typed value of current row in all + visible columns + z, gz, select rows matching display value of current cell/row + in current column/all visible columns + + Row Sorting/Filtering + [ ] sort ascending/descending by current column; replace any + existing sort criteria + g[ g] sort ascending/descending by all key columns; replace + any existing sort criteria + z[ z] sort ascending/descending by current column; add to ex‐ + isting sort criteria + gz[ gz] sort ascending/descending by all key columns; add to ex‐ + isting sort criteria + " open duplicate sheet with only selected rows + g" open duplicate sheet with all rows + gz" open duplicate sheet with deepcopy of selected rows + + Editing Rows and Cells + a za append a blank row/column; appended columns cannot be + copied to clipboard + ga gza number append number blank rows/columns + d gd delete (cut) current/selected row(s) and move to clip‐ + board + y gy yank (copy) current/all selected row(s) to clipboard + zy gzy yank (copy) contents of current column for current/se‐ + lected row(s) to clipboard + zd gzd delete (cut) contents of current column for current/se‐ + lected row(s) and move to clipboard + p P paste clipboard rows after/before current row + zp gzp set cells of current column for current/selected row(s) + to last clipboard value + Y gY yank (copy) current/all selected row(s) to system + clipboard (using options.clipboard_copy_cmd) + zY gzY yank (copy) contents of current column for + current/selected row(s) to system clipboard (using + options.clipboard_copy_cmd) + f fill null cells in current column with contents of non- + null cells up the current column + e text edit contents of current cell + ge text set contents of current column for selected rows to text + + Commands While Editing Input + Enter ^C accept/abort input + ^O open external $EDITOR to edit contents + ^R reload initial value + ^A ^E go to beginning/end of line + ^B ^F go back/forward one character + ^← ^→ (arrow) go back/forward one word + ^H ^D delete previous/current character + ^T transpose previous and current characters + ^U ^K clear from cursor to beginning/end of line + ^Y paste from cell clipboard + Backspace Del delete previous/current character + Insert toggle insert mode + Up Down set contents to previous/next in history + Tab Shift+Tab autocomplete input (when available) + Shift+Arrow move cursor in direction of Arrow and re-enter edit + mode + + Data Toolkit + o input open input in VisiData + ^S g^S filename save current/all sheet(s) to filename in format + determined by extension (default .tsv) + Note: if the format does not support multisave, or the + filename ends in a /, a directory will be created. + z^S filename save current column only to filename in format + determined by extension (default .tsv) + ^D filename.vd save CommandLog to filename.vd file + A open new blank sheet with one column + T open new sheet that has rows and columns of current + sheet transposed + + + aggregator add aggregator to current column (see Frequency Table) + z+ aggregator display result of aggregator over values in selected + rows for current column + & concatenate top two sheets in Sheets Stack + g& concatenate all sheets in Sheets Stack + + Data Visualization + . (dot) plot current numeric column vs key columns. The numeric + key column is used for the x-axis; categorical key column + values determine color. + g. plot a graph of all visible numeric columns vs key + columns. + + If rows on the current sheet represent plottable coordinates (as in .shp + or vector .mbtiles sources), . plots the current row, and g. plots all + selected rows (or all rows if none selected). + + Canvas-specific Commands + + - increase/decrease zoom level, centered on cursor + _ (underscore) zoom to fit full extent + z_ (underscore) set aspect ratio + x xmin xmax set xmin/xmax on graph + y ymin ymax set ymin/ymax on graph + s t u select/toggle/unselect rows on source sheet con‐ + tained within canvas cursor + gs gt gu select/toggle/unselect rows on source sheet visi‐ + ble on screen + d delete rows on source sheet contained within can‐ + vas cursor + gd delete rows on source sheet visible on screen + Enter open sheet of source rows contained within canvas + cursor + gEnter open sheet of source rows visible on screen + 1 - 9 toggle display of layers + ^L redraw all pixels on canvas + v toggle show_graph_labels option + mouse scrollwheel zoom in/out of canvas + left click-drag set canvas cursor + right click-drag scroll canvas + + Split Screen + Z split screen in half, so that second sheet on the stack is + visible in a second pane + zZ split screen, and queries for height of second pane + + Split Window specific Commands + gZ close an already split screen, current pane full + screens + Tab jump to other pane + ^^ (Ctrl+^) swap which sheet is in current pane + + Other Commands + Q quit current sheet and remove it from the CommandLog + v toggle sheet-specific visibility (multi-line rows on + Sheet, legends/axes on Graph) + + ^E g^E view traceback for most recent error(s) + z^E view traceback for error in current cell + + ^L refresh screen + ^R reload current sheet + z^R clear cache for current column + ^Z suspend VisiData process + ^G show cursor position and bounds of current sheet on sta‐ + tus line + ^V show version and copyright information on status line + ^P open Status History + m keystroke first, begin recording macro; second, prompt for + keystroke No, and complete recording. Macro can then be + executed everytime provided keystroke is used. Will + override existing keybinding. Macros will run on current + row, column, sheet. + gm open an index of all existing macros. Can be directly + viewed with Enter, and then modified with ^S. + + ^Y z^Y g^Y open current row/cell/sheet as Python object + ^X expr evaluate Python expr and opens result as Python object + z^X expr evaluate Python expr, in context of current row, and + open result as Python object + g^X stmt execute Python stmt in the global scope + + Internal Sheets List + . VisiDataMenu (Shift+V) browse list of core sheets + . Directory Sheet browse properties of files in a directory + . Plugins Sheet browse, install, and (de)activate plugins + + Metasheets + . Columns Sheet (Shift+C) edit column properties + . Sheets Sheet (Shift+S) jump between sheets or join them together + . Options Sheet (Shift+O) edit configuration options + . Commandlog (Shift+D) modify and save commands for replay + . Error Sheet (Ctrl+E) view last error + . Status History (Ctrl+P) view history of status messages + . Threads Sheet (Ctrl+T) view, cancel, and profile + asynchronous threads + + Derived Sheets + . Frequency Table (Shift+F) group rows by column value, with + aggregations of other columns + . Describe Sheet (Shift+I) view summary statistics for each column + . Pivot Table (Shift+W) group rows by key and summarize current + column + . Melted Sheet (Shift+M) unpivot non-key columns into + variable/value columns + . Transposed Sheet (Shift+T) open new sheet with rows and columns + transposed + + INTERNAL SHEETS + VisiDataMenu (Shift+V) + (sheet-specific commands) + Enter load sheet in current row + + Directory Sheet + (global commands) + Space open-dir-current + open the Directory Sheet for the current directory + (sheet-specific commands) + Enter gEnter open current/selected file(s) as new sheet(s) + ^O g^O open current/selected file(s) in external $EDITOR + ^R z^R gz^R reload information for all/current/selected file(s) + + Plugins Sheet + Browse through a list of available plugins. VisiData needs to be + restarted before plugin activation takes effect. Installation may require + internet access. + (global commands) + Space open-plugins + open the Plugins Sheet + (sheet-specific commands) + a install and activate current plugin + d deactivate current plugin + + METASHEETS + Columns Sheet (Shift+C) + Properties of columns on the source sheet can be changed with standard + editing commands (e ge g= Del) on the Columns Sheet. Multiple aggregators + can be set by listing them (separated by spaces) in the aggregators + column. The 'g' commands affect the selected rows, which are the literal + columns on the source sheet. + (global commands) + gC open Columns Sheet with all visible columns from all + sheets + (sheet-specific commands) + & add column from concatenating selected source columns + g! gz! toggle/unset selected columns as key columns on + source sheet + g+ aggregator add Ar aggregator No to selected source columns + g- (hyphen) hide selected columns on source sheet + g~ g# g% g$ g@ gz# z% + set type of selected columns on source sheet to + str/int/float/currency/date/len/floatsi + Enter open a Frequency Table sheet grouped by column + referenced in current row + + Sheets Sheet (Shift+S) + open Sheets Stack, which contains only the active sheets on the current + stack + (global commands) + gS open Sheets Sheet, which contains all sheets from + current session, active and inactive + Alt number jump to sheet number + (sheet-specific commands) + Enter jump to sheet referenced in current row + gEnter push selected sheets to top of sheet stack + a add row to reference a new blank sheet + gC gI open Columns Sheet/Describe Sheet with all visible + columns from selected sheets + g^R reload all selected sheets + z^C gz^C abort async threads for current/selected sheets(s) + g^S save selected or all sheets + & jointype merge selected sheets with visible columns from all, + keeping rows according to jointype: + . inner keep only rows which match keys on all + sheets + . outer keep all rows from first selected sheet + . full keep all rows from all sheets (union) + . diff keep only rows NOT in all sheets + . append keep all rows from all sheets + (concatenation) + . extend copy first selected sheet, keeping all rows + and sheet type, and extend with columns from other + sheets + . merge mostly keep all rows from first selected + sheet, except prioritise cells with non-null/non- + error values + + Options Sheet (Shift+O) + (global commands) + Shift+O edit global options (apply to all sheets) + zO edit sheet options (apply to current sheet only) + gO open options.config as TextSheet + (sheet-specific commands) + Enter e edit option at current row + d remove option override for this context + + CommandLog (Shift+D) + (global commands) + D open current sheet's CommandLog with all other loose + ends removed; includes commands from parent sheets + gD open global CommandLog for all commands executed in + the current session + zD open current sheet's CommandLog with the parent + sheets commands' removed + (sheet-specific commands) + x replay command in current row + gx replay contents of entire CommandLog + ^C abort replay + + DERIVED SHEETS + Frequency Table (Shift+F) + A Frequency Table groups rows by one or more columns, and includes + summary columns for those with aggregators. + (global commands) + gF open Frequency Table, grouped by all key columns on + source sheet + zF open one-line summary for all rows and selected rows + (sheet-specific commands) + s t u select/toggle/unselect these entries in source sheet + Enter gEnter open copy of source sheet with rows that are grouped + in current cell / selected rows + + Describe Sheet (Shift+I) + A Describe Sheet contains descriptive statistics for all visible columns. + (global commands) + gI open Describe Sheet for all visible columns on all + sheets + (sheet-specific commands) + zs zu select/unselect rows on source sheet that are being + described in current cell + ! toggle/unset current column as a key column on source + sheet + Enter open a Frequency Table sheet grouped on column + referenced in current row + zEnter open copy of source sheet with rows described in cur‐ + rent cell + + Pivot Table (Shift+W) + Set key column(s) and aggregators on column(s) before pressing Shift+W on + the column to pivot. + (sheet-specific commands) + Enter open sheet of source rows aggregated in current pivot + row + zEnter open sheet of source rows aggregated in current pivot + cell + + Melted Sheet (Shift+M) + Open Melted Sheet (unpivot), with key columns retained and all non-key + columns reduced to Variable-Value rows. + (global commands) + gM regex open Melted Sheet (unpivot), with key columns + retained and regex capture groups determining how the + non-key columns will be reduced to Variable-Value + rows. + + Python Object Sheet (^X ^Y g^Y z^Y) + (sheet-specific commands) + Enter dive further into Python object + v toggle show/hide for methods and hidden properties + gv zv show/hide methods and hidden properties + +COMMANDLINE OPTIONS + -P=longname preplay longname before replay or regular + launch; limited to Base Sheet bound commands + +toplevel:subsheet:col:row launch vd with subsheet of toplevel at + top-of-stack, and cursor at col and row; all + arguments are optional + + -f, --filetype=filetype tsv set loader to use for + filetype instead of file extension + -y, --confirm-overwrite=F True overwrite existing files + without confirmation + --mouse-interval=int 1 max time between + press/release for click + (ms) + --null-value=NoneType None a value to be counted as + null + --undo=bool True enable undo/redo + --col-cache-size=int 0 max number of cache en‐ + tries in each cached col‐ + umn + --clean-names False clean column/sheet names + to be valid Python iden‐ + tifiers + --default-width=int 20 default column width + --default-height=int 10 default column height + --textwrap-cells=bool True wordwrap text for multi‐ + line rows + --quitguard False confirm before quitting + last sheet + --debug False exit on error and display + stacktrace + --skip=int 0 skip N rows before header + --header=int 1 parse first N rows as + column names + --force-256-colors False use 256 colors even if + curses reports fewer + --use-default-colors False curses use default termi‐ + nal colors + --note-pending=str ⌛ note to display for pend‐ + ing cells + --note-format-exc=str ? cell note for an excep‐ + tion during formatting + --note-getter-exc=str ! cell note for an excep‐ + tion during computation + --note-type-exc=str ! cell note for an excep‐ + tion during type conver‐ + sion + --scroll-incr=int 3 amount to scroll with + scrollwheel + --name-joiner=str _ string to join sheet or + column names + --value-joiner=str string to join display + values + --wrap False wrap text to fit window + width on TextSheet + --save-filetype=str tsv specify default file type + to save as + --profile=str filename to save binary + profiling data + --min-memory-mb=int 0 minimum memory to con‐ + tinue loading and async + processing + --encoding=str utf-8 encoding passed to + codecs.open + --encoding-errors=str surrogateescape encoding_errors passed to + codecs.open + --bulk-select-clear False clear selected rows be‐ + fore new bulk selections + --some-selected-rows False if no rows selected, if + True, someSelectedRows + returns all rows; if + False, fails + --delimiter=str field delimiter to use + for tsv/usv filetype + --row-delimiter=str " row delimiter to use + for tsv/usv filetype + --tsv-safe-newline=str replacement for newline + character when saving to + tsv + --tsv-safe-tab=str replacement for tab char‐ + acter when saving to tsv + --visibility=int 0 visibility level (0=low, + 1=high) + --expand-col-scanrows=int 1000 number of rows to check + when expanding columns (0 + = all) + --json-indent=NoneType None indent to use when saving + json + --json-sort-keys False sort object keys when + saving to json + --default-colname=str column name to use for + non-dict rows + --filetype=str specify file type + --confirm-overwrite=bool True whether to prompt for + overwrite confirmation on + save + --safe-error=str #ERR error string to use while + saving + --clipboard-copy-cmd=str command to copy stdin to + system clipboard + --clipboard-paste-cmd=str command to get contents + of system clipboard + --fancy-chooser=bool True a nicer selection inter‐ + face for aggregators and + jointype + --describe-aggrs=str mean stdev numeric aggregators to + calculate on Describe + sheet + --histogram-bins=int 0 number of bins for his‐ + togram of numeric columns + --numeric-binning False bin numeric columns into + ranges + --replay-wait=float 0.0 time to wait between re‐ + played commands, in sec‐ + onds + --replay-movement False insert movements during + replay + --visidata-dir=str ~/.visidata/ directory to load and + store additional files + --rowkey-prefix=str キ string prefix for rowkey + in the cmdlog + --cmdlog-histfile=str file to autorecord each + cmdlog action to + --regex-flags=str I flags to pass to re.com‐ + pile() [AILMSUX] + --regex-maxsplit=int 0 maxsplit to pass to + regex.split + --default-sample-size=int 100 number of rows to sample + for regex.split + --show-graph-labels=bool True show axes and legend on + graph + --plot-colors=str list of distinct colors + to use for plotting dis‐ + tinct objects + --zoom-incr=float 2.0 amount to multiply cur‐ + rent zoomlevel when zoom‐ + ing + --motd-url=str source of randomized + startup messages + --dir-recurse False walk source path recur‐ + sively on DirSheet + --dir-hidden False load hidden files on + DirSheet + --config=str ~/.visidatarc config file to exec in + Python + --play=str + --batch False replay in batch mode + (with no interface and + all status sent to std‐ + out) + --output=NoneType None save the final visible + sheet to output at the + end of replay + --preplay=str longnames to preplay be‐ + fore replay + --imports=str plugins imports to preload before + .visidatarc (command-line + only) + --incr-base=float 1.0 start value for column + increments + --csv-dialect=str excel dialect passed to + csv.reader + --csv-delimiter=str , delimiter passed to + csv.reader + --csv-quotechar=str " quotechar passed to + csv.reader + --csv-skipinitialspace=bool True skipinitialspace passed + to csv.reader + --csv-escapechar=NoneType None escapechar passed to + csv.reader + --csv-lineterminator=str " lineterminator passed + to csv.writer + --safety-first False sanitize input/output to + handle edge cases, with a + performance cost + --fixed-rows=int 1000 number of rows to check + for fixed width columns + --fixed-maxcols=int 0 max number of fixed-width + columns to create (0 is + no max) + --postgres-schema=str public The desired schema for + the Postgres database + --html-title=str

{sheet.name}

+ table header when saving + to html + --pcap-internet=str n (y/s/n) if save_dot in‐ + cludes all internet hosts + separately (y), combined + (s), or does not include + the internet (n) + --graphviz-edge-labels=bool True whether to include edge + labels on graphviz dia‐ + grams + --pdf-tables False parse PDF for tables in‐ + stead of pages of text + --plugins-url=str https://visidata.org/plugins/plugins.jsonl + source of plugins sheet + + DISPLAY OPTIONS + Display options can only be set via the Options Sheet or a .visidatarc + (see FILES). + + disp_splitwin_pct 0 height of second sheet on screen + disp_currency_fmt %.02f default fmtstr to format for cur‐ + rency values + disp_float_fmt {:.02f} default fmtstr to format for + float values + disp_int_fmt {:.0f} default fmtstr to format for int + values + disp_date_fmt %Y-%m-%d default fmtstr to strftime for + date values + disp_note_none ⌀ visible contents of a cell whose + value is None + disp_truncator … indicator that the contents are + only partially visible + disp_oddspace · displayable character for odd + whitespace + disp_more_left < header note indicating more col‐ + umns to the left + disp_more_right > header note indicating more col‐ + umns to the right + disp_error_val displayed contents for computa‐ + tion exception + disp_ambig_width 1 width to use for unicode chars + marked ambiguous + disp_pending string to display in pending + cells + color_note_pending bold magenta color of note in pending cells + color_note_type 226 yellow color of cell note for non-str + types in anytype columns + color_note_row 220 yellow color of row note on left edge + disp_column_sep | separator between columns + disp_keycol_sep ║ separator between key columns and + rest of columns + disp_rowtop_sep | + disp_rowmid_sep ⁝ + disp_rowbot_sep ⁝ + disp_rowend_sep ║ + disp_keytop_sep ║ + disp_keymid_sep ║ + disp_keybot_sep ║ + disp_endtop_sep ║ + disp_endmid_sep ║ + disp_endbot_sep ║ + disp_selected_note • + disp_sort_asc ↑↟⇞⇡⇧⇑ characters for ascending sort + disp_sort_desc ↓↡⇟⇣⇩⇓ characters for descending sort + color_default normal the default color + color_default_hdr bold color of the column headers + color_bottom_hdr underline color of the bottom header row + color_current_row reverse color of the cursor row + color_current_col bold color of the cursor column + color_current_hdr bold reverse color of the header for the cur‐ + sor column + color_column_sep 246 blue color of column separators + color_key_col 81 cyan color of key columns + color_hidden_col 8 color of hidden columns on + metasheets + color_selected_row 215 yellow color of selected rows + disp_rstatus_fmt {sheet.longname} {sheet.nRows:9d} {sheet.rowtype} + right-side status format string + disp_status_fmt {sheet.shortcut}› {sheet.name}| + status line prefix + disp_lstatus_max 0 maximum length of left status + line + disp_status_sep | separator between statuses + color_keystrokes white color of input keystrokes on sta‐ + tus line + color_status bold status line color + color_error red error message color + color_warning yellow warning message color + color_top_status underline top window status bar color + color_active_status bold active window status bar color + color_inactive_status 8 inactive window status bar color + color_working green color of system running smoothly + color_edit_cell normal cell color to use when editing + cell + disp_edit_fill _ edit field fill character + disp_unprintable · substitute character for unprint‐ + ables + disp_histogram * histogram element character + disp_histolen 50 width of histogram column + disp_replay_play ▶ status indicator for active re‐ + play + disp_replay_pause ‖ status indicator for paused re‐ + play + color_status_replay green color of replay status indicator + disp_pixel_random False randomly choose attr from set of + pixels instead of most common + color_graph_hidden 238 blue color of legend for hidden attri‐ + bute + color_graph_selected bold color of selected graph points + color_graph_axis bold color for graph axis labels + color_add_pending green color for rows pending add + color_change_pending reverse yellow color for cells pending modifica‐ + tion + color_delete_pending red color for rows pending delete + color_xword_active green color of active clue + +EXAMPLES + vd foo.tsv + open the file foo.tsv in the current directory + + vd -f sqlite bar.db + open the file bar.db as a sqlite database + + vd -b countries.fixed -o countries.tsv + convert countries.fixed (in fixed width format) to countries.tsv (in tsv + format) + + vd postgres://username:password@hostname:port/database + open a connection to the given postgres database + + vd --play tests/pivot.vd --replay-wait 1 --output tests/pivot.tsv + replay tests/pivot.vd, waiting 1 second between commands, and output the + final sheet to test/pivot.tsv + + ls -l | vd -f fixed --skip 1 --header 0 + parse the output of ls -l into usable data + + ls | vd | lpr + interactively select a list of filenames to send to the printer + + vd newfile.tsv + open a blank sheet named newfile if file does not exist + + vd sample.xlsx +:sheet1:2:3 + launch with sheet1 at top-of-stack, and cursor at column 2 and row 3 + + vd -P open-plugins + preplay longname open-plugins before starting the session + +FILES + At the start of every session, VisiData looks for $HOME/.visidatarc, and + calls Python exec() on its contents if it exists. For example: + + options.min_memory_mb=100 # stop processing without 100MB free + + bindkey('0', 'go-leftmost') # alias '0' to go to first column, like vim + + def median(values): + L = sorted(values) + return L[len(L)//2] + + aggregator('median', median) + + Functions defined in .visidatarc are available in python expressions + (e.g. in derived columns). + +SUPPORTED SOURCES + Core VisiData includes these sources: + + tsv (tab-separated value) + Plain and simple. VisiData writes tsv format by default. See the + --tsv-delimiter option. + + csv (comma-separated value) + .csv files are a scourge upon the earth, and still regrettably + common. + See the --csv-dialect, --csv-delimiter, --csv-quotechar, and + --csv-skipinitialspace options. + Accepted dialects are excel-tab, unix, and excel. + + fixed (fixed width text) + Columns are autodetected from the first 1000 rows (adjustable with + --fixed-rows). + + json (single object) and jsonl/ndjson/ldjson (one object per line). + Cells containing lists (e.g. [3]) or dicts ({3}) can be expanded + into new columns with ( and unexpanded with ). + + sqlite + May include multiple tables. The initial sheet is the table + directory; Enter loaders the entrie table into memory. z^S saves + modifications to source. + + URL schemes are also supported: + http (requires requests); can be used as transport for with another + filetype + + For a list of all remaining formats supported by VisiData, see + https://visidata.org/formats. + + In addition, .zip, .gz, .bz2, and .xz files are decompressed on the fly. + +SUPPORTED OUTPUT FORMATS + These are the supported savers: + + tsv (tab-separated value) + csv (comma-separated value) + json (one object with all rows) + jsonl/ndjson/ldjson (one object per line/row) + All expanded subcolumns must be closed (with )) to retain the same + structure. + sqlite (save to source with z^S) + md (markdown table) + +AUTHOR + VisiData was made by Saul Pwanson . + +Linux/MacOS Dec 5, 2020 Linux/MacOS diff --git a/visidata/menu.py b/visidata/menu.py new file mode 100644 index 000000000..e6cd8f240 --- /dev/null +++ b/visidata/menu.py @@ -0,0 +1,29 @@ +from visidata import * + +def open_mnu(p): + return MenuSheet(p.name, source=p) + + +vd.save_mnu=vd.save_tsv + +class MenuSheet(VisiDataMetaSheet): + rowtype='labels' # { .x .y .text .color .command .input } + + +class MenuCanvas(BaseSheet): + rowtype='labels' + def click(self, r): + vd.replayOne(vd.cmdlog.newRow(sheet=self.name, col='', row='', longname=r.command, input=r.input)) + + def reload(self): + self.rows = self.source.rows + + def draw(self, scr): + vd.clearCaches() + for r in Progress(self.source.rows): + x, y = map(int, (r.x, r.y)) + clipdraw(scr, y, x, r.text, colors[r.color]) + vd.onMouse(scr, y, x, 1, len(r.text), BUTTON1_RELEASED=lambda y,x,key,r=r,sheet=self: sheet.click(r)) + + +MenuSheet.addCommand('z.', 'disp-menu', 'vd.push(MenuCanvas(name, "disp", source=sheet))', '') diff --git a/visidata/metasheets.py b/visidata/metasheets.py index 59f95f86b..87ac1396e 100644 --- a/visidata/metasheets.py +++ b/visidata/metasheets.py @@ -33,6 +33,7 @@ def setValue(self, srcCol, val): columns = [ ColumnAttr('sheet', type=str), ColumnAttr('name', width=options.default_width), + ColumnAttr('keycol', type=int, width=0), ColumnAttr('width', type=int), ColumnAttr('height', type=int), ColumnAttr('hoffset', type=int, width=0), @@ -163,7 +164,7 @@ def allColumnsSheet(vd): @ColumnsSheet.command('&', 'join-cols', 'add column from concatenating selected source columns') def join_cols(sheet): - cols = sheet.someSelectedRows + cols = sheet.onlySelectedRows destSheet = cols[0].sheet if len(set(c.sheet for c in cols)) > 1: @@ -186,20 +187,21 @@ def join_cols(sheet): Sheet.addCommand('C', 'columns-sheet', 'vd.push(ColumnsSheet(name+"_columns", source=[sheet]))', 'open Columns Sheet: edit column properties for current sheet') # used ColumnsSheet, affecting the 'row' (source column) -ColumnsSheet.addCommand('g!', 'key-selected', 'for c in someSelectedRows: c.sheet.setKeys([c])', 'toggle selected rows as key columns on source sheet') -ColumnsSheet.addCommand('gz!', 'key-off-selected', 'for c in someSelectedRows: c.sheet.unsetKeys([c])', 'unset selected rows as key columns on source sheet') +ColumnsSheet.addCommand('g!', 'key-selected', 'for c in onlySelectedRows: c.sheet.setKeys([c])', 'toggle selected rows as key columns on source sheet') +ColumnsSheet.addCommand('gz!', 'key-off-selected', 'for c in onlySelectedRows: c.sheet.unsetKeys([c])', 'unset selected rows as key columns on source sheet') -ColumnsSheet.addCommand('g-', 'hide-selected', 'someSelectedRows.hide()', 'hide selected columns on source sheet') +ColumnsSheet.addCommand('g-', 'hide-selected', 'onlySelectedRows.hide()', 'hide selected columns on source sheet') ColumnsSheet.addCommand(None, 'resize-source-rows-max', 'for c in selectedRows or [cursorRow]: c.setWidth(c.getMaxWidth(c.sheet.visibleRows))', 'adjust widths of selected source columns') -ColumnsSheet.addCommand('g%', 'type-float-selected', 'someSelectedRows.type=float', 'set type of selected columns to float') -ColumnsSheet.addCommand('g#', 'type-int-selected', 'someSelectedRows.type=int', 'set type of selected columns to int') -ColumnsSheet.addCommand('gz#', 'type-len-selected', 'someSelectedRows.type=vlen', 'set type of selected columns to len') -ColumnsSheet.addCommand('g@', 'type-date-selected', 'someSelectedRows.type=date', 'set type of selected columns to date') -ColumnsSheet.addCommand('g$', 'type-currency-selected', 'someSelectedRows.type=currency', 'set type of selected columns to currency') -ColumnsSheet.addCommand('g~', 'type-string-selected', 'someSelectedRows.type=str', 'set type of selected columns to str') -ColumnsSheet.addCommand('gz~', 'type-any-selected', 'someSelectedRows.type=anytype', 'set type of selected columns to anytype') +ColumnsSheet.addCommand('g%', 'type-float-selected', 'onlySelectedRows.type=float', 'set type of selected columns to float') +ColumnsSheet.addCommand('g#', 'type-int-selected', 'onlySelectedRows.type=int', 'set type of selected columns to int') +ColumnsSheet.addCommand('gz#', 'type-len-selected', 'onlySelectedRows.type=vlen', 'set type of selected columns to len') +ColumnsSheet.addCommand('g@', 'type-date-selected', 'onlySelectedRows.type=date', 'set type of selected columns to date') +ColumnsSheet.addCommand('g$', 'type-currency-selected', 'onlySelectedRows.type=currency', 'set type of selected columns to currency') +ColumnsSheet.addCommand('g~', 'type-string-selected', 'onlySelectedRows.type=str', 'set type of selected columns to str') +ColumnsSheet.addCommand('gz~', 'type-any-selected', 'onlySelectedRows.type=anytype', 'set type of selected columns to anytype') +OptionsSheet.addCommand('d', 'unset-option', 'options.unset(cursorRow.name, str(source))', 'remove option override for this context') OptionsSheet.addCommand(None, 'edit-option', 'editOption(cursorRow)', 'edit option at current row') OptionsSheet.bindkey('e', 'edit-option') OptionsSheet.bindkey(ENTER, 'edit-option') diff --git a/visidata/modify.py b/visidata/modify.py index 7d70e9747..366467019 100644 --- a/visidata/modify.py +++ b/visidata/modify.py @@ -120,6 +120,7 @@ def deleteBy(sheet, func, commit=False): if r is row: sheet.cursorRowIndex = len(sheet.rows)-1 else: + sheet.deleteSourceRow(r) ndeleted += 1 if not commit: @@ -181,25 +182,7 @@ def commitMods(self): @Sheet.api def commitDeletes(self): 'Return the number of rows that have been marked for deletion. Delete the rows. Clear the marking.' - ndeleted = 0 - - dest_row = None # row to re-place cursor after - oldidx = self.cursorRowIndex - while oldidx < len(self.rows): - if not self.isDeleted(self.rows[oldidx]): - dest_row = self.rows[oldidx] - break - oldidx += 1 - - newidx = 0 - for r in Progress(list(self.rows), gerund='deleting'): - if self.isDeleted(self.rows[newidx]): - self.deleteSourceRow(newidx) - ndeleted += 1 - else: - if r is dest_row: - self.cursorRowIndex = newidx - newidx += 1 + ndeleted = self.deleteBy(self.isDeleted, commit=True) if ndeleted: vd.status('deleted %s %s' % (ndeleted, self.rowtype)) @@ -269,17 +252,14 @@ def commit(sheet, *rows): cstr = sheet.changestr(adds, mods, deletes) path = sheet.source - if not cstr: - vd.fail('no diffs') - - if options.confirm_overwrite: - confirm('really %s? ' % cstr) + if sheet.options.confirm_overwrite: + vd.confirm('really %s? ' % cstr) sheet.putChanges() Sheet.addCommand('a', 'add-row', 'addNewRows(1, cursorRowIndex); cursorDown(1)', 'append a blank row') -Sheet.addCommand('ga', 'add-rows', 'addNewRows(int(input("add rows: ", value=1)), cursorRowIndex)', 'append N blank rows') -Sheet.addCommand('za', 'addcol-new', 'addColumnAtCursor(SettableColumn()); cursorRight(1)', 'append an empty column') -Sheet.addCommand('gza', 'addcol-bulk', 'addColumnAtCursor(*(SettableColumn() for c in range(int(input("add columns: ")))))', 'append N empty columns') +Sheet.addCommand('ga', 'add-rows', 'addNewRows(int(input("add rows: ", value=1)), cursorRowIndex); cursorDown(1)', 'append N blank rows') +Sheet.addCommand('za', 'addcol-new', 'addColumnAtCursor(SettableColumn(input("column name: "))); cursorRight(1)', 'append an empty column') +Sheet.addCommand('gza', 'addcol-bulk', 'addColumnAtCursor(*(SettableColumn() for c in range(int(input("add columns: "))))); cursorRight(1)', 'append N empty columns') Sheet.addCommand('z^S', 'commit-sheet', 'commit()') diff --git a/visidata/movement.py b/visidata/movement.py index d035ab948..eddb8fee3 100644 --- a/visidata/movement.py +++ b/visidata/movement.py @@ -87,12 +87,20 @@ def nextColRegex(sheet, colregex): vd.fail('no column name matches /%s/' % colregex) + +@Column.property +def visibleWidth(self): + 'Width of column as is displayed in terminal' + vcolidx = self.sheet.visibleCols.index(self) + return self.sheet._visibleColLayout[vcolidx][1] + + Sheet.addCommand(None, 'go-left', 'cursorRight(-1)', 'go left'), Sheet.addCommand(None, 'go-down', 'cursorDown(+1)', 'go down'), Sheet.addCommand(None, 'go-up', 'cursorDown(-1)', 'go up'), Sheet.addCommand(None, 'go-right', 'cursorRight(+1)', 'go right'), -Sheet.addCommand(None, 'go-pagedown', 'cursorDown(nScreenRows); sheet.topRowIndex += nScreenRows', 'scroll one page forward'), -Sheet.addCommand(None, 'go-pageup', 'cursorDown(-nScreenRows); sheet.topRowIndex -= nScreenRows', 'scroll one page backward'), +Sheet.addCommand(None, 'go-pagedown', 'cursorDown(bottomRowIndex-topRowIndex); sheet.topRowIndex = bottomRowIndex', 'scroll one page forward'), +Sheet.addCommand(None, 'go-pageup', 'cursorDown(topRowIndex-bottomRowIndex); sheet.bottomRowIndex = topRowIndex', 'scroll one page backward'), Sheet.addCommand(None, 'go-leftmost', 'sheet.cursorVisibleColIndex = sheet.leftVisibleColIndex = 0', 'go all the way to the left of sheet'), Sheet.addCommand(None, 'go-top', 'sheet.cursorRowIndex = sheet.topRowIndex = 0', 'go all the way to the top of sheet'), @@ -160,13 +168,13 @@ def go_mouse(sheet): Sheet.addCommand(None, 'scroll-leftmost', 'sheet.leftVisibleColIndex = cursorVisibleColIndex', 'scroll sheet to leftmost column') Sheet.addCommand(None, 'scroll-rightmost', 'tmp = cursorVisibleColIndex; pageLeft(); sheet.cursorVisibleColIndex = tmp', 'scroll sheet to rightmost column') -Sheet.addCommand('zl', 'scroll-cells-right', 'cursorCol.hoffset += cursorCol.width-2', 'scroll display of current column to the right') -Sheet.addCommand('zh', 'scroll-cells-left', 'cursorCol.hoffset -= cursorCol.width-2', 'scroll display of current column to the left') -Sheet.addCommand('gzl', 'scroll-cells-rightmost', 'cursorCol.hoffset = -cursorCol.width+1', 'scroll display of current column to the end') +Sheet.addCommand('zl', 'scroll-cells-right', 'cursorCol.hoffset += cursorCol.visibleWidth-2', 'scroll display of current column to the right') +Sheet.addCommand('zh', 'scroll-cells-left', 'cursorCol.hoffset -= cursorCol.visibleWidth-2', 'scroll display of current column to the left') +Sheet.addCommand('gzl', 'scroll-cells-rightmost', 'cursorCol.hoffset = -cursorCol.visibleWidth+2', 'scroll display of current column to the end') Sheet.addCommand('gzh', 'scroll-cells-leftmost', 'cursorCol.hoffset = 0', 'scroll display of current column to the beginning') -Sheet.addCommand('zj', 'scroll-cells-down', 'cursorCol.voffset += 1', 'scroll display of current column down one line') -Sheet.addCommand('zk', 'scroll-cells-up', 'cursorCol.voffset -= 1', 'scroll display of current column up one line') +Sheet.addCommand('zj', 'scroll-cells-down', 'cursorCol.voffset += 1 if cursorCol.height > 1 else fail("multiline column needed for scrolling")', 'scroll display of current column down one line') +Sheet.addCommand('zk', 'scroll-cells-up', 'cursorCol.voffset -= 1 if cursorCol.height > 1 else fail("multiline column needed for scrolling")', 'scroll display of current column up one line') Sheet.addCommand('gzj', 'scroll-cells-bottom', 'cursorCol.voffset = -1', 'scroll display of current column to the bottom') Sheet.addCommand('gzk', 'scroll-cells-top', 'cursorCol.voffset = 0', 'scroll display of current column to the top') diff --git a/visidata/path.py b/visidata/path.py index 5edfe211a..212267940 100644 --- a/visidata/path.py +++ b/visidata/path.py @@ -238,7 +238,8 @@ def read(self, n=None): return r def seek(self, n): - assert n == 0, 'RepeatFile can only seek to beginning' + if n != 0: + vd.error('RepeatFile can only seek to beginning') self.iter = RepeatFileIter(self) def __iter__(self): diff --git a/visidata/pivot.py b/visidata/pivot.py index 86f943a20..234730d27 100644 --- a/visidata/pivot.py +++ b/visidata/pivot.py @@ -45,7 +45,7 @@ def __init__(self, name, groupByCols, pivotCols, **kwargs): self.groupByCols = groupByCols # whose values become rows def isNumericRange(self, col): - return isNumeric(col) and self.source.options.numeric_binning + return vd.isNumeric(col) and self.source.options.numeric_binning def initCols(self): self.columns = [] @@ -167,7 +167,7 @@ def groupRows(self, rowfunc=None): numericCols = [c for c in self.groupByCols if self.isNumericRange(c)] if len(numericCols) > 1: - vd.error('only one numeric column can be binned') + vd.fail('only one numeric column can be binned') numericBins = [] degenerateBinning = False @@ -181,10 +181,11 @@ def groupRows(self, rowfunc=None): if width == 0: # only one value (and maybe errors) numericBins = [(minval, maxval)] - elif numericCols[0].type in (int, vlen) and nbins > width: - # more bins than int vals, just use the vals + elif (numericCols[0].type in (int, vlen) and nbins > (maxval - minval)) or (width == 1): + # (more bins than int vals) or (if bins are of width 1), just use the vals as bins degenerateBinning = True - numericBins = [(minval+i, minval+i) for i in range(maxval-minval+1)] + numericBins = [(val, val) for val in sorted(set(vals))] + nbins = len(numericBins) else: numericBins = [(minval+width*i, minval+width*(i+1)) for i in range(nbins)] @@ -214,7 +215,8 @@ def groupRows(self, rowfunc=None): if not width: binidx = 0 elif degenerateBinning: - binidx = val-minval + # in degenerate binning, each val has its own bin + binidx = numericBins.index((val, val)) else: binidx = int((val-minval)//width) groupRow = numericGroupRows[formatRange(numericCols[0], numericBins[min(binidx, nbins-1)])] diff --git a/visidata/plugins.py b/visidata/plugins.py index 4c82f15d2..8748a3530 100644 --- a/visidata/plugins.py +++ b/visidata/plugins.py @@ -46,9 +46,18 @@ def _checkHash(data, sha): import hashlib return hashlib.sha256(data.strip().encode('utf-8')).hexdigest() == sha +def _pluginColorizer(s,c,r,v): + if not r: return None + ver = _loadedVersion(r) + if not ver: return None + if ver != r.latest_ver: return 'color_warning' + return 'color_working' class PluginsSheet(JsonLinesSheet): - rowtype = "plugins" + rowtype = "plugins" # rowdef: AttrDict of json dict + colorizers = [ + CellColorizer(3, None, _pluginColorizer) + ] def iterload(self): for r in JsonLinesSheet.iterload(self): diff --git a/visidata/pyobj.py b/visidata/pyobj.py index 05babaf01..e135a7d9c 100644 --- a/visidata/pyobj.py +++ b/visidata/pyobj.py @@ -2,7 +2,7 @@ from visidata import * -__all__ = ['PythonSheet', 'expand_cols_deep', 'deduceType', 'closeColumn', 'ListOfDictSheet', 'SheetDict', 'PyobjSheet'] +__all__ = ['PythonSheet', 'expand_cols_deep', 'deduceType', 'closeColumn', 'ListOfDictSheet', 'SheetDict', 'PyobjSheet', 'view'] option('visibility', 0, 'visibility level') option('expand_col_scanrows', 1000, 'number of rows to check when expanding columns (0 = all)') diff --git a/visidata/regex.py b/visidata/regex.py index 152a642dc..e7bb0c615 100644 --- a/visidata/regex.py +++ b/visidata/regex.py @@ -26,7 +26,7 @@ def makeRegexMatcher(regex, origcol): def _regexMatcher(row): m = regex.search(origcol.getDisplayValue(row)) if m: - return m.groups() + return m.groupdict() if m.groupdict() else m.groups() return _regexMatcher @asyncthread @@ -43,7 +43,7 @@ def addRegexColumns(regexMaker, vs, origcol, regexstr): else: exampleRows = vs.rows - cols = [] + cols = {} ncols = 0 # number of new columns added already for r in Progress(exampleRows + [vs.cursorRow]): try: @@ -53,12 +53,22 @@ def addRegexColumns(regexMaker, vs, origcol, regexstr): except Exception as e: vd.exceptionCaught(e) - for _ in range(len(m)-len(cols)): - cols.append(Column(origcol.name+'_re'+str(len(cols)), - getter=lambda col,row,i=len(cols),func=func: func(row)[i], - origCol=origcol)) + if isinstance(m, dict): + for name in m: + if name in cols: + continue + cols[name] = Column(origcol.name+'_'+str(name), + getter=lambda col,row,name=name,func=func: func(row)[name], + origCol=origcol) + elif isinstance(m, (tuple, list)): + for _ in range(len(m)-len(cols)): + cols[len(cols)] = Column(origcol.name+'_re'+str(len(cols)), + getter=lambda col,row,i=len(cols),func=func: func(row)[i], + origCol=origcol) + else: + raise TypeError("addRegexColumns() expects a dict, list, or tuple from regexMaker, but got a "+type(m).__name__) - vs.addColumnAtCursor(*cols) + vs.addColumnAtCursor(*cols.values()) def regexTransform(origcol, instr): @@ -103,5 +113,5 @@ def regex_flags(sheet): Sheet.addCommand(':', 'split-col', 'addRegexColumns(makeRegexSplitter, sheet, cursorCol, input("split regex: ", type="regex-split"))', 'add new columns from regex split; number of columns determined by example row at cursor') Sheet.addCommand(';', 'capture-col', 'addRegexColumns(makeRegexMatcher, sheet, cursorCol, input("match regex: ", type="regex-capture"))', 'add new column from capture groups of regex; requires example row') Sheet.addCommand('*', 'addcol-subst', 'addColumnAtCursor(Column(cursorCol.name + "_re", getter=regexTransform(cursorCol, input("transform column by regex: ", type="regex-subst"))))', 'add column derived from current column, replacing regex with subst (may include \1 backrefs)') -Sheet.addCommand('g*', 'setcol-subst', 'setSubst([cursorCol], selectedRows)', 'regex/subst - modify selected rows in current column, replacing regex with subst, (may include backreferences \\1 etc)') -Sheet.addCommand('gz*', 'setcol-subst-all', 'setSubst(visibleCols, selectedRows)', 'modify selected rows in all visible columns, replacing regex with subst (may include \\1 backrefs)') +Sheet.addCommand('g*', 'setcol-subst', 'setSubst([cursorCol], someSelectedRows)', 'regex/subst - modify selected rows in current column, replacing regex with subst, (may include backreferences \\1 etc)') +Sheet.addCommand('gz*', 'setcol-subst-all', 'setSubst(visibleCols, someSelectedRows)', 'modify selected rows in all visible columns, replacing regex with subst (may include \\1 backrefs)') diff --git a/visidata/save.py b/visidata/save.py index 5048d6802..c419f468f 100644 --- a/visidata/save.py +++ b/visidata/save.py @@ -112,7 +112,8 @@ def saveSheets(vd, givenpath, *vsheets, confirm_overwrite=False): except FileExistsError: pass - assert givenpath.is_dir(), filetype + ' cannot save multiple sheets to non-dir' + if not givenpath.is_dir(): + vd.fail(f'cannot save multiple {filetype} sheets to non-dir') # get save function to call for vs in vsheets: @@ -135,5 +136,6 @@ def save_txt(vd, p, *vsheets): Sheet.addCommand('^S', 'save-sheet', 'vd.saveSheets(inputPath("save to: ", value=getDefaultSaveName()), sheet, confirm_overwrite=options.confirm_overwrite)', 'save current sheet to filename in format determined by extension (default .tsv)') BaseSheet.addCommand('g^S', 'save-all', 'vd.saveSheets(inputPath("save all sheets to: "), *vd.sheets, confirm_overwrite=options.confirm_overwrite)', 'save all sheets to given file or directory)') +IndexSheet.addCommand('g^S', 'save-sheets-selected', 'vd.saveSheets(inputPath("save all sheets to: "), *selectedRows, confirm_overwrite=options.confirm_overwrite)', 'save all sheets to given file or directory)') Sheet.addCommand('', 'save-col', 'save_cols([cursorCol])', 'save current column only to filename in format determined by extension (default .tsv)') Sheet.addCommand('', 'save-col-keys', 'save_cols(keyCols + [cursorCol])', 'save key columns and current column to filename in format determined by extension (default .tsv)') diff --git a/visidata/search.py b/visidata/search.py index 2bc10b5e1..17a56c4dd 100644 --- a/visidata/search.py +++ b/visidata/search.py @@ -74,8 +74,8 @@ def search_expr(sheet, expr, reverse=False): Sheet.addCommand('r', 'search-keys', 'tmp=cursorVisibleColIndex; vd.moveRegex(sheet, regex=input("row key regex: ", type="regex-row", defaultLast=True), columns=keyCols or [visibleCols[0]]); sheet.cursorVisibleColIndex=tmp', 'go to next row with key matching regex') Sheet.addCommand('/', 'search-col', 'vd.moveRegex(sheet, regex=input("/", type="regex", defaultLast=True), columns="cursorCol", backward=False)', 'search for regex forwards in current column'), Sheet.addCommand('?', 'searchr-col', 'vd.moveRegex(sheet, regex=input("?", type="regex", defaultLast=True), columns="cursorCol", backward=True)', 'search for regex backwards in current column'), -Sheet.addCommand('n', 'next-search', 'vd.moveRegex(sheet, reverse=False)', 'go to next match from last regex search'), -Sheet.addCommand('N', 'search-prev', 'vd.moveRegex(sheet, reverse=True)', 'go to previous match from last regex search'), +Sheet.addCommand('n', 'search-next', 'vd.moveRegex(sheet, reverse=False)', 'go to next match from last regex search'), +Sheet.addCommand('N', 'searchr-next', 'vd.moveRegex(sheet, reverse=True)', 'go to previous match from last regex search'), Sheet.addCommand('g/', 'search-cols', 'vd.moveRegex(sheet, regex=input("g/", type="regex", defaultLast=True), backward=False, columns="visibleCols")', 'search for regex forwards over all visible columns'), Sheet.addCommand('g?', 'searchr-cols', 'vd.moveRegex(sheet, regex=input("g?", type="regex", defaultLast=True), backward=True, columns="visibleCols")', 'search for regex backwards over all visible columns'), diff --git a/visidata/selection.py b/visidata/selection.py index 424f047e3..0d0e07361 100644 --- a/visidata/selection.py +++ b/visidata/selection.py @@ -1,5 +1,6 @@ from visidata import vd, Sheet, Progress, option, asyncthread, options, rotateRange, Fanout, undoAttrCopyFunc, copy option('bulk_select_clear', False, 'clear selected rows before new bulk selections', replay=True) +option('some_selected_rows', False, 'if no rows selected, if True, someSelectedRows returns all rows; if False, fails') Sheet.init('_selectedRows', dict) # rowid(row) -> row @@ -94,12 +95,24 @@ def selectedRows(self): return Fanout((r for r in self.rows if self.rowid(r) in self._selectedRows)) @Sheet.property -def someSelectedRows(self): +def onlySelectedRows(self): 'List of selected rows in sheet order. Fail if no rows are selected.' if self.nSelectedRows == 0: vd.fail('no rows selected') return self.selectedRows +@Sheet.property +def someSelectedRows(self): + '''Return a list of rows: + (a) in batch mode, always return selectedRows + (b) in interactive mode, if options.some_selected_rows is True, return selectedRows or all rows if none selected + (c) in interactive mode, if options.some_selected_rows is False, return selectedRows or fail if none selected''' + if options.batch: + return self.selectedRows + if options.some_selected_rows: + return self.selectedRows or self.rows + return self.onlySelectedRows + @Sheet.property def nSelectedRows(self): 'Number of selected rows.' @@ -141,8 +154,10 @@ def addUndoSelection(sheet): Sheet.addCommand('g|', 'select-cols-regex', 'selectByIdx(vd.searchRegex(sheet, regex=input("select regex: ", type="regex", defaultLast=True), columns="visibleCols"))', 'select rows matching regex in any visible column') Sheet.addCommand('g\\', 'unselect-cols-regex', 'unselectByIdx(vd.searchRegex(sheet, regex=input("unselect regex: ", type="regex", defaultLast=True), columns="visibleCols"))', 'unselect rows matching regex in any visible column') -Sheet.addCommand(',', 'select-equal-cell', 'select(gatherBy(lambda r,c=cursorCol,v=cursorTypedValue: c.getTypedValue(r) == v), progress=False)', 'select rows matching current cell in current column') -Sheet.addCommand('g,', 'select-equal-row', 'select(gatherBy(lambda r,currow=cursorRow,vcols=visibleCols: all([c.getTypedValue(r) == c.getTypedValue(currow) for c in vcols])), progress=False)', 'select rows matching current row in all visible columns') +Sheet.addCommand(',', 'select-equal-cell', 'select(gatherBy(lambda r,c=cursorCol,v=cursorDisplay: c.getDisplayValue(r) == v), progress=False)', 'select rows matching current cell in current column') +Sheet.addCommand('g,', 'select-equal-row', 'select(gatherBy(lambda r,currow=cursorRow,vcols=visibleCols: all([c.getDisplayValue(r) == c.getDisplayValue(currow) for c in vcols])), progress=False)', 'select rows matching current row in all visible columns') +Sheet.addCommand('z,', 'select-exact-cell', 'select(gatherBy(lambda r,c=cursorCol,v=cursorTypedValue: c.getTypedValue(r) == v), progress=False)', 'select rows matching current cell in current column') +Sheet.addCommand('gz,', 'select-exact-row', 'select(gatherBy(lambda r,currow=cursorRow,vcols=visibleCols: all([c.getTypedValue(r) == c.getTypedValue(currow) for c in vcols])), progress=False)', 'select rows matching current row in all visible columns') Sheet.addCommand('z|', 'select-expr', 'expr=inputExpr("select by expr: "); select(gatherBy(lambda r, sheet=sheet, expr=expr: sheet.evalExpr(expr, r)), progress=False)', 'select rows matching Python expression in any visible column') Sheet.addCommand('z\\', 'unselect-expr', 'expr=inputExpr("unselect by expr: "); unselect(gatherBy(lambda r, sheet=sheet, expr=expr: sheet.evalExpr(expr, r)), progress=False)', 'unselect rows matching Python expression in any visible column') diff --git a/visidata/settings.py b/visidata/settings.py index 1cdf1ae92..897302cf6 100644 --- a/visidata/settings.py +++ b/visidata/settings.py @@ -140,7 +140,7 @@ def _get(self, k, obj=None): return opt def _set(self, k, v, obj=None, helpstr=''): - self._cache.clear() # invalidate entire cache on any set() + self._cache.clear() # invalidate entire cache on any change return self._opts.set(k, Option(k, v, helpstr), obj) def is_set(self, k, obj=None): @@ -149,7 +149,7 @@ def is_set(self, k, obj=None): return d.get(self._opts.objname(obj), None) def get(self, optname, default=None): - 'Return the value of the given optname option in the options context. `default` is only returned if the option is not defined. An Exception is never raised.' + 'Return the value of the given *optname* option in the options context. *default* is only returned if the option is not defined. An Exception is never raised.' d = self._get(optname, None) if d: return d.value @@ -178,7 +178,7 @@ def set(self, optname, value, obj='override'): curval = opt.value t = type(curval) if value is None and curval is not None: - value = t() # empty value + return self.unset(optname, obj=obj) elif isinstance(value, str) and t is bool: # special case for bool options value = value and (value[0] not in "0fFnN") # ''/0/false/no are false, everything else is true elif type(value) is t: # if right type, no conversion @@ -189,17 +189,29 @@ def set(self, optname, value, obj='override'): value = t(value) if curval != value and self._get(optname, 'global').replayable: - if obj != 'global' and type(obj) is not type: # options set on init aren't recorded - vd.setOption(optname, value, obj) + if obj != 'global' and type(obj) is not type: # global and class options set on init aren't recorded + if vd.cmdlog: + objname = self._opts.objname(obj) + vd.cmdlog.addRow(vd.cmdlog.newRow(sheet=objname, row=optname, + keystrokes='', input=str(value), + longname='set-option')) else: curval = None vd.warning('setting unknown option %s' % optname) return self._set(optname, value, obj) - def unset(self, optname, obj='override'): + def unset(self, optname, obj=None): 'Remove setting value for given context.' - self._opts.unset(optname, obj) + v = self._opts.unset(optname, obj) + opt = self._get(optname) + if vd.cmdlog and opt and opt.replayable: + objname = self._opts.objname(obj) + vd.cmdlog.addRow(vd.cmdlog.newRow(sheet=objname, row=optname, + keystrokes='', input='', + longname='unset-option')) + self._cache.clear() # invalidate entire cache on any change + return v def setdefault(self, optname, value, helpstr): return self._set(optname, value, 'global', helpstr=helpstr) @@ -299,21 +311,23 @@ def unbindkey(cls, keystrokes): @BaseSheet.api def getCommand(sheet, cmd): - 'Return the Command for the given arg, which may be keystrokes, longname, or a Command itself, within the context of `sheet`.' + 'Return the Command for the given *cmd*, which may be keystrokes, longname, or a Command itself, within the context of `sheet`.' if isinstance(cmd, Command): return cmd - longname = vd.bindkeys._get(cmd, obj=sheet) - try: - if longname: - return vd.commands._get(longname, obj=sheet) or vd.fail('no command "%s"' % longname) - else: - return vd.commands._get(cmd, obj=sheet) or vd.fail('no binding for %s' % cmd) - except Exception as exc: - return None + longname = cmd + while vd.bindkeys._get(longname, obj=sheet): + longname = vd.bindkeys._get(longname, obj=sheet) + + if not longname: + vd.fail('no binding for %s' % cmd) + + return vd.commands._get(longname, obj=sheet) or vd.fail('no command "%s"' % longname) def loadConfigFile(fnrc, _globals=None): + if not fnrc: + return p = visidata.Path(fnrc) if _globals is None: _globals = globals() @@ -342,8 +356,8 @@ def addOptions(parser): @VisiData.api def loadConfigAndPlugins(vd, args): # set visidata_dir and config manually before loading config file, so visidata_dir can be set from cli or from $VD_DIR - options.visidata_dir = args.visidata_dir or os.getenv('VD_DIR', '') or options.visidata_dir - options.config = args.config or os.getenv('VD_CONFIG', '') or options.config + options.visidata_dir = args.visidata_dir if args.visidata_dir is not None else os.getenv('VD_DIR', '') or options.visidata_dir + options.config = args.config if args.config is not None else os.getenv('VD_CONFIG', '') or options.config sys.path.append(str(visidata.Path(options.visidata_dir))) sys.path.append(str(visidata.Path(options.visidata_dir)/"plugins-deps")) @@ -359,4 +373,5 @@ def loadConfigAndPlugins(vd, args): loadConfigFile(options.config, vd.getGlobals()) +BaseSheet.bindkey('^M', '^J') # for windows ENTER BaseSheet.addCommand('gO', 'open-config', 'vd.push(open_txt(Path(options.config)))', 'open options.config as text sheet') diff --git a/visidata/sheets.py b/visidata/sheets.py index ae6ce6023..9769c41ee 100644 --- a/visidata/sheets.py +++ b/visidata/sheets.py @@ -4,7 +4,7 @@ import textwrap from visidata import VisiData, Extensible, globalCommand, ColumnAttr, ColumnItem, vd, ENTER, EscapeException, drawcache, drawcache_property, LazyChainMap, asyncthread, ExpectedException -from visidata import (options, theme, isNumeric, Column, option, namedlist, SettableColumn, +from visidata import (options, theme, Column, option, namedlist, SettableColumn, TypedExceptionWrapper, BaseSheet, UNLOADED, vd, clipdraw, ColorAttr, update_attr, colors, undoAttrFunc) import visidata @@ -17,7 +17,6 @@ option('default_height', 10, 'default column height') option('textwrap_cells', True, 'wordwrap text for multiline rows') -option('cmd_after_edit', 'go-down', 'command longname to execute after successful edit') option('quitguard', False, 'confirm before quitting last sheet') option('debug', False, 'exit on error and display stacktrace') option('skip', 0, 'skip N rows before header', replay=True) @@ -56,7 +55,8 @@ theme('disp_endmid_sep', '║', '') # ╽╿┃ theme('disp_endbot_sep', '║', '') # ╽╿┃╜‖ theme('disp_selected_note', '•', '') # - +theme('disp_sort_asc', '↑↟⇞⇡⇧⇑', 'characters for ascending sort') # ↑▲↟↥↾↿⇞⇡⇧⇈⤉⤒⥔⥘⥜⥠⍏˄ˆ +theme('disp_sort_desc', '↓↡⇟⇣⇩⇓', 'characters for descending sort') # ↓▼↡↧⇂⇃⇟⇣⇩⇊⤈⤓⥕⥙⥝⥡⍖˅ˇ theme('color_default', 'normal', 'the default color') theme('color_default_hdr', 'bold', 'color of the column headers') theme('color_bottom_hdr', 'underline', 'color of the bottom header row') @@ -95,11 +95,12 @@ class RecursiveExprException(Exception): class LazyComputeRow: 'Calculate column values as needed.' - def __init__(self, sheet, row): + def __init__(self, sheet, row, col=None): self.row = row + self.col = col self.sheet = sheet if not hasattr(self.sheet, '_lcm'): - self.sheet._lcm = LazyChainMap(sheet, vd) + self.sheet._lcm = LazyChainMap(sheet, vd, col) else: self.sheet._lcm.clear() # reset locals on lcm @@ -107,7 +108,7 @@ def __init__(self, sheet, row): self._keys = [c.name for c in self.sheet.columns] def keys(self): - return self._keys + self.sheet._lcm.keys() + ['row', 'sheet'] + return self._keys + self.sheet._lcm.keys() + ['row', 'sheet', 'col'] def __str__(self): return str(self.as_dict()) @@ -122,26 +123,34 @@ def __getitem__(self, colid): try: i = self._keys.index(colid) c = self.sheet.columns[i] + if c is self.col: + j = self._keys[i+1:].index(colid) + c = self.sheet.columns[i+j+1] - if c in self._usedcols: - raise RecursiveExprException() - self._usedcols.add(c) - ret = c.getTypedValue(self.row) - self._usedcols.remove(c) - return ret except ValueError: try: - return self.sheet._lcm[colid] + c = self.sheet._lcm[colid] except (KeyError, AttributeError): - if colid == 'row': - return self - elif colid == 'sheet': - return self.sheet - raise KeyError(colid) + if colid == 'sheet': return self.sheet + elif colid == 'row': c = self.row + elif colid == 'col': c = self.col + else: + raise KeyError(colid) + + if not isinstance(c, Column): # columns calc in the context of the row of the cell being calc'ed + return c + + if c in self._usedcols: + raise RecursiveExprException() + + self._usedcols.add(c) + ret = c.getTypedValue(self.row) + self._usedcols.remove(c) + return ret class BasicRow(collections.defaultdict): - def __init__(self): - super().__init__(lambda: None) + def __init__(self, *args): + collections.defaultdict.__init__(self, lambda: None, *args) def __bool__(self): return True @@ -310,7 +319,19 @@ def __copy__(self): @property def bottomRowIndex(self): - return max(self._rowLayout.keys()) + return max(self._rowLayout.keys()) if self._rowLayout else self.topRowIndex+self.nScreenRows + + @bottomRowIndex.setter + def bottomRowIndex(self, newidx): + 'Set topRowIndex, by getting height of *newidx* row and going backwards until more than nScreenRows is allocated.' + nrows = 0 + i = 0 + while nrows < self.nScreenRows and newidx-i >= 0: + h = self.calc_height(self.rows[newidx-i]) + nrows += h + i += 1 + + self._topRowIndex = newidx-i+2 def __deepcopy__(self, memo): 'same as __copy__' @@ -321,9 +342,10 @@ def __deepcopy__(self, memo): def __repr__(self): return self.name - def evalExpr(self, expr, row=None): + def evalExpr(self, expr, row=None, col=None): if row: - contexts = vd._evalcontexts.setdefault((self, self.rowid(row)), LazyComputeRow(self, row)) + # contexts are cached by sheet/rowid for duration of drawcycle + contexts = vd._evalcontexts.setdefault((self, self.rowid(row), col), LazyComputeRow(self, row, col=col)) else: contexts = None @@ -525,14 +547,10 @@ def checkCursor(self): # check bounds, scroll if necessary if self.topRowIndex > self.cursorRowIndex: self.topRowIndex = self.cursorRowIndex - else: - if self.topRowIndex < self.cursorRowIndex-self.nScreenRows+1: - self.topRowIndex = self.cursorRowIndex-self.nScreenRows+1 - elif self._rowLayout: # only check this if topRowIndex has not been set (which clears _rowLayout) - bottomRowIndex = self.bottomRowIndex - y, h = self._rowLayout[bottomRowIndex] - if self.cursorRowIndex > bottomRowIndex and y+h > self.nScreenRows: - self._topRowIndex += bottomRowIndex-self.cursorRowIndex+2 + elif self.bottomRowIndex < self.cursorRowIndex: + self.bottomRowIndex = self.cursorRowIndex + elif self.bottomRowIndex == self.cursorRowIndex and self._rowLayout and self._rowLayout[self.bottomRowIndex][1] > 1: + self.bottomRowIndex = self.cursorRowIndex if self.cursorCol and self.cursorCol.keycol: return @@ -611,7 +629,14 @@ def drawColHeader(self, scr, y, h, vcolidx): hdrs = col.name.split('\n') for i in range(h): - name = ' ' # save room at front for LeftMore + name = ' ' # save room at front for LeftMore or sorted arrow + for j, (sortcol, sortdir) in enumerate(self._ordering): + if col is sortcol: + try: + name = self.options.disp_sort_desc[j] if sortdir else self.options.disp_sort_asc[j] + except IndexError: + pass + if h-i-1 < len(hdrs): name += hdrs[::-1][h-i-1] @@ -667,6 +692,7 @@ def draw(self, scr): 'colsep': options.disp_column_sep, 'keysep': options.disp_keycol_sep, 'selectednote': options.disp_selected_note, + 'disp_truncator': options.disp_truncator, } self._rowLayout = {} # [rowidx] -> (y, height) @@ -690,41 +716,16 @@ def draw(self, scr): rowcattr = self._colorize(None, row) - y += self.drawRow(scr, row, self.topRowIndex+rowidx, y, rowcattr, maxheight=self.windowHeight-y, **drawparams) + y += self.drawRow(scr, row, self.topRowIndex+rowidx, y, rowcattr, maxheight=self.windowHeight-y-1, **drawparams) if vcolidx+1 < self.nVisibleCols: scr.addstr(headerRow, self.windowWidth-2, options.disp_more_right, colors.color_column_sep) scr.refresh() - def drawRow(self, scr, row, rowidx, ybase, rowcattr: ColorAttr, maxheight, - isNull='', - topsep='', - midsep='', - botsep='', - endsep='', - keytopsep='', - keymidsep='', - keybotsep='', - endtopsep='', - endmidsep='', - endbotsep='', - colsep='', - keysep='', - selectednote='' - ): - - # sepattr is the attr between cell/columns - sepcattr = update_attr(rowcattr, colors.color_column_sep, 1) - - # apply current row here instead of in a colorizer, because it needs to know dispRowIndex - if rowidx == self.cursorRowIndex: - color_current_row = colors.get_color('color_current_row', 5) - basecellcattr = sepcattr = update_attr(rowcattr, color_current_row) - else: - basecellcattr = rowcattr - - displines = {} # [vcolidx] -> list of lines in that cell + def calc_height(self, row, displines=None, isNull=None): + if displines is None: + displines = {} # [vcolidx] -> list of lines in that cell for vcolidx, (x, colwidth) in sorted(self._visibleColLayout.items()): if x < self.windowWidth: # only draw inside window @@ -733,11 +734,11 @@ def drawRow(self, scr, row, rowidx, ybase, rowcattr: ColorAttr, maxheight, continue col = vcols[vcolidx] cellval = col.getCell(row) - if colwidth > 1 and isNumeric(col): + if colwidth > 1 and vd.isNumeric(col): cellval.display = cellval.display.rjust(colwidth-2) try: - if isNull(cellval.value): + if isNull and isNull(cellval.value): cellval.note = options.disp_note_none cellval.notecolor = 'color_note_type' except (TypeError, ValueError): @@ -754,8 +755,37 @@ def drawRow(self, scr, row, rowidx, ybase, rowcattr: ColorAttr, maxheight, h = len(lines) # of this cell heights.append(min(col.height, h)) - height = min(max(heights), maxheight) or 1 # display even empty rows + return max(heights) + + def drawRow(self, scr, row, rowidx, ybase, rowcattr: ColorAttr, maxheight, + isNull='', + topsep='', + midsep='', + botsep='', + endsep='', + keytopsep='', + keymidsep='', + keybotsep='', + endtopsep='', + endmidsep='', + endbotsep='', + colsep='', + keysep='', + selectednote='', + disp_truncator='' + ): + # sepattr is the attr between cell/columns + sepcattr = update_attr(rowcattr, colors.color_column_sep, 1) + + # apply current row here instead of in a colorizer, because it needs to know dispRowIndex + if rowidx == self.cursorRowIndex: + color_current_row = colors.get_color('color_current_row', 5) + basecellcattr = sepcattr = update_attr(rowcattr, color_current_row) + else: + basecellcattr = rowcattr + displines = {} # [vcolidx] -> list of lines in that cell + height = min(self.calc_height(row, displines), maxheight) or 1 # display even empty rows self._rowLayout[rowidx] = (ybase, height) for vcolidx, (col, cellval, lines) in displines.items(): @@ -821,11 +851,12 @@ def drawRow(self, scr, row, rowidx, ybase, rowcattr: ColorAttr, maxheight, else: sepchars = midsep - clipdraw(scr, y, x, (disp_column_fill if colwidth > 2 else '')+line[hoffset:], cattr.attr, w=colwidth-(1 if note else 0)) + pre = disp_truncator if hoffset != 0 else disp_column_fill + clipdraw(scr, y, x, (pre if colwidth > 2 else '')+line[hoffset:], cattr.attr, w=colwidth-(1 if note else 0)) vd.onMouse(scr, y, x, 1, colwidth, BUTTON3_RELEASED='edit-cell') if x+colwidth+len(sepchars) <= self.windowWidth: - scr.addstr(y, x+colwidth, sepchars, basecellcattr.attr) + scr.addstr(y, x+colwidth, sepchars, sepcattr.attr) for notefunc in vd.rowNoters: ch = notefunc(self, row) @@ -937,6 +968,8 @@ class SheetsSheet(IndexSheet): def reload(self): self.rows = self.source + def sort(self): + self.rows[1:] = sorted(self.rows[1:], key=self.sortkey) @VisiData.property @drawcache @@ -1035,7 +1068,7 @@ def updateColNames(sheet, rows, cols, overwrite=False): Sheet.addCommand('!', 'key-col', 'toggleKeys([cursorCol])', 'toggle current column as a key column') Sheet.addCommand('z!', 'key-col-off', 'unsetKeys([cursorCol])', 'unset current column as a key column') -Sheet.addCommand('e', 'edit-cell', 'cursorCol.setValues([cursorRow], editCell(cursorVisibleColIndex)); options.cmd_after_edit and sheet.execCommand(options.cmd_after_edit)', 'edit contents of current cell') +Sheet.addCommand('e', 'edit-cell', 'cursorCol.setValues([cursorRow], editCell(cursorVisibleColIndex)) if not (cursorRow is None) else fail("no rows to edit")', 'edit contents of current cell') Sheet.addCommand('ge', 'setcol-input', 'cursorCol.setValuesTyped(selectedRows, input("set selected to: ", value=cursorDisplay))', 'set contents of current column for selected rows to same input') Sheet.addCommand('"', 'dup-selected', 'vs=copy(sheet); vs.name += "_selectedref"; vs.reload=lambda vs=vs,rows=selectedRows: setattr(vs, "rows", list(rows)); vd.push(vs)', 'open duplicate sheet with only selected rows'), diff --git a/visidata/shell.py b/visidata/shell.py index 34fa227e6..4a59acc0f 100644 --- a/visidata/shell.py +++ b/visidata/shell.py @@ -65,7 +65,7 @@ def calcValue(self, row): else: args.append(arg) - p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + p = subprocess.Popen(args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) return p.communicate() except Exception as e: vd.exceptionCaught(e) @@ -146,9 +146,8 @@ def removeFile(self, path): else: path.unlink() - def deleteSourceRow(self, rowidx): - self.removeFile(self.rows[rowidx]) - self.rows.pop(rowidx) + def deleteSourceRow(self, r): + self.removeFile(r) def iterload(self): def _walkfiles(p): @@ -188,6 +187,15 @@ def preloadHook(self): def restat(self): vstat.cache_clear() + @asyncthread + def putChanges(self): + self.commitAdds() + self.commitMods() + self.commitDeletes() + + self._deferredDels.clear() + self.reload() + class FileListSheet(DirSheet): _ordering = [] @@ -195,9 +203,17 @@ def iterload(self): for fn in self.source.open_text(): yield Path(fn.rstrip()) + +@VisiData.api +def inputShell(vd): + cmd = vd.input("sh$ ", type="sh") + if '$' not in cmd: + vd.warning('no $column in command') + return cmd + globalCommand('', 'open-dir-current', 'vd.push(vd.currentDirSheet)', 'open Directory Sheet: browse properties of files in current directory') -Sheet.addCommand('z;', 'addcol-sh', 'cmd=input("sh$ ", type="sh"); addShellColumns(cmd, sheet)', 'create new column from bash expression, with $columnNames as variables') +Sheet.addCommand('z;', 'addcol-sh', 'cmd=inputShell(); addShellColumns(cmd, sheet)', 'create new column from bash expression, with $columnNames as variables') DirSheet.addCommand(ENTER, 'open-row', 'vd.push(openSource(cursorRow or fail("no row"), filetype="dir" if cursorRow.is_dir() else LazyComputeRow(sheet, cursorRow).ext))', 'open current file as a new sheet') DirSheet.addCommand('g'+ENTER, 'open-rows', 'for r in selectedRows: vd.push(openSource(r))', 'open selected files as new sheets') diff --git a/visidata/sort.py b/visidata/sort.py index f50f5ad0e..b5972239e 100644 --- a/visidata/sort.py +++ b/visidata/sort.py @@ -37,6 +37,20 @@ def __eq__(self, other): def __lt__(self, other): return other.obj < self.obj + +@Sheet.api +def sortkey(self, r, prog=None): + ret = [] + for col, reverse in self._ordering: + if isinstance(col, str): + col = self.column(col) + val = col.getTypedValue(r) + ret.append(Reversor(val) if reverse else val) + + if prog: + prog.addProgress(1) + return ret + @Sheet.api @asyncthread def sort(self): @@ -45,19 +59,8 @@ def sort(self): return try: with Progress(gerund='sorting', total=self.nRows) as prog: - def sortkey(r): - ret = [] - for col, reverse in self._ordering: - if isinstance(col, str): - col = self.column(col) - val = col.getTypedValue(r) - ret.append(Reversor(val) if reverse else val) - - prog.addProgress(1) - return ret - # must not reassign self.rows: use .sort() instead of sorted() - self.rows.sort(key=sortkey) + self.rows.sort(key=lambda r,self=self,prog=prog: self.sortkey(r, prog=prog)) except TypeError as e: vd.warning('sort incomplete due to TypeError; change column type') vd.exceptionCaught(e, status=False) diff --git a/visidata/statusbar.py b/visidata/statusbar.py index 4d95e3e37..d18219963 100644 --- a/visidata/statusbar.py +++ b/visidata/statusbar.py @@ -1,7 +1,7 @@ import collections import curses -from visidata import vd, VisiData, BaseSheet, Sheet, ColumnItem, Column, RowColorizer, options, colors, wrmap, clipdraw, ExpectedException, update_attr, theme +from visidata import vd, VisiData, BaseSheet, Sheet, ColumnItem, Column, RowColorizer, options, colors, wrmap, clipdraw, ExpectedException, update_attr, theme, MissingAttrFormatter __all__ = ['StatusSheet', 'status', 'error', 'fail', 'warning', 'debug'] @@ -39,7 +39,7 @@ def status(self, *args, priority=0): if not args: return True - k = (priority, args) + k = (priority, tuple(map(str, args))) self.statuses[k] = self.statuses.get(k, 0) + 1 if self.statusHistory: @@ -97,7 +97,7 @@ def leftStatus(sheet): def drawLeftStatus(vd, scr, vs): 'Draw left side of status bar.' cattr = colors.get_color('color_status') - active = vs is vd.sheets[0] # active sheet + active = (vs is vd.sheets[0]) if vd.sheets else False # active sheet if active: cattr = update_attr(cattr, colors.color_active_status, 0) else: @@ -153,7 +153,7 @@ def drawLeftStatus(vd, scr, vs): @VisiData.api def rightStatus(vd, sheet): 'Return right side of status bar. Overrideable.' - return options.disp_rstatus_fmt.format(sheet=sheet, vd=vd) + return MissingAttrFormatter().format(sheet.options.disp_rstatus_fmt, sheet=sheet, vd=vd) @VisiData.api @@ -166,7 +166,7 @@ def drawRightStatus(vd, scr, vs): (vd.rightStatus(vs), 'color_status'), ] - active = vs is vd.sheets[0] # active sheet + active = vs is vd.activeSheet if active: statcolors.append((vd.keystrokes or '', 'color_keystrokes')) @@ -177,7 +177,7 @@ def drawRightStatus(vd, scr, vs): gerund = vs.progresses[0].gerund else: gerund = 'processing' - statcolors.insert(1, (' %s %s…' % (vs.progressPct, gerund), 'color_status')) + statcolors.insert(1, (' %s %s…' % (vs.progressPct, gerund), 'color_working')) if active and vd.currentReplay: statcolors.insert(0, (vd.replayStatus, 'color_status_replay')) diff --git a/visidata/utils.py b/visidata/utils.py index 7c566940e..44780befc 100644 --- a/visidata/utils.py +++ b/visidata/utils.py @@ -1,8 +1,9 @@ import operator +import string 'Various helper classes and functions.' -__all__ = ['AlwaysDict', 'AttrDict', 'moveListItem', 'namedlist', 'classproperty'] +__all__ = ['AlwaysDict', 'AttrDict', 'moveListItem', 'namedlist', 'classproperty', 'MissingAttrFormatter'] class AlwaysDict(dict): @@ -20,6 +21,8 @@ def __getattr__(self, k): try: return self[k] except KeyError: + if k.startswith("__"): + raise AttributeError return None def __setattr__(self, k, v): @@ -96,3 +99,17 @@ def __setattr__(self, k, v): super().__setattr__(k, v) return NamedListTemplate + +class MissingAttrFormatter(string.Formatter): + "formats {} fields with `''`, that would normally result in a raised KeyError or AttributeError; intended for user customisable format strings." + def get_field(self, field_name, *args, **kwargs): + try: + return super().get_field(field_name, *args, **kwargs) + except (KeyError, AttributeError): + return (None, field_name) + + def format_field(self, value, format_spec): + # value is missing + if not value: + return '' + return super().format_field(value, format_spec)