From 73a8b7ea8dfe04d439e0f647c1836e6ed74fff97 Mon Sep 17 00:00:00 2001 From: Saul Pwanson Date: Mon, 12 Oct 2020 19:17:34 -0700 Subject: [PATCH 001/162] [motd-] display motd on status --- visidata/motd.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/visidata/motd.py b/visidata/motd.py index 815c059cf..fc0c7b8a7 100644 --- a/visidata/motd.py +++ b/visidata/motd.py @@ -10,7 +10,7 @@ import random -from visidata import option, options, asyncsingle, urlcache +from visidata import option, options, asyncsingle, urlcache, vd from visidata import __version__ option('motd_url', 'https://visidata.org/motd-'+__version__, 'source of randomized startup messages', sheettype=None) From 8a9ddb66e3418ce0a162b1f31c2b433795c038bc Mon Sep 17 00:00:00 2001 From: anjakefala Date: Mon, 12 Oct 2020 19:46:10 -0700 Subject: [PATCH 002/162] [readme] fix links --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 22bcdecf7..5da1eb3ea 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ A terminal interface for exploring and arranging tabular data. ## Dependencies -- Linux or OS/X +- Linux, OS/X or Windows - Python 3.6+ - python-dateutil - other modules may be required for opening particular data sources @@ -42,7 +42,7 @@ Please see [/install](https://visidata.org/install) for detailed instructions, a $ vd [] ... $ | vd [] -VisiData supports tsv, csv, xlsx, hdf5, sqlite, json and more (see the [list of supported sources](https://visidata.org/man#sources)). +VisiData supports tsv, csv, xlsx, hdf5, sqlite, json and more (see the [list of supported sources](https://visidata.org/formats)). Use `-f ` to force a particular filetype. From e074b75b8d65662aceda80a16bf8effadd082fe8 Mon Sep 17 00:00:00 2001 From: anjakefala Date: Mon, 12 Oct 2020 19:46:27 -0700 Subject: [PATCH 003/162] [docs] remove unnecessary opener --- docs/index.md | 9 --------- 1 file changed, 9 deletions(-) diff --git a/docs/index.md b/docs/index.md index 815d2b121..0194999e3 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,14 +1,5 @@ # VisiData Documentation -## How the documentation is organised - -Inspired by [Daniele Procida](https://www.divio.com/blog/documentation/), the documentation is structured into the following categories: -* Tutorials take you on a guided tour of features in VisiData. This is the recommended place to start if you are new. -* Reference guides contain a comprehensive overview of all out-of-box available commands and options. -* How-to guides are recipes. They will guide you through the steps of common workflows in VisiData. -* [The VisiData book]() is a high-level overview of fundamental concepts and provides background information on VisiData design decisions. - - ## Getting started * [VisiData video demo](https://youtu.be/N1CBDTgGtOU) From 7e75e8752c92d786bea1649cc28af16475932831 Mon Sep 17 00:00:00 2001 From: Saul Pwanson Date: Mon, 12 Oct 2020 19:48:33 -0700 Subject: [PATCH 004/162] [nfc] minor text changes --- CHANGELOG.md | 1 - setup.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dcda915e7..636cdf923 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -89,7 +89,6 @@ ## Dev niceties - Fully automate dev setup with Gitpod.io (thanks @ajkerrigan for PR #673) -## Additions and Improvements # v2.-4 (2020-07-27) diff --git a/setup.py b/setup.py index 9dc51bb6e..dccac273c 100755 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ setup(name='visidata', version=__version__, - description='curses interface for exploring and arranging tabular data', + description='terminal interface for exploring and arranging tabular data', long_description=open('README.md').read(), long_description_content_type='text/markdown', author='Saul Pwanson', From bead7ca123820dc75612fa12e2aed1402e871a05 Mon Sep 17 00:00:00 2001 From: anjakefala Date: Mon, 12 Oct 2020 19:53:02 -0700 Subject: [PATCH 005/162] [2.0.1] bump version to 2.0.1 --- CHANGELOG.md | 5 +++++ README.md | 2 +- setup.py | 2 +- visidata/__init__.py | 2 +- visidata/main.py | 2 +- 5 files changed, 9 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 636cdf923..125403380 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # VisiData version history +# v2.0.1 (2020-10-13) + +## Bugfixes + - Fix printing of motd to status + # v2.0 (2020-10-12) ## Additions and Improvements diff --git a/README.md b/README.md index 5da1eb3ea..2d5f87c28 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ [![Gitpod ready-to-code](https://img.shields.io/badge/Gitpod-ready--to--code-blue?logo=gitpod)](https://gitpod.io/#https://github.com/saulpw/visidata) -# VisiData v2.0 [![CircleCI](https://circleci.com/gh/saulpw/visidata/tree/develop.svg?style=svg)](https://circleci.com/gh/saulpw/visidata/tree/develop) +# VisiData v2.0.1 [![CircleCI](https://circleci.com/gh/saulpw/visidata/tree/develop.svg?style=svg)](https://circleci.com/gh/saulpw/visidata/tree/develop) A terminal interface for exploring and arranging tabular data. diff --git a/setup.py b/setup.py index dccac273c..9833bf4de 100755 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ from setuptools import setup # tox can't actually run python3 setup.py: https://github.com/tox-dev/tox/issues/96 #from visidata import __version__ -__version__ = '2.0' +__version__ = '2.0.1' setup(name='visidata', version=__version__, diff --git a/visidata/__init__.py b/visidata/__init__.py index 55a95030b..683ac419a 100644 --- a/visidata/__init__.py +++ b/visidata/__init__.py @@ -1,6 +1,6 @@ 'VisiData: a curses interface for exploring and arranging tabular data' -__version__ = '2.0' +__version__ = '2.0.1' __version_info__ = 'VisiData v' + __version__ __author__ = 'Saul Pwanson ' __status__ = 'Production/Stable' diff --git a/visidata/main.py b/visidata/main.py index 5ebda7dd4..fda66ce34 100755 --- a/visidata/main.py +++ b/visidata/main.py @@ -2,7 +2,7 @@ # Usage: $0 [] [ ...] # $0 [] --play [--batch] [-w ] [-o ] [field=value ...] -__version__ = '2.0' +__version__ = '2.0.1' __version_info__ = 'saul.pw/VisiData v' + __version__ from copy import copy From 7a2d8a1e00a4804a394f8f9b30bb36ad1c6e6240 Mon Sep 17 00:00:00 2001 From: AJ Kerrigan Date: Tue, 13 Oct 2020 10:05:07 -0400 Subject: [PATCH 006/162] add traceback for deprecated calls Also import some globals to avoid "not found" errors. --- visidata/deprecated.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/visidata/deprecated.py b/visidata/deprecated.py index cb383566f..4c32de24a 100644 --- a/visidata/deprecated.py +++ b/visidata/deprecated.py @@ -1,4 +1,4 @@ -from visidata import VisiData +from visidata import Column, PyobjSheet, VisiData, vd import visidata alias = visidata.BaseSheet.bindkey @@ -6,11 +6,15 @@ def deprecated(ver, instead=''): def decorator(func): def wrapper(*args, **kwargs): - # ideally would include a stacktrace + import traceback + + for line in reversed(traceback.extract_stack(limit=6)[:-1]): + vd.warning(f' file {line.filename} at line {line.lineno} in {line.name}') + vd.warning(f'Deprecated call traceback (most recent last):') msg = f'{func.__name__} deprecated since v{ver}' if instead: msg += f'; use {instead}' - visidata.warning(msg) + vd.warning(msg) return func(*args, **kwargs) return wrapper return decorator @@ -30,7 +34,7 @@ def copyToClipboard(value): @deprecated('1.6') def replayableOption(optname, default, helpstr): - option(optname, default, helpstr, replay=True) + vd.option(optname, default, helpstr, replay=True) @deprecated('1.6') def SubrowColumn(*args, **kwargs): From 58d6eabec7ec7090acf45c79c836db78508f49e6 Mon Sep 17 00:00:00 2001 From: anjakefala Date: Tue, 13 Oct 2020 07:45:07 -0700 Subject: [PATCH 007/162] [man] remove redundant normal_text Addresses #718 --- visidata/man/vd.1 | 4 ++-- visidata/man/vd.inc | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/visidata/man/vd.1 b/visidata/man/vd.1 index ccd6b026a..738d03d24 100644 --- a/visidata/man/vd.1 +++ b/visidata/man/vd.1 @@ -369,9 +369,9 @@ open .It Ic "z+" Ar aggregator .No display result of Ar aggregator No over values in selected rows for current column .It Ic " &" -.No concatenate top two sheets in Sy Sheets Stack No +.No concatenate top two sheets in Sy Sheets Stack .It Ic "g&" -.No concatenate all sheets in Sy Sheets Stack No +.No concatenate all sheets in Sy Sheets Stack .Pp .El .Ss Data Visualization diff --git a/visidata/man/vd.inc b/visidata/man/vd.inc index a9dbbe98c..565a74268 100644 --- a/visidata/man/vd.inc +++ b/visidata/man/vd.inc @@ -369,9 +369,9 @@ open .It Ic "z+" Ar aggregator .No display result of Ar aggregator No over values in selected rows for current column .It Ic " &" -.No concatenate top two sheets in Sy Sheets Stack No +.No concatenate top two sheets in Sy Sheets Stack .It Ic "g&" -.No concatenate all sheets in Sy Sheets Stack No +.No concatenate all sheets in Sy Sheets Stack .Pp .El .Ss Data Visualization @@ -654,7 +654,7 @@ abort async threads for current/selected sheets(s) .It Ic zO .No edit sheet options (apply to Sy current sheet No only) .It Ic gO -.No open Sy options.config No as Sy TextSheet No +.No open Sy options.config No as Sy TextSheet .El .Bl -inset -compact .It (sheet-specific commands) From 6fd3355169efea5c8621c1c6a207c6c44af02df5 Mon Sep 17 00:00:00 2001 From: anjakefala Date: Tue, 13 Oct 2020 20:36:33 -0700 Subject: [PATCH 008/162] [man] fix all build warnings Closes #718 --- visidata/man/vd.1 | 7 ++++--- visidata/man/vd.inc | 5 +++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/visidata/man/vd.1 b/visidata/man/vd.1 index 738d03d24..614c6c4ad 100644 --- a/visidata/man/vd.1 +++ b/visidata/man/vd.1 @@ -1,4 +1,4 @@ -.Dd July 23, 2020 +.Dd Oct 13, 2020 .Dt vd \&1 "Quick Reference Guide" .Os Linux/MacOS . @@ -654,7 +654,7 @@ abort async threads for current/selected sheets(s) .It Ic zO .No edit sheet options (apply to Sy current sheet No only) .It Ic gO -.No open Sy options.config No as Sy TextSheet No +.No open Sy options.config No as Sy TextSheet .El .Bl -inset -compact .It (sheet-specific commands) @@ -1121,7 +1121,7 @@ color of active clue .No open a blank sheet named Ar newfile No if file does not exist .Pp .Dl Ic vd sample.xlsx +:sheet1:2:3 -.No launch with Sy sheet1 No at top-of-stack, and cursor at column Sy 2 No and row Sy 3 No +.No launch with Sy sheet1 No at top-of-stack, and cursor at column Sy 2 No and row Sy 3 .Pp .Dl Ic vd -P open-plugins .No preplay longname Sy open-plugins No before starting the session @@ -1181,6 +1181,7 @@ These are the supported sources: .Bl -inset -compact -offset xxx .It Uses the safe YAML loader, which supports the most common uses of YAML. .El +.El .Pp .Bl -inset -compact -offset xxx .It Sy pcap No ( requires Sy dpkt Ns , Sy dnslib Ns ) diff --git a/visidata/man/vd.inc b/visidata/man/vd.inc index 565a74268..237bb79b5 100644 --- a/visidata/man/vd.inc +++ b/visidata/man/vd.inc @@ -1,4 +1,4 @@ -.Dd July 23, 2020 +.Dd Oct 13, 2020 .Dt vd \&1 "Quick Reference Guide" .Os Linux/MacOS . @@ -819,7 +819,7 @@ overwrite existing files without confirmation .No open a blank sheet named Ar newfile No if file does not exist .Pp .Dl Ic vd sample.xlsx +:sheet1:2:3 -.No launch with Sy sheet1 No at top-of-stack, and cursor at column Sy 2 No and row Sy 3 No +.No launch with Sy sheet1 No at top-of-stack, and cursor at column Sy 2 No and row Sy 3 .Pp .Dl Ic vd -P open-plugins .No preplay longname Sy open-plugins No before starting the session @@ -879,6 +879,7 @@ These are the supported sources: .Bl -inset -compact -offset xxx .It Uses the safe YAML loader, which supports the most common uses of YAML. .El +.El .Pp .Bl -inset -compact -offset xxx .It Sy pcap No ( requires Sy dpkt Ns , Sy dnslib Ns ) From 7f8a15fc213ff82ac6f3dde7c321b1c96d3813b2 Mon Sep 17 00:00:00 2001 From: anjakefala Date: Tue, 13 Oct 2020 20:50:22 -0700 Subject: [PATCH 009/162] [docs 2.0] link to both versions of manpage --- docs/index.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/index.md b/docs/index.md index 0194999e3..8f8e40799 100644 --- a/docs/index.md +++ b/docs/index.md @@ -13,9 +13,10 @@ ## References -* [quick reference guide](/man) +* [quick reference guide for 2.0](/man) * all available commands and options * also available as a manpage via `man vd` and from inside VisiData with `Ctrl+H` +* [quick reference guide for 1.5.2](/docs/v1.5.2/man) * [keyboard layout of commands](/docs/kblayout) ## 'How to' recipes From 30fe91c7cc4069b23ce976a72230c3e892d51bc2 Mon Sep 17 00:00:00 2001 From: anjakefala Date: Tue, 13 Oct 2020 21:14:03 -0700 Subject: [PATCH 010/162] [man] replace optional sources with link to /formats --- visidata/man/vd.1 | 65 ++++++--------------------------------------- visidata/man/vd.inc | 65 ++++++--------------------------------------- 2 files changed, 16 insertions(+), 114 deletions(-) diff --git a/visidata/man/vd.1 b/visidata/man/vd.1 index 614c6c4ad..d5ad85025 100644 --- a/visidata/man/vd.1 +++ b/visidata/man/vd.1 @@ -1144,7 +1144,7 @@ For example: Functions defined in .visidatarc are available in python expressions (e.g. in derived columns). . .Sh SUPPORTED SOURCES -These are the supported sources: +Core VisiData includes these sources: .Pp .Bl -inset -compact -offset xxx .It Sy tsv No (tab-separated value) @@ -1177,68 +1177,24 @@ These are the supported sources: .El .Pp .Bl -inset -compact -offset xxx -.It Sy yaml Ns / Ns Sy yml No (requires Sy PyYAML Ns ) -.Bl -inset -compact -offset xxx -.It Uses the safe YAML loader, which supports the most common uses of YAML. -.El -.El -.Pp -.Bl -inset -compact -offset xxx -.It Sy pcap No ( requires Sy dpkt Ns , Sy dnslib Ns ) +.It Sy sqlite .Bl -inset -compact -offset xxx -.It View and investigate captured network traffic in a tabular format. +.It May include multiple tables. The initial sheet is the table directory; +.Sy Enter No loaders the entrie table into memory. Sy z^S No saves modifications to source. .El .El .Pp -.Bl -inset -compact -offset xxx -.It Sy png No (requires Sy pypng Ns ) -.Bl -inset -compact -offset xxx -.It Pixels can be edited and saved in data form. Images can be plotted with Ic "\&." No (dot). -.El -.El -. -.Pp -The following URL schemes are supported: +URL schemes are also supported: .Bl -inset -compact -offset xxx .It Sy http No (requires Sy requests Ns ); can be used as transport for with another filetype -.It Sy postgres No (requires Sy psycopg2 Ns ) -.El -. -.Pp -.Bl -inset -compact -.It The following sources may include multiple tables. The initial sheet is the table directory; -.Sy Enter No loads the entire table into memory. .El . .Pp -.Bl -inset -compact -offset xxx -.It Sy sqlite -.It Sy xlsx No (requires Sy openpyxl Ns ) -.It Sy xls No (requires Sy xlrd Ns ) -.It Sy hdf5 No (requires Sy h5py Ns ) -.It Sy ttf Ns / Ns Sy otf No (requires Sy fonttools Ns ) -.It Sy mbtiles No (requires Sy mapbox-vector-tile Ns ) -.It Sy htm Ns / Ns Sy html No (requires Sy lxml Ns ) -.It Sy xml No (requires Sy lxml Ns ) -.Bl -tag -width XXXX -compact -offset XXX -.It Sy " v" -show only columns in current row attributes -.It Sy za -add column for xml attribute -.El -.It Sy xpt No (SAS; requires Sy xport Ns ) -.It Sy sas7bdat No (SAS; requires Sy sas7bdat Ns ) -.It Sy sav No (SPSS; requires Sy savReaderWriter Ns ) -.It Sy dta No (Stata; requires Sy pandas Ns ) -.It Sy shp No (requires Sy pyshp Ns ) -.El +For a list of all remaining formats supported by VisiData, see https://visidata.org/formats. .Pp In addition, .Sy .zip Ns , Sy .gz Ns , Sy .bz2 Ns , and Sy .xz No files are decompressed on the fly. .Pp -.No VisiData has an adapter for Sy pandas Ns . To load a file format which is supported by Sy pandas Ns , pass Sy -f pandas data.foo Ns . This will call Sy pandas.read_foo() Ns . -.Pp -.No For example, Sy vd -f pandas data.parquet No loads a parquet file. Note that when using the Sy pandas No loader, the Sy .fileformat No file extension is mandatory . .Sh SUPPORTED OUTPUT FORMATS These are the supported savers: @@ -1248,18 +1204,13 @@ These are the supported savers: .It Sy csv No (comma-separated value) .It Sy json No (one object with all rows) .It Sy jsonl Ns / Ns Sy ndjson Ns / Ns Sy ldjson No (one object per line/row) -.It Sy sqlite No (multisave capable) .Bl -inset -compact -offset xxx .It All expanded subcolumns must be closed (with Sy "\&)" Ns ) to retain the same structure. -.It Sy .shp No files can be saved as Sy geoJSON Ns . .El -.It Sy md No (org-mode compatible markdown table) -.It Sy htm Ns / Ns Sy html No (requires Sy lxml Ns ) -.It Sy png No (requires Sy pypng Ns ) +.It Sy sqlite No (save to source with Sy z^S Ns ) +.It Sy md No (markdown table) .El .Pp -.No Multisave is supported by Sy html Ns , Sy md Ns , and Sy txt Ns ; Sy g^S No will save all sheets into a single output file. -.Pp . .Sh AUTHOR .Nm VisiData diff --git a/visidata/man/vd.inc b/visidata/man/vd.inc index 237bb79b5..027699b9d 100644 --- a/visidata/man/vd.inc +++ b/visidata/man/vd.inc @@ -842,7 +842,7 @@ For example: Functions defined in .visidatarc are available in python expressions (e.g. in derived columns). . .Sh SUPPORTED SOURCES -These are the supported sources: +Core VisiData includes these sources: .Pp .Bl -inset -compact -offset xxx .It Sy tsv No (tab-separated value) @@ -875,68 +875,24 @@ These are the supported sources: .El .Pp .Bl -inset -compact -offset xxx -.It Sy yaml Ns / Ns Sy yml No (requires Sy PyYAML Ns ) -.Bl -inset -compact -offset xxx -.It Uses the safe YAML loader, which supports the most common uses of YAML. -.El -.El -.Pp -.Bl -inset -compact -offset xxx -.It Sy pcap No ( requires Sy dpkt Ns , Sy dnslib Ns ) +.It Sy sqlite .Bl -inset -compact -offset xxx -.It View and investigate captured network traffic in a tabular format. +.It May include multiple tables. The initial sheet is the table directory; +.Sy Enter No loaders the entrie table into memory. Sy z^S No saves modifications to source. .El .El .Pp -.Bl -inset -compact -offset xxx -.It Sy png No (requires Sy pypng Ns ) -.Bl -inset -compact -offset xxx -.It Pixels can be edited and saved in data form. Images can be plotted with Ic "\&." No (dot). -.El -.El -. -.Pp -The following URL schemes are supported: +URL schemes are also supported: .Bl -inset -compact -offset xxx .It Sy http No (requires Sy requests Ns ); can be used as transport for with another filetype -.It Sy postgres No (requires Sy psycopg2 Ns ) -.El -. -.Pp -.Bl -inset -compact -.It The following sources may include multiple tables. The initial sheet is the table directory; -.Sy Enter No loads the entire table into memory. .El . .Pp -.Bl -inset -compact -offset xxx -.It Sy sqlite -.It Sy xlsx No (requires Sy openpyxl Ns ) -.It Sy xls No (requires Sy xlrd Ns ) -.It Sy hdf5 No (requires Sy h5py Ns ) -.It Sy ttf Ns / Ns Sy otf No (requires Sy fonttools Ns ) -.It Sy mbtiles No (requires Sy mapbox-vector-tile Ns ) -.It Sy htm Ns / Ns Sy html No (requires Sy lxml Ns ) -.It Sy xml No (requires Sy lxml Ns ) -.Bl -tag -width XXXX -compact -offset XXX -.It Sy " v" -show only columns in current row attributes -.It Sy za -add column for xml attribute -.El -.It Sy xpt No (SAS; requires Sy xport Ns ) -.It Sy sas7bdat No (SAS; requires Sy sas7bdat Ns ) -.It Sy sav No (SPSS; requires Sy savReaderWriter Ns ) -.It Sy dta No (Stata; requires Sy pandas Ns ) -.It Sy shp No (requires Sy pyshp Ns ) -.El +For a list of all remaining formats supported by VisiData, see https://visidata.org/formats. .Pp In addition, .Sy .zip Ns , Sy .gz Ns , Sy .bz2 Ns , and Sy .xz No files are decompressed on the fly. .Pp -.No VisiData has an adapter for Sy pandas Ns . To load a file format which is supported by Sy pandas Ns , pass Sy -f pandas data.foo Ns . This will call Sy pandas.read_foo() Ns . -.Pp -.No For example, Sy vd -f pandas data.parquet No loads a parquet file. Note that when using the Sy pandas No loader, the Sy .fileformat No file extension is mandatory . .Sh SUPPORTED OUTPUT FORMATS These are the supported savers: @@ -946,18 +902,13 @@ These are the supported savers: .It Sy csv No (comma-separated value) .It Sy json No (one object with all rows) .It Sy jsonl Ns / Ns Sy ndjson Ns / Ns Sy ldjson No (one object per line/row) -.It Sy sqlite No (multisave capable) .Bl -inset -compact -offset xxx .It All expanded subcolumns must be closed (with Sy "\&)" Ns ) to retain the same structure. -.It Sy .shp No files can be saved as Sy geoJSON Ns . .El -.It Sy md No (org-mode compatible markdown table) -.It Sy htm Ns / Ns Sy html No (requires Sy lxml Ns ) -.It Sy png No (requires Sy pypng Ns ) +.It Sy sqlite No (save to source with Sy z^S Ns ) +.It Sy md No (markdown table) .El .Pp -.No Multisave is supported by Sy html Ns , Sy md Ns , and Sy txt Ns ; Sy g^S No will save all sheets into a single output file. -.Pp . .Sh AUTHOR .Nm VisiData From b92f009150b9e5249670cded614cd03ca6ec844b Mon Sep 17 00:00:00 2001 From: Saul Pwanson Date: Tue, 13 Oct 2020 12:26:11 -0700 Subject: [PATCH 011/162] [texttables-] bind fmt in scope #723 --- visidata/loaders/texttables.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/visidata/loaders/texttables.py b/visidata/loaders/texttables.py index aaccd2112..39a086b60 100644 --- a/visidata/loaders/texttables.py +++ b/visidata/loaders/texttables.py @@ -4,7 +4,7 @@ try: import tabulate for fmt in tabulate.tabulate_formats: - def save_table(path, *sheets): + def save_table(path, *sheets, fmt=fmt): import tabulate with path.open_text(mode='w') as fp: From b15d97071828ad58bd7c5c427e5390df1bd878cb Mon Sep 17 00:00:00 2001 From: Saul Pwanson Date: Wed, 14 Oct 2020 20:04:29 -0700 Subject: [PATCH 012/162] [xlsx] add "active" column #726 --- visidata/loaders/xlsx.py | 1 + 1 file changed, 1 insertion(+) diff --git a/visidata/loaders/xlsx.py b/visidata/loaders/xlsx.py index 8391288c5..7b4987194 100644 --- a/visidata/loaders/xlsx.py +++ b/visidata/loaders/xlsx.py @@ -14,6 +14,7 @@ class XlsxIndexSheet(IndexSheet): ColumnAttr('name', width=0), # visidata Sheet name ColumnAttr('nRows', type=int), ColumnAttr('nCols', type=int), + Column('active', getter=lambda col,row: row.source is col.sheet.workbook.active), ] nKeys = 1 From 5ff10fb890ce0f348186fe24570a706e5263a0ec Mon Sep 17 00:00:00 2001 From: Saul Pwanson Date: Wed, 14 Oct 2020 22:22:05 -0700 Subject: [PATCH 013/162] [dev] update changelog --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 125403380..8120d2301 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # VisiData version history +# v2.?.? (????) + + - [xlsx] add active column #726 + +## Bugfixes + + - [tabulate] fix savers to save in their own format #723 + # v2.0.1 (2020-10-13) ## Bugfixes From 7ade58d97377eb4f67adec22bd709924b342cd56 Mon Sep 17 00:00:00 2001 From: anjakefala Date: Wed, 14 Oct 2020 22:56:00 -0700 Subject: [PATCH 014/162] [tabulate] update with correct rst Related to #723 --- tests/golden/save-benchmarks.rst | 103 ++++++++++++++++--------------- 1 file changed, 53 insertions(+), 50 deletions(-) diff --git a/tests/golden/save-benchmarks.rst b/tests/golden/save-benchmarks.rst index 6debff9d4..8288cb9d4 100644 --- a/tests/golden/save-benchmarks.rst +++ b/tests/golden/save-benchmarks.rst @@ -1,50 +1,53 @@ -|| Date || Customer || SKU || Item || Quantity || Unit || Paid || -| 2018-07-03 | Robert Armstrong | FOOD213 | BFF Oh My Gravy! Beef & Salmon 2.8oz | 4 | 12.95 | $51.8 | -| 2018-07-03 | Kyle Kennedy | FOOD121 | Food, Adult Cat - 3.5 oz | 1 | 4.22 | $4.22 | -| 2018-07-05 | Douglas "Dougie" Powers | FOOD121 | Food, Adult Cat 3.5 oz | 1 | 4.22 | $4.22 | -| 2018-07-06 | 桜 高橋 (Sakura Takahashi) | FOOD122 | Food, Senior Wet Cat - 3 oz | 12 | 1.29 | 157¥ | -| 2018-07-10 | David Attenborough | NSCT201 | Food, Salamander | 30 | 0.05 | $1.5 | -| 2018-07-10 | Susan Ashworth | CAT060 | Cat, Korat (Felis catus) | 1 | 720.42 | $720.42 | -| 2018-07-10 | Susan Ashworth | FOOD130 | Food, Kitten 3kg | 1 | 14.94 | $14.94 | -| 2018-07-13 | Wil Wheaton | NSCT523 | Monster, Rust (Monstrus gygaxus) | 1 | 39.95 | $39.95 | -| 2018-07-13 | Robert Armstrong | FOOD216 | BFF Oh My Gravy! Chicken & Shrimp 2.8oz | 4 | 12.95 | $51.8 | -| 2018-07-17 | Robert Armstrong | FOOD217 | BFF Oh My Gravy! Duck & Tuna 2.8oz | 4 | 12.95 | $51.8 | -| 2018-07-17 | Helen Halestorm | LAGO342 | Rabbit (Oryctolagus cuniculus) | 2 | 32.94 | $65.88 | -| 2018-07-18 | 桜 高橋 (Sakura Takahashi) | FOOD122 | Food, Senior Wet Cat - 3 oz | 6 | 1.29 | 157¥ | -| 2018-07-19 | Rubeus Hagrid | FOOD170 | Food, Dog - 5kg | 5 | 44.95 | $224.75 | -| 2018-07-20 | Jon Arbuckle | FOOD167 | Food, Premium Wet Cat - 3.5 oz | 50 | 3.95 | $197.5 | -| 2018-07-23 | Robert Armstrong | FOOD215 | BFF Oh My Gravy! Lamb & Tuna 2.8oz | 4 | 12.95 | $51.8 | -| 2018-07-23 | Douglas "Dougie" Powers | TOY235 | Laser Pointer | 1 | 16.12 | $16.12 | -| 2018-07-24 | 桜 高橋 (Sakura Takahashi) | FOOD122 | Food, Senior Wet Cat - 3 oz | 3 | 1.29 | 157¥ | -| 2018-07-26 | Douglas "Dougie" Powers | FOOD420 | Food, Shark - 10 kg | 1 | 15.7 | $15.7 | -| 2018-07-27 | 桜 高橋 (Sakura Takahashi) | FOOD122 | Food, Senior Wet Cat - 3 oz | 3 | 1.29 | 157¥ | -| 2018-07-30 | 桜 高橋 (Sakura Takahashi) | RETURN | Food, Senior Wet Cat - 3 oz | 1 | 1.29 | 157¥ | -| 2018-07-31 | Rubeus Hagrid | CAT060 | Food, Dragon - 50kg | 5 | 720.42 | $3602.1 | -| 2018-08-01 | David Attenborough | FOOD360 | Food, Rhinocerous - 50kg | 4 | 5.72 | $22.88 | -| 2018-08-02 | Susan Ashworth | CAT110 | Cat, Maine Coon (Felix catus) | 1 | 1309.68 | $1309.68 | -| 2018-08-02 | Susan Ashworth | FOOD130 | Food, Kitten 3kg | 3 | 14.94 | $44.82 | -| 2018-08-06 | Robert Armstrong | FOOD212 | BFF Oh My Gravy! Beef & Chicken 2.8oz | 4 | 12.95 | $51.8 | -| 2018-08-07 | Juan Johnson | REPT082 | Kingsnake, California (Lampropeltis getula) | 1 | 89.95 | $89.95 | -| 2018-08-07 | Juan Johnson | RDNT443 | Mouse, Pinky (Mus musculus) | 1 | 1.49 | $1.49 | -| 2018-08-10 | Robert Armstrong | FOOD211 | BFF Oh My Gravy! Chicken & Turkey 2.8oz | 4 | 12.95 | $51.8 | -| 2018-08-13 | Monica Johnson | RDNT443 | Mouse, Pinky (Mus musculus) | 1 | 1.49 | $1.49 | -| 2018-08-13 | María Fernández | FOOD146 | Forti Diet Prohealth Mouse/Rat 3lbs | 2 | 2 | $4.0 | -| 2018-08-15 | Mr. Praline | RETURN | Parrot, Norwegian Blue (Mopsitta tanta) | 1 | 2300 | -$2300.0 | -| 2018-08-15 | Kyle Kennedy | FOOD121 | Food, Adult Cat - 3.5 oz | 2 | 4.22 | $8.44 | -| 2018-08-16 | Helen Halestorm | RETURN | Rabbit (Oryctolagus cuniculus) | 6 | 0 | $0.0 | -| 2018-08-16 | Kyle Kennedy | DOG010 | Dog, Golden Retriever (Canis lupus familiaris) | 1 | 2495.99 | $2495.99 | -| 2018-08-16 | Michael Smith | BIRD160 | Parakeet, Blue (Melopsittacus undulatus) | 1 | 29.95 | $31.85 | -| 2018-08-17 | Rubeus Hagrid | NSCT201 | Food, Spider | 5 | 0.05 | $0.25 | -| 2018-08-20 | Kyle Kennedy | RETURN | Dog, Golden Retriever (Canis lupus familiaris) | 1 | 1247.99 | -$1247.99 | -| 2018-08-20 | Monica Johnson | NSCT201 | Crickets, Adult Live (Gryllus assimilis) | 30 | 0.05 | $1.5 | -| 2018-08-20 | David Attenborough | NSCT084 | Food, Pangolin | 30 | 0.17 | $5.10 | -| 2018-08-21 | Robert Armstrong | FOOD214 | BFF Oh My Gravy! Duck & Salmon 2.8oz | 4 | 12.95 | $51.8 | -| 2018-08-22 | David Attenborough | BIRD160 | Food, Quoll | 1 | 29.95 | $29.95 | -| 2018-08-22 | Jon Arbuckle | FOOD170 | Food, Adult Dog - 5kg | 1 | 44.95 | $44.95 | -| 2018-08-24 | Robert Armstrong | FOOD218 | BFF Oh My Gravy! Chicken & Salmon 2.8oz | 4 | 12.95 | $51.8 | -| 2018-08-27 | Monica Johnson | NSCT443 | Mealworms, Large (Tenebrio molitor) 100ct | 1 | 1.99 | $1.99 | -| 2018-08-28 | Susan Ashworth | CAT020 | Cat, Scottish Fold (Felis catus) | 1 | 1964.53 | $1964.53 | -| 2018-08-28 | Susan Ashworth | FOOD130 | Food, Kitten 3kg | 2 | 14.94 | $29.88 | -| 2018-08-29 | Robert Armstrong | FOOD219 | BFF Oh My Gravy! Chicken & Pumpkin 2.8oz | 4 | 12.95 | $51.8 | -| 2018-08-31 | Robert Armstrong | FOOD219 | BFF Oh My Gravy! Chicken & Pumpkin 2.8oz | 144 | 12.95 | $1864.8 | -| 2018-08-31 | Juan Johnson | REPT217 | Lizard, Spinytail (Uromastyx ornatus) | 1 | 99.95 | $99.95 | \ No newline at end of file +================ ========================== ======= ============================================== ========== ========= ========= +Date Customer SKU Item Quantity Unit Paid +================ ========================== ======= ============================================== ========== ========= ========= +7/3/2018 1:47p Robert Armstrong FOOD213 BFF Oh My Gravy! Beef & Salmon 2.8oz 4 $12.95 $51.8 +7/3/2018 3:32p Kyle Kennedy FOOD121 Food, Adult Cat - 3.5 oz 1 $4.22 $4.22 +7/5/2018 4:15p Douglas "Dougie" Powers FOOD121 Food, Adult Cat 3.5 oz 1 $4.22 $4.22 +7/6/2018 12:15p 桜 高橋 (Sakura Takahashi) FOOD122 Food, Senior Wet Cat - 3 oz 12 $1.29 157¥ +7/10/2018 10:28a David Attenborough NSCT201 Food, Salamander 30 $.05 $1.5 +7/10/2018 5:23p Susan Ashworth CAT060 Cat, Korat (Felis catus) 1 $720.42 $720.42 +7/10/2018 5:23p Susan Ashworth FOOD130 Food, Kitten 3kg 1 $14.94 $14.94 +7/13/2018 10:26a Wil Wheaton NSCT523 Monster, Rust (Monstrus gygaxus) 1 $39.95 $39.95 +7/13/2018 3:49p Robert Armstrong FOOD216 BFF Oh My Gravy! Chicken & Shrimp 2.8oz 4 $12.95 $51.8 +7/17/2018 9:01a Robert Armstrong FOOD217 BFF Oh My Gravy! Duck & Tuna 2.8oz 4 $12.95 $51.8 +7/17/2018 11:30a Helen Halestorm LAGO342 Rabbit (Oryctolagus cuniculus) 2 $32.94 $65.88 +7/18/2018 12:16p 桜 高橋 (Sakura Takahashi) FOOD122 Food, Senior Wet Cat - 3 oz 6 $1.29 157¥ +7/19/2018 10:28a Rubeus Hagrid FOOD170 Food, Dog - 5kg 5 $44.95 $224.75 +7/20/2018 2:13p Jon Arbuckle FOOD167 Food, Premium Wet Cat - 3.5 oz 50 $3.95 $197.5 +7/23/2018 1:41p Robert Armstrong FOOD215 BFF Oh My Gravy! Lamb & Tuna 2.8oz 4 $12.95 $51.8 +7/23/2018 4:23p Douglas "Dougie" Powers TOY235 Laser Pointer 1 $16.12 $16.12 +7/24/2018 12:16p 桜 高橋 (Sakura Takahashi) FOOD122 Food, Senior Wet Cat - 3 oz 3 $1.29 157¥ +7/26/2018 4:39p Douglas "Dougie" Powers FOOD420 Food, Shark - 10 kg 1 $15.70 $15.7 +7/27/2018 12:16p 桜 高橋 (Sakura Takahashi) FOOD122 Food, Senior Wet Cat - 3 oz 3 $1.29 157¥ +7/30/2018 12:17p 桜 高橋 (Sakura Takahashi) RETURN Food, Senior Wet Cat - 3 oz 1 $1.29 157¥ +7/31/2018 5:42p Rubeus Hagrid CAT060 Food, Dragon - 50kg 5 $720.42 $3602.1 +8/1/2018 2:44p David Attenborough FOOD360 Food, Rhinocerous - 50kg 4 $5.72 $22.88 +8/2/2018 5:12p Susan Ashworth CAT110 Cat, Maine Coon (Felix catus) 1 $1,309.68 $1309.68 +8/2/2018 5:12p Susan Ashworth FOOD130 Food, Kitten 3kg 3 $14.94 $44.82 +8/6/2018 10:21a Robert Armstrong FOOD212 BFF Oh My Gravy! Beef & Chicken 2.8oz 4 $12.95 $51.8 +8/7/2018 4:12p Juan Johnson REPT082 Kingsnake, California (Lampropeltis getula) 1 $89.95 $89.95 +8/7/2018 4:12p Juan Johnson RDNT443 Mouse, Pinky (Mus musculus) 1 $1.49 $1.49 +8/10/2018 4:31p Robert Armstrong FOOD211 BFF Oh My Gravy! Chicken & Turkey 2.8oz 4 $12.95 $51.8 +8/13/2018 2:07p Monica Johnson RDNT443 Mouse, Pinky (Mus musculus) 1 $1.49 $1.49 +8/13/2018 2:08p María Fernández FOOD146 Forti Diet Prohealth Mouse/Rat 3lbs 2 $2.00 $4.0 +8/15/2018 11:57a Mr. Praline RETURN Parrot, Norwegian Blue (Mopsitta tanta) 1 $2300.00 -$2300.0 +8/15/2018 3:48p Kyle Kennedy FOOD121 Food, Adult Cat - 3.5 oz 2 $4.22 $8.44 +8/16/2018 11:50a Helen Halestorm RETURN Rabbit (Oryctolagus cuniculus) 6 $0 $0.0 +8/16/2018 4:00p Kyle Kennedy DOG010 Dog, Golden Retriever (Canis lupus familiaris) 1 $2,495.99 $2495.99 +8/16/2018 5:15p Michael Smith BIRD160 Parakeet, Blue (Melopsittacus undulatus) 1 29.95 $31.85 +8/17/2018 9:26a Rubeus Hagrid NSCT201 Food, Spider 5 $.05 $0.25 +8/20/2018 9:36a Kyle Kennedy RETURN Dog, Golden Retriever (Canis lupus familiaris) 1 $1,247.99 -$1247.99 +8/20/2018 3:31p Monica Johnson NSCT201 Crickets, Adult Live (Gryllus assimilis) 30 $.05 $1.5 +8/20/2018 5:12p David Attenborough NSCT084 Food, Pangolin 30 $.17 $5.10 +8/21/2018 12:13p Robert Armstrong FOOD214 BFF Oh My Gravy! Duck & Salmon 2.8oz 4 $12.95 $51.8 +8/22/2018 9:38a David Attenborough BIRD160 Food, Quoll 1 29.95 $29.95 +8/22/2018 2:13p Jon Arbuckle FOOD170 Food, Adult Dog - 5kg 1 $44.95 $44.95 +8/24/2018 11:42a Robert Armstrong FOOD218 BFF Oh My Gravy! Chicken & Salmon 2.8oz 4 $12.95 $51.8 +8/27/2018 3:05p Monica Johnson NSCT443 Mealworms, Large (Tenebrio molitor) 100ct 1 $1.99 $1.99 +8/28/2018 5:32p Susan Ashworth CAT020 Cat, Scottish Fold (Felis catus) 1 $1,964.53 $1964.53 +8/28/2018 5:32p Susan Ashworth FOOD130 Food, Kitten 3kg 2 $14.94 $29.88 +8/29/2018 10:07a Robert Armstrong FOOD219 BFF Oh My Gravy! Chicken & Pumpkin 2.8oz 4 $12.95 $51.8 +8/31/2018 12:00a Robert Armstrong FOOD219 BFF Oh My Gravy! Chicken & Pumpkin 2.8oz 144 $12.95 $1864.8 +8/31/2018 5:57p Juan Johnson REPT217 Lizard, Spinytail (Uromastyx ornatus) 1 $99.95 $99.95 +================ ========================== ======= ============================================== ========== ========= ========= \ No newline at end of file From 8c3907c88c4c12e1ac8d5e40db932bef516e1b58 Mon Sep 17 00:00:00 2001 From: AJ Kerrigan Date: Thu, 15 Oct 2020 02:30:53 -0400 Subject: [PATCH 015/162] trim visidata import list, fix typo --- visidata/deprecated.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/visidata/deprecated.py b/visidata/deprecated.py index 4c32de24a..b0ae77f6c 100644 --- a/visidata/deprecated.py +++ b/visidata/deprecated.py @@ -1,4 +1,4 @@ -from visidata import Column, PyobjSheet, VisiData, vd +from visidata import VisiData, vd import visidata alias = visidata.BaseSheet.bindkey @@ -42,7 +42,7 @@ def SubrowColumn(*args, **kwargs): @deprecated('1.6') def DeferredSetColumn(*args, **kwargs): - return Column(*args, defer=True, **kwargs) + return visidata.Column(*args, defer=True, **kwargs) @deprecated('2.0') def bindkey_override(keystrokes, longname): @@ -62,7 +62,7 @@ def exec_keystrokes(self, keystrokes, vdglobals=None): @VisiData.api def filetype(vd, ext, constructor): 'Add constructor to handle the given file type/extension.' - globals().setdefault('open_'+ext, lambda p,ext=ext: constructor(p,name, source=p, filetype=ext)) + globals().setdefault('open_'+ext, lambda p,ext=ext: constructor(p.name, source=p, filetype=ext)) @deprecated('2.0', 'Sheet(namepart1, namepart2, ...)') @VisiData.global_api @@ -73,12 +73,12 @@ def joinSheetnames(vd, *sheetnames): @deprecated('2.0', 'PyobjSheet') @VisiData.global_api def load_pyobj(*names, **kwargs): - return PyobjSheet(*names, **kwargs) + return visidata.PyobjSheet(*names, **kwargs) @deprecated('2.0', 'PyobjSheet') @VisiData.global_api def push_pyobj(name, pyobj): - vs = PyobjSheet(name, source=pyobj) + vs = visidata.PyobjSheet(name, source=pyobj) if vs: return vd.push(vs) else: From 171e05d1a4c413a78e3570cc8d0856679c040679 Mon Sep 17 00:00:00 2001 From: anjakefala Date: Wed, 14 Oct 2020 23:36:02 -0700 Subject: [PATCH 016/162] [tests] fix golden for test --- tests/golden/save-benchmarks.rst | 106 +++++++++++++++---------------- 1 file changed, 53 insertions(+), 53 deletions(-) diff --git a/tests/golden/save-benchmarks.rst b/tests/golden/save-benchmarks.rst index 8288cb9d4..bcab632d5 100644 --- a/tests/golden/save-benchmarks.rst +++ b/tests/golden/save-benchmarks.rst @@ -1,53 +1,53 @@ -================ ========================== ======= ============================================== ========== ========= ========= -Date Customer SKU Item Quantity Unit Paid -================ ========================== ======= ============================================== ========== ========= ========= -7/3/2018 1:47p Robert Armstrong FOOD213 BFF Oh My Gravy! Beef & Salmon 2.8oz 4 $12.95 $51.8 -7/3/2018 3:32p Kyle Kennedy FOOD121 Food, Adult Cat - 3.5 oz 1 $4.22 $4.22 -7/5/2018 4:15p Douglas "Dougie" Powers FOOD121 Food, Adult Cat 3.5 oz 1 $4.22 $4.22 -7/6/2018 12:15p 桜 高橋 (Sakura Takahashi) FOOD122 Food, Senior Wet Cat - 3 oz 12 $1.29 157¥ -7/10/2018 10:28a David Attenborough NSCT201 Food, Salamander 30 $.05 $1.5 -7/10/2018 5:23p Susan Ashworth CAT060 Cat, Korat (Felis catus) 1 $720.42 $720.42 -7/10/2018 5:23p Susan Ashworth FOOD130 Food, Kitten 3kg 1 $14.94 $14.94 -7/13/2018 10:26a Wil Wheaton NSCT523 Monster, Rust (Monstrus gygaxus) 1 $39.95 $39.95 -7/13/2018 3:49p Robert Armstrong FOOD216 BFF Oh My Gravy! Chicken & Shrimp 2.8oz 4 $12.95 $51.8 -7/17/2018 9:01a Robert Armstrong FOOD217 BFF Oh My Gravy! Duck & Tuna 2.8oz 4 $12.95 $51.8 -7/17/2018 11:30a Helen Halestorm LAGO342 Rabbit (Oryctolagus cuniculus) 2 $32.94 $65.88 -7/18/2018 12:16p 桜 高橋 (Sakura Takahashi) FOOD122 Food, Senior Wet Cat - 3 oz 6 $1.29 157¥ -7/19/2018 10:28a Rubeus Hagrid FOOD170 Food, Dog - 5kg 5 $44.95 $224.75 -7/20/2018 2:13p Jon Arbuckle FOOD167 Food, Premium Wet Cat - 3.5 oz 50 $3.95 $197.5 -7/23/2018 1:41p Robert Armstrong FOOD215 BFF Oh My Gravy! Lamb & Tuna 2.8oz 4 $12.95 $51.8 -7/23/2018 4:23p Douglas "Dougie" Powers TOY235 Laser Pointer 1 $16.12 $16.12 -7/24/2018 12:16p 桜 高橋 (Sakura Takahashi) FOOD122 Food, Senior Wet Cat - 3 oz 3 $1.29 157¥ -7/26/2018 4:39p Douglas "Dougie" Powers FOOD420 Food, Shark - 10 kg 1 $15.70 $15.7 -7/27/2018 12:16p 桜 高橋 (Sakura Takahashi) FOOD122 Food, Senior Wet Cat - 3 oz 3 $1.29 157¥ -7/30/2018 12:17p 桜 高橋 (Sakura Takahashi) RETURN Food, Senior Wet Cat - 3 oz 1 $1.29 157¥ -7/31/2018 5:42p Rubeus Hagrid CAT060 Food, Dragon - 50kg 5 $720.42 $3602.1 -8/1/2018 2:44p David Attenborough FOOD360 Food, Rhinocerous - 50kg 4 $5.72 $22.88 -8/2/2018 5:12p Susan Ashworth CAT110 Cat, Maine Coon (Felix catus) 1 $1,309.68 $1309.68 -8/2/2018 5:12p Susan Ashworth FOOD130 Food, Kitten 3kg 3 $14.94 $44.82 -8/6/2018 10:21a Robert Armstrong FOOD212 BFF Oh My Gravy! Beef & Chicken 2.8oz 4 $12.95 $51.8 -8/7/2018 4:12p Juan Johnson REPT082 Kingsnake, California (Lampropeltis getula) 1 $89.95 $89.95 -8/7/2018 4:12p Juan Johnson RDNT443 Mouse, Pinky (Mus musculus) 1 $1.49 $1.49 -8/10/2018 4:31p Robert Armstrong FOOD211 BFF Oh My Gravy! Chicken & Turkey 2.8oz 4 $12.95 $51.8 -8/13/2018 2:07p Monica Johnson RDNT443 Mouse, Pinky (Mus musculus) 1 $1.49 $1.49 -8/13/2018 2:08p María Fernández FOOD146 Forti Diet Prohealth Mouse/Rat 3lbs 2 $2.00 $4.0 -8/15/2018 11:57a Mr. Praline RETURN Parrot, Norwegian Blue (Mopsitta tanta) 1 $2300.00 -$2300.0 -8/15/2018 3:48p Kyle Kennedy FOOD121 Food, Adult Cat - 3.5 oz 2 $4.22 $8.44 -8/16/2018 11:50a Helen Halestorm RETURN Rabbit (Oryctolagus cuniculus) 6 $0 $0.0 -8/16/2018 4:00p Kyle Kennedy DOG010 Dog, Golden Retriever (Canis lupus familiaris) 1 $2,495.99 $2495.99 -8/16/2018 5:15p Michael Smith BIRD160 Parakeet, Blue (Melopsittacus undulatus) 1 29.95 $31.85 -8/17/2018 9:26a Rubeus Hagrid NSCT201 Food, Spider 5 $.05 $0.25 -8/20/2018 9:36a Kyle Kennedy RETURN Dog, Golden Retriever (Canis lupus familiaris) 1 $1,247.99 -$1247.99 -8/20/2018 3:31p Monica Johnson NSCT201 Crickets, Adult Live (Gryllus assimilis) 30 $.05 $1.5 -8/20/2018 5:12p David Attenborough NSCT084 Food, Pangolin 30 $.17 $5.10 -8/21/2018 12:13p Robert Armstrong FOOD214 BFF Oh My Gravy! Duck & Salmon 2.8oz 4 $12.95 $51.8 -8/22/2018 9:38a David Attenborough BIRD160 Food, Quoll 1 29.95 $29.95 -8/22/2018 2:13p Jon Arbuckle FOOD170 Food, Adult Dog - 5kg 1 $44.95 $44.95 -8/24/2018 11:42a Robert Armstrong FOOD218 BFF Oh My Gravy! Chicken & Salmon 2.8oz 4 $12.95 $51.8 -8/27/2018 3:05p Monica Johnson NSCT443 Mealworms, Large (Tenebrio molitor) 100ct 1 $1.99 $1.99 -8/28/2018 5:32p Susan Ashworth CAT020 Cat, Scottish Fold (Felis catus) 1 $1,964.53 $1964.53 -8/28/2018 5:32p Susan Ashworth FOOD130 Food, Kitten 3kg 2 $14.94 $29.88 -8/29/2018 10:07a Robert Armstrong FOOD219 BFF Oh My Gravy! Chicken & Pumpkin 2.8oz 4 $12.95 $51.8 -8/31/2018 12:00a Robert Armstrong FOOD219 BFF Oh My Gravy! Chicken & Pumpkin 2.8oz 144 $12.95 $1864.8 -8/31/2018 5:57p Juan Johnson REPT217 Lizard, Spinytail (Uromastyx ornatus) 1 $99.95 $99.95 -================ ========================== ======= ============================================== ========== ========= ========= \ No newline at end of file +========== ========================== ======= ============================================== ========== ======= ========= +Date Customer SKU Item Quantity Unit Paid +========== ========================== ======= ============================================== ========== ======= ========= +2018-07-03 Robert Armstrong FOOD213 BFF Oh My Gravy! Beef & Salmon 2.8oz 4 12.95 $51.8 +2018-07-03 Kyle Kennedy FOOD121 Food, Adult Cat - 3.5 oz 1 4.22 $4.22 +2018-07-05 Douglas "Dougie" Powers FOOD121 Food, Adult Cat 3.5 oz 1 4.22 $4.22 +2018-07-06 桜 高橋 (Sakura Takahashi) FOOD122 Food, Senior Wet Cat - 3 oz 12 1.29 157¥ +2018-07-10 David Attenborough NSCT201 Food, Salamander 30 0.05 $1.5 +2018-07-10 Susan Ashworth CAT060 Cat, Korat (Felis catus) 1 720.42 $720.42 +2018-07-10 Susan Ashworth FOOD130 Food, Kitten 3kg 1 14.94 $14.94 +2018-07-13 Wil Wheaton NSCT523 Monster, Rust (Monstrus gygaxus) 1 39.95 $39.95 +2018-07-13 Robert Armstrong FOOD216 BFF Oh My Gravy! Chicken & Shrimp 2.8oz 4 12.95 $51.8 +2018-07-17 Robert Armstrong FOOD217 BFF Oh My Gravy! Duck & Tuna 2.8oz 4 12.95 $51.8 +2018-07-17 Helen Halestorm LAGO342 Rabbit (Oryctolagus cuniculus) 2 32.94 $65.88 +2018-07-18 桜 高橋 (Sakura Takahashi) FOOD122 Food, Senior Wet Cat - 3 oz 6 1.29 157¥ +2018-07-19 Rubeus Hagrid FOOD170 Food, Dog - 5kg 5 44.95 $224.75 +2018-07-20 Jon Arbuckle FOOD167 Food, Premium Wet Cat - 3.5 oz 50 3.95 $197.5 +2018-07-23 Robert Armstrong FOOD215 BFF Oh My Gravy! Lamb & Tuna 2.8oz 4 12.95 $51.8 +2018-07-23 Douglas "Dougie" Powers TOY235 Laser Pointer 1 16.12 $16.12 +2018-07-24 桜 高橋 (Sakura Takahashi) FOOD122 Food, Senior Wet Cat - 3 oz 3 1.29 157¥ +2018-07-26 Douglas "Dougie" Powers FOOD420 Food, Shark - 10 kg 1 15.7 $15.7 +2018-07-27 桜 高橋 (Sakura Takahashi) FOOD122 Food, Senior Wet Cat - 3 oz 3 1.29 157¥ +2018-07-30 桜 高橋 (Sakura Takahashi) RETURN Food, Senior Wet Cat - 3 oz 1 1.29 157¥ +2018-07-31 Rubeus Hagrid CAT060 Food, Dragon - 50kg 5 720.42 $3602.1 +2018-08-01 David Attenborough FOOD360 Food, Rhinocerous - 50kg 4 5.72 $22.88 +2018-08-02 Susan Ashworth CAT110 Cat, Maine Coon (Felix catus) 1 1309.68 $1309.68 +2018-08-02 Susan Ashworth FOOD130 Food, Kitten 3kg 3 14.94 $44.82 +2018-08-06 Robert Armstrong FOOD212 BFF Oh My Gravy! Beef & Chicken 2.8oz 4 12.95 $51.8 +2018-08-07 Juan Johnson REPT082 Kingsnake, California (Lampropeltis getula) 1 89.95 $89.95 +2018-08-07 Juan Johnson RDNT443 Mouse, Pinky (Mus musculus) 1 1.49 $1.49 +2018-08-10 Robert Armstrong FOOD211 BFF Oh My Gravy! Chicken & Turkey 2.8oz 4 12.95 $51.8 +2018-08-13 Monica Johnson RDNT443 Mouse, Pinky (Mus musculus) 1 1.49 $1.49 +2018-08-13 María Fernández FOOD146 Forti Diet Prohealth Mouse/Rat 3lbs 2 2 $4.0 +2018-08-15 Mr. Praline RETURN Parrot, Norwegian Blue (Mopsitta tanta) 1 2300 -$2300.0 +2018-08-15 Kyle Kennedy FOOD121 Food, Adult Cat - 3.5 oz 2 4.22 $8.44 +2018-08-16 Helen Halestorm RETURN Rabbit (Oryctolagus cuniculus) 6 0 $0.0 +2018-08-16 Kyle Kennedy DOG010 Dog, Golden Retriever (Canis lupus familiaris) 1 2495.99 $2495.99 +2018-08-16 Michael Smith BIRD160 Parakeet, Blue (Melopsittacus undulatus) 1 29.95 $31.85 +2018-08-17 Rubeus Hagrid NSCT201 Food, Spider 5 0.05 $0.25 +2018-08-20 Kyle Kennedy RETURN Dog, Golden Retriever (Canis lupus familiaris) 1 1247.99 -$1247.99 +2018-08-20 Monica Johnson NSCT201 Crickets, Adult Live (Gryllus assimilis) 30 0.05 $1.5 +2018-08-20 David Attenborough NSCT084 Food, Pangolin 30 0.17 $5.10 +2018-08-21 Robert Armstrong FOOD214 BFF Oh My Gravy! Duck & Salmon 2.8oz 4 12.95 $51.8 +2018-08-22 David Attenborough BIRD160 Food, Quoll 1 29.95 $29.95 +2018-08-22 Jon Arbuckle FOOD170 Food, Adult Dog - 5kg 1 44.95 $44.95 +2018-08-24 Robert Armstrong FOOD218 BFF Oh My Gravy! Chicken & Salmon 2.8oz 4 12.95 $51.8 +2018-08-27 Monica Johnson NSCT443 Mealworms, Large (Tenebrio molitor) 100ct 1 1.99 $1.99 +2018-08-28 Susan Ashworth CAT020 Cat, Scottish Fold (Felis catus) 1 1964.53 $1964.53 +2018-08-28 Susan Ashworth FOOD130 Food, Kitten 3kg 2 14.94 $29.88 +2018-08-29 Robert Armstrong FOOD219 BFF Oh My Gravy! Chicken & Pumpkin 2.8oz 4 12.95 $51.8 +2018-08-31 Robert Armstrong FOOD219 BFF Oh My Gravy! Chicken & Pumpkin 2.8oz 144 12.95 $1864.8 +2018-08-31 Juan Johnson REPT217 Lizard, Spinytail (Uromastyx ornatus) 1 99.95 $99.95 +========== ========================== ======= ============================================== ========== ======= ========= \ No newline at end of file From e1d1234713c86264dfc0d545d56d0eb013a5eb21 Mon Sep 17 00:00:00 2001 From: anjakefala Date: Wed, 14 Oct 2020 23:44:57 -0700 Subject: [PATCH 017/162] [dev] set developing version --- setup.py | 2 +- visidata/__init__.py | 2 +- visidata/main.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 9833bf4de..0a416c25f 100755 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ from setuptools import setup # tox can't actually run python3 setup.py: https://github.com/tox-dev/tox/issues/96 #from visidata import __version__ -__version__ = '2.0.1' +__version__ = '2.0+' setup(name='visidata', version=__version__, diff --git a/visidata/__init__.py b/visidata/__init__.py index 683ac419a..faaa3f8b7 100644 --- a/visidata/__init__.py +++ b/visidata/__init__.py @@ -1,6 +1,6 @@ 'VisiData: a curses interface for exploring and arranging tabular data' -__version__ = '2.0.1' +__version__ = '2.0+' __version_info__ = 'VisiData v' + __version__ __author__ = 'Saul Pwanson ' __status__ = 'Production/Stable' diff --git a/visidata/main.py b/visidata/main.py index fda66ce34..ca25c9513 100755 --- a/visidata/main.py +++ b/visidata/main.py @@ -2,7 +2,7 @@ # Usage: $0 [] [ ...] # $0 [] --play [--batch] [-w ] [-o ] [field=value ...] -__version__ = '2.0.1' +__version__ = '2.0+' __version_info__ = 'saul.pw/VisiData v' + __version__ from copy import copy From f9412846d6d070d752af9ad7a3ee59e293bc6d4b Mon Sep 17 00:00:00 2001 From: Andreas Motl Date: Thu, 15 Oct 2020 00:17:17 +0200 Subject: [PATCH 018/162] Fix HDF5 loader --- visidata/loaders/hdf5.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/visidata/loaders/hdf5.py b/visidata/loaders/hdf5.py index 8994eaa0f..873c2efb3 100644 --- a/visidata/loaders/hdf5.py +++ b/visidata/loaders/hdf5.py @@ -32,7 +32,7 @@ def iterload(self): elif len(source.shape) == 2: # matrix ncols = source.shape[1] for i in range(ncols): - self.addColumns(ColumnItem('', i, width=8), index=i) + self.addColumn(ColumnItem('', i, width=8), index=i) self.recalc() yield from source # copy else: @@ -42,10 +42,16 @@ def iterload(self): def openRow(self, row): + import h5py if isinstance(row, BaseSheet): return row - if isinstance(row, h5py.Object): - return H5ObjSheet(row) + if isinstance(row, h5py.HLObject): + return Hdf5ObjSheet(row) + + import numpy + from .npy import NpySheet + if isinstance(row, numpy.ndarray): + return NpySheet(None, npy=row) Hdf5ObjSheet.addCommand('A', 'dive-metadata', 'vd.push(SheetDict(cursorRow.name + "_attrs", source=cursorRow.attrs))', 'open metadata sheet for object referenced in current row') From 07f4e985cc52437cf79e39c75c02f593052fac11 Mon Sep 17 00:00:00 2001 From: anjakefala Date: Thu, 15 Oct 2020 21:56:10 -0700 Subject: [PATCH 019/162] [api] expose visidata.view Addresses #732 --- visidata/pyobj.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/visidata/pyobj.py b/visidata/pyobj.py index 05babaf01..e135a7d9c 100644 --- a/visidata/pyobj.py +++ b/visidata/pyobj.py @@ -2,7 +2,7 @@ from visidata import * -__all__ = ['PythonSheet', 'expand_cols_deep', 'deduceType', 'closeColumn', 'ListOfDictSheet', 'SheetDict', 'PyobjSheet'] +__all__ = ['PythonSheet', 'expand_cols_deep', 'deduceType', 'closeColumn', 'ListOfDictSheet', 'SheetDict', 'PyobjSheet', 'view'] option('visibility', 0, 'visibility level') option('expand_col_scanrows', 1000, 'number of rows to check when expanding columns (0 = all)') From d432c67c7cd16b694eb565ad044e4a57c3f58432 Mon Sep 17 00:00:00 2001 From: Saul Pwanson Date: Mon, 19 Oct 2020 00:35:06 -0700 Subject: [PATCH 020/162] [select] differentiate select-equal- and select-exact- #734 --- CHANGELOG.md | 4 ++++ visidata/selection.py | 6 ++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8120d2301..6b3aa8689 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,10 @@ # v2.?.? (????) - [xlsx] add active column #726 + - [select] differentiate select-equal- and select-exact- #734 + - previous select-equal- matched type value + - now select-equal- matches display value + - add `z,` and `gz,` bindings for select-exact-cell/-row ## Bugfixes diff --git a/visidata/selection.py b/visidata/selection.py index 424f047e3..7d175e89c 100644 --- a/visidata/selection.py +++ b/visidata/selection.py @@ -141,8 +141,10 @@ def addUndoSelection(sheet): Sheet.addCommand('g|', 'select-cols-regex', 'selectByIdx(vd.searchRegex(sheet, regex=input("select regex: ", type="regex", defaultLast=True), columns="visibleCols"))', 'select rows matching regex in any visible column') Sheet.addCommand('g\\', 'unselect-cols-regex', 'unselectByIdx(vd.searchRegex(sheet, regex=input("unselect regex: ", type="regex", defaultLast=True), columns="visibleCols"))', 'unselect rows matching regex in any visible column') -Sheet.addCommand(',', 'select-equal-cell', 'select(gatherBy(lambda r,c=cursorCol,v=cursorTypedValue: c.getTypedValue(r) == v), progress=False)', 'select rows matching current cell in current column') -Sheet.addCommand('g,', 'select-equal-row', 'select(gatherBy(lambda r,currow=cursorRow,vcols=visibleCols: all([c.getTypedValue(r) == c.getTypedValue(currow) for c in vcols])), progress=False)', 'select rows matching current row in all visible columns') +Sheet.addCommand(',', 'select-equal-cell', 'select(gatherBy(lambda r,c=cursorCol,v=cursorDisplayValue: c.getDisplayValue(r) == v), progress=False)', 'select rows matching current cell in current column') +Sheet.addCommand('g,', 'select-equal-row', 'select(gatherBy(lambda r,currow=cursorRow,vcols=visibleCols: all([c.getDisplayValue(r) == c.getDisplayValue(currow) for c in vcols])), progress=False)', 'select rows matching current row in all visible columns') +Sheet.addCommand('z,', 'select-exact-cell', 'select(gatherBy(lambda r,c=cursorCol,v=cursorTypedValue: c.getTypedValue(r) == v), progress=False)', 'select rows matching current cell in current column') +Sheet.addCommand('gz,', 'select-exact-row', 'select(gatherBy(lambda r,currow=cursorRow,vcols=visibleCols: all([c.getTypedValue(r) == c.getTypedValue(currow) for c in vcols])), progress=False)', 'select rows matching current row in all visible columns') Sheet.addCommand('z|', 'select-expr', 'expr=inputExpr("select by expr: "); select(gatherBy(lambda r, sheet=sheet, expr=expr: sheet.evalExpr(expr, r)), progress=False)', 'select rows matching Python expression in any visible column') Sheet.addCommand('z\\', 'unselect-expr', 'expr=inputExpr("unselect by expr: "); unselect(gatherBy(lambda r, sheet=sheet, expr=expr: sheet.evalExpr(expr, r)), progress=False)', 'unselect rows matching Python expression in any visible column') From 7e5c69ef796175f59fb6bf3bd66ae16b936914a2 Mon Sep 17 00:00:00 2001 From: Saul Pwanson Date: Mon, 19 Oct 2020 00:53:30 -0700 Subject: [PATCH 021/162] [clipboard] require some selected rows for clipboard ops #681 --- CHANGELOG.md | 1 + visidata/clipboard.py | 14 +++++++------- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6b3aa8689..aee09d4ac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ - previous select-equal- matched type value - now select-equal- matches display value - add `z,` and `gz,` bindings for select-exact-cell/-row + - [clipboard] clipboard commands now require some selected rows #681 ## Bugfixes diff --git a/visidata/clipboard.py b/visidata/clipboard.py index 6498aaea9..6c0e9220a 100644 --- a/visidata/clipboard.py +++ b/visidata/clipboard.py @@ -181,26 +181,26 @@ def saveToClipboard(sheet, rows, filetype=None): Sheet.addCommand('p', 'paste-after', 'paste_after(cursorRowIndex)', 'paste clipboard rows after current row') Sheet.addCommand('P', 'paste-before', 'paste_before(cursorRowIndex)', 'paste clipboard rows before current row') -Sheet.addCommand('gd', 'delete-selected', 'copyRows(selectedRows); deleteSelected()', 'delete (cut) selected rows and move them to clipboard') -Sheet.addCommand('gy', 'copy-selected', 'copyRows(selectedRows)', 'yank (copy) selected rows to clipboard') +Sheet.addCommand('gd', 'delete-selected', 'copyRows(someSelectedRows); deleteSelected()', 'delete (cut) selected rows and move them to clipboard') +Sheet.addCommand('gy', 'copy-selected', 'copyRows(someSelectedRows)', 'yank (copy) selected rows to clipboard') Sheet.addCommand('zy', 'copy-cell', 'copyCells(cursorCol, [cursorRow])', 'yank (copy) current cell to clipboard') Sheet.addCommand('zp', 'paste-cell', 'cursorCol.setValuesTyped([cursorRow], vd.clipcells[0]) if vd.clipcells else warning("no cells to paste")', 'set contents of current cell to last clipboard value') Sheet.addCommand('zd', 'delete-cell', 'vd.clipcells = [cursorDisplay]; cursorCol.setValues([cursorRow], None)', 'delete (cut) current cell and move it to clipboard') -Sheet.addCommand('gzd', 'delete-cells', 'vd.clipcells = list(vd.sheet.cursorCol.getDisplayValue(r) for r in selectedRows); cursorCol.setValues(selectedRows, None)', 'delete (cut) contents of current column for selected rows and move them to clipboard') +Sheet.addCommand('gzd', 'delete-cells', 'vd.clipcells = list(vd.sheet.cursorCol.getDisplayValue(r) for r in someSelectedRows); cursorCol.setValues(someSelectedRows, None)', 'delete (cut) contents of current column for selected rows and move them to clipboard') Sheet.bindkey('BUTTON2_PRESSED', 'go-mouse') Sheet.addCommand('BUTTON2_RELEASED', 'syspaste-cells', 'pasteFromClipboard(visibleCols[cursorVisibleColIndex:], rows[cursorRowIndex:])', 'paste into VisiData from system clipboard') Sheet.bindkey('BUTTON2_CLICKED', 'go-mouse') -Sheet.addCommand('gzy', 'copy-cells', 'copyCells(cursorCol, selectedRows)', 'yank (copy) contents of current column for selected rows to clipboard') -Sheet.addCommand('gzp', 'setcol-clipboard', 'for r, v in zip(selectedRows, itertools.cycle(vd.clipcells)): cursorCol.setValuesTyped([r], v)', 'set cells of current column for selected rows to last clipboard value') +Sheet.addCommand('gzy', 'copy-cells', 'copyCells(cursorCol, someSelectedRows)', 'yank (copy) contents of current column for selected rows to clipboard') +Sheet.addCommand('gzp', 'setcol-clipboard', 'for r, v in zip(someSelectedRows, itertools.cycle(vd.clipcells)): cursorCol.setValuesTyped([r], v)', 'set cells of current column for selected rows to last clipboard value') Sheet.addCommand('Y', 'syscopy-row', 'syscopyRows([cursorRow])', 'yank (copy) current row to system clipboard (using options.clipboard_copy_cmd)') -Sheet.addCommand('gY', 'syscopy-selected', 'syscopyRows(selectedRows)', 'yank (copy) selected rows to system clipboard (using options.clipboard_copy_cmd)') +Sheet.addCommand('gY', 'syscopy-selected', 'syscopyRows(someSelectedRows)', 'yank (copy) selected rows to system clipboard (using options.clipboard_copy_cmd)') Sheet.addCommand('zY', 'syscopy-cell', 'syscopyCells(cursorCol, [cursorRow])', 'yank (copy) current cell to system clipboard (using options.clipboard_copy_cmd)') -Sheet.addCommand('gzY', 'syscopy-cells', 'syscopyCells(cursorCol, selectedRows)', 'yank (copy) contents of current column from selected rows to system clipboard (using options.clipboard_copy_cmd') +Sheet.addCommand('gzY', 'syscopy-cells', 'syscopyCells(cursorCol, someSelectedRows)', 'yank (copy) contents of current column from selected rows to system clipboard (using options.clipboard_copy_cmd') Sheet.bindkey('KEY_DC', 'delete-cell'), Sheet.bindkey('gKEY_DC', 'delete-cells'), From 6a304df4de6bf008bce06d9821a3ef6bbca4f4bc Mon Sep 17 00:00:00 2001 From: Saul Pwanson Date: Mon, 19 Oct 2020 01:36:14 -0700 Subject: [PATCH 022/162] [select-] cursorDisplay, not cursorDisplayValue #734 --- docs/api/interface.rst | 2 +- visidata/selection.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/api/interface.rst b/docs/api/interface.rst index 0d81fcdf4..2337ec314 100644 --- a/docs/api/interface.rst +++ b/docs/api/interface.rst @@ -91,7 +91,7 @@ Examples passwd = vd.input("password: ", display=False) # initial value is the formatted value under the cursor - vd.status(vd.input("text to show: ", value=cursorDisplayValue)) + vd.status(vd.input("text to show: ", value=cursorDisplay)) .. _colors: diff --git a/visidata/selection.py b/visidata/selection.py index 7d175e89c..fc0ff8c73 100644 --- a/visidata/selection.py +++ b/visidata/selection.py @@ -141,7 +141,7 @@ def addUndoSelection(sheet): Sheet.addCommand('g|', 'select-cols-regex', 'selectByIdx(vd.searchRegex(sheet, regex=input("select regex: ", type="regex", defaultLast=True), columns="visibleCols"))', 'select rows matching regex in any visible column') Sheet.addCommand('g\\', 'unselect-cols-regex', 'unselectByIdx(vd.searchRegex(sheet, regex=input("unselect regex: ", type="regex", defaultLast=True), columns="visibleCols"))', 'unselect rows matching regex in any visible column') -Sheet.addCommand(',', 'select-equal-cell', 'select(gatherBy(lambda r,c=cursorCol,v=cursorDisplayValue: c.getDisplayValue(r) == v), progress=False)', 'select rows matching current cell in current column') +Sheet.addCommand(',', 'select-equal-cell', 'select(gatherBy(lambda r,c=cursorCol,v=cursorDisplay: c.getDisplayValue(r) == v), progress=False)', 'select rows matching current cell in current column') Sheet.addCommand('g,', 'select-equal-row', 'select(gatherBy(lambda r,currow=cursorRow,vcols=visibleCols: all([c.getDisplayValue(r) == c.getDisplayValue(currow) for c in vcols])), progress=False)', 'select rows matching current row in all visible columns') Sheet.addCommand('z,', 'select-exact-cell', 'select(gatherBy(lambda r,c=cursorCol,v=cursorTypedValue: c.getTypedValue(r) == v), progress=False)', 'select rows matching current cell in current column') Sheet.addCommand('gz,', 'select-exact-row', 'select(gatherBy(lambda r,currow=cursorRow,vcols=visibleCols: all([c.getTypedValue(r) == c.getTypedValue(currow) for c in vcols])), progress=False)', 'select rows matching current row in all visible columns') From d3f9fb40c086c094e8f28782f26bee3527d268a1 Mon Sep 17 00:00:00 2001 From: anjakefala Date: Mon, 19 Oct 2020 20:25:05 -0700 Subject: [PATCH 023/162] [dev] update changelog --- CHANGELOG.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index aee09d4ac..e24a7b4bc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ # v2.?.? (????) - - [xlsx] add active column #726 + - [xlsx] add active column (thanks @kbd for feature request #726) - [select] differentiate select-equal- and select-exact- #734 - previous select-equal- matched type value - now select-equal- matches display value @@ -11,7 +11,10 @@ ## Bugfixes - - [tabulate] fix savers to save in their own format #723 + - [api] expose visidata.view (thanks @alekibango for bug report #732) + - [loaders hdf5] misc bugfixes to hdf5 dataset loading (thanks @amotl for PR #728) + - [man] fix warnings with manpage (thanks @jsvine for the bug report #718) + - [tabulate] fix savers to save in their own format (thanks @frosencrantz for bug report #723) # v2.0.1 (2020-10-13) From 8afa2f98cad94612581a18df6d13e98bf390cc6c Mon Sep 17 00:00:00 2001 From: anjakefala Date: Mon, 19 Oct 2020 20:25:34 -0700 Subject: [PATCH 024/162] [select- pandas] nSelected is now nSelectedRows --- visidata/loaders/_pandas.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/visidata/loaders/_pandas.py b/visidata/loaders/_pandas.py index 5bfcd7d77..bef76609e 100644 --- a/visidata/loaders/_pandas.py +++ b/visidata/loaders/_pandas.py @@ -180,7 +180,7 @@ def unselectRow(self, row): return is_selected @property - def nSelected(self): + def nSelectedRows(self): self._checkSelectedIndex() return self._selectedMask.sum() From fcaefaa28ab632095d6c1dd671707c3e66cddb88 Mon Sep 17 00:00:00 2001 From: anjakefala Date: Mon, 19 Oct 2020 22:30:31 -0700 Subject: [PATCH 025/162] [defer delete-] fix commitDeletes --- visidata/modify.py | 20 +------------------- 1 file changed, 1 insertion(+), 19 deletions(-) diff --git a/visidata/modify.py b/visidata/modify.py index 7d70e9747..38a652ae1 100644 --- a/visidata/modify.py +++ b/visidata/modify.py @@ -181,25 +181,7 @@ def commitMods(self): @Sheet.api def commitDeletes(self): 'Return the number of rows that have been marked for deletion. Delete the rows. Clear the marking.' - ndeleted = 0 - - dest_row = None # row to re-place cursor after - oldidx = self.cursorRowIndex - while oldidx < len(self.rows): - if not self.isDeleted(self.rows[oldidx]): - dest_row = self.rows[oldidx] - break - oldidx += 1 - - newidx = 0 - for r in Progress(list(self.rows), gerund='deleting'): - if self.isDeleted(self.rows[newidx]): - self.deleteSourceRow(newidx) - ndeleted += 1 - else: - if r is dest_row: - self.cursorRowIndex = newidx - newidx += 1 + ndeleted = self.deleteBy(self.isDeleted, commit=True) if ndeleted: vd.status('deleted %s %s' % (ndeleted, self.rowtype)) From 67d32329a0303d53ba5e582ff389f206146ec126 Mon Sep 17 00:00:00 2001 From: Saul Pwanson Date: Tue, 20 Oct 2020 00:46:10 -0700 Subject: [PATCH 026/162] [api] add options.unset(); add unset-option command #733 --- CHANGELOG.md | 2 ++ docs/api/options.rst | 1 + visidata/cmdlog.py | 18 +++++------------- visidata/metasheets.py | 1 + visidata/settings.py | 26 +++++++++++++++++++------- 5 files changed, 28 insertions(+), 20 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e24a7b4bc..c09ce06ae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,8 @@ - now select-equal- matches display value - add `z,` and `gz,` bindings for select-exact-cell/-row - [clipboard] clipboard commands now require some selected rows #681 + - [api] add options.unset() + - [commands] add unset-option command bound to `d` on OptionsSheet #733 ## Bugfixes diff --git a/docs/api/options.rst b/docs/api/options.rst index 4288ca0c3..03c9c4964 100644 --- a/docs/api/options.rst +++ b/docs/api/options.rst @@ -66,6 +66,7 @@ Options API .. autofunction:: visidata.vd.options.get .. autofunction:: visidata.vd.options.set +.. autofunction:: visidata.vd.options.unset .. autofunction:: visidata.vd.options.getall The dict returned by ``options.getall('foo_')`` is designed to be used as kwargs to other loaders, so that their options can be passed through VisiData transparently. diff --git a/visidata/cmdlog.py b/visidata/cmdlog.py index bea4a565d..70fdffb44 100644 --- a/visidata/cmdlog.py +++ b/visidata/cmdlog.py @@ -280,12 +280,13 @@ def replayOne(vd, r): vs = None longname = getattr(r, 'longname', None) - if longname == 'set-option': + if longname in ['set-option', 'unset-option']: try: - if r.col: - options.set(r.row, r.input, r.col) + context = vs if r.sheet and vs else vd + if longname == 'set-option': + context.options.set(r.row, r.input, r.sheet or r.col) else: - options[r.row] = r.input + context.options.unset(r.row, r.sheet or r.col) escaped = False except Exception as e: @@ -376,15 +377,6 @@ def replayStatus(vd): return ' │ %s %s/%s' % (x, vd.currentReplay.cursorRowIndex, len(vd.currentReplay.rows)) -@VisiData.api -def setOption(vd, optname, optval, obj=None): - if vd.cmdlog: - objname = options._opts.objname(obj) - vd.cmdlog.addRow(vd.cmdlog.newRow(col=objname, row=optname, - keystrokes='', input=str(optval), - longname='set-option')) - - @BaseSheet.property def cmdlog(sheet): rows = sheet.cmdlog_sheet.rows diff --git a/visidata/metasheets.py b/visidata/metasheets.py index 59f95f86b..a51d320cd 100644 --- a/visidata/metasheets.py +++ b/visidata/metasheets.py @@ -200,6 +200,7 @@ def join_cols(sheet): ColumnsSheet.addCommand('g~', 'type-string-selected', 'someSelectedRows.type=str', 'set type of selected columns to str') ColumnsSheet.addCommand('gz~', 'type-any-selected', 'someSelectedRows.type=anytype', 'set type of selected columns to anytype') +OptionsSheet.addCommand('d', 'unset-option', 'options.unset(cursorRow.name, str(source))', 'remove option override for this context') OptionsSheet.addCommand(None, 'edit-option', 'editOption(cursorRow)', 'edit option at current row') OptionsSheet.bindkey('e', 'edit-option') OptionsSheet.bindkey(ENTER, 'edit-option') diff --git a/visidata/settings.py b/visidata/settings.py index 1cdf1ae92..85ae31b9e 100644 --- a/visidata/settings.py +++ b/visidata/settings.py @@ -140,7 +140,7 @@ def _get(self, k, obj=None): return opt def _set(self, k, v, obj=None, helpstr=''): - self._cache.clear() # invalidate entire cache on any set() + self._cache.clear() # invalidate entire cache on any change return self._opts.set(k, Option(k, v, helpstr), obj) def is_set(self, k, obj=None): @@ -149,7 +149,7 @@ def is_set(self, k, obj=None): return d.get(self._opts.objname(obj), None) def get(self, optname, default=None): - 'Return the value of the given optname option in the options context. `default` is only returned if the option is not defined. An Exception is never raised.' + 'Return the value of the given *optname* option in the options context. *default* is only returned if the option is not defined. An Exception is never raised.' d = self._get(optname, None) if d: return d.value @@ -178,7 +178,7 @@ def set(self, optname, value, obj='override'): curval = opt.value t = type(curval) if value is None and curval is not None: - value = t() # empty value + return self.unset(optname, obj=obj) elif isinstance(value, str) and t is bool: # special case for bool options value = value and (value[0] not in "0fFnN") # ''/0/false/no are false, everything else is true elif type(value) is t: # if right type, no conversion @@ -189,17 +189,29 @@ def set(self, optname, value, obj='override'): value = t(value) if curval != value and self._get(optname, 'global').replayable: - if obj != 'global' and type(obj) is not type: # options set on init aren't recorded - vd.setOption(optname, value, obj) + if obj != 'global' and type(obj) is not type: # global and class options set on init aren't recorded + if vd.cmdlog: + objname = self._opts.objname(obj) + vd.cmdlog.addRow(vd.cmdlog.newRow(sheet=objname, row=optname, + keystrokes='', input=str(value), + longname='set-option')) else: curval = None vd.warning('setting unknown option %s' % optname) return self._set(optname, value, obj) - def unset(self, optname, obj='override'): + def unset(self, optname, obj=None): 'Remove setting value for given context.' - self._opts.unset(optname, obj) + v = self._opts.unset(optname, obj) + opt = self._get(optname) + if vd.cmdlog and opt and opt.replayable: + objname = self._opts.objname(obj) + vd.cmdlog.addRow(vd.cmdlog.newRow(sheet=objname, row=optname, + keystrokes='', input='', + longname='unset-option')) + self._cache.clear() # invalidate entire cache on any change + return v def setdefault(self, optname, value, helpstr): return self._set(optname, value, 'global', helpstr=helpstr) From a20f8d3e6a47593c50be892bc30aed37843d2cb7 Mon Sep 17 00:00:00 2001 From: Saul Pwanson Date: Tue, 20 Oct 2020 01:03:39 -0700 Subject: [PATCH 027/162] [color-] use separator color for sep chars --- CHANGELOG.md | 1 + visidata/sheets.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c09ce06ae..63d21e934 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ - [loaders hdf5] misc bugfixes to hdf5 dataset loading (thanks @amotl for PR #728) - [man] fix warnings with manpage (thanks @jsvine for the bug report #718) - [tabulate] fix savers to save in their own format (thanks @frosencrantz for bug report #723) + - [color] use `color_column_sep` for sep chars (thanks @geekscrapy for bug report) # v2.0.1 (2020-10-13) diff --git a/visidata/sheets.py b/visidata/sheets.py index ae6ce6023..9bf3559cd 100644 --- a/visidata/sheets.py +++ b/visidata/sheets.py @@ -825,7 +825,7 @@ def drawRow(self, scr, row, rowidx, ybase, rowcattr: ColorAttr, maxheight, vd.onMouse(scr, y, x, 1, colwidth, BUTTON3_RELEASED='edit-cell') if x+colwidth+len(sepchars) <= self.windowWidth: - scr.addstr(y, x+colwidth, sepchars, basecellcattr.attr) + scr.addstr(y, x+colwidth, sepchars, sepcattr.attr) for notefunc in vd.rowNoters: ch = notefunc(self, row) From 5ee164096425bcabf0ae4e10d9f9e92ecae13093 Mon Sep 17 00:00:00 2001 From: Saul Pwanson Date: Tue, 20 Oct 2020 15:49:51 -0700 Subject: [PATCH 028/162] [windows] add Ctrl+M as alias for Ctrl+J #741 - Make getCommand iteratively search for bindings, so that bindkey() can be used to bind a keystroke to another keystroke (instead of a longname). --- CHANGELOG.md | 1 + visidata/settings.py | 19 ++++++++++--------- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 63d21e934..fdce5916d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ - [man] fix warnings with manpage (thanks @jsvine for the bug report #718) - [tabulate] fix savers to save in their own format (thanks @frosencrantz for bug report #723) - [color] use `color_column_sep` for sep chars (thanks @geekscrapy for bug report) + - [windows] add Ctrl+M as alias for Ctrl+J #741 (thanks @bob-u for bug report #741) # v2.0.1 (2020-10-13) diff --git a/visidata/settings.py b/visidata/settings.py index 85ae31b9e..c7166e494 100644 --- a/visidata/settings.py +++ b/visidata/settings.py @@ -311,18 +311,18 @@ def unbindkey(cls, keystrokes): @BaseSheet.api def getCommand(sheet, cmd): - 'Return the Command for the given arg, which may be keystrokes, longname, or a Command itself, within the context of `sheet`.' + 'Return the Command for the given *cmd*, which may be keystrokes, longname, or a Command itself, within the context of `sheet`.' if isinstance(cmd, Command): return cmd - longname = vd.bindkeys._get(cmd, obj=sheet) - try: - if longname: - return vd.commands._get(longname, obj=sheet) or vd.fail('no command "%s"' % longname) - else: - return vd.commands._get(cmd, obj=sheet) or vd.fail('no binding for %s' % cmd) - except Exception as exc: - return None + longname = cmd + while vd.bindkeys._get(longname, obj=sheet): + longname = vd.bindkeys._get(longname, obj=sheet) + + if not longname: + vd.fail('no binding for %s' % cmd) + + return vd.commands._get(longname, obj=sheet) or vd.fail('no command "%s"' % longname) def loadConfigFile(fnrc, _globals=None): @@ -371,4 +371,5 @@ def loadConfigAndPlugins(vd, args): loadConfigFile(options.config, vd.getGlobals()) +BaseSheet.bindkey('^M', '^J') # for windows ENTER BaseSheet.addCommand('gO', 'open-config', 'vd.push(open_txt(Path(options.config)))', 'open options.config as text sheet') From b9f7d8f75f71a3f752bf11321caa42f41afc4760 Mon Sep 17 00:00:00 2001 From: anjakefala Date: Tue, 20 Oct 2020 20:26:41 -0700 Subject: [PATCH 029/162] [alias-] alias should be deprecated first, then actual --- visidata/deprecated.py | 2 +- visidata/loaders/pandas_freqtbl.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/visidata/deprecated.py b/visidata/deprecated.py index cb383566f..05712418c 100644 --- a/visidata/deprecated.py +++ b/visidata/deprecated.py @@ -90,7 +90,7 @@ def push_pyobj(name, pyobj): alias('frequency-rows', 'frequency-summary') alias('dup-cell', 'dive-cell') alias('dup-row', 'dive-row') -alias('next-search', 'search-next') +alias('search-next', 'next-search') alias('prev-search', 'search-prev') alias('prev-sheet', 'jump-prev') alias('prev-value', 'go-prev-value') diff --git a/visidata/loaders/pandas_freqtbl.py b/visidata/loaders/pandas_freqtbl.py index cf5e4f153..8ccef5473 100644 --- a/visidata/loaders/pandas_freqtbl.py +++ b/visidata/loaders/pandas_freqtbl.py @@ -172,6 +172,6 @@ def expand_source_rows(source, vd, cursorRow): PandasFreqTableSheet.addCommand('t', 'stoggle-row', 'toggle([cursorRow]); cursorDown(1)', 'toggle selection of rows grouped in current row in source sheet') PandasFreqTableSheet.addCommand('s', 'select-row', 'select([cursorRow]); cursorDown(1)', 'select rows grouped in current row in source sheet') PandasFreqTableSheet.addCommand('u', 'unselect-row', 'unselect([cursorRow]); cursorDown(1)', 'unselect rows grouped in current row in source sheet') -PandasFreqTableSheet.addCommand(ENTER, 'dup-row', 'expand_source_rows(source, vd, cursorRow)', 'open copy of source sheet with rows that are grouped in current row') +PandasFreqTableSheet.addCommand(ENTER, 'open-row', 'expand_source_rows(source, vd, cursorRow)', 'open copy of source sheet with rows that are grouped in current row') PandasFreqTableSheet.class_options.numeric_binning = False From 4c30a4673d48e1f34037635ef94036127b766751 Mon Sep 17 00:00:00 2001 From: anjakefala Date: Tue, 20 Oct 2020 20:56:26 -0700 Subject: [PATCH 030/162] [set-option-] blank sheet or col fallback to 'override' --- tests/freq-fmtstr.vd | 1 - visidata/cmdlog.py | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/freq-fmtstr.vd b/tests/freq-fmtstr.vd index c742631cd..a3c27ce4e 100644 --- a/tests/freq-fmtstr.vd +++ b/tests/freq-fmtstr.vd @@ -4,6 +4,5 @@ sheet col row longname input keystrokes comment benchmark Date type-date @ benchmark columns-sheet C benchmark_columns fmtstr キDate edit-cell %Y-%m e -benchmark_columns quit-sheet q benchmark Date freq-col F benchmark_Date_freq histogram hide-col - diff --git a/visidata/cmdlog.py b/visidata/cmdlog.py index 70fdffb44..9c9ae6472 100644 --- a/visidata/cmdlog.py +++ b/visidata/cmdlog.py @@ -284,9 +284,9 @@ def replayOne(vd, r): try: context = vs if r.sheet and vs else vd if longname == 'set-option': - context.options.set(r.row, r.input, r.sheet or r.col) + context.options.set(r.row, r.input, r.sheet or r.col or 'override') else: - context.options.unset(r.row, r.sheet or r.col) + context.options.unset(r.row, r.sheet or r.col or 'override') escaped = False except Exception as e: From 6cac4c12d5e5b08091c64e3a8f2623c0764ef72d Mon Sep 17 00:00:00 2001 From: anjakefala Date: Tue, 20 Oct 2020 21:06:39 -0700 Subject: [PATCH 031/162] [search] next-search and searchr-next are now bound to n and N (was next-search and search-prev) --- visidata/deprecated.py | 3 ++- visidata/search.py | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/visidata/deprecated.py b/visidata/deprecated.py index 05712418c..17bc3c378 100644 --- a/visidata/deprecated.py +++ b/visidata/deprecated.py @@ -90,8 +90,9 @@ def push_pyobj(name, pyobj): alias('frequency-rows', 'frequency-summary') alias('dup-cell', 'dive-cell') alias('dup-row', 'dive-row') -alias('search-next', 'next-search') +alias('next-search', 'search-next') alias('prev-search', 'search-prev') +alias('search-prev', 'searchr-next') alias('prev-sheet', 'jump-prev') alias('prev-value', 'go-prev-value') alias('next-value', 'go-next-value') diff --git a/visidata/search.py b/visidata/search.py index 2bc10b5e1..17a56c4dd 100644 --- a/visidata/search.py +++ b/visidata/search.py @@ -74,8 +74,8 @@ def search_expr(sheet, expr, reverse=False): Sheet.addCommand('r', 'search-keys', 'tmp=cursorVisibleColIndex; vd.moveRegex(sheet, regex=input("row key regex: ", type="regex-row", defaultLast=True), columns=keyCols or [visibleCols[0]]); sheet.cursorVisibleColIndex=tmp', 'go to next row with key matching regex') Sheet.addCommand('/', 'search-col', 'vd.moveRegex(sheet, regex=input("/", type="regex", defaultLast=True), columns="cursorCol", backward=False)', 'search for regex forwards in current column'), Sheet.addCommand('?', 'searchr-col', 'vd.moveRegex(sheet, regex=input("?", type="regex", defaultLast=True), columns="cursorCol", backward=True)', 'search for regex backwards in current column'), -Sheet.addCommand('n', 'next-search', 'vd.moveRegex(sheet, reverse=False)', 'go to next match from last regex search'), -Sheet.addCommand('N', 'search-prev', 'vd.moveRegex(sheet, reverse=True)', 'go to previous match from last regex search'), +Sheet.addCommand('n', 'search-next', 'vd.moveRegex(sheet, reverse=False)', 'go to next match from last regex search'), +Sheet.addCommand('N', 'searchr-next', 'vd.moveRegex(sheet, reverse=True)', 'go to previous match from last regex search'), Sheet.addCommand('g/', 'search-cols', 'vd.moveRegex(sheet, regex=input("g/", type="regex", defaultLast=True), backward=False, columns="visibleCols")', 'search for regex forwards over all visible columns'), Sheet.addCommand('g?', 'searchr-cols', 'vd.moveRegex(sheet, regex=input("g?", type="regex", defaultLast=True), backward=True, columns="visibleCols")', 'search for regex backwards over all visible columns'), From 175a4b5cd0128673d98f772e29c89489a18f641e Mon Sep 17 00:00:00 2001 From: anjakefala Date: Wed, 21 Oct 2020 20:15:58 -0700 Subject: [PATCH 032/162] [vdj] explicitly use JsonLineSheet's newRow, instead of CommandLog's --- visidata/cmdlog.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/visidata/cmdlog.py b/visidata/cmdlog.py index 9c9ae6472..590a8118e 100644 --- a/visidata/cmdlog.py +++ b/visidata/cmdlog.py @@ -212,6 +212,10 @@ class CommandLog(_CommandLog, VisiDataMetaSheet): pass class CommandLogJsonl(_CommandLog, JsonLinesSheet): + + def newRow(self): + return JsonLinesSheet.newRow(self) + def iterload(self): for r in JsonLinesSheet.iterload(self): if isinstance(r, TypedWrapper): From d959f7d0dd0b4c6dba4e18295d4c2c61aca39d7a Mon Sep 17 00:00:00 2001 From: anjakefala Date: Wed, 21 Oct 2020 20:16:36 -0700 Subject: [PATCH 033/162] [AttrDict-] raise AttributeError for missing dunders - copy() requests a bunch of different dunders, and expects AttributeErrors to be raised when they do not exists. fixes copy() for JsonLinesSheet --- visidata/utils.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/visidata/utils.py b/visidata/utils.py index 7c566940e..9b2bcb2cf 100644 --- a/visidata/utils.py +++ b/visidata/utils.py @@ -20,6 +20,8 @@ def __getattr__(self, k): try: return self[k] except KeyError: + if k.startswith("__"): + raise AttributeError return None def __setattr__(self, k, v): From e2314f3c91a648ce1d404ad533fc791d0dac06d7 Mon Sep 17 00:00:00 2001 From: Jeremy Singer-Vine Date: Thu, 22 Oct 2020 23:22:23 -0400 Subject: [PATCH 034/162] Update info for @jsvine's plugins --- plugins/plugins.jsonl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/plugins/plugins.jsonl b/plugins/plugins.jsonl index a9f7389b8..965df4adb 100644 --- a/plugins/plugins.jsonl +++ b/plugins/plugins.jsonl @@ -1,7 +1,7 @@ {"name": "vfake", "description": "replace column with fake values", "maintainer": "Saul Pwanson @saulpw", "latest_release": "2020-10-06", "url": "https://raw.githubusercontent.com/saulpw/visidata/0d8af5a226818aa2f99a2f74e08543152b22a36d/plugins/vfake.py", "latest_ver": "1.1", "visidata_ver": "2.0", "pydeps": "Faker", "vdplugindeps": "", "sha256": "8153b04a364ac3db2ee310503f762924c4dc472445477af9e027c4d0e921af7f"} -{"name": "vddedupe", "description": "adds commands for selection and removal of rows which are duplicates of prior rows", "maintainer": "Jeremy Singer-Vine @jsvine", "latest_release": "2019-01-01", "url": "https://raw.githubusercontent.com/jsvine/visidata-plugins/fc070edde97c543ed345667cba58110d08b4895d/plugins/vddedupe.py", "latest_ver": "0.0.1", "visidata_ver": "1.5.2", "pydeps": "", "vdplugindeps": "", "sha256": "c9adf5323e7fbdc537ceedb58de30af2c145141eba1d53288a1e187cfd28452d"} -{"name": "vdnormcol", "description": "normalises column names in any given sheet so that the names are unique, valid Python identifiers, and only composed of lowercase letters, numbers, and underscores", "maintainer": "Jeremy Singer-Vine @jsvine", "latest_release": "2018-12-31", "url": "https://raw.githubusercontent.com/jsvine/visidata-plugins/fc070edde97c543ed345667cba58110d08b4895d/plugins/vdnormcol.py", "latest_ver": "0.0.0", "visidata_ver": "2.0", "pydeps": "", "vdplugindeps": "", "sha256": "77d1d40c31cb7d66c47decffa85c5db120e81969ab10c6dd41d436992995e72e"} -{"name": "vdfec", "description": "loader for .fec files from the Federal Election Commission", "maintainer": "Jeremy Singer-Vine @jsvine", "latest_release": "2019-04-21", "url": "https://raw.githubusercontent.com/jsvine/visidata-plugins/fc070edde97c543ed345667cba58110d08b4895d/plugins/vdfec.py", "latest_ver": "0.0.0", "visidata_ver": "1.5.2", "pydeps": "fecfile", "vdplugindeps": "", "sha256": "e917170e5bb74ac6242f81113aa48ba7e36763ad32cb190420acb93c3f62765c"} +{"name": "dedupe", "description": "adds commands for selection and removal of rows which are duplicates of prior rows", "maintainer": "Jeremy Singer-Vine @jsvine", "latest_release": "2020-10-11", "url": "https://raw.githubusercontent.com/jsvine/visidata-plugins/37cba82ed2cc49aedc1a377af984acfe1ef3d5cd/plugins/dedupe.py", "latest_ver": "0.1.0", "visidata_ver": "2.0", "pydeps": "", "vdplugindeps": "", "sha256": "ac74e898fd8492d9dc96f09ab7852a4057ff853565061709907b52bf12f2ac25"} +{"name": "normcol", "description": "normalizes column names in any given sheet so that the names are unique, valid Python identifiers, and only composed of lowercase letters, numbers, and underscores", "maintainer": "Jeremy Singer-Vine @jsvine", "latest_release": "2020-10-11", "url": "https://raw.githubusercontent.com/jsvine/visidata-plugins/37cba82ed2cc49aedc1a377af984acfe1ef3d5cd/plugins/normcol.py", "latest_ver": "0.1.0", "visidata_ver": "2.0", "pydeps": "", "vdplugindeps": "", "sha256": "018e364d03f0eeda808005e6ad2564d7e9df945820e9d0c4549c15c2cfbfec08"} +{"name": "fec", "description": "loader for .fec files from the Federal Election Commission", "maintainer": "Jeremy Singer-Vine @jsvine", "latest_release": "2019-04-21", "url": "https://raw.githubusercontent.com/jsvine/visidata-plugins/37cba82ed2cc49aedc1a377af984acfe1ef3d5cd/plugins/fec.py", "latest_ver": "0.0.0", "visidata_ver": "1.5.2", "pydeps": "fecfile", "vdplugindeps": "", "sha256": "a4f06a2e6589433dad583c66b6566073c236b7838b6c11d317b7be29ec0ed199"} {"name": "livesearch", "description": "filter rows as you search", "maintainer": "Saul Pwanson @saulpw", "latest_release": "2019-06-09", "url": "https://raw.githubusercontent.com/saulpw/visidata/a185cb5f734a58d14478cf13477246c7773245f8/plugins/livesearch.py", "latest_ver": "0.9", "visidata_ver": "2.-1", "pydeps": "", "vdplugindeps": "", "sha256": "57ba2685252f0a4659d2582e05e7d9bab9e680314883328a202ed220dbd0bb6f"} {"name": "sparkline", "description": "add a sparkline column to visualise trends of numeric cells in a row", "maintainer": "Lucas Messenger @layertwo", "latest_release": "2020-09-13", "url": "https://raw.githubusercontent.com/saulpw/visidata/e4a373cd74b49b2b91876b17d7a1cd0cd0303848/plugins/sparkline.py", "latest_ver": "0.1", "visidata_ver": "2.0", "pydeps": "", "vdplugindeps": "", "sha256": "4256bea5535405a2478cb6827df4be86f28c69b3cb7f30cc08e2cd655e48b592"} {"name": "rownum", "description": "add column of original row ordering", "maintainer": "Saul Pwanson @saulpw", "latest_release": "2019-11-07", "url": "https://raw.githubusercontent.com/saulpw/visidata/a185cb5f734a58d14478cf13477246c7773245f8/plugins/rownum.py", "latest_ver": "0.9", "visidata_ver": "2.0", "pydeps": "", "vdplugindeps": "", "sha256": "caf896841639ecdef25c2f227f3370a4c4f0b0b31bdd72ef422dfe94ec5664f0"} From 96449ec06b7702241149b4846e63c72557389628 Mon Sep 17 00:00:00 2001 From: anjakefala Date: Thu, 22 Oct 2020 21:15:39 -0700 Subject: [PATCH 035/162] [plugins] fix-up shas --- plugins/plugins.jsonl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/plugins/plugins.jsonl b/plugins/plugins.jsonl index 965df4adb..f3bbdf830 100644 --- a/plugins/plugins.jsonl +++ b/plugins/plugins.jsonl @@ -1,7 +1,7 @@ {"name": "vfake", "description": "replace column with fake values", "maintainer": "Saul Pwanson @saulpw", "latest_release": "2020-10-06", "url": "https://raw.githubusercontent.com/saulpw/visidata/0d8af5a226818aa2f99a2f74e08543152b22a36d/plugins/vfake.py", "latest_ver": "1.1", "visidata_ver": "2.0", "pydeps": "Faker", "vdplugindeps": "", "sha256": "8153b04a364ac3db2ee310503f762924c4dc472445477af9e027c4d0e921af7f"} -{"name": "dedupe", "description": "adds commands for selection and removal of rows which are duplicates of prior rows", "maintainer": "Jeremy Singer-Vine @jsvine", "latest_release": "2020-10-11", "url": "https://raw.githubusercontent.com/jsvine/visidata-plugins/37cba82ed2cc49aedc1a377af984acfe1ef3d5cd/plugins/dedupe.py", "latest_ver": "0.1.0", "visidata_ver": "2.0", "pydeps": "", "vdplugindeps": "", "sha256": "ac74e898fd8492d9dc96f09ab7852a4057ff853565061709907b52bf12f2ac25"} -{"name": "normcol", "description": "normalizes column names in any given sheet so that the names are unique, valid Python identifiers, and only composed of lowercase letters, numbers, and underscores", "maintainer": "Jeremy Singer-Vine @jsvine", "latest_release": "2020-10-11", "url": "https://raw.githubusercontent.com/jsvine/visidata-plugins/37cba82ed2cc49aedc1a377af984acfe1ef3d5cd/plugins/normcol.py", "latest_ver": "0.1.0", "visidata_ver": "2.0", "pydeps": "", "vdplugindeps": "", "sha256": "018e364d03f0eeda808005e6ad2564d7e9df945820e9d0c4549c15c2cfbfec08"} -{"name": "fec", "description": "loader for .fec files from the Federal Election Commission", "maintainer": "Jeremy Singer-Vine @jsvine", "latest_release": "2019-04-21", "url": "https://raw.githubusercontent.com/jsvine/visidata-plugins/37cba82ed2cc49aedc1a377af984acfe1ef3d5cd/plugins/fec.py", "latest_ver": "0.0.0", "visidata_ver": "1.5.2", "pydeps": "fecfile", "vdplugindeps": "", "sha256": "a4f06a2e6589433dad583c66b6566073c236b7838b6c11d317b7be29ec0ed199"} +{"name": "dedupe", "description": "adds commands for selection and removal of rows which are duplicates of prior rows", "maintainer": "Jeremy Singer-Vine @jsvine", "latest_release": "2020-10-11", "url": "https://raw.githubusercontent.com/jsvine/visidata-plugins/37cba82ed2cc49aedc1a377af984acfe1ef3d5cd/plugins/dedupe.py", "latest_ver": "0.1.0", "visidata_ver": "2.0", "pydeps": "", "vdplugindeps": "", "sha256": "2e077b8d62bc8ec2235d22d7d9711d99e7366c8c0ff20405f70647044cca67a1"} +{"name": "normcol", "description": "normalizes column names in any given sheet so that the names are unique, valid Python identifiers, and only composed of lowercase letters, numbers, and underscores", "maintainer": "Jeremy Singer-Vine @jsvine", "latest_release": "2020-10-11", "url": "https://raw.githubusercontent.com/jsvine/visidata-plugins/37cba82ed2cc49aedc1a377af984acfe1ef3d5cd/plugins/normcol.py", "latest_ver": "0.1.0", "visidata_ver": "2.0", "pydeps": "", "vdplugindeps": "", "sha256": "ede6b9e508e1ce7842e00d308c372837c9b4d4906b88e0ba62efd5b481f816f6"} +{"name": "fec", "description": "loader for .fec files from the Federal Election Commission", "maintainer": "Jeremy Singer-Vine @jsvine", "latest_release": "2019-04-21", "url": "https://raw.githubusercontent.com/jsvine/visidata-plugins/37cba82ed2cc49aedc1a377af984acfe1ef3d5cd/plugins/fec.py", "latest_ver": "0.0.0", "visidata_ver": "1.5.2", "pydeps": "fecfile", "vdplugindeps": "", "sha256": "e917170e5bb74ac6242f81113aa48ba7e36763ad32cb190420acb93c3f62765c"} {"name": "livesearch", "description": "filter rows as you search", "maintainer": "Saul Pwanson @saulpw", "latest_release": "2019-06-09", "url": "https://raw.githubusercontent.com/saulpw/visidata/a185cb5f734a58d14478cf13477246c7773245f8/plugins/livesearch.py", "latest_ver": "0.9", "visidata_ver": "2.-1", "pydeps": "", "vdplugindeps": "", "sha256": "57ba2685252f0a4659d2582e05e7d9bab9e680314883328a202ed220dbd0bb6f"} {"name": "sparkline", "description": "add a sparkline column to visualise trends of numeric cells in a row", "maintainer": "Lucas Messenger @layertwo", "latest_release": "2020-09-13", "url": "https://raw.githubusercontent.com/saulpw/visidata/e4a373cd74b49b2b91876b17d7a1cd0cd0303848/plugins/sparkline.py", "latest_ver": "0.1", "visidata_ver": "2.0", "pydeps": "", "vdplugindeps": "", "sha256": "4256bea5535405a2478cb6827df4be86f28c69b3cb7f30cc08e2cd655e48b592"} {"name": "rownum", "description": "add column of original row ordering", "maintainer": "Saul Pwanson @saulpw", "latest_release": "2019-11-07", "url": "https://raw.githubusercontent.com/saulpw/visidata/a185cb5f734a58d14478cf13477246c7773245f8/plugins/rownum.py", "latest_ver": "0.9", "visidata_ver": "2.0", "pydeps": "", "vdplugindeps": "", "sha256": "caf896841639ecdef25c2f227f3370a4c4f0b0b31bdd72ef422dfe94ec5664f0"} From f4a9a7026be33f47d662d3cf9347a819c7a9d689 Mon Sep 17 00:00:00 2001 From: Saul Pwanson Date: Wed, 21 Oct 2020 00:42:02 -0700 Subject: [PATCH 036/162] [cosmetic] max aggregator helpstr --- visidata/aggregators.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/visidata/aggregators.py b/visidata/aggregators.py index 03e40aabb..63a7d51c7 100644 --- a/visidata/aggregators.py +++ b/visidata/aggregators.py @@ -96,7 +96,7 @@ def quantiles(q, helpstr): return [percentile(round(100*i/q), helpstr) for i in range(1, q)] vd.aggregator('min', min, 'minimum value') -vd.aggregator('max', max, 'minimum value') +vd.aggregator('max', max, 'maximum value') vd.aggregator('avg', mean, 'arithmetic mean of values', type=float) vd.aggregator('mean', mean, 'arithmetic mean of values', type=float) vd.aggregator('median', median, 'median of values') From 3dabc0a9c00a812952e925c4cbebb6eb7d5e1525 Mon Sep 17 00:00:00 2001 From: Saul Pwanson Date: Wed, 21 Oct 2020 00:47:19 -0700 Subject: [PATCH 037/162] [expr-] fix `row` to return row object --- visidata/sheets.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/visidata/sheets.py b/visidata/sheets.py index 9bf3559cd..ed887ccd0 100644 --- a/visidata/sheets.py +++ b/visidata/sheets.py @@ -134,7 +134,7 @@ def __getitem__(self, colid): return self.sheet._lcm[colid] except (KeyError, AttributeError): if colid == 'row': - return self + return self.row elif colid == 'sheet': return self.sheet raise KeyError(colid) From 8afd4222c94352bd4a54da8dda4ed04633f8f793 Mon Sep 17 00:00:00 2001 From: Saul Pwanson Date: Thu, 22 Oct 2020 11:01:48 -0700 Subject: [PATCH 038/162] [input] add ^M to accept value on windows #744 --- visidata/_input.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/visidata/_input.py b/visidata/_input.py index 507b4327a..3aaf99edd 100644 --- a/visidata/_input.py +++ b/visidata/_input.py @@ -6,7 +6,7 @@ from visidata import EscapeException, ExpectedException, clipdraw, Sheet, VisiData from visidata import vd, options, theme, colors -from visidata import launchExternalEditor, suspend, ColumnItem, ENTER +from visidata import launchExternalEditor, suspend, ColumnItem __all__ = ['confirm', 'CompleteKey'] @@ -124,7 +124,6 @@ def editline(vd, scr, y, x, w, i=0, attr=curses.A_NORMAL, value='', fillchar=' ' 'A better curses line editing widget.' with EnableCursor(): ESC='^[' - ENTER='^J' TAB='^I' history_state = HistoryState(history) @@ -196,7 +195,7 @@ def find_nonword(s, a, b, incr): elif ch in ('^H', 'KEY_BACKSPACE', '^?'): i -= 1; v = delchar(v, i) elif ch == TAB: v, i = complete_state.complete(v, i, +1) elif ch == 'KEY_BTAB': v, i = complete_state.complete(v, i, -1) - elif ch == ENTER: break + elif ch in ['^J', '^M']: break # ENTER to accept value elif ch == '^K': v = v[:i] # ^Kill to end-of-line elif ch == '^O': v = launchExternalEditor(v) elif ch == '^R': v = str(value) # ^Reload initial value From 6e52abec387b7222c4d2c1453c94713f3a4f7286 Mon Sep 17 00:00:00 2001 From: Saul Pwanson Date: Thu, 22 Oct 2020 20:46:19 -0700 Subject: [PATCH 039/162] [docs] add man/vd.txt as plaintext manpage --- visidata/man/vd.txt | 878 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 878 insertions(+) create mode 100644 visidata/man/vd.txt diff --git a/visidata/man/vd.txt b/visidata/man/vd.txt new file mode 100644 index 000000000..995e141a5 --- /dev/null +++ b/visidata/man/vd.txt @@ -0,0 +1,878 @@ +vd(1) Quick Reference Guide vd(1) + +NAME + VisiData — a terminal utility for exploring and arranging tabular data + +SYNOPSIS + vd [options] [input ...] + vd [options] --play cmdlog [-w waitsecs] [--batch] [-o output] + [field=value] + vd [options] [input ...] +toplevel:subsheet:col:row + +DESCRIPTION + VisiData is an easy-to-use multipurpose tool to explore, clean, edit, and + restructure data. Rows can be selected, filtered, and grouped; columns + can be rearranged, transformed, and derived via regex or Python expres‐ + sions; and workflows can be saved, documented, and replayed. + + REPLAY MODE + -p, --play=cmdlog replay a saved cmdlog within the interface + -w, --replay-wait=seconds + wait seconds between commands + -b, --batch replay in batch mode (with no interface) + -o, --output=file save final visible sheet to file as .tsv + --replay-movement toggle --play to move cursor cell-by-cell + field=value replace "{field}" in cmdlog contents with value + + Commands During Replay + ^U pause/resume replay + ^N execute next row in replaying sheet + ^K cancel current replay + + GLOBAL COMMANDS + All keystrokes are case sensitive. The ^ prefix is shorthand for Ctrl. + + Keystrokes to start off with + ^Q abort program immediately + ^C cancel user input or abort all async threads on current + sheet + g^C abort all secondary threads + q quit current sheet + gq quit all sheets (clean exit) + + ^H view this man page + z^H view sheet of command longnames and keybindings + Space longname execute command by its longname + + U undo the most recent modification (requires enabled + options.undo) + R redo the most recent undo (requires enabled + options.undo) + + Cursor Movement + Arrow PgUp Home go as expected + h j k l go left/down/up/right + gh gj gk gl go all the way to the left/bottom/top/right of sheet + G gg go all the way to the bottom/top of sheet + ^B ^F scroll one page back/forward + zz scroll current row to center of screen + + ^^ (Ctrl+^) jump to previous sheet (swaps with current sheet) + + / ? regex search for regex forward/backward in current column + g/ g? regex search for regex forward/backward over all visible + columns + z/ z? expr search by Python expr forward/backward in current column + (with column names as variables) + n N go to next/previous match from last regex search + + < > go up/down current column to next value + z< z> go up/down current column to next null value + { } go up/down current column to next selected row + + c regex go to next column with name matching regex + r regex go to next row with key matching regex + zc zr number go to column/row number (0-based) + + H J K L slide current row/column left/down/up/right + gH gJ gK gL slide current row/column all the way to the left/bot‐ + tom/top/right of sheet + zH zJ zK zK number + slide current row/column number positions to the + left/down/up/right + + zh zj zk zl scroll one left/down/up/right + + Column Manipulation + _ (underscore) + toggle width of current column between full and default + width + g_ toggle widths of all visible columns between full and + default width + z_ number adjust width of current column to number + gz_ number adjust widths of all visible columns to Ar number + + - (hyphen) hide current column + z- reduce width of current column by half + gv unhide all columns + + ! z! toggle/unset current column as a key column + ~ # % $ @ z# + set type of current column to str/int/float/cur‐ + rency/date/len + ^ edit name of current column + g^ set names of all unnamed visible columns to contents of + selected rows (or current row) + z^ set name of current column to combined contents of cur‐ + rent cell in selected rows (or current row) + gz^ set name of all visible columns to combined contents of + current column for selected rows (or current row) + + = expr create new column from Python expr, with column names as + variables + g= expr set current column for selected rows to result of Python + expr + gz= expr set current column for selected rows to the items in + result of Python sequence expr + z= expr evaluate Python expression on current row and set + current cell with result of Python expr + + i (iota) add column with incremental values + gi set current column for selected rows to incremental + values + zi step add column with values at increment step + gzi step set current column for selected rows at increment step + + ' (tick) add a frozen copy of current column with all cells eval‐ + uated + g' open a frozen copy of current sheet with all visible + columns evaluated + z' gz' add/reset cache for current/all visible column(s) + + : regex add new columns from regex split; number of columns + determined by example row at cursor + ; regex add new columns from capture groups of regex (also + requires example row) + z; expr create new column from bash expr, with $columnNames as + variables + * regex/subst add column derived from current column, replacing regex + with subst (may include \1 backrefs) + g* gz* regex/subst + modify selected rows in current/all visible column(s), + replacing regex with subst (may include \1 backrefs) + + ( g( expand current/all visible column(s) of lists (e.g. [3]) + or dicts (e.g. {3}) fully + z( gz( depth expand current/all visible column(s) of lists (e.g. [3]) + or dicts (e.g. {3}) to given depth (0= fully) + ) unexpand current column; restore original column and re‐ + move other columns at this level + zM row-wise expand current column of lists (e.g. [3]) or + dicts (e.g. {3}) within that column + + Row Selection + s t u select/toggle/unselect current row + gs gt gu select/toggle/unselect all rows + zs zt zu select/toggle/unselect all rows from top to cursor + gzs gzt gzu select/toggle/unselect all rows from cursor to bottom + | \ regex select/unselect rows matching regex in current column + g| g\ regex select/unselect rows matching regex in any visible + column + z| z\ expr select/unselect rows matching Python expr in any visible + column + , (comma) select rows matching current cell in current column + g, select rows matching current row in all visible columns + + Row Sorting/Filtering + [ ] sort ascending/descending by current column; replace any + existing sort criteria + g[ g] sort ascending/descending by all key columns; replace + any existing sort criteria + z[ z] sort ascending/descending by current column; add to ex‐ + isting sort criteria + gz[ gz] sort ascending/descending by all key columns; add to ex‐ + isting sort criteria + " open duplicate sheet with only selected rows + g" open duplicate sheet with all rows + gz" open duplicate sheet with deepcopy of selected rows + + Editing Rows and Cells + a za append a blank row/column; appended columns cannot be + copied to clipboard + ga gza number append number blank rows/columns + d gd delete (cut) current/selected row(s) and move to clip‐ + board + y gy yank (copy) current/all selected row(s) to clipboard + zy gzy yank (copy) contents of current column for current/se‐ + lected row(s) to clipboard + zd gzd delete (cut) contents of current column for current/se‐ + lected row(s) and move to clipboard + p P paste clipboard rows after/before current row + zp gzp set cells of current column for current/selected row(s) + to last clipboard value + Y gY yank (copy) current/all selected row(s) to system + clipboard (using options.clipboard_copy_cmd) + zY gzY yank (copy) contents of current column for + current/selected row(s) to system clipboard (using + options.clipboard_copy_cmd) + f fill null cells in current column with contents of non- + null cells up the current column + e text edit contents of current cell + ge text set contents of current column for selected rows to text + + Commands While Editing Input + Enter ^C accept/abort input + ^O open external $EDITOR to edit contents + ^R reload initial value + ^A ^E go to beginning/end of line + ^B ^F go back/forward one character + ^← ^→ (arrow) go back/forward one word + ^H ^D delete previous/current character + ^T transpose previous and current characters + ^U ^K clear from cursor to beginning/end of line + ^Y paste from cell clipboard + Backspace Del delete previous/current character + Insert toggle insert mode + Up Down set contents to previous/next in history + Tab Shift+Tab autocomplete input (when available) + + Data Toolkit + o input open input in VisiData + ^S g^S filename save current/all sheet(s) to filename in format + determined by extension (default .tsv) + Note: if the format does not support multisave, or the + filename ends in a /, a directory will be created. + z^S filename save current column only to filename in format + determined by extension (default .tsv) + ^D filename.vd save CommandLog to filename.vd file + A open new blank sheet with one column + T open new sheet that has rows and columns of current + sheet transposed + + + aggregator add aggregator to current column (see Frequency Table) + z+ aggregator display result of aggregator over values in selected + rows for current column + & concatenate top two sheets in Sheets Stack + g& concatenate all sheets in Sheets Stack + + Data Visualization + . (dot) plot current numeric column vs key columns. The numeric + key column is used for the x-axis; categorical key column + values determine color. + g. plot a graph of all visible numeric columns vs key + columns. + + If rows on the current sheet represent plottable coordinates (as in .shp + or vector .mbtiles sources), . plots the current row, and g. plots all + selected rows (or all rows if none selected). + + Canvas-specific Commands + + - increase/decrease zoom level, centered on cursor + _ (underscore) zoom to fit full extent + z_ (underscore) set aspect ratio + x xmin xmax set xmin/xmax on graph + y ymin ymax set ymin/ymax on graph + s t u select/toggle/unselect rows on source sheet con‐ + tained within canvas cursor + gs gt gu select/toggle/unselect rows on source sheet visi‐ + ble on screen + d delete rows on source sheet contained within can‐ + vas cursor + gd delete rows on source sheet visible on screen + Enter open sheet of source rows contained within canvas + cursor + gEnter open sheet of source rows visible on screen + 1 - 9 toggle display of layers + ^L redraw all pixels on canvas + v toggle show_graph_labels option + mouse scrollwheel zoom in/out of canvas + left click-drag set canvas cursor + right click-drag scroll canvas + + Split Screen + Z split screen in half, so that second sheet on the stack is + visible in a second pane + zZ split screen, and queries for height of second pane + + Split Window specific Commands + gZ close an already split screen, current pane full + screens + Tab jump to other pane + ^^ (Ctrl+^) swap which sheet is in current pane + + Other Commands + Q quit current sheet and remove it from the CommandLog + v toggle sheet-specific visibility (multi-line rows on + Sheet, legends/axes on Graph) + + ^E g^E view traceback for most recent error(s) + z^E view traceback for error in current cell + + ^L refresh screen + ^R reload current sheet + z^R clear cache for current column + ^Z suspend VisiData process + ^G show cursor position and bounds of current sheet on sta‐ + tus line + ^V show version and copyright information on status line + ^P open Status History + + ^Y z^Y g^Y open current row/cell/sheet as Python object + ^X expr evaluate Python expr and opens result as Python object + z^X expr evaluate Python expr, in context of current row, and + open result as Python object + g^X stmt execute Python stmt in the global scope + + Internal Sheets List + . VisiDataMenu (Shift+V) browse list of core sheets + . Directory Sheet browse properties of files in a directory + . Plugins Sheet browse, install, and (de)activate plugins + + Metasheets + . Columns Sheet (Shift+C) edit column properties + . Sheets Sheet (Shift+S) jump between sheets or join them together + . Options Sheet (Shift+O) edit configuration options + . Commandlog (Shift+D) modify and save commands for replay + . Error Sheet (Ctrl+E) view last error + . Status History (Ctrl+P) view history of status messages + . Threads Sheet (Ctrl+T) view, cancel, and profile + asynchronous threads + + Derived Sheets + . Frequency Table (Shift+F) group rows by column value, with + aggregations of other columns + . Describe Sheet (Shift+I) view summary statistics for each column + . Pivot Table (Shift+W) group rows by key and summarize current + column + . Melted Sheet (Shift+M) unpivot non-key columns into + variable/value columns + . Transposed Sheet (Shift+T) open new sheet with rows and columns + transposed + + INTERNAL SHEETS + VisiDataMenu (Shift+V) + (sheet-specific commands) + Enter load sheet in current row + + Directory Sheet + (global commands) + Space open-dir-current + open the Directory Sheet for the current directory + (sheet-specific commands) + Enter gEnter open current/selected file(s) as new sheet(s) + ^O g^O open current/selected file(s) in external $EDITOR + ^R z^R gz^R reload information for all/current/selected file(s) + + Plugins Sheet + Browse through a list of available plugins. VisiData needs to be + restarted before plugin activation takes effect. Installation may require + internet access. + (global commands) + Space open-plugins + open the Plugins Sheet + (sheet-specific commands) + a install and activate current plugin + d deactivate current plugin + + METASHEETS + Columns Sheet (Shift+C) + Properties of columns on the source sheet can be changed with standard + editing commands (e ge g= Del) on the Columns Sheet. Multiple aggregators + can be set by listing them (separated by spaces) in the aggregators + column. The 'g' commands affect the selected rows, which are the literal + columns on the source sheet. + (global commands) + gC open Columns Sheet with all visible columns from all + sheets + (sheet-specific commands) + & add column from concatenating selected source columns + g! gz! toggle/unset selected columns as key columns on + source sheet + g+ aggregator add Ar aggregator No to selected source columns + g- (hyphen) hide selected columns on source sheet + g~ g# g% g$ g@ gz# + set type of selected columns on source sheet to + str/int/float/currency/date/len + Enter open a Frequency Table sheet grouped by column + referenced in current row + + Sheets Sheet (Shift+S) + open Sheets Stack, which contains only the active sheets on the current + stack + (global commands) + gS open Sheets Sheet, which contains all sheets from + current session, active and inactive + Alt number jump to sheet number + (sheet-specific commands) + Enter jump to sheet referenced in current row + gEnter push selected sheets to top of sheet stack + a add row to reference a new blank sheet + gC gI open Columns Sheet/Describe Sheet with all visible + columns from selected sheets + g^R reload all selected sheets + z^C gz^C abort async threads for current/selected sheets(s) + & jointype merge selected sheets with visible columns from all, + keeping rows according to jointype: + . inner keep only rows which match keys on all + sheets + . outer keep all rows from first selected sheet + . full keep all rows from all sheets (union) + . diff keep only rows NOT in all sheets + . append keep all rows from all sheets + (concatenation) + . extend copy first selected sheet, keeping all rows + and sheet type, and extend with columns from other + sheets + . merge mostly keep all rows from first selected + sheet, except prioritise cells with non-null/non- + error values + + Options Sheet (Shift+O) + (global commands) + Shift+O edit global options (apply to all sheets) + zO edit sheet options (apply to current sheet only) + gO open options.config as TextSheet + (sheet-specific commands) + Enter e edit option at current row + + CommandLog (Shift+D) + (global commands) + D open current sheet's CommandLog with all other loose + ends removed; includes commands from parent sheets + gD open global CommandLog for all commands executed in + the current session + zD open current sheet's CommandLog with the parent + sheets commands' removed + (sheet-specific commands) + x replay command in current row + gx replay contents of entire CommandLog + ^C abort replay + + DERIVED SHEETS + Frequency Table (Shift+F) + A Frequency Table groups rows by one or more columns, and includes + summary columns for those with aggregators. + (global commands) + gF open Frequency Table, grouped by all key columns on + source sheet + zF open one-line summary for all rows and selected rows + (sheet-specific commands) + s t u select/toggle/unselect these entries in source sheet + Enter gEnter open copy of source sheet with rows that are grouped + in current cell / selected rows + + Describe Sheet (Shift+I) + A Describe Sheet contains descriptive statistics for all visible columns. + (global commands) + gI open Describe Sheet for all visible columns on all + sheets + (sheet-specific commands) + zs zu select/unselect rows on source sheet that are being + described in current cell + ! toggle/unset current column as a key column on source + sheet + Enter open a Frequency Table sheet grouped on column + referenced in current row + zEnter open copy of source sheet with rows described in cur‐ + rent cell + + Pivot Table (Shift+W) + Set key column(s) and aggregators on column(s) before pressing Shift+W on + the column to pivot. + (sheet-specific commands) + Enter open sheet of source rows aggregated in current pivot + row + zEnter open sheet of source rows aggregated in current pivot + cell + + Melted Sheet (Shift+M) + Open Melted Sheet (unpivot), with key columns retained and all non-key + columns reduced to Variable-Value rows. + (global commands) + gM regex open Melted Sheet (unpivot), with key columns + retained and regex capture groups determining how the + non-key columns will be reduced to Variable-Value + rows. + + Python Object Sheet (^X ^Y g^Y z^Y) + (sheet-specific commands) + Enter dive further into Python object + v toggle show/hide for methods and hidden properties + gv zv show/hide methods and hidden properties + +COMMANDLINE OPTIONS + -P=longname preplay longname before replay or regular + launch; limited to Base Sheet bound commands + +toplevel:subsheet:col:row launch vd with subsheet of toplevel at + top-of-stack, and cursor at col and row; all + arguments are optional + + -f, --filetype=filetype tsv set loader to use for + filetype instead of file extension + -y, --confirm-overwrite=F True overwrite existing files + without confirmation + --mouse-interval=int 1 max time between + press/release for click + (ms) + --null-value=NoneType None a value to be counted as + null + --undo=bool True enable undo/redo + --col-cache-size=int 0 max number of cache en‐ + tries in each cached col‐ + umn + --clean-names False clean column/sheet names + to be valid Python iden‐ + tifiers + --default-width=int 20 default column width + --default-height=int 10 default column height + --textwrap-cells=bool True wordwrap text for multi‐ + line rows + --cmd-after-edit=str go-down command longname to exe‐ + cute after successful + edit + --quitguard False confirm before quitting + last sheet + --debug False exit on error and display + stacktrace + --skip=int 0 skip N rows before header + --header=int 1 parse first N rows as + column names + --force-256-colors False use 256 colors even if + curses reports fewer + --use-default-colors False curses use default termi‐ + nal colors + --note-pending=str ⌛ note to display for pend‐ + ing cells + --note-format-exc=str ? cell note for an excep‐ + tion during formatting + --note-getter-exc=str ! cell note for an excep‐ + tion during computation + --note-type-exc=str ! cell note for an excep‐ + tion during type conver‐ + sion + --scroll-incr=int 3 amount to scroll with + scrollwheel + --name-joiner=str _ string to join sheet or + column names + --value-joiner=str string to join display + values + --wrap False wrap text to fit window + width on TextSheet + --save-filetype=str tsv specify default file type + to save as + --profile=str filename to save binary + profiling data + --min-memory-mb=int 0 minimum memory to con‐ + tinue loading and async + processing + --encoding=str utf-8 encoding passed to + codecs.open + --encoding-errors=str surrogateescape encoding_errors passed to + codecs.open + --bulk-select-clear False clear selected rows be‐ + fore new bulk selections + --delimiter=str field delimiter to use + for tsv/usv filetype + --row-delimiter=str " row delimiter to use + for tsv/usv filetype + --tsv-safe-newline=str replacement for newline + character when saving to + tsv + --tsv-safe-tab=str replacement for tab char‐ + acter when saving to tsv + --visibility=int 0 visibility level (0=low, + 1=high) + --expand-col-scanrows=int 1000 number of rows to check + when expanding columns (0 + = all) + --json-indent=NoneType None indent to use when saving + json + --json-sort-keys False sort object keys when + saving to json + --default-colname=str column name to use for + non-dict rows + --filetype=str specify file type + --confirm-overwrite=bool True whether to prompt for + overwrite confirmation on + save + --safe-error=str #ERR error string to use while + saving + --clipboard-copy-cmd=str command to copy stdin to + system clipboard + --clipboard-paste-cmd=str command to get contents + of system clipboard + --fancy-chooser=bool True a nicer selection inter‐ + face for aggregators and + jointype + --describe-aggrs=str mean stdev numeric aggregators to + calculate on Describe + sheet + --histogram-bins=int 0 number of bins for his‐ + togram of numeric columns + --numeric-binning=bool True bin numeric columns into + ranges + --replay-wait=float 0.0 time to wait between re‐ + played commands, in sec‐ + onds + --replay-movement False insert movements during + replay + --visidata-dir=str ~/.visidata/ directory to load and + store additional files + --rowkey-prefix=str キ string prefix for rowkey + in the cmdlog + --cmdlog-histfile=str file to autorecord each + cmdlog action to + --regex-flags=str I flags to pass to re.com‐ + pile() [AILMSUX] + --regex-maxsplit=int 0 maxsplit to pass to + regex.split + --default-sample-size=int 100 number of rows to sample + for regex.split + --show-graph-labels=bool True show axes and legend on + graph + --plot-colors=str list of distinct colors + to use for plotting dis‐ + tinct objects + --zoom-incr=float 2.0 amount to multiply cur‐ + rent zoomlevel when zoom‐ + ing + --motd-url=str source of randomized + startup messages + --dir-recurse False walk source path recur‐ + sively on DirSheet + --dir-hidden False load hidden files on + DirSheet + --config=str ~/.visidatarc config file to exec in + Python + --play=str + --batch False replay in batch mode + (with no interface and + all status sent to std‐ + out) + --output=NoneType None save the final visible + sheet to output at the + end of replay + --preplay=str longnames to preplay be‐ + fore replay + --imports=str plugins imports to preload before + .visidatarc (command-line + only) + --incr-base=float 1.0 start value for column + increments + --csv-dialect=str excel dialect passed to + csv.reader + --csv-delimiter=str , delimiter passed to + csv.reader + --csv-quotechar=str " quotechar passed to + csv.reader + --csv-skipinitialspace=bool True skipinitialspace passed + to csv.reader + --csv-escapechar=NoneType None escapechar passed to + csv.reader + --csv-lineterminator=str " lineterminator passed + to csv.writer + --safety-first False sanitize input/output to + handle edge cases, with a + performance cost + --fixed-rows=int 1000 number of rows to check + for fixed width columns + --fixed-maxcols=int 0 max number of fixed-width + columns to create (0 is + no max) + --postgres-schema=str public The desired schema for + the Postgres database + --html-title=str

{sheet.name}

+ table header when saving + to html + --pcap-internet=str n (y/s/n) if save_dot in‐ + cludes all internet hosts + separately (y), combined + (s), or does not include + the internet (n) + --graphviz-edge-labels=bool True whether to include edge + labels on graphviz dia‐ + grams + --plugins-url=str https://visidata.org/plugins/plugins.jsonl + source of plugins sheet + + DISPLAY OPTIONS + Display options can only be set via the Options Sheet or a .visidatarc + (see FILES). + + disp_splitwin_pct 0 height of second sheet on screen + disp_currency_fmt %.02f default fmtstr to format for cur‐ + rency values + disp_float_fmt {:.02f} default fmtstr to format for + float values + disp_int_fmt {:.0f} default fmtstr to format for int + values + disp_date_fmt %Y-%m-%d default fmtstr to strftime for + date values + disp_note_none ⌀ visible contents of a cell whose + value is None + disp_truncator … indicator that the contents are + only partially visible + disp_oddspace · displayable character for odd + whitespace + disp_more_left < header note indicating more col‐ + umns to the left + disp_more_right > header note indicating more col‐ + umns to the right + disp_error_val displayed contents for computa‐ + tion exception + disp_ambig_width 1 width to use for unicode chars + marked ambiguous + disp_pending string to display in pending + cells + color_note_pending bold magenta color of note in pending cells + color_note_type 226 yellow color of cell note for non-str + types in anytype columns + color_note_row 220 yellow color of row note on left edge + disp_column_sep | separator between columns + disp_keycol_sep ║ separator between key columns and + rest of columns + disp_rowtop_sep | + disp_rowmid_sep ⁝ + disp_rowbot_sep ⁝ + disp_rowend_sep ║ + disp_keytop_sep ║ + disp_keymid_sep ║ + disp_keybot_sep ║ + disp_endtop_sep ║ + disp_endmid_sep ║ + disp_endbot_sep ║ + disp_selected_note • + color_default normal the default color + color_default_hdr bold color of the column headers + color_bottom_hdr underline color of the bottom header row + color_current_row reverse color of the cursor row + color_current_col bold color of the cursor column + color_current_hdr bold reverse color of the header for the cur‐ + sor column + color_column_sep 246 blue color of column separators + color_key_col 81 cyan color of key columns + color_hidden_col 8 color of hidden columns on + metasheets + color_selected_row 215 yellow color of selected rows + disp_rstatus_fmt {sheet.longname} {sheet.nRows:9d} {sheet.rowtype} + right-side status format string + disp_status_fmt {sheet.shortcut}› {sheet.name}| + status line prefix + disp_lstatus_max 0 maximum length of left status + line + disp_status_sep | separator between statuses + color_keystrokes white color of input keystrokes on sta‐ + tus line + color_status bold status line color + color_error red error message color + color_warning yellow warning message color + color_top_status underline top window status bar color + color_active_status bold active window status bar color + color_inactive_status 8 inactive window status bar color + color_working green color of system running smoothly + color_edit_cell normal cell color to use when editing + cell + disp_edit_fill _ edit field fill character + disp_unprintable · substitute character for unprint‐ + ables + disp_histogram * histogram element character + disp_histolen 50 width of histogram column + disp_replay_play ▶ status indicator for active re‐ + play + disp_replay_pause ‖ status indicator for paused re‐ + play + color_status_replay green color of replay status indicator + disp_pixel_random False randomly choose attr from set of + pixels instead of most common + color_graph_hidden 238 blue color of legend for hidden attri‐ + bute + color_graph_selected bold color of selected graph points + color_graph_axis bold color for graph axis labels + color_add_pending green color for rows pending add + color_change_pending reverse yellow color for cells pending modifica‐ + tion + color_delete_pending red color for rows pending delete + color_xword_active green color of active clue + +EXAMPLES + vd foo.tsv + open the file foo.tsv in the current directory + + vd -f sqlite bar.db + open the file bar.db as a sqlite database + + vd -b countries.fixed -o countries.tsv + convert countries.fixed (in fixed width format) to countries.tsv (in tsv + format) + + vd postgres://username:password@hostname:port/database + open a connection to the given postgres database + + vd --play tests/pivot.vd --replay-wait 1 --output tests/pivot.tsv + replay tests/pivot.vd, waiting 1 second between commands, and output the + final sheet to test/pivot.tsv + + ls -l | vd -f fixed --skip 1 --header 0 + parse the output of ls -l into usable data + + ls | vd | lpr + interactively select a list of filenames to send to the printer + + vd newfile.tsv + open a blank sheet named newfile if file does not exist + + vd sample.xlsx +:sheet1:2:3 + launch with sheet1 at top-of-stack, and cursor at column 2 and row 3 + + vd -P open-plugins + preplay longname open-plugins before starting the session + +FILES + At the start of every session, VisiData looks for $HOME/.visidatarc, and + calls Python exec() on its contents if it exists. For example: + + options.min_memory_mb=100 # stop processing without 100MB free + + bindkey('0', 'go-leftmost') # alias '0' to go to first column, like vim + + def median(values): + L = sorted(values) + return L[len(L)//2] + + aggregator('median', median) + + Functions defined in .visidatarc are available in python expressions + (e.g. in derived columns). + +SUPPORTED SOURCES + Core VisiData includes these sources: + + tsv (tab-separated value) + Plain and simple. VisiData writes tsv format by default. See the + --tsv-delimiter option. + + csv (comma-separated value) + .csv files are a scourge upon the earth, and still regrettably + common. + See the --csv-dialect, --csv-delimiter, --csv-quotechar, and + --csv-skipinitialspace options. + Accepted dialects are excel-tab, unix, and excel. + + fixed (fixed width text) + Columns are autodetected from the first 1000 rows (adjustable with + --fixed-rows). + + json (single object) and jsonl/ndjson/ldjson (one object per line). + Cells containing lists (e.g. [3]) or dicts ({3}) can be expanded + into new columns with ( and unexpanded with ). + + sqlite + May include multiple tables. The initial sheet is the table + directory; Enter loaders the entrie table into memory. z^S saves + modifications to source. + + URL schemes are also supported: + http (requires requests); can be used as transport for with another + filetype + + For a list of all remaining formats supported by VisiData, see + https://visidata.org/formats. + + In addition, .zip, .gz, .bz2, and .xz files are decompressed on the fly. + +SUPPORTED OUTPUT FORMATS + These are the supported savers: + + tsv (tab-separated value) + csv (comma-separated value) + json (one object with all rows) + jsonl/ndjson/ldjson (one object per line/row) + All expanded subcolumns must be closed (with )) to retain the same + structure. + sqlite (save to source with z^S) + md (markdown table) + +AUTHOR + VisiData was made by Saul Pwanson . + +Linux/MacOS Oct 13, 2020 Linux/MacOS From 931978e452b1168debbca70b7de8c4bc90426f71 Mon Sep 17 00:00:00 2001 From: anjakefala Date: Thu, 22 Oct 2020 21:37:53 -0700 Subject: [PATCH 040/162] [docs] package man/vd.txt as a fallback for when man is not available on os Closes #745 --- MANIFEST.in | 1 + dev/mkman.sh | 1 + setup.py | 2 +- visidata/help.py | 3 ++- 4 files changed, 5 insertions(+), 2 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index 731f3a3a0..ce2d37747 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,3 +1,4 @@ include README.md include LICENSE.gpl3 include visidata/man/vd.1 +include visidata/man/vd.txt diff --git a/dev/mkman.sh b/dev/mkman.sh index 558572494..f6cbf5ea5 100755 --- a/dev/mkman.sh +++ b/dev/mkman.sh @@ -21,3 +21,4 @@ $MAN/parse_options.py $BUILD/vd-cli.inc $BUILD/vd-opts.inc soelim -rt -I $BUILD $BUILD/vd.inc > $BUILD/vd-pre.1 preconv -r -e utf8 $BUILD/vd-pre.1 > $MAN/vd.1 +MANWIDTH=80 man $MAN/vd.1 > $MAN/vd.txt diff --git a/setup.py b/setup.py index 0a416c25f..3d5557072 100755 --- a/setup.py +++ b/setup.py @@ -25,7 +25,7 @@ packages=['visidata', 'visidata.loaders', 'visidata.tests'], include_package_data=True, data_files = [('share/man/man1', ['visidata/man/vd.1'])], - package_data={'visidata': ['man/vd.1']}, + package_data={'visidata': ['man/vd.1', 'man/vd.txt']}, license='GPLv3', classifiers=[ 'Development Status :: 5 - Production/Stable', diff --git a/visidata/help.py b/visidata/help.py index 536dc040e..fded3ac51 100644 --- a/visidata/help.py +++ b/visidata/help.py @@ -63,7 +63,8 @@ def openManPage(vd): from pkg_resources import resource_filename import os with SuspendCurses(): - os.system(' '.join(['man', resource_filename(__name__, 'man/vd.1')])) + if os.system(' '.join(['man', resource_filename(__name__, 'man/vd.1')])) != 0: + vd.push(TextSheet('man_vd', source=Path(resource_filename(__name__, 'man/vd.txt')))) # in VisiData, ^H refers to the man page From a9953b62de8c976605d95aa5f4b4404f530dd50b Mon Sep 17 00:00:00 2001 From: Saul Pwanson Date: Fri, 23 Oct 2020 00:17:25 -0700 Subject: [PATCH 041/162] [http] automatic API pagination #480 --- visidata/loaders/http.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/visidata/loaders/http.py b/visidata/loaders/http.py index 1a5a63682..abb81bfd5 100644 --- a/visidata/loaders/http.py +++ b/visidata/loaders/http.py @@ -30,8 +30,16 @@ def openurl_http(path, filetype=None): if not response.encoding: response.encoding = options.encoding + # Automatically paginate if a 'next' URL is given + def _iter_lines(path=path, response=response): + while response: + yield from response.iter_lines(decode_unicode=True) + + src = response.links.get('next', {}).get('url', None) + response = requests.get(src, stream=True) if src else None + # create resettable iterator over contents - fp = RepeatFile(iter_lines=response.iter_lines(decode_unicode=True)) + fp = RepeatFile(iter_lines=_iter_lines()) # call open_ with a usable Path return vd.openSource(Path(path.given, fp=fp), filetype=filetype) From 12b0bc2fe2dbc9d65e454a314b3c2ec250afd471 Mon Sep 17 00:00:00 2001 From: Saul Pwanson Date: Fri, 23 Oct 2020 00:35:26 -0700 Subject: [PATCH 042/162] [docs api] updates choices text --- docs/api/columns.rst | 1 + visidata/choose.py | 9 +++++---- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/docs/api/columns.rst b/docs/api/columns.rst index 0e55970d6..605f82117 100644 --- a/docs/api/columns.rst +++ b/docs/api/columns.rst @@ -161,6 +161,7 @@ Types API Examples ~~~~~~~~~ +:: # Add an ip_address type. vd.addType(ipaddress.ip_address, icon=':', formatter=lambda fmt,ip: str(ip)) diff --git a/visidata/choose.py b/visidata/choose.py index edeb890e1..33117d09a 100644 --- a/visidata/choose.py +++ b/visidata/choose.py @@ -5,13 +5,14 @@ option('fancy_chooser', True, 'a nicer selection interface for aggregators and jointype') @VisiData.api -def chooseOne(vd, L): - return vd.choose(L, 1) +def chooseOne(vd, choices): + 'Return one user-selected key from *choices*.' + return vd.choose(choices, 1)[0] @VisiData.api def choose(vd, choices, n=None): - 'Return *n* (default 1) of *choices* elements (if list) or values (if dict).' + 'Return a list of 1 to *n* "key" from elements of *choices* (see chooseMany).' ret = vd.chooseMany(choices) or vd.fail('no choice made') if n and len(ret) > n: vd.fail('can only choose %s' % n) @@ -39,7 +40,7 @@ def chooseFancy(vd, choices): @VisiData.api def chooseMany(vd, choices): - '''*choices* is a list of dicts; each dict must have a unique "key" whose value has no spaces. Return a list of 1 or more keys, as chosen by the user. Handle replay correctly.''' + 'Return a list of 1 or more keys from *choices*, which is a list of dicts. Each element dict must have a unique "key", which must be typed directly by the user in non-fancy mode (therefore no spaces). All other items in the dicts are also shown in fancy chooser mode. Use previous choices from the replay input if available. Add chosen keys (space-separated) to the cmdlog as input for the current command.''' if vd.cmdlog: v = vd.getLastArgs() if v is not None: From b7b3a3e8735e07a03396f560bb3b4b9c5ad04bd8 Mon Sep 17 00:00:00 2001 From: anjakefala Date: Sat, 24 Oct 2020 23:18:20 -0700 Subject: [PATCH 043/162] [choices-] remove duplicate indexing --- visidata/choose.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/visidata/choose.py b/visidata/choose.py index 33117d09a..5eaf276c9 100644 --- a/visidata/choose.py +++ b/visidata/choose.py @@ -7,7 +7,7 @@ @VisiData.api def chooseOne(vd, choices): 'Return one user-selected key from *choices*.' - return vd.choose(choices, 1)[0] + return vd.choose(choices, 1) @VisiData.api From c1fe447b2ae852ec7994c9884b234e2d83b18bb1 Mon Sep 17 00:00:00 2001 From: anjakefala Date: Sat, 24 Oct 2020 23:32:50 -0700 Subject: [PATCH 044/162] [numeric binning] perform degenerate binning when nbins greater than number of values --- visidata/pivot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/visidata/pivot.py b/visidata/pivot.py index 86f943a20..bf8ccee37 100644 --- a/visidata/pivot.py +++ b/visidata/pivot.py @@ -181,7 +181,7 @@ def groupRows(self, rowfunc=None): if width == 0: # only one value (and maybe errors) numericBins = [(minval, maxval)] - elif numericCols[0].type in (int, vlen) and nbins > width: + elif numericCols[0].type in (int, vlen) and nbins > (maxval - minval): # more bins than int vals, just use the vals degenerateBinning = True numericBins = [(minval+i, minval+i) for i in range(maxval-minval+1)] From 7683783d2542e66de8be68782db2ed6afe376d00 Mon Sep 17 00:00:00 2001 From: anjakefala Date: Sat, 24 Oct 2020 23:33:17 -0700 Subject: [PATCH 045/162] [pivot] switch error to fail when more than one numericCol --- visidata/pivot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/visidata/pivot.py b/visidata/pivot.py index bf8ccee37..b9070d898 100644 --- a/visidata/pivot.py +++ b/visidata/pivot.py @@ -167,7 +167,7 @@ def groupRows(self, rowfunc=None): numericCols = [c for c in self.groupByCols if self.isNumericRange(c)] if len(numericCols) > 1: - vd.error('only one numeric column can be binned') + vd.fail('only one numeric column can be binned') numericBins = [] degenerateBinning = False From 691e0525a5733c3d332e1b627bef2a9386dc679b Mon Sep 17 00:00:00 2001 From: anjakefala Date: Sun, 25 Oct 2020 00:01:56 -0700 Subject: [PATCH 046/162] [dev] update changelog --- CHANGELOG.md | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fdce5916d..92134567c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,23 +2,30 @@ # v2.?.? (????) - - [xlsx] add active column (thanks @kbd for feature request #726) - - [select] differentiate select-equal- and select-exact- #734 + - [api] add options.unset() + - [clipboard] clipboard commands now require some selected rows #681 + - [commands] add unset-option command bound to `d` on OptionsSheet #733 + - [loaders http] have automatic API pagination (thanks @geekscrapy for feature request #480) + - [loaders xlsx] add active column (thanks @kbd for feature request #726) + - [search] `search-next` and `searchr-next` are now bound to n and N (was `next-search` and `search-prev`) + - [select] differentiate select-equal- and select-exact- (thanks @geekscrapy for feature request #734) - previous select-equal- matched type value - now select-equal- matches display value - add `z,` and `gz,` bindings for select-exact-cell/-row - - [clipboard] clipboard commands now require some selected rows #681 - - [api] add options.unset() - - [commands] add unset-option command bound to `d` on OptionsSheet #733 ## Bugfixes - [api] expose visidata.view (thanks @alekibango for bug report #732) - [loaders hdf5] misc bugfixes to hdf5 dataset loading (thanks @amotl for PR #728) - [man] fix warnings with manpage (thanks @jsvine for the bug report #718) + - [numeric binning] perform degenerate binning when number of bins greater than number of values + - (instead of when greater than width of bins) - [tabulate] fix savers to save in their own format (thanks @frosencrantz for bug report #723) - [color] use `color_column_sep` for sep chars (thanks @geekscrapy for bug report) - [windows] add Ctrl+M as alias for Ctrl+J #741 (thanks @bob-u for bug report #741) + - [windows man] package man/vd.txt as a fallback for when manis not available on os (thanks @bob-u for bug report #745) + - [loaders jsonl] fix copy-rows + - [loaders vdj] fix add-row # v2.0.1 (2020-10-13) From f9b6b9c5703edefa9bb88ad734886fa94c3c7911 Mon Sep 17 00:00:00 2001 From: anjakefala Date: Mon, 26 Oct 2020 22:07:28 -0700 Subject: [PATCH 047/162] [macros] add improved macro system - `m` (`macro-record`) begins recording macro; `m` prompts for keystroke, and completes recording - macro can then be executed everytime provided keystroke is used, will override existing keybinding - `gm` opens an index of all existing macros, can be directly viewed with `Enter` and then modified with `Ctrl+S` - macros will run command on current row, column sheet - remove deprecated `z Ctrl+D` older iteration of macro system Closes #755 Co-authored-by: saulpw --- visidata/__init__.py | 1 + visidata/cmdlog.py | 2 +- visidata/macros.py | 42 +++++++++++++++++++++++++++++++++++++----- 3 files changed, 39 insertions(+), 6 deletions(-) diff --git a/visidata/__init__.py b/visidata/__init__.py index faaa3f8b7..f5d662486 100644 --- a/visidata/__init__.py +++ b/visidata/__init__.py @@ -95,6 +95,7 @@ def getGlobals(): import visidata.incr import visidata.customdate import visidata.misc +from .macros import * from .loaders.csv import * from .loaders.archive import * diff --git a/visidata/cmdlog.py b/visidata/cmdlog.py index 590a8118e..67d0b8fe2 100644 --- a/visidata/cmdlog.py +++ b/visidata/cmdlog.py @@ -13,7 +13,7 @@ # prefixes which should not be logged nonLogged = '''forget exec-longname undo redo quit show error errors statuses options threads jump -replay cancel save-cmdlog +replay cancel save-cmdlog macro go- search scroll prev next page start end zoom resize visibility mouse suspend redraw no-op help syscopy sysopen profile toggle'''.split() diff --git a/visidata/macros.py b/visidata/macros.py index 4c4ab3dea..c48bfe1df 100644 --- a/visidata/macros.py +++ b/visidata/macros.py @@ -1,30 +1,62 @@ from visidata import * +from functools import wraps +vd.macroMode = None +vd.macrobindings = {} @VisiData.lazy_property def macrosheet(vd): macrospath = Path(os.path.join(options.visidata_dir, 'macros.tsv')) macrosheet = vd.loadInternalSheet(TsvSheet, macrospath, columns=(ColumnItem('command', 0), ColumnItem('filename', 1))) or vd.error('error loading macros') + real_macrosheet = IndexSheet('user_macros', rows=[]) for ks, fn in macrosheet.rows: vs = vd.loadInternalSheet(CommandLog, Path(fn)) + vd.status(f"setting {ks}") setMacro(ks, vs) + real_macrosheet.addRow(vs) - return macrosheet + return real_macrosheet + +@VisiData.api +def runMacro(vd, macro): + vd.replay_sync(macro, live=True) def setMacro(ks, vs): - vd.bindkeys.set(ks, vs.name, 'override') - vd.commands.set(vs.name, vs, 'override') + vd.macrobindings[ks] = vs + BaseSheet.addCommand(ks, vs.name, 'runMacro(vd.macrobindings[keystrokes])') @CommandLog.api def saveMacro(self, rows, ks): vs = copy(self) - vs.rows = self.selectedRows + vs.rows = rows macropath = Path(fnSuffix(options.visidata_dir+"macro")) vd.save_vd(macropath, vs) setMacro(ks, vs) append_tsv_row(vd.macrosheet, (ks, macropath)) +@CommandLog.api +@wraps(CommandLog.afterExecSheet) +def afterExecSheet(cmdlog, sheet, escaped, err): + cmdlog.afterExecSheet.__wrapped__(cmdlog, sheet, escaped, err) + if vd.macroMode and (vd.activeCommand is not None) and (vd.activeCommand is not UNLOADED): + cmd = copy(vd.activeCommand) + cmd.row = cmd.col = cmd.sheet = '' + vd.macroMode.addRow(cmd) + +@CommandLog.api +def startMacro(cmdlog): + if vd.macroMode: + ks = vd.input('save macro for keystroke: ') + vd.cmdlog.saveMacro(vd.macroMode.rows, ks) + vd.macroMode = None + else: + vd.status("recording macro") + vd.macroMode = CommandLog('current_macro', rows=[]) + + +vd.status(vd.macrosheet) -CommandLog.addCommand('z^D', 'save-macro', 'sheet.saveMacro(selectedRows or fail("no rows selected"), input("save macro for keystroke: "))', 'save selected rows to macro mapped to keystroke') +Sheet.addCommand('m', 'macro-start', 'vd.cmdlog.startMacro()') +Sheet.addCommand('gm', 'macro-sheet', 'vd.push(vd.macrosheet)') From 950ad3bfc5154917bf5f5ea6fdc43a3430b72ed1 Mon Sep 17 00:00:00 2001 From: anjakefala Date: Tue, 27 Oct 2020 20:00:08 -0700 Subject: [PATCH 048/162] [docs] update join Closes #656 --- docs/join.md | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/docs/join.md b/docs/join.md index f6dd3bfc6..9c0b829f9 100644 --- a/docs/join.md +++ b/docs/join.md @@ -1,25 +1,22 @@ -- Update: 2018-08-19 -- Version: VisiData 1.3.1 +- Update: 2020-10-27 +- Version: VisiData 2.0.1 # Combining datasets ## How to join two datasets -1. Load the datasets into VisiData. +1. Open the datasets in VisiData. a. `vd d1.tsv d2.tsv` **or** b. Press `o` and enter a filepath for each file. - -2. Press `Shift+S` to open the **Sheets sheet**. -3. Use `s` or `t` to select the sheets to merge. -4. Press `g Shift+C` to open a **Columns sheet** with all of the columns from selected sheets. -5. Press `g!` on the rows that reference the indices on which the join will be performed. At least one key column, per sheet, should be set. -6. Press `Ctrl+^` or `S` to return to the **Sheets sheet**. -7. Optional: If performing a left outer join, use `Shift+J` or `Shift+K` to reorder the sheets. The first sheet will be the one for whom all rows will be retained. -8. Type `&` followed by the *jointype* to execute the join +2. Press `S` to open up the **Sheets Sheet**. Through here, you can navigate to every sheet by pressing `Enter` on the row it is referenced in. +3. Navigate to the sheets you want the join, and set their shared columns as key columns with `!`. +4. Press `S` to return to the **Sheets sheet**. Select the sheets you want to merge with `s`. +5. Optional: If performing a left outer join, use `Shift+J` or `Shift+K` to reorder the sheets. The first sheet will be the one for whom all rows will be retained. +6. Type `&` to open the join-chooser, and select your desired jointype with `Enter`. jointype description --------- ------------- @@ -28,10 +25,11 @@ jointype description `full` keeps all rows from all sheets (union) `diff` keeps only rows NOT in all sheets `extend` keeps all rows and retain **SheetType** from first selected sheet +`merge` Merges differences from other sheets into first sheet ## How to append two datasets -1. Load the datasets into VisiData. +1. Open the datasets with VisiData. 2. Press `Shift+S` to open the **Sheets sheet**. 3. Use `s` or `t` to select the sheets to merge. -4. Type `&` followed by `append` to concatenate the selected datasets. +4. Type `&` and press `Enter` on `append` to concatenate the selected datasets. From f01de7c951657929076c74e96b1107e2f038f5e3 Mon Sep 17 00:00:00 2001 From: anjakefala Date: Wed, 28 Oct 2020 20:29:57 -0700 Subject: [PATCH 049/162] [macros] rename macro-start to macro-record --- visidata/macros.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/visidata/macros.py b/visidata/macros.py index c48bfe1df..b4970ef51 100644 --- a/visidata/macros.py +++ b/visidata/macros.py @@ -58,5 +58,5 @@ def startMacro(cmdlog): vd.status(vd.macrosheet) -Sheet.addCommand('m', 'macro-start', 'vd.cmdlog.startMacro()') +Sheet.addCommand('m', 'macro-record', 'vd.cmdlog.startMacro()') Sheet.addCommand('gm', 'macro-sheet', 'vd.push(vd.macrosheet)') From 57ca324cabe2ebce37774e6bd40bbc223ee675db Mon Sep 17 00:00:00 2001 From: saulpw Date: Wed, 28 Oct 2020 21:41:51 -0700 Subject: [PATCH 050/162] [json] try loading as jsonl before json (inverted) - jsonl is a streamable format, so this way it doesn't have to wait for the entire contents to be loaded before failing to parse as json and then trying to parse as jsonl - fixes api loading with http so that contents of each response are added as they happen - unfurl toplevel lists - functionally now jsonl and json are identical --- visidata/loaders/json.py | 52 +++++++++++++++++++--------------------- 1 file changed, 25 insertions(+), 27 deletions(-) diff --git a/visidata/loaders/json.py b/visidata/loaders/json.py index 040efc9bd..356b29de0 100644 --- a/visidata/loaders/json.py +++ b/visidata/loaders/json.py @@ -9,13 +9,13 @@ option('default_colname', '', 'column name to use for non-dict rows') -def open_json(p): +def open_jsonobj(p): return JsonSheet(p.name, source=p) def open_jsonl(p): - return JsonLinesSheet(p.name, source=p) + return JsonSheet(p.name, source=p) -open_ndjson = open_ldjson = open_jsonl +open_ndjson = open_ldjson = open_json = open_jsonl class JsonSheet(PythonSheet): @@ -23,18 +23,28 @@ def iterload(self): self.colnames = {} # [colname] -> Column self.columns = [] - try: - with self.source.open_text() as fp: - ret = json.load(fp, object_pairs_hook=OrderedDict) - - if isinstance(ret, list): - yield from Progress(ret) - else: - yield ret + with self.source.open_text() as fp: + for L in fp: + try: + ret = json.loads(L, object_pairs_hook=OrderedDict) + if isinstance(ret, list): + yield from Progress(ret) + else: + yield ret + + except ValueError as e: + if self.rows: # if any rows have been added already + e.stacktrace = stacktrace() + yield TypedExceptionWrapper(json.loads, L, exception=e) # an error on one line + else: + with self.source.open_text() as fp: + ret = json.load(fp, object_pairs_hook=OrderedDict) + if isinstance(ret, list): + yield from Progress(ret) + else: + yield ret + break - except ValueError as e: - vd.status('trying jsonl') - yield from JsonLinesSheet.iterload(self) def addRow(self, row, index=None): # Wrap non-dict rows in a dummy object with a predictable key name. @@ -56,19 +66,7 @@ def addRow(self, row, index=None): def newRow(self): return {} - -class JsonLinesSheet(JsonSheet): - def iterload(self): - self.colnames = {} # [colname] -> Column - self.columns = [] - with self.source.open_text() as fp: - for L in fp: - try: - yield json.loads(L, object_pairs_hook=OrderedDict) - except Exception as e: - e.stacktrace = stacktrace() - yield TypedExceptionWrapper(json.loads, L, exception=e) - +JsonLinesSheet=JsonSheet ## saving json and jsonl From 693399d1bd1fb14105b1e23e86e0a4e57b440516 Mon Sep 17 00:00:00 2001 From: saulpw Date: Wed, 28 Oct 2020 22:07:12 -0700 Subject: [PATCH 051/162] [json] try parsing options.json_indent as int #753 - this means json output can't be indented with a number. this seems like an uncommon use case. --- visidata/loaders/json.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/visidata/loaders/json.py b/visidata/loaders/json.py index 356b29de0..02ca2fc02 100644 --- a/visidata/loaders/json.py +++ b/visidata/loaders/json.py @@ -114,7 +114,12 @@ def save_json(vd, p, *vsheets): else: it = {vs.name: [_rowdict(vs.visibleCols, row) for row in vs.iterrows()] for vs in vsheets} - jsonenc = _vjsonEncoder(indent=options.json_indent) + try: + indent = int(options.json_indent) + except Exception: + indent = options.json_indent + + jsonenc = _vjsonEncoder(indent=indent) with Progress(gerund='saving'): for chunk in jsonenc.iterencode(it): fp.write(chunk) From dd916a3839c6ca274211b820d95e88f8640aa1e8 Mon Sep 17 00:00:00 2001 From: saulpw Date: Wed, 28 Oct 2020 22:25:34 -0700 Subject: [PATCH 052/162] [dev] add a CoC #737 --- CODE_OF_CONDUCT.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 CODE_OF_CONDUCT.md diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 000000000..7c5bd336b --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1 @@ +don't be a dick From 00779f3eb3dba09c48616ffe90b3ba1958886998 Mon Sep 17 00:00:00 2001 From: saulpw Date: Wed, 28 Oct 2020 23:28:46 -0700 Subject: [PATCH 053/162] [docs] update README --- README.md | 73 ++++++++++++++++++++----------------------------------- 1 file changed, 27 insertions(+), 46 deletions(-) diff --git a/README.md b/README.md index 2d5f87c28..3a3addd14 100644 --- a/README.md +++ b/README.md @@ -1,82 +1,58 @@ [![Gitpod ready-to-code](https://img.shields.io/badge/Gitpod-ready--to--code-blue?logo=gitpod)](https://gitpod.io/#https://github.com/saulpw/visidata) +[![twitter @VisiData][1.1]][1] + # VisiData v2.0.1 [![CircleCI](https://circleci.com/gh/saulpw/visidata/tree/develop.svg?style=svg)](https://circleci.com/gh/saulpw/visidata/tree/develop) A terminal interface for exploring and arranging tabular data. ![Frequency table](http://visidata.org/freq-move-row.gif) -## Dependencies +VisiData supports tsv, csv, sqlite, json, xlsx (Excel), hdf5, and [many other formats](https://visidata.org/formats)). -- Linux, OS/X or Windows -- Python 3.6+ -- python-dateutil -- other modules may be required for opening particular data sources - - see [requirements.txt](https://github.com/saulpw/visidata/blob/stable/requirements.txt) or the [supported sources](https://visidata.org/formats) in the vd manpage +## Platform requirements -## Getting started +- Linux, OS/X, or Windows (with WSL) +- Python 3.6+ +- additional Python modules are required for certain formats and sources -### Installation +## Install -Each package contains the full loader suite but differs in which loader dependencies will get installed by default. +To install the latest release from PyPi: -The base VisiData package concerns loaders whose dependencies are covered by the Python3 standard library. + pip3 install visidata -Base loaders: tsv, csv, json, sqlite, and fixed width text. +To install the cutting edge `develop` branch (no warranty expressed or implied): -|Platform |Package Manager | Command | Out-of-box Loaders | -|-------------------|----------------------------------------|----------------------------------------------|----------------------| -|all |[pip3](https://visidata.org/install#pip3) | `pip3 install visidata` | Base | -|all |[conda](https://visidata.org/install#conda) | `conda install --channel conda-forge visidata` | Base, http, html, xls(x) | -|MacOS |[Homebrew](https://visidata.org/install#brew) | `brew install saulpw/vd/visidata` | Base, http, html, xls(x) | -|Linux (Debian/Ubuntu) |[apt](https://visidata.org/install#apt) | [full instructions](https://visidata.org/install#apt) | Base, http, html, xls(x) | -|Linux (Debian/Ubuntu) |[dpkg](https://visidata.org/install#dpkg) | [full instructions](https://visidata.org/install#dpkg) | Base, http, html, xls(x) | -|Windows |[WSL](https://visidata.org/install#wsl) | Windows is not directly supported (use WSL) | N/A | -|all |[github](https://visidata.org/install#git) | `pip3 install git+https://github.com/saulpw/visidata.git@stable` | Base | -|Linux (NixOS)|[nix](https://visidata.org/install#nix)| `nix-env -i visidata`|Base, yaml, xls(x), hdf5, html, pandas, shp | + pip3 install git+https://github.com/saulpw/visidata.git@develop -Please see [/install](https://visidata.org/install) for detailed instructions, additional information, and troubleshooting. +See [visidata.org/install](https://visidata.org/install) for detailed instructions for all available platforms and package managers. ### Usage - $ vd [] ... - $ | vd [] + $ vd + $ | vd -VisiData supports tsv, csv, xlsx, hdf5, sqlite, json and more (see the [list of supported sources](https://visidata.org/formats)). - -Use `-f ` to force a particular filetype. +Press `Ctrl+Q` to quit. +Hundreds of other commands and options are also available; see the documentation. ### Documentation +* [VisiData documentation](https://visidata.org/docs) +* [Plugin Author's Guide and API Reference](https://visidata.org/docs/api) +* [Quick reference](https://visidata.org/man) (available within `vd` with `Ctrl+H`), which has a list of commands and options. * [Intro to VisiData Tutorial](https://jsvine.github.io/intro-to-visidata/) by [Jeremy Singer-Vine](https://www.jsvine.com/) -* Quick reference: `Ctrl+H` within `vd` will open the [man page](https://visidata.org/man), which has a list of all commands and options. -* [/docs](https://visidata.org/docs) contains a collection of howto recipes. ### Help and Support If you have a question, issue, or suggestion regarding VisiData, please [create an issue on Github](https://github.com/saulpw/visidata/issues) or chat with us at #visidata on [freenode.net](https://webchat.freenode.net/). -Here are some concrete ways you can help make VisiData even more awesome: - -* Write a blogpost (or tweet or whatever) about a VisiData command or feature you use frequently, and share it with us! -* Expand VisiData to support .xyz proprietary data format. Creating a loader [is really straightforward](https://www.visidata.org/docs/api/loaders.html). -* Create and maintain [new installation packages](https://github.com/saulpw/visidata/labels/packaging). -* Acknowledge the realities of late-stage capitalism and [give regular old money](https://www.patreon.com/saulpw). - -## Other applications within the VisiData ecosystem - -The core interface paradigm--rows and columns--can be used to create efficient terminal workflows with a minimum of effort for almost any application. These have been prototyped as proof of this concept: - -- [vgit](https://github.com/saulpw/visidata/tree/stable/plugins/vgit): a git interface -- [vsh](https://github.com/saulpw/vsh): a collection of utilities like `vping` and `vtop`. -- [vdgalcon](https://github.com/saulpw/vdgalcon): a port of the classic game [Galactic Conquest](https://www.galcon.com) - -Other workflows can also be created as separate apps using the visidata module. These apps can be very small and provide a lot of functionality; for example, see the included [viewtsv](https://visidata.org/docs/viewtsv). +If you use VisiData regularly, please [support me on Patreon](https://www.patreon.com/saulpw)! ## License -VisiData, including the main `vd` application, addons, loaders, and other code in this repository, is available for use and redistribution under GPLv3. +Code in the `stable` branch of this repository, including the main `vd` application, loaders, and plugins, is available for use and redistribution under GPLv3. ## Credits @@ -85,3 +61,8 @@ VisiData is conceived and developed by Saul Pwanson ``. Anja Kefala `` maintains the documentation and packages for all platforms. Many thanks to numerous other [contributors](https://visidata.org/credits/), and to those wonderful users who provide feedback, for helping to make VisiData the awesome tool that it is. + +[1.1]: http://i.imgur.com/tXSoThF.png +[1]: http://www.twitter.com/VisiData + + From e0b105238f08decd3817b888e82710f531b94fcd Mon Sep 17 00:00:00 2001 From: anjakefala Date: Wed, 28 Oct 2020 23:33:28 -0700 Subject: [PATCH 054/162] [README] place icons in single line --- README.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/README.md b/README.md index 3a3addd14..394f32496 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,6 @@ -[![Gitpod ready-to-code](https://img.shields.io/badge/Gitpod-ready--to--code-blue?logo=gitpod)](https://gitpod.io/#https://github.com/saulpw/visidata) -[![twitter @VisiData][1.1]][1] -# VisiData v2.0.1 [![CircleCI](https://circleci.com/gh/saulpw/visidata/tree/develop.svg?style=svg)](https://circleci.com/gh/saulpw/visidata/tree/develop) +# VisiData v2.0.1 [![twitter @VisiData][1.1]][1] [![CircleCI](https://circleci.com/gh/saulpw/visidata/tree/develop.svg?style=svg)](https://circleci.com/gh/saulpw/visidata/tree/develop) [![Gitpod ready-to-code](https://img.shields.io/badge/Gitpod-ready--to--code-blue?logo=gitpod)](https://gitpod.io/#https://github.com/saulpw/visidata) A terminal interface for exploring and arranging tabular data. From 7e286cd957f8013c0d5ddef6e5ad2fd6e415591f Mon Sep 17 00:00:00 2001 From: anjakefala Date: Fri, 30 Oct 2020 19:07:56 -0700 Subject: [PATCH 055/162] [dir commit-sheet-] fix commit-sheet and delete-row on DirSheet --- visidata/modify.py | 1 + visidata/shell.py | 14 +++++++++++--- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/visidata/modify.py b/visidata/modify.py index 38a652ae1..0e30a0f5f 100644 --- a/visidata/modify.py +++ b/visidata/modify.py @@ -120,6 +120,7 @@ def deleteBy(sheet, func, commit=False): if r is row: sheet.cursorRowIndex = len(sheet.rows)-1 else: + sheet.deleteSourceRow(r) ndeleted += 1 if not commit: diff --git a/visidata/shell.py b/visidata/shell.py index 34fa227e6..85d2024dc 100644 --- a/visidata/shell.py +++ b/visidata/shell.py @@ -146,9 +146,8 @@ def removeFile(self, path): else: path.unlink() - def deleteSourceRow(self, rowidx): - self.removeFile(self.rows[rowidx]) - self.rows.pop(rowidx) + def deleteSourceRow(self, r): + self.removeFile(r) def iterload(self): def _walkfiles(p): @@ -188,6 +187,15 @@ def preloadHook(self): def restat(self): vstat.cache_clear() + @asyncthread + def putChanges(self): + self.commitAdds() + self.commitMods() + self.commitDeletes() + + self._deferredDels.clear() + self.reload() + class FileListSheet(DirSheet): _ordering = [] From 1f7abfc38b2f9afbe98e603c9535743dee29d5b0 Mon Sep 17 00:00:00 2001 From: anjakefala Date: Sat, 31 Oct 2020 18:03:38 -0700 Subject: [PATCH 056/162] [join] fix append description --- visidata/join.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/visidata/join.py b/visidata/join.py index c3e33dc2b..52c473736 100644 --- a/visidata/join.py +++ b/visidata/join.py @@ -31,7 +31,7 @@ def createJoinedSheet(sheets, jointype=''): 'outer': 'all rows from first selected sheet', 'full': 'all rows from all sheets (union)', 'diff': 'only rows NOT in all sheets', - 'append': 'only columns from first sheet; extend with rows from all sheets', + 'append': 'columns all sheets; extend with rows from all sheets', 'extend': 'only rows from first sheet; extend with columns from all sheets', 'merge': 'merge differences from other sheets into first sheet', }.items()] From 31c2a96c40ce2050ef8f906cee4f2dad18ea5616 Mon Sep 17 00:00:00 2001 From: anjakefala Date: Sat, 31 Oct 2020 18:19:16 -0700 Subject: [PATCH 057/162] [freeze-] freeze-sheet with errors should replace with null --- visidata/freeze.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/visidata/freeze.py b/visidata/freeze.py index 26d2d61a4..2bba46b70 100644 --- a/visidata/freeze.py +++ b/visidata/freeze.py @@ -44,10 +44,11 @@ def reload(self): row = [] self.addRow(row) for col in self.source.visibleCols: - try: - row.append(col.getTypedValue(r)) - except Exception as e: + val = col.getTypedValue(r) + if isinstance(val, TypedExceptionWrapper): row.append(None) + else: + row.append(val) Sheet.addCommand("'", 'freeze-col', 'sheet.addColumnAtCursor(StaticColumn(cursorCol))', 'add a frozen copy of current column with all cells evaluated') From c72201bed5cf6afcbff97dab777f8000926e85b0 Mon Sep 17 00:00:00 2001 From: anjakefala Date: Sun, 1 Nov 2020 19:50:45 -0800 Subject: [PATCH 058/162] [docs] update docs --- CONTRIBUTING.md | 1 + docs/async.md | 83 ------------------------------------------------- docs/columns.md | 25 +++++++++++++-- docs/index.md | 4 +-- 4 files changed, 24 insertions(+), 89 deletions(-) delete mode 100644 docs/async.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 89b5b7961..7be5bb5ec 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -58,6 +58,7 @@ Some examples of great bug reports: ## Submitting Source Code +Check out the [Plugin Authors Guide](https://visidata.org/docs/api) for an overview of the API. Code in `plugins/` or `visidata/loaders/` is welcome, as long as it is useful to someone and safe for everyone. Updates or additions to the core code should be proposed via an [Github Issue](https://github.com/saulpw/visidata/issues/new/choose) before submitting a PR. diff --git a/docs/async.md b/docs/async.md deleted file mode 100644 index 1ad704cfb..000000000 --- a/docs/async.md +++ /dev/null @@ -1,83 +0,0 @@ -- Date: 2017-12-27 -- VisiData v1.0 - -# Maintaining a responsive interface - -## `@asyncthread` - -Use the `@asyncthread` decorator on a function to make it execute in a thread. -The return value is the spawned thread (which can often be ignored by the caller); the return value of the original function is effectively lost. - -Cells which are being computed in a separate thread should have that thread as their value until their result is available. -This will show the `options.disp_pending` notation and allow the user to interact with the specific thread (via e.g. `z^Y` and others). - -Each thread is added to `Sheet.currentThreads` for the current sheet. -Note that a thread spawned by calling a function on a different sheet will add the thread to the currentThreads for the topmost/current sheet instead. - -### Canceling threads - -The user can cancel all `Sheet.currentThreads` with `^C`. - -Internally, `cancelThread(*threads)` will send each thread an `EscapeException`, which percolates up the stack to be caught by the thread entry point. -EscapeException inherits from BaseException instead of Exception, so that threads can still have catch-all try blocks with `except Exception:`. -An unqualified `except:` clause is bad practice (as always); when used in an async function, it will make the thread uncancelable. - -### Wait for threads to finish - -`sync(expectedThreads)` will wait for all but some number of `expectedThreads` to finish. - -This will only rarely be useful. - -# Threads Sheet (`^T`) - -All threads (active, aborted, and completed) are added to `VisiData.threads`, which can be viewed as the ThreadsSheet via `^T`. -Threads which take less than `min_thread_time_s` (hardcoded in `asyncthread.py` to 10ms) are removed, to reduce clutter. - -- Press `ENTER` (on the Threads Sheet) on a thread to view its performance profile (if `options.profile_threads` was True when the thread started). -- Press `^_` (anywhere) to toggle profiling of the main thread. - -## Profiling - -The view of a performance profile in VisiData is the output from `pstats.Stats.print_stats()`. - -- `z^S` on the performance profile will call `dump_stats()` and save the profile data to the given filename, for analysis with e.g. [pyprof2calltree]() and [kcachegrind](). -- (`z^S` because the raw text can be saved with `^S` as usual. Ideally, `^S` to a file with a `.pyprof` extension on a profile sheet would do this instead.) - -# Progress counters - -In all `@asyncthread` functions, a `Progress` counter should be used to provide a progress percentage, which appears in the right-hand status. - -## Progress as iterable - -When iterating over a potentially large sequence: - - for item in Progress(iterable): - -This is just like `for item in iterable`, but it also keeps track of progress, to display on the right status line. - -- This only displays if used in another thread (but is harmless if not). -- Use Progress around the innermost iterations for maximum granularity and apparent responsiveness. -- But this incurs a small amount of overhead, so if a tight loop needs every last optimization, use it with an outer iterator instead (if there is one). -- Multiple Progress objects used in parallel will stack properly. -- Multiple Progress objects used serially will make the progress indicator reset (which is better than having no indicator at all). - -If `iterable` does not know its own length, it (or an approximation) should be passed as the `total` keyword argument: - - for item in Progress(iterable, total=approx_size): - -The `Progress` object contributes 1 towards the total for each iteration. -To contribute a different amount, use `Progress.addProgress(n)` (n-1 if being used as an iterable, as 1 will be added automatically). - -## Progress as context manager - -To manage `Progress` without wrapping an iterable, use it as a context manager with only a `total` keyword argument, and call `addProgress` as progress is made: - - with Progress(total=amt) as prog: - while amt > 0: - some_amount = some_work() - prog.addProgress(some_amount) - amt -= some_amount - -- Using `Progress()` other than as an iterable or a context manager will have no effect. - ---- diff --git a/docs/columns.md b/docs/columns.md index f8379eddc..057fbd7db 100644 --- a/docs/columns.md +++ b/docs/columns.md @@ -1,5 +1,5 @@ -- Update: 2020-06-17 -- Version: VisiData 2.0 +- Update: 2020-10-01 +- Version: VisiData 2.0.1 # Columns @@ -9,7 +9,7 @@ Commands(s) Operation ------------ ----------- `!` pins the current column on the left as a key column `H` `L` slides the current column **one position** to the left/right -`gH` `gL` slides the current column **all the way** to the left/right of the sheet +`gH` `gL` slides the current column **all the way** to the left/right of its section --- @@ -29,6 +29,10 @@ Commands(s) Operation ###### How to unhide columns +1. Press `gv` to unhide all columns on current sheet. + +**or** + 1. Press `Shift+C` on the source sheet to open its **Columns sheet**. 2. Move the cursor right to the **width** column. 3. Move the cursor down to the row which represents the column you wish to unhide. Currently, that cell should contain the value **0**. @@ -135,6 +139,21 @@ uses the commands for column splitting and transformation with [xd/puzzles.tsv]( - `;` adds new columns derived from pulling the contents of the current column which match the *regex within capture groups*. This command also requires an example row. - `*` followed by *regex*`/`*substring* replaces the text which matches the capture groups in *regex* with the contents of *substring*. *substring* may include backreferences (*\1* etc). +## [How do I substitute text in my column] + +The `*` command can be used to do content transformations of cells. The `g*` variant transforms in-place, instead of creating a new column. + +The following example uses [benchmarks.csv](https://raw.githubusercontent.com/saulpw/visidata/stable/sample_data/benchmarks.csv). + +**Question** Transform the **SKU** values of *food* to *nutri*. + +1. Move cursor to **SKU** column. +2. Press `gs` to select all rows. +3. Type `g*` folowed by *food/nutri*. + +- tests/transform-cols.vd + + --- ## [How to expand columns that contain nested data](#expand) {#expand} diff --git a/docs/index.md b/docs/index.md index 8f8e40799..5ccad4af6 100644 --- a/docs/index.md +++ b/docs/index.md @@ -80,8 +80,6 @@ # For developers VisiData can interact with data from any source or in any format. - +* [VisiData API Guide](/docs/api) * [guide to contributing](/contributing) * [viewtsv annotated](/docs/viewtsv) -* [async](/docs/async) -* [graphics reference](/docs/graphics) From 66d18fe20b51370cfcd346ae7683bade5f59f6e9 Mon Sep 17 00:00:00 2001 From: anjakefala Date: Mon, 2 Nov 2020 18:21:31 -0800 Subject: [PATCH 059/162] [aggregator] add checklist #754 --- dev/checklists/add-aggregator.md | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 dev/checklists/add-aggregator.md diff --git a/dev/checklists/add-aggregator.md b/dev/checklists/add-aggregator.md new file mode 100644 index 000000000..9ae8e384a --- /dev/null +++ b/dev/checklists/add-aggregator.md @@ -0,0 +1,4 @@ +- API for adding aggregators can be found [here](https://www.visidata.org/docs/api/columns.html#aggregators). A description is necessary. +- [ ] Review whether new aggregator should be included in **DescribeSheet**. If so, add it to `describe_aggrs`. +- [ ] Check if aggregator replaces an existing **DescribeSheet** column, remove it from the **DescribeSheet**, if so. +- [ ] add the aggregator to visidata:docs/group.md From 4c2927f3084fbe7d22eb93871322f1900c99fbea Mon Sep 17 00:00:00 2001 From: Jeremy Singer-Vine Date: Sun, 25 Oct 2020 21:53:33 -0400 Subject: [PATCH 060/162] Add mode and stdev to aggregator options --- visidata/aggregators.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/visidata/aggregators.py b/visidata/aggregators.py index 63a7d51c7..116d51db9 100644 --- a/visidata/aggregators.py +++ b/visidata/aggregators.py @@ -1,6 +1,7 @@ import math import functools import collections +from statistics import mode, stdev from visidata import Progress, Column from visidata import * @@ -100,10 +101,12 @@ def quantiles(q, helpstr): vd.aggregator('avg', mean, 'arithmetic mean of values', type=float) vd.aggregator('mean', mean, 'arithmetic mean of values', type=float) vd.aggregator('median', median, 'median of values') +vd.aggregator('mode', mode, 'mode of values') vd.aggregator('sum', sum, 'sum of values') vd.aggregator('distinct', set, 'distinct values', type=vlen) vd.aggregator('count', lambda values: sum(1 for v in values), 'number of values', type=int) vd.aggregator('list', list, 'list of values') +vd.aggregator('stdev', stdev, 'standard deviation of values', type=float) vd.aggregators['q3'] = quantiles(3, 'tertiles (33/66th pctile)') vd.aggregators['q4'] = quantiles(4, 'quartiles (25/50/75th pctile)') From b765fc22a88277e85f87f800a2c4c47c99142f1d Mon Sep 17 00:00:00 2001 From: anjakefala Date: Mon, 2 Nov 2020 18:33:26 -0800 Subject: [PATCH 061/162] [docs] add mode and stdev to list of aggregators --- docs/group.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/group.md b/docs/group.md index 82cbccb65..4af59201b 100644 --- a/docs/group.md +++ b/docs/group.md @@ -19,6 +19,7 @@ Aggregator Description `min` smallest value in the group `max` largest value in the group `avg`/`mean` average value of the group +`mode` most frequently appearing value in group `median` median value in the group `q3/q4/q5/q10` add quantile aggregators to group (e.g. q4 adds p25, p50, p75) `sum` total summation of all numbers in the group @@ -26,6 +27,7 @@ Aggregator Description `count` number of values in the group `keymax` key of the row with the largest value in the group `list` gathers values in column into a list +`stdev` standard deviation of values The follow howtos will have examples of workflows involving grouping of data and statistical aggregation. From 9fa8dd0a290a2741208e7f10547b368dedaf70f3 Mon Sep 17 00:00:00 2001 From: anjakefala Date: Mon, 2 Nov 2020 19:18:29 -0800 Subject: [PATCH 062/162] [expr] remove duplicate tabbing suggestions Closes #747 --- visidata/expr.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/visidata/expr.py b/visidata/expr.py index 243478457..435d25b16 100644 --- a/visidata/expr.py +++ b/visidata/expr.py @@ -22,6 +22,10 @@ def __call__(self, val, state): varnames = [] varnames.extend(sorted((base+col.name) for col in self.sheet.columns if col.name.startswith(partial))) varnames.extend(sorted((base+x) for x in globals() if x.startswith(partial))) + + # Remove duplicate tabbing suggestions + varnames_dict = {var:None for var in varnames} + varnames = list(varnames_dict.keys()) return varnames[state%len(varnames)] From 7ba66d19bb969bf036fcfc65bb97365bf27d8a2a Mon Sep 17 00:00:00 2001 From: anjakefala Date: Mon, 2 Nov 2020 19:45:48 -0800 Subject: [PATCH 063/162] [input] do not duplicate most recent lastInputs --- visidata/_input.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/visidata/_input.py b/visidata/_input.py index 3aaf99edd..0e56aa505 100644 --- a/visidata/_input.py +++ b/visidata/_input.py @@ -311,6 +311,8 @@ def input(self, prompt, type=None, defaultLast=False, history=[], **kwargs): if ret: if isinstance(type, str): + if self.lastInputs[type] and self.lastInputs[type][-1] == ret: + return ret self.lastInputs[type].append(ret) elif defaultLast: history or vd.fail("no previous input") From 40a986785b8dff5282135f08d30d252e0f3e0816 Mon Sep 17 00:00:00 2001 From: anjakefala Date: Tue, 3 Nov 2020 21:00:22 -0800 Subject: [PATCH 064/162] [sqlite-] getSavedValue has been removed from API --- visidata/loaders/sqlite.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/visidata/loaders/sqlite.py b/visidata/loaders/sqlite.py index af430e8bb..60e037e7f 100644 --- a/visidata/loaders/sqlite.py +++ b/visidata/loaders/sqlite.py @@ -84,7 +84,7 @@ def values(row, cols): sql += ', '.join('%s=?' % c.name for c, _ in rowmods.items()) sql += ' WHERE %s' % ' AND '.join('"%s"=?' % c.name for c in wherecols) self.execute(conn, sql, - parms=values(row, [c for c, _ in rowmods.items()]) + list(c.getSavedValue(row) for c in wherecols)) + parms=values(row, [c for c, _ in rowmods.items()]) + list(Column.calcValue(c, row) for c in wherecols)) for r in dels.values(): sql = 'DELETE FROM "%s" ' % self.tableName From 3913f7a9585b40e168c23514f3ebe26131563070 Mon Sep 17 00:00:00 2001 From: Saul Pwanson Date: Tue, 27 Oct 2020 15:19:04 -0700 Subject: [PATCH 065/162] [dev] replace asserts with vd.fail/error --- visidata/canvas.py | 6 ++++-- visidata/cmdlog.py | 2 -- visidata/loaders/_pandas.py | 4 +++- visidata/loaders/mbtiles.py | 2 +- visidata/loaders/pandas_freqtbl.py | 13 ++++++++++--- visidata/path.py | 3 ++- visidata/save.py | 3 ++- 7 files changed, 22 insertions(+), 11 deletions(-) diff --git a/visidata/canvas.py b/visidata/canvas.py index 8697e340e..f2204b8e2 100644 --- a/visidata/canvas.py +++ b/visidata/canvas.py @@ -417,8 +417,10 @@ def polygon(self, vertexes, attr=0, row=None): self.polylines.append((vertexes + [vertexes[0]], attr, row)) def qcurve(self, vertexes, attr=0, row=None): - 'quadratic curve from vertexes[0] to vertexes[2] with control point at vertexes[1]' - assert len(vertexes) == 3, len(vertexes) + 'Draw quadratic curve from vertexes[0] to vertexes[2] with control point at vertexes[1]' + if len(vertexes) != 3: + vd.fail('need exactly 3 points for qcurve (got %d)' % len(vertexes)) + x1, y1 = vertexes[0] x2, y2 = vertexes[1] x3, y3 = vertexes[2] diff --git a/visidata/cmdlog.py b/visidata/cmdlog.py index 67d0b8fe2..011618216 100644 --- a/visidata/cmdlog.py +++ b/visidata/cmdlog.py @@ -159,12 +159,10 @@ def beforeExecHook(self, sheet, cmd, args, keystrokes): colname = sheet.cursorCol.name or sheet.visibleCols.index(sheet.cursorCol) if contains(cmd.execstr, 'plotterCursorBox'): - assert not colname and not rowname bb = sheet.cursorBox colname = '%s %s' % (sheet.formatX(bb.xmin), sheet.formatX(bb.xmax)) rowname = '%s %s' % (sheet.formatY(bb.ymin), sheet.formatY(bb.ymax)) elif contains(cmd.execstr, 'plotterVisibleBox'): - assert not colname and not rowname bb = sheet.visibleBox colname = '%s %s' % (sheet.formatX(bb.xmin), sheet.formatX(bb.xmax)) rowname = '%s %s' % (sheet.formatY(bb.ymin), sheet.formatY(bb.ymax)) diff --git a/visidata/loaders/_pandas.py b/visidata/loaders/_pandas.py index bef76609e..340090b04 100644 --- a/visidata/loaders/_pandas.py +++ b/visidata/loaders/_pandas.py @@ -17,7 +17,9 @@ class DataFrameAdapter: def __init__(self, df): import pandas as pd - assert isinstance(df, pd.DataFrame) + if not isinstance(df, pd.DataFrame): + vd.fail('%s is not a dataframe' % type(df).__name__) + self.df = df def __len__(self): diff --git a/visidata/loaders/mbtiles.py b/visidata/loaders/mbtiles.py index d8d834375..e762e3bff 100644 --- a/visidata/loaders/mbtiles.py +++ b/visidata/loaders/mbtiles.py @@ -111,7 +111,7 @@ def iterpolylines(self, r): for poly in mpoly: yield poly+[poly[0]], self.plotColor(key), r else: - assert False, t + vd.warning('unknown geometry type %s' % t) @asyncthread def reload(self): diff --git a/visidata/loaders/pandas_freqtbl.py b/visidata/loaders/pandas_freqtbl.py index 8ccef5473..0e3f3d6cd 100644 --- a/visidata/loaders/pandas_freqtbl.py +++ b/visidata/loaders/pandas_freqtbl.py @@ -13,8 +13,13 @@ class DataFrameRowSliceAdapter: def __init__(self, df, mask): import pandas as pd import numpy as np - assert isinstance(df, pd.DataFrame) - assert isinstance(mask, pd.Series) and df.shape[0] == mask.shape[0] + if not isinstance(df, pd.DataFrame): + vd.fail('%s is not a dataframe' % type(df).__name__) + if not isinstance(mask, pd.Series): + vd.fail('mask %s is not a Series' % type(mask).__name__) + if df.shape[0] != mask.shape[0]: + vd.fail('dataframe and mask have different shapes (%s vs %s)' % (df.shape[0], mask.shape[0])) + self.df = df self.mask_bool = mask # boolean mask self.mask_iloc = np.where(mask.values)[0] # integer indexes corresponding to mask @@ -145,7 +150,9 @@ def reload(self): for element in Progress(value_counts.index): if len(self.groupByCols) == 1: element = (element,) - assert len(element) == len(self.groupByCols) + elif len(element) != len(self.groupByCols): + vd.fail('different number of index cols and groupby cols (%s vs %s)' % (len(element), len(self.groupByCols))) + mask = df[self.groupByCols[0].name] == element[0] for i in range(1, len(self.groupByCols)): mask = mask & (df[self.groupByCols[i].name] == element[i]) diff --git a/visidata/path.py b/visidata/path.py index 5edfe211a..212267940 100644 --- a/visidata/path.py +++ b/visidata/path.py @@ -238,7 +238,8 @@ def read(self, n=None): return r def seek(self, n): - assert n == 0, 'RepeatFile can only seek to beginning' + if n != 0: + vd.error('RepeatFile can only seek to beginning') self.iter = RepeatFileIter(self) def __iter__(self): diff --git a/visidata/save.py b/visidata/save.py index 5048d6802..06034b05b 100644 --- a/visidata/save.py +++ b/visidata/save.py @@ -112,7 +112,8 @@ def saveSheets(vd, givenpath, *vsheets, confirm_overwrite=False): except FileExistsError: pass - assert givenpath.is_dir(), filetype + ' cannot save multiple sheets to non-dir' + if not givenpath.is_dir(): + vd.fail(f'cannot save multiple {filetype} sheets to non-dir') # get save function to call for vs in vsheets: From 63be1fb92c9d26425988881d86e5632c031ccd65 Mon Sep 17 00:00:00 2001 From: Saul Pwanson Date: Tue, 27 Oct 2020 21:16:08 -0700 Subject: [PATCH 066/162] [draw] sort indicators in column headers #582 --- CHANGELOG.md | 1 + tests/sort-levels.tsv | 50 +++++++++++++++++++++++++++++++++++++++++++ tests/sort-levels.vd | 5 +++++ visidata/sheets.py | 12 +++++++++-- 4 files changed, 66 insertions(+), 2 deletions(-) create mode 100644 tests/sort-levels.tsv create mode 100644 tests/sort-levels.vd diff --git a/CHANGELOG.md b/CHANGELOG.md index 92134567c..acac57892 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ - previous select-equal- matched type value - now select-equal- matches display value - add `z,` and `gz,` bindings for select-exact-cell/-row + - [display] sort indication #582 ## Bugfixes diff --git a/tests/sort-levels.tsv b/tests/sort-levels.tsv new file mode 100644 index 000000000..8e4853904 --- /dev/null +++ b/tests/sort-levels.tsv @@ -0,0 +1,50 @@ +primary secondary tertiary +1 1 1 +2 2 4 +1 3 2 +1 1 1 +1 3 2 +2 2 4 +1 1 1 +2 2 4 +2 2 4 +1 1 1 +1 3 2 +2 2 3 +2 2 4 +1 3 2 +2 2 4 +2 2 4 +1 1 1 +2 2 3 +1 1 1 +1 1 1 +1 3 2 +2 1 3 +2 1 3 +1 3 2 +1 3 2 +2 1 3 +2 2 4 +1 3 1 +2 1 3 +1 1 1 +2 3 3 +2 2 3 +2 1 3 +2 1 3 +1 1 1 +1 1 1 +1 3 2 +2 2 3 +2 2 4 +1 3 1 +1 1 1 +2 2 4 +1 3 2 +1 1 1 +1 3 1 +2 3 2 +1 3 2 +2 2 3 +1 1 1 diff --git a/tests/sort-levels.vd b/tests/sort-levels.vd new file mode 100644 index 000000000..9c41de817 --- /dev/null +++ b/tests/sort-levels.vd @@ -0,0 +1,5 @@ +sheet col row longname input keystrokes comment + open-file tests/sort-levels.tsv o +sort-levels primary sort-asc [ sort ascending by current column; replace any existing sort criteria +sort-levels secondary sort-asc-add z[ sort ascending by current column; add to existing sort criteria +sort-levels tertiary sort-desc-add z] sort descending by current column; add to existing sort criteria diff --git a/visidata/sheets.py b/visidata/sheets.py index ed887ccd0..654665757 100644 --- a/visidata/sheets.py +++ b/visidata/sheets.py @@ -56,7 +56,8 @@ theme('disp_endmid_sep', '║', '') # ╽╿┃ theme('disp_endbot_sep', '║', '') # ╽╿┃╜‖ theme('disp_selected_note', '•', '') # - +theme('disp_sort_asc', '↑↟⇞⇡⇧⇑', 'characters for ascending sort') # ↑▲↟↥↾↿⇞⇡⇧⇈⤉⤒⥔⥘⥜⥠⍏˄ˆ +theme('disp_sort_desc', '↓↡⇟⇣⇩⇓', 'characters for descending sort') # ↓▼↡↧⇂⇃⇟⇣⇩⇊⤈⤓⥕⥙⥝⥡⍖˅ˇ theme('color_default', 'normal', 'the default color') theme('color_default_hdr', 'bold', 'color of the column headers') theme('color_bottom_hdr', 'underline', 'color of the bottom header row') @@ -611,7 +612,14 @@ def drawColHeader(self, scr, y, h, vcolidx): hdrs = col.name.split('\n') for i in range(h): - name = ' ' # save room at front for LeftMore + name = ' ' # save room at front for LeftMore or sorted arrow + for j, (sortcol, sortdir) in enumerate(self._ordering): + if col is sortcol: + try: + name = self.options.disp_sort_desc[j] if sortdir else self.options.disp_sort_asc[j] + except IndexError: + pass + if h-i-1 < len(hdrs): name += hdrs[::-1][h-i-1] From 99dce900cc70ce3fe9e73cd6336075b2052f1a50 Mon Sep 17 00:00:00 2001 From: Saul Pwanson Date: Wed, 28 Oct 2020 01:39:20 -0700 Subject: [PATCH 067/162] [dev] update roadmap --- dev/ROADMAP | 70 ++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 48 insertions(+), 22 deletions(-) diff --git a/dev/ROADMAP b/dev/ROADMAP index 33b7ef91e..dfaf42f43 100644 --- a/dev/ROADMAP +++ b/dev/ROADMAP @@ -1,35 +1,61 @@ -# Roadmap to 2.0 +# Roadmap for 2.x -## API +1. holdovers from 2.0 + - [canvas] API docs + - [options] option enums + - [options] user-defined option aliases + - [splitpane] File preview in directory view + - [defermods] 'modified since last save' indicator on status bar + - [scroll cell] zh/zl Left hand side of a cell with content hidden doesn't show ellipsis #751 -The primary goal of 2.0 is a stable, documented Python API that can be used to create an broad ecosystem of loaders, plugins, and apps. +2. Persistence + - input history #736 #468 + - key indexes for better joining and lookups -The Book of VisiData will describe the api and all functions and objects available to plugins and command execstrs. +3. Interface discoverability for commands #247 #742 + - clickable menu canvas + - more clickable affordances all around + - clickable motd + - possibly add popup modals -## Split panes +4. more expressive expressions + - Memory sheet; can give things names and use in expressions. #392 + - access column values from execstr #655 + - shortcut name for current column #659 -+ horizontal split -- one pane can be dependent on the other; e.g. move cursor in one, see preview update in other -+ use for choosing aggregators, command help, other choose() +5. better asynchronicity + - async thread pool + - streaming architecture #366 #656 -- DirSheet file preview #309 +6. Loaders/Savers + - frictionless saver #237 + - RSS reader #157 + - toml loader #735 + - HexSheet for unknown/binary files #548 + - .ods loader (LibreOffice/OpenOffice spreadsheet) #473 -## add jsonl as more robust system format -+ add .vdj format (visidata cmdlog as jsonl) -+ migrate plugins.tsv to be jsonl format + - jsonl load and save round-trip (minimal diff) #429 -## Minor features, refactors, cleanup +## other features -- 'modified' indicator on status bar, also enables quitguard -- remove all asserts; replace with error() on case-by-case basis +a) generate non-terminal graphs (ggplot) +b) automatic reload into time series +c) intra-cell coloring (for search results, markup) -- [options] allow list or dict of enum values for default -- [options] for options that have been renamed, add aliases for compatibility - - e.g. `tsv_delimiter` and `tsv_row_delimiter` become 'proper' options with 'delimiter', 'delim', and 'd' becoming aliases -# Features +# major plugin projects -- Memory sheet; can give things names and use in expressions. - - 'show' commands are copied there by default. +1. fully operational SQL viewer/editor -- [columns] add helpstr for all specialized-sheet columns + - #282: Select starting table in postgres from command-line + - #579: [Postgres] Allow inserting / deleting rows + - #522: [postgres] parms in options + - #586: SQL query data + - #727: [postgres] Transaction error when viewing table + - #729: Integrate generic SQL loader + +2. web scraping + + - #480: HTTP API pagination loader + - #465: Ability to load from the contents of a cell. + - #505: [html] Provide way to access non-table elements From 229177f344572ed5eb79fb9f241f7d52590dbf1d Mon Sep 17 00:00:00 2001 From: anjakefala Date: Sun, 8 Nov 2020 15:23:06 -0800 Subject: [PATCH 068/162] [docs] add VisiData cheat sheet to references Addresses #770 --- docs/index.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/index.md b/docs/index.md index 5ccad4af6..a1ac73de5 100644 --- a/docs/index.md +++ b/docs/index.md @@ -17,6 +17,7 @@ * all available commands and options * also available as a manpage via `man vd` and from inside VisiData with `Ctrl+H` * [quick reference guide for 1.5.2](/docs/v1.5.2/man) +* [VisiData Cheat Sheet](https://jsvine.github.io/visidata-cheat-sheet/en/) * [keyboard layout of commands](/docs/kblayout) ## 'How to' recipes From 4a98cc25e4780b164fb4f2dc5ba2d1920180d98b Mon Sep 17 00:00:00 2001 From: Saul Pwanson Date: Thu, 5 Nov 2020 21:59:07 -0800 Subject: [PATCH 069/162] [sheets] sorting on SheetsSheet does not move SheetsSheet itself #761 #518 --- visidata/sheets.py | 2 ++ visidata/sort.py | 27 +++++++++++++++------------ 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/visidata/sheets.py b/visidata/sheets.py index 654665757..f9adf9828 100644 --- a/visidata/sheets.py +++ b/visidata/sheets.py @@ -945,6 +945,8 @@ class SheetsSheet(IndexSheet): def reload(self): self.rows = self.source + def sort(self): + self.rows[1:] = sorted(self.rows[1:], key=self.sortkey) @VisiData.property @drawcache diff --git a/visidata/sort.py b/visidata/sort.py index f50f5ad0e..e2b34f2b4 100644 --- a/visidata/sort.py +++ b/visidata/sort.py @@ -37,6 +37,20 @@ def __eq__(self, other): def __lt__(self, other): return other.obj < self.obj + +@Sheet.api +def sortkey(self, r, prog=None): + ret = [] + for col, reverse in self._ordering: + if isinstance(col, str): + col = self.column(col) + val = col.getTypedValue(r) + ret.append(Reversor(val) if reverse else val) + + if prog: + prog.addProgress(1) + return ret + @Sheet.api @asyncthread def sort(self): @@ -45,19 +59,8 @@ def sort(self): return try: with Progress(gerund='sorting', total=self.nRows) as prog: - def sortkey(r): - ret = [] - for col, reverse in self._ordering: - if isinstance(col, str): - col = self.column(col) - val = col.getTypedValue(r) - ret.append(Reversor(val) if reverse else val) - - prog.addProgress(1) - return ret - # must not reassign self.rows: use .sort() instead of sorted() - self.rows.sort(key=sortkey) + self.rows.sort(key=lambda r,self=self,prog=prog: self.sortkey(r, prog=prog)) except TypeError as e: vd.warning('sort incomplete due to TypeError; change column type') vd.exceptionCaught(e, status=False) From 030e66a7cb5fbfe5fb84c1b472a72f4bce4a6a7f Mon Sep 17 00:00:00 2001 From: Saul Pwanson Date: Thu, 5 Nov 2020 22:16:42 -0800 Subject: [PATCH 070/162] [zip] add extract-* commands --- visidata/loaders/archive.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/visidata/loaders/archive.py b/visidata/loaders/archive.py index 3fac973a0..f1620ba88 100644 --- a/visidata/loaders/archive.py +++ b/visidata/loaders/archive.py @@ -36,14 +36,21 @@ def openZipFile(self, fp, *args, **kwargs): vd.error(err) def openRow(self, fi): - zfp = zipfile.ZipFile(str(self.source), 'r') - decodedfp = codecs.iterdecode(self.openZipFile(zfp, fi), + decodedfp = codecs.iterdecode(self.openZipFile(self.zfp, fi), encoding=options.encoding, errors=options.encoding_errors) return vd.openSource(Path(fi.filename, fp=decodedfp, filesize=fi.file_size), filetype=options.filetype) + @asyncthread + def extract(self, *rows, path=None): + self.zfp.extractall(members=[r.filename for r in rows], path=path) + + @property + def zfp(self): + return zipfile.ZipFile(str(self.source), 'r') + def iterload(self): - with zipfile.ZipFile(str(self.source), 'r') as zf: + with self.zfp as zf: for zi in Progress(zf.infolist()): yield zi @@ -72,3 +79,9 @@ def iterload(self): with tarfile.open(name=str(self.source)) as tf: for ti in Progress(tf.getmembers()): yield ti + + +ZipSheet.addCommand('x', 'extract-file', 'extract(cursorRow)') +ZipSheet.addCommand('gx', 'extract-selected', 'extract(*someSelectedRows)') +ZipSheet.addCommand('zx', 'extract-file-to', 'extract(cursorRow, path=inputPath("extract to: "))') +ZipSheet.addCommand('gzx', 'extract-selected-to', 'extract(*someSelectedRows, path=inputPath("extract %d files to: " % nSelected))') From 55d323f2ef221e44b60daa099dc355f65cd847f0 Mon Sep 17 00:00:00 2001 From: Saul Pwanson Date: Thu, 5 Nov 2020 22:30:04 -0800 Subject: [PATCH 071/162] [tests] add golden for sort-levels --- tests/golden/sort-levels.tsv | 50 ++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 tests/golden/sort-levels.tsv diff --git a/tests/golden/sort-levels.tsv b/tests/golden/sort-levels.tsv new file mode 100644 index 000000000..e4cbd54d6 --- /dev/null +++ b/tests/golden/sort-levels.tsv @@ -0,0 +1,50 @@ +primary secondary tertiary +1 1 1 +1 1 1 +1 1 1 +1 1 1 +1 1 1 +1 1 1 +1 1 1 +1 1 1 +1 1 1 +1 1 1 +1 1 1 +1 1 1 +1 1 1 +1 3 2 +1 3 2 +1 3 2 +1 3 2 +1 3 2 +1 3 2 +1 3 2 +1 3 2 +1 3 2 +1 3 2 +1 3 1 +1 3 1 +1 3 1 +2 1 3 +2 1 3 +2 1 3 +2 1 3 +2 1 3 +2 1 3 +2 2 4 +2 2 4 +2 2 4 +2 2 4 +2 2 4 +2 2 4 +2 2 4 +2 2 4 +2 2 4 +2 2 4 +2 2 3 +2 2 3 +2 2 3 +2 2 3 +2 2 3 +2 3 3 +2 3 2 From 5bfe64a0cf447576b314621d6df2f753154fdb10 Mon Sep 17 00:00:00 2001 From: Saul Pwanson Date: Sun, 8 Nov 2020 15:39:35 -0800 Subject: [PATCH 072/162] [docs] add filename for example --- docs/api/plugins.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/api/plugins.rst b/docs/api/plugins.rst index b1ff06ad3..257251ef7 100644 --- a/docs/api/plugins.rst +++ b/docs/api/plugins.rst @@ -41,6 +41,9 @@ Multiple plugins can be defined in the same manifest; one line per plugin in the Complete "Hello world" plugin example ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +hello.py +^^^^^^^^^^^^^^^^ + :: '''This plugin adds the ``hello-world`` command to all sheets. From e5fe3c78c715d9a59ba28ef8b518a0d1de9335dd Mon Sep 17 00:00:00 2001 From: anjakefala Date: Sun, 8 Nov 2020 16:29:43 -0800 Subject: [PATCH 073/162] [sort-] fix tabbing bug --- visidata/sort.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/visidata/sort.py b/visidata/sort.py index e2b34f2b4..b5972239e 100644 --- a/visidata/sort.py +++ b/visidata/sort.py @@ -49,7 +49,7 @@ def sortkey(self, r, prog=None): if prog: prog.addProgress(1) - return ret + return ret @Sheet.api @asyncthread From 61e8faf45329cd4ecc9d2c13e69029cac4b20edd Mon Sep 17 00:00:00 2001 From: anjakefala Date: Wed, 11 Nov 2020 22:07:43 -0500 Subject: [PATCH 074/162] [open-] additionally search for open_filetype within the vd scope --- docs/api/loaders.rst | 10 ++++++++++ visidata/_open.py | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/docs/api/loaders.rst b/docs/api/loaders.rst index 5b1388ea7..90e0a7bcc 100644 --- a/docs/api/loaders.rst +++ b/docs/api/loaders.rst @@ -30,6 +30,16 @@ The *p* argument is a :ref:`visidata.Path`. The actual loading happens in the Sheet. An existing :ref:`sheet type` can be used, or a new sheet type can be created. +If the loader is within a plugin, the ``open_`` should be decorated with a ``@VisiData,api`` in order to make them available through the ``vd`` object's scope. + +:: + + @VisiData.api + def open_readme(vd, p): + return ReadmeSheet(p.name, source=p) + +Note, the change in the ``open_`` function signature, when decorated. + Step 2. Create a Sheet subclass ------------------------------- diff --git a/visidata/_open.py b/visidata/_open.py index a6c7bbbed..6cf029621 100644 --- a/visidata/_open.py +++ b/visidata/_open.py @@ -56,7 +56,7 @@ def openPath(vd, p, filetype=None): filetype = filetype.lower() - openfunc = vd.getGlobals().get('open_' + filetype) + openfunc = getattr(vd, 'open_' + filetype, vd.getGlobals().get('open_' + filetype)) if not openfunc: vd.warning('unknown "%s" filetype' % filetype) filetype = 'txt' From 3b2089af9651a079ce2db4cb9bf4bb38bcf02343 Mon Sep 17 00:00:00 2001 From: anjakefala Date: Wed, 11 Nov 2020 22:36:43 -0500 Subject: [PATCH 075/162] [config] --config='' now ignores visidatarc Closes #777 --- visidata/settings.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/visidata/settings.py b/visidata/settings.py index c7166e494..897302cf6 100644 --- a/visidata/settings.py +++ b/visidata/settings.py @@ -326,6 +326,8 @@ def getCommand(sheet, cmd): def loadConfigFile(fnrc, _globals=None): + if not fnrc: + return p = visidata.Path(fnrc) if _globals is None: _globals = globals() @@ -354,8 +356,8 @@ def addOptions(parser): @VisiData.api def loadConfigAndPlugins(vd, args): # set visidata_dir and config manually before loading config file, so visidata_dir can be set from cli or from $VD_DIR - options.visidata_dir = args.visidata_dir or os.getenv('VD_DIR', '') or options.visidata_dir - options.config = args.config or os.getenv('VD_CONFIG', '') or options.config + options.visidata_dir = args.visidata_dir if args.visidata_dir is not None else os.getenv('VD_DIR', '') or options.visidata_dir + options.config = args.config if args.config is not None else os.getenv('VD_CONFIG', '') or options.config sys.path.append(str(visidata.Path(options.visidata_dir))) sys.path.append(str(visidata.Path(options.visidata_dir)/"plugins-deps")) From 25a3858bae825da90b970e4c5231e251d50c1188 Mon Sep 17 00:00:00 2001 From: Paul O'Leary McCann Date: Mon, 9 Nov 2020 22:35:39 +0900 Subject: [PATCH 076/162] Add conll loader to plugins file --- plugins/plugins.jsonl | 1 + 1 file changed, 1 insertion(+) diff --git a/plugins/plugins.jsonl b/plugins/plugins.jsonl index f3bbdf830..37d2f1258 100644 --- a/plugins/plugins.jsonl +++ b/plugins/plugins.jsonl @@ -11,3 +11,4 @@ {"name": "genericSQL", "description": "add loaders for SQL databases (Oracle, MySQL)", "maintainer": "Andrew Swanson @aswan89", "latest_release": "2020-04-15", "url": "https://raw.githubusercontent.com/aswan89/visidata_plugin_genericSQL/1.0.0/generic_sql.py", "latest_ver": "1.0.0", "visidata_ver": "2.-3", "pydeps": "sqlalchemy cx_oracle mysqlclient pyodbc", "vdplugindeps": "", "sha256": "71bbaabaa4ffb973ef7745dc3eadca108b34a7e0f0bf6cf6631eca3fa4532513"} {"name": "diff", "description": "adds command to create diff sheets", "maintainer": "Anja Kefala @anjakefala", "latest_release": "2020-10-09", "url": "https://raw.githubusercontent.com/saulpw/visidata/492d558ff4aba55a192965a27ae6499f8d219072/plugins/diff.py", "latest_ver": "0.9", "visidata_ver": "2.0", "sha256": "0085bf8afdc9abf74b7dd52c251f7b7f7befc507aae262bbb656c2ab7379ddd1"} {"name": "marks", "description": "adds commands for marking selected rows, and selecting + viewing marked rows", "maintainer": "Saul Pwanson @saulpw", "latest_release": "2020-10-09", "url": "https://raw.githubusercontent.com/saulpw/visidata/54e49feee97a607cce5355bf0e3a79b2466c3d8a/plugins/marks.py", "latest_ver": "0.1", "visidata_ver": "2.0", "sha256": "f15977f327ccc387be208922b66afe4b65666912e8066d02352c53b1aaff941f"} +{"name":"conll", "description":"CoNLL data loader", "maintainer": "Paul McCann ", "latest_release": "2020-11-09", "latest_ver": "0.1.0", "url": "https://github.com/polm/visidata-conll", "visidata_ver": "v2.0", "pydeps": "pyconll", "sha256": "a757ba4907baf5f9c001d817e1cefe38f6bbebed31f1e79783a94091e39810c1"} From 49d5a622ed94e7709b3f37363ecdc7f154b6b301 Mon Sep 17 00:00:00 2001 From: Paul O'Leary McCann Date: Thu, 12 Nov 2020 12:42:56 +0900 Subject: [PATCH 077/162] Fix jsonl --- plugins/plugins.jsonl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/plugins.jsonl b/plugins/plugins.jsonl index 37d2f1258..855bdd8e1 100644 --- a/plugins/plugins.jsonl +++ b/plugins/plugins.jsonl @@ -11,4 +11,4 @@ {"name": "genericSQL", "description": "add loaders for SQL databases (Oracle, MySQL)", "maintainer": "Andrew Swanson @aswan89", "latest_release": "2020-04-15", "url": "https://raw.githubusercontent.com/aswan89/visidata_plugin_genericSQL/1.0.0/generic_sql.py", "latest_ver": "1.0.0", "visidata_ver": "2.-3", "pydeps": "sqlalchemy cx_oracle mysqlclient pyodbc", "vdplugindeps": "", "sha256": "71bbaabaa4ffb973ef7745dc3eadca108b34a7e0f0bf6cf6631eca3fa4532513"} {"name": "diff", "description": "adds command to create diff sheets", "maintainer": "Anja Kefala @anjakefala", "latest_release": "2020-10-09", "url": "https://raw.githubusercontent.com/saulpw/visidata/492d558ff4aba55a192965a27ae6499f8d219072/plugins/diff.py", "latest_ver": "0.9", "visidata_ver": "2.0", "sha256": "0085bf8afdc9abf74b7dd52c251f7b7f7befc507aae262bbb656c2ab7379ddd1"} {"name": "marks", "description": "adds commands for marking selected rows, and selecting + viewing marked rows", "maintainer": "Saul Pwanson @saulpw", "latest_release": "2020-10-09", "url": "https://raw.githubusercontent.com/saulpw/visidata/54e49feee97a607cce5355bf0e3a79b2466c3d8a/plugins/marks.py", "latest_ver": "0.1", "visidata_ver": "2.0", "sha256": "f15977f327ccc387be208922b66afe4b65666912e8066d02352c53b1aaff941f"} -{"name":"conll", "description":"CoNLL data loader", "maintainer": "Paul McCann ", "latest_release": "2020-11-09", "latest_ver": "0.1.0", "url": "https://github.com/polm/visidata-conll", "visidata_ver": "v2.0", "pydeps": "pyconll", "sha256": "a757ba4907baf5f9c001d817e1cefe38f6bbebed31f1e79783a94091e39810c1"} +{"name":"conll", "description":"CoNLL data loader", "maintainer": "Paul McCann ", "latest_release": "2020-11-09", "latest_ver": "v0.1.0", "url": "https://raw.githubusercontent.com/polm/visidata-conll/83579a939813b3a3bca638bbb15bb9e8cf4e08ac/conll.py", "visidata_ver": "v2.0", "pydeps": "pyconll","sha256":"b7165a62eb3279cd6c7580df9d449475b1744b2fc692e7ed6ff1ca92cd544ada"} From 880e4b32eea12a18ce1cdc5aa9fa36391734669b Mon Sep 17 00:00:00 2001 From: anjakefala Date: Thu, 12 Nov 2020 21:32:48 -0500 Subject: [PATCH 078/162] [plugins] ensure consistent version style --- plugins/plugins.jsonl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/plugins.jsonl b/plugins/plugins.jsonl index 855bdd8e1..b7c0f669d 100644 --- a/plugins/plugins.jsonl +++ b/plugins/plugins.jsonl @@ -11,4 +11,4 @@ {"name": "genericSQL", "description": "add loaders for SQL databases (Oracle, MySQL)", "maintainer": "Andrew Swanson @aswan89", "latest_release": "2020-04-15", "url": "https://raw.githubusercontent.com/aswan89/visidata_plugin_genericSQL/1.0.0/generic_sql.py", "latest_ver": "1.0.0", "visidata_ver": "2.-3", "pydeps": "sqlalchemy cx_oracle mysqlclient pyodbc", "vdplugindeps": "", "sha256": "71bbaabaa4ffb973ef7745dc3eadca108b34a7e0f0bf6cf6631eca3fa4532513"} {"name": "diff", "description": "adds command to create diff sheets", "maintainer": "Anja Kefala @anjakefala", "latest_release": "2020-10-09", "url": "https://raw.githubusercontent.com/saulpw/visidata/492d558ff4aba55a192965a27ae6499f8d219072/plugins/diff.py", "latest_ver": "0.9", "visidata_ver": "2.0", "sha256": "0085bf8afdc9abf74b7dd52c251f7b7f7befc507aae262bbb656c2ab7379ddd1"} {"name": "marks", "description": "adds commands for marking selected rows, and selecting + viewing marked rows", "maintainer": "Saul Pwanson @saulpw", "latest_release": "2020-10-09", "url": "https://raw.githubusercontent.com/saulpw/visidata/54e49feee97a607cce5355bf0e3a79b2466c3d8a/plugins/marks.py", "latest_ver": "0.1", "visidata_ver": "2.0", "sha256": "f15977f327ccc387be208922b66afe4b65666912e8066d02352c53b1aaff941f"} -{"name":"conll", "description":"CoNLL data loader", "maintainer": "Paul McCann ", "latest_release": "2020-11-09", "latest_ver": "v0.1.0", "url": "https://raw.githubusercontent.com/polm/visidata-conll/83579a939813b3a3bca638bbb15bb9e8cf4e08ac/conll.py", "visidata_ver": "v2.0", "pydeps": "pyconll","sha256":"b7165a62eb3279cd6c7580df9d449475b1744b2fc692e7ed6ff1ca92cd544ada"} +{"name": "conll", "description": "CoNLL data loader", "maintainer": "Paul McCann ", "latest_release": "2020-11-09", "url": "https://raw.githubusercontent.com/polm/visidata-conll/83579a939813b3a3bca638bbb15bb9e8cf4e08ac/conll.py", "latest_ver": "0.1.0", "visidata_ver": "2.0", "pydeps": "pyconll", "sha256": "b7165a62eb3279cd6c7580df9d449475b1744b2fc692e7ed6ff1ca92cd544ada"} From ecd5f2be28ba32dfdf459ffca826331f12adf995 Mon Sep 17 00:00:00 2001 From: anjakefala Date: Thu, 12 Nov 2020 21:35:52 -0500 Subject: [PATCH 079/162] [plugins] fix sha --- plugins/plugins.jsonl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/plugins.jsonl b/plugins/plugins.jsonl index b7c0f669d..7eadc832c 100644 --- a/plugins/plugins.jsonl +++ b/plugins/plugins.jsonl @@ -11,4 +11,4 @@ {"name": "genericSQL", "description": "add loaders for SQL databases (Oracle, MySQL)", "maintainer": "Andrew Swanson @aswan89", "latest_release": "2020-04-15", "url": "https://raw.githubusercontent.com/aswan89/visidata_plugin_genericSQL/1.0.0/generic_sql.py", "latest_ver": "1.0.0", "visidata_ver": "2.-3", "pydeps": "sqlalchemy cx_oracle mysqlclient pyodbc", "vdplugindeps": "", "sha256": "71bbaabaa4ffb973ef7745dc3eadca108b34a7e0f0bf6cf6631eca3fa4532513"} {"name": "diff", "description": "adds command to create diff sheets", "maintainer": "Anja Kefala @anjakefala", "latest_release": "2020-10-09", "url": "https://raw.githubusercontent.com/saulpw/visidata/492d558ff4aba55a192965a27ae6499f8d219072/plugins/diff.py", "latest_ver": "0.9", "visidata_ver": "2.0", "sha256": "0085bf8afdc9abf74b7dd52c251f7b7f7befc507aae262bbb656c2ab7379ddd1"} {"name": "marks", "description": "adds commands for marking selected rows, and selecting + viewing marked rows", "maintainer": "Saul Pwanson @saulpw", "latest_release": "2020-10-09", "url": "https://raw.githubusercontent.com/saulpw/visidata/54e49feee97a607cce5355bf0e3a79b2466c3d8a/plugins/marks.py", "latest_ver": "0.1", "visidata_ver": "2.0", "sha256": "f15977f327ccc387be208922b66afe4b65666912e8066d02352c53b1aaff941f"} -{"name": "conll", "description": "CoNLL data loader", "maintainer": "Paul McCann ", "latest_release": "2020-11-09", "url": "https://raw.githubusercontent.com/polm/visidata-conll/83579a939813b3a3bca638bbb15bb9e8cf4e08ac/conll.py", "latest_ver": "0.1.0", "visidata_ver": "2.0", "pydeps": "pyconll", "sha256": "b7165a62eb3279cd6c7580df9d449475b1744b2fc692e7ed6ff1ca92cd544ada"} +{"name": "conll", "description": "CoNLL data loader", "maintainer": "Paul McCann ", "latest_release": "2020-11-09", "url": "https://raw.githubusercontent.com/polm/visidata-conll/83579a939813b3a3bca638bbb15bb9e8cf4e08ac/conll.py", "latest_ver": "0.1.0", "visidata_ver": "2.0", "pydeps": "pyconll", "sha256": "ff0df4c817121e57780ddd3e16ae67703fa69129c9e33e83fe40dd5220ad93e2"} From db753e641d4f892f83555355978f57fbbe4d8ef8 Mon Sep 17 00:00:00 2001 From: Saul Pwanson Date: Thu, 12 Nov 2020 15:24:20 -0800 Subject: [PATCH 080/162] [status] allow non-hashable status msgs by deduping based on stringified contents --- visidata/statusbar.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/visidata/statusbar.py b/visidata/statusbar.py index 4d95e3e37..cc9f7a2ba 100644 --- a/visidata/statusbar.py +++ b/visidata/statusbar.py @@ -39,7 +39,7 @@ def status(self, *args, priority=0): if not args: return True - k = (priority, args) + k = (priority, tuple(map(str, args))) self.statuses[k] = self.statuses.get(k, 0) + 1 if self.statusHistory: From 36cf8cbca90d283636aa0929e25a1f986abd4ad4 Mon Sep 17 00:00:00 2001 From: Saul Pwanson Date: Thu, 12 Nov 2020 19:48:31 -0800 Subject: [PATCH 081/162] [sqlite] use rowid to update and delete rows - note that this will not work with WITHOUT ROWID sqlite tables --- visidata/loaders/sqlite.py | 31 +++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/visidata/loaders/sqlite.py b/visidata/loaders/sqlite.py index 60e037e7f..fbec5de7a 100644 --- a/visidata/loaders/sqlite.py +++ b/visidata/loaders/sqlite.py @@ -37,8 +37,9 @@ def iterload(self): tblname = self.tableName if not isinstance(self, SqliteIndexSheet): self.columns = [] + self.addColumn(ColumnItem('rowid', 0, type=int, width=0)) for i, r in enumerate(self.execute(conn, 'PRAGMA TABLE_INFO("%s")' % tblname)): - c = ColumnItem(r[1], i, type=sqltypes.get(r[2].upper(), anytype)) + c = ColumnItem(r[1], i+1, type=sqltypes.get(r[2].upper(), anytype)) self.addColumn(c) if r[-1]: @@ -46,7 +47,7 @@ def iterload(self): r = self.execute(conn, 'SELECT COUNT(*) FROM "%s"' % tblname).fetchall() rowcount = r[0][0] - for row in Progress(self.execute(conn, 'SELECT * FROM "%s"' % tblname), total=rowcount-1): + for row in Progress(self.execute(conn, 'SELECT rowid, * FROM "%s"' % tblname), total=rowcount-1): yield list(row) @asyncthread @@ -71,25 +72,34 @@ def values(row, cols): return vals with self.conn() as conn: - wherecols = self.keyCols or self.visibleCols + wherecols = [self.columns[0]] # self.column("rowid") for r in adds.values(): cols = self.visibleCols sql = 'INSERT INTO "%s" ' % self.tableName sql += '(%s)' % ','.join(c.name for c in cols) sql += 'VALUES (%s)' % ','.join('?' for c in cols) - self.execute(conn, sql, parms=values(r, cols)) + res = self.execute(conn, sql, parms=values(r, cols)) + if res.rowcount != res.arraysize: + vd.warning('not all rows inserted') # f'{res.rowcount}/{res.arraysize} rows inserted' for row, rowmods in mods.values(): sql = 'UPDATE "%s" SET ' % self.tableName sql += ', '.join('%s=?' % c.name for c, _ in rowmods.items()) sql += ' WHERE %s' % ' AND '.join('"%s"=?' % c.name for c in wherecols) - self.execute(conn, sql, - parms=values(row, [c for c, _ in rowmods.items()]) + list(Column.calcValue(c, row) for c in wherecols)) + newvals=values(row, [c for c, _ in rowmods.items()]) + # calcValue gets the 'previous' value (before update) + wherevals=list(Column.calcValue(c, row) or '' for c in wherecols) + res = self.execute(conn, sql, parms=newvals+wherevals) + if res.rowcount != res.arraysize: + vd.warning('not all rows updated') # f'{res.rowcount}/{res.arraysize} rows updated' for r in dels.values(): sql = 'DELETE FROM "%s" ' % self.tableName sql += ' WHERE %s' % ' AND '.join('"%s"=?' % c.name for c in wherecols) - self.execute(conn, sql, parms=list(c.getTypedValue(r) for c in wherecols)) + wherevals=list(Column.calcValue(c, row) for c in wherecols) + res = self.execute(conn, sql, parms=wherevals) + if res.rowcount != res.arraysize: + vd.warning('not all rows deleted') # f'{res.rowcount}/{res.arraysize} rows deleted' conn.commit() @@ -101,8 +111,8 @@ class SqliteIndexSheet(SqliteSheet, IndexSheet): tableName = 'sqlite_master' def iterload(self): for row in SqliteSheet.iterload(self): - if row[0] != 'index': - tblname = row[1] + if row[1] != 'index': + tblname = row[2] yield SqliteSheet(tblname, source=self, tableName=tblname, row=row) @@ -110,10 +120,11 @@ class SqliteQuerySheet(SqliteSheet): def iterload(self): with self.conn() as conn: self.columns = [] + self.addColumn(ColumnItem('rowid', 0, type=int)) self.result = self.execute(conn, self.query, parms=getattr(self, 'parms', [])) for i, desc in enumerate(self.result.description): - self.addColumn(ColumnItem(desc[0], i)) + self.addColumn(ColumnItem(desc[0], i+1)) for row in self.result: yield row From f07c93bb831662a7c315bb0152b74690941ec622 Mon Sep 17 00:00:00 2001 From: anjakefala Date: Fri, 13 Nov 2020 18:50:52 -0800 Subject: [PATCH 082/162] [defer] commit changes, even if no defer --- visidata/modify.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/visidata/modify.py b/visidata/modify.py index 0e30a0f5f..abff8765a 100644 --- a/visidata/modify.py +++ b/visidata/modify.py @@ -252,9 +252,6 @@ def commit(sheet, *rows): cstr = sheet.changestr(adds, mods, deletes) path = sheet.source - if not cstr: - vd.fail('no diffs') - if options.confirm_overwrite: confirm('really %s? ' % cstr) From 433f9c10ac335917b90ae025664f11eb40ba57e3 Mon Sep 17 00:00:00 2001 From: Anatoli Babenia Date: Sun, 15 Nov 2020 12:19:32 +0300 Subject: [PATCH 083/162] Support running as `python -m visidata` --- visidata/__main__.py | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 visidata/__main__.py diff --git a/visidata/__main__.py b/visidata/__main__.py new file mode 100644 index 000000000..fbba325bc --- /dev/null +++ b/visidata/__main__.py @@ -0,0 +1,3 @@ +from .main import vd_cli + +vd_cli() From 411de01ecfa9b6393df437449d8fc0aa800742eb Mon Sep 17 00:00:00 2001 From: anjakefala Date: Mon, 16 Nov 2020 20:24:45 -0500 Subject: [PATCH 084/162] [type-] fix recursive undo for typing --- visidata/column.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/visidata/column.py b/visidata/column.py index 2beb4eb2e..e36dc5276 100644 --- a/visidata/column.py +++ b/visidata/column.py @@ -164,7 +164,7 @@ def type(self): @type.setter def type(self, t): if self._type != t: - vd.addUndo(setattr, self, 'type', self._type) + vd.addUndo(setattr, self, '_type', self.type) self._type = t @property From 61037e8d51661bca08ff9fe4e91066295993838e Mon Sep 17 00:00:00 2001 From: Saul Pwanson Date: Fri, 13 Nov 2020 19:45:53 -0800 Subject: [PATCH 085/162] [dev nfc] README cosmetic --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index a4176290b..91f218e7e 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ A terminal interface for exploring and arranging tabular data. ![Frequency table](http://visidata.org/freq-move-row.gif) -VisiData supports tsv, csv, sqlite, json, xlsx (Excel), hdf5, and [many other formats](https://visidata.org/formats)). +VisiData supports tsv, csv, sqlite, json, xlsx (Excel), hdf5, and [many other formats](https://visidata.org/formats). ## Platform requirements @@ -30,7 +30,7 @@ See [visidata.org/install](https://visidata.org/install) for detailed instructio $ vd $ | vd -Press `Ctrl+Q` to quit. +Press `Ctrl+Q` to quit at any time. Hundreds of other commands and options are also available; see the documentation. From 0ac52acf6bf4600ed52a2774631bb854a311c756 Mon Sep 17 00:00:00 2001 From: Saul Pwanson Date: Sat, 14 Nov 2020 23:26:25 -0800 Subject: [PATCH 086/162] [sqlite-] fix commit deletes --- visidata/loaders/sqlite.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/visidata/loaders/sqlite.py b/visidata/loaders/sqlite.py index fbec5de7a..a514a2496 100644 --- a/visidata/loaders/sqlite.py +++ b/visidata/loaders/sqlite.py @@ -93,7 +93,7 @@ def values(row, cols): if res.rowcount != res.arraysize: vd.warning('not all rows updated') # f'{res.rowcount}/{res.arraysize} rows updated' - for r in dels.values(): + for row in dels.values(): sql = 'DELETE FROM "%s" ' % self.tableName sql += ' WHERE %s' % ' AND '.join('"%s"=?' % c.name for c in wherecols) wherevals=list(Column.calcValue(c, row) for c in wherecols) From b2b1069440a515fe012318b398b1ea269c700965 Mon Sep 17 00:00:00 2001 From: Saul Pwanson Date: Sun, 15 Nov 2020 15:14:03 -0800 Subject: [PATCH 087/162] [options-] fix confirm_overwrite in batch mode - fix -y to set confirm_overwrite to False (meaning, no confirmation necessary for overwrite) - make confirm() always fail in batch mode - make confirm_overwrite a sheet-specific option --- dev/ROADMAP | 4 ++-- visidata/_input.py | 3 +++ visidata/main.py | 8 ++++---- visidata/modify.py | 4 ++-- 4 files changed, 11 insertions(+), 8 deletions(-) diff --git a/dev/ROADMAP b/dev/ROADMAP index dfaf42f43..b414be5b5 100644 --- a/dev/ROADMAP +++ b/dev/ROADMAP @@ -41,7 +41,7 @@ a) generate non-terminal graphs (ggplot) b) automatic reload into time series c) intra-cell coloring (for search results, markup) - +d) Shift+Arrow in cell edit to accept and move cursor that direction, keeping in edit mode # major plugin projects @@ -54,7 +54,7 @@ c) intra-cell coloring (for search results, markup) - #727: [postgres] Transaction error when viewing table - #729: Integrate generic SQL loader -2. web scraping +2. web scraper - #480: HTTP API pagination loader - #465: Ability to load from the contents of a cell. diff --git a/visidata/_input.py b/visidata/_input.py index 0e56aa505..f6f4a2a85 100644 --- a/visidata/_input.py +++ b/visidata/_input.py @@ -324,6 +324,9 @@ def input(self, prompt, type=None, defaultLast=False, history=[], **kwargs): @VisiData.global_api def confirm(vd, prompt, exc=EscapeException): 'Display *prompt* on status line and demand input that starts with "Y" or "y" to proceed. Raise *exc* otherwise. Return True.' + if options.batch: + return vd.fail('cannot confirm in batch mode: ' + prompt) + yn = vd.input(prompt, value='no', record=False)[:1] if not yn or yn not in 'Yy': msg = 'disconfirmed: ' + prompt diff --git a/visidata/main.py b/visidata/main.py index ca25c9513..7f4666254 100755 --- a/visidata/main.py +++ b/visidata/main.py @@ -59,15 +59,15 @@ def duptty(): return stdin, stdout option_aliases = {} -def optalias(abbr, name): - option_aliases[abbr] = name +def optalias(abbr, name, val=None): + option_aliases[abbr] = (name, val) optalias('f', 'filetype') optalias('p', 'play') optalias('b', 'batch') optalias('P', 'preplay') -optalias('y', 'confirm_overwrite') +optalias('y', 'confirm_overwrite', False) optalias('o', 'output') optalias('w', 'replay_wait') optalias('d', 'delimiter') @@ -130,7 +130,7 @@ def main_vd(): pass optname = optname.replace('-', '_') - optname = option_aliases.get(optname, optname) + optname, optval = option_aliases.get(optname, (optname, optval)) if optval is None: opt = options._get(optname) diff --git a/visidata/modify.py b/visidata/modify.py index abff8765a..58d10b2b5 100644 --- a/visidata/modify.py +++ b/visidata/modify.py @@ -252,8 +252,8 @@ def commit(sheet, *rows): cstr = sheet.changestr(adds, mods, deletes) path = sheet.source - if options.confirm_overwrite: - confirm('really %s? ' % cstr) + if sheet.options.confirm_overwrite: + vd.confirm('really %s? ' % cstr) sheet.putChanges() From f859dcbd4d99f9a224b493a28ffb8b29b8db0221 Mon Sep 17 00:00:00 2001 From: Saul Pwanson Date: Sun, 15 Nov 2020 15:18:12 -0800 Subject: [PATCH 088/162] [add] add bulk rows and cols should leave cursor on first added (like add singles) --- visidata/modify.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/visidata/modify.py b/visidata/modify.py index 58d10b2b5..2bf8475f5 100644 --- a/visidata/modify.py +++ b/visidata/modify.py @@ -258,8 +258,8 @@ def commit(sheet, *rows): sheet.putChanges() Sheet.addCommand('a', 'add-row', 'addNewRows(1, cursorRowIndex); cursorDown(1)', 'append a blank row') -Sheet.addCommand('ga', 'add-rows', 'addNewRows(int(input("add rows: ", value=1)), cursorRowIndex)', 'append N blank rows') +Sheet.addCommand('ga', 'add-rows', 'addNewRows(int(input("add rows: ", value=1)), cursorRowIndex); cursorDown(1)', 'append N blank rows') Sheet.addCommand('za', 'addcol-new', 'addColumnAtCursor(SettableColumn()); cursorRight(1)', 'append an empty column') -Sheet.addCommand('gza', 'addcol-bulk', 'addColumnAtCursor(*(SettableColumn() for c in range(int(input("add columns: ")))))', 'append N empty columns') +Sheet.addCommand('gza', 'addcol-bulk', 'addColumnAtCursor(*(SettableColumn() for c in range(int(input("add columns: "))))); cursorRight(1)', 'append N empty columns') Sheet.addCommand('z^S', 'commit-sheet', 'commit()') From d21c163b296139c02588ab5e35d637ee935d1285 Mon Sep 17 00:00:00 2001 From: Saul Pwanson Date: Sun, 15 Nov 2020 22:50:59 -0800 Subject: [PATCH 089/162] [json] skip lines starting with # - add `#!vd -p` as first line of .vdj for executable vd script --- visidata/loaders/json.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/visidata/loaders/json.py b/visidata/loaders/json.py index 02ca2fc02..f66bea967 100644 --- a/visidata/loaders/json.py +++ b/visidata/loaders/json.py @@ -26,6 +26,8 @@ def iterload(self): with self.source.open_text() as fp: for L in fp: try: + if L.startswith('#'): # skip commented lines + continue ret = json.loads(L, object_pairs_hook=OrderedDict) if isinstance(ret, list): yield from Progress(ret) From ac2fb0cfdea449249fd3f4f0b5f4949ccd610dc3 Mon Sep 17 00:00:00 2001 From: Saul Pwanson Date: Mon, 16 Nov 2020 15:37:49 -0800 Subject: [PATCH 090/162] [cli] allow = in .vd replay parameters --- visidata/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/visidata/main.py b/visidata/main.py index 7f4666254..28a64d781 100755 --- a/visidata/main.py +++ b/visidata/main.py @@ -167,7 +167,7 @@ def main_vd(): elif current_args.get('play', None) and '=' in arg: # parse 'key=value' pairs for formatting cmdlog template in replay mode - k, v = arg.split('=') + k, v = arg.split('=', maxsplit=1) fmtkwargs[k] = v else: inputs.append((arg, copy(current_args))) From e08be842c7d9d65d727809a1e28ca40946ac9465 Mon Sep 17 00:00:00 2001 From: Saul Pwanson Date: Mon, 16 Nov 2020 15:39:24 -0800 Subject: [PATCH 091/162] [sqlite] add cosmetic whitespace in sql statement --- visidata/loaders/sqlite.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/visidata/loaders/sqlite.py b/visidata/loaders/sqlite.py index a514a2496..388267f77 100644 --- a/visidata/loaders/sqlite.py +++ b/visidata/loaders/sqlite.py @@ -77,7 +77,7 @@ def values(row, cols): cols = self.visibleCols sql = 'INSERT INTO "%s" ' % self.tableName sql += '(%s)' % ','.join(c.name for c in cols) - sql += 'VALUES (%s)' % ','.join('?' for c in cols) + sql += ' VALUES (%s)' % ','.join('?' for c in cols) res = self.execute(conn, sql, parms=values(r, cols)) if res.rowcount != res.arraysize: vd.warning('not all rows inserted') # f'{res.rowcount}/{res.arraysize} rows inserted' From 98d4911c93163fc59123c27ba66959b4b13ed74b Mon Sep 17 00:00:00 2001 From: Saul Pwanson Date: Wed, 18 Nov 2020 19:35:09 -0800 Subject: [PATCH 092/162] [input] Shift+Arrow within edit-cell to move cursor and reenter edit mode --- dev/ROADMAP | 1 - visidata/_input.py | 35 ++++++++++++++++++++++++++++++++++- visidata/mainloop.py | 4 ++++ visidata/sheets.py | 3 +-- 4 files changed, 39 insertions(+), 4 deletions(-) diff --git a/dev/ROADMAP b/dev/ROADMAP index b414be5b5..c371e23c4 100644 --- a/dev/ROADMAP +++ b/dev/ROADMAP @@ -41,7 +41,6 @@ a) generate non-terminal graphs (ggplot) b) automatic reload into time series c) intra-cell coloring (for search results, markup) -d) Shift+Arrow in cell edit to accept and move cursor that direction, keeping in edit mode # major plugin projects diff --git a/visidata/_input.py b/visidata/_input.py index f6f4a2a85..522b03b8b 100644 --- a/visidata/_input.py +++ b/visidata/_input.py @@ -16,6 +16,22 @@ VisiData.init('lastInputs', lambda: collections.defaultdict(list)) # [input_type] -> list of prevInputs +class AcceptInput(Exception): + '*args[0]* is the input to be accepted' + +visidata.vd._nextCommands = [] + +@VisiData.api +def queueCommand(vd, longname): #, input=None, sheet=None, col=None, row=None): + vd._nextCommands.append(longname) + +def acceptThenFunc(*longnames): + def _acceptthen(v, i): + for longname in longnames: + vd.queueCommand(longname) + raise AcceptInput(v) + return _acceptthen + # editline helpers class EnableCursor: @@ -240,7 +256,10 @@ def editText(vd, y, x, w, record=True, display=True, **kwargs): v = vd.getLastArgs() if v is None: - v = vd.editline(vd.sheets[0]._scr, y, x, w, display=display, **kwargs) + try: + v = vd.editline(vd.sheets[0]._scr, y, x, w, display=display, **kwargs) + except AcceptInput as e: + v = e.args[0] # clear keyboard buffer to neutralize multi-line pastes (issue#585) curses.flushinp() @@ -370,11 +389,25 @@ def editCell(self, vcolidx=None, rowidx=None, value=None, **kwargs): y, h = self._rowLayout.get(rowidx, (0, 0)) value = value or col.getDisplayValue(self.rows[self.cursorRowIndex]) + bindings={ + 'kUP': acceptThenFunc('go-up', 'rename-col' if rowidx < 0 else 'edit-cell'), + 'KEY_SR': acceptThenFunc('go-up', 'rename-col' if rowidx < 0 else 'edit-cell'), + 'kDN': acceptThenFunc('go-down', 'rename-col' if rowidx < 0 else 'edit-cell'), + 'KEY_SF': acceptThenFunc('go-down', 'rename-col' if rowidx < 0 else 'edit-cell'), + 'KEY_SRIGHT': acceptThenFunc('go-right', 'rename-col' if rowidx < 0 else 'edit-cell'), + 'KEY_SLEFT': acceptThenFunc('go-left', 'rename-col' if rowidx < 0 else 'edit-cell'), + } + + bindings.update(kwargs.get('bindings', {})) + kwargs['bindings'] = bindings + editargs = dict(value=value, fillchar=options.disp_edit_fill, truncchar=options.disp_truncator) + editargs.update(kwargs) # update with user-specified args r = vd.editText(y, x, w, **editargs) + if rowidx >= 0: # if not header r = col.type(r) # convert input to column type, let exceptions be raised diff --git a/visidata/mainloop.py b/visidata/mainloop.py index 4846b3a8e..66540781d 100644 --- a/visidata/mainloop.py +++ b/visidata/mainloop.py @@ -117,6 +117,10 @@ def mainloop(self, scr): self.draw_all() + if vd._nextCommands: + sheet.execCommand(vd._nextCommands.pop(0), keystrokes=self.keystrokes) + continue + keystroke = self.getkeystroke(scr, sheet) if not keystroke and prefixWaiting and ESC in self.keystrokes: # timeout ESC diff --git a/visidata/sheets.py b/visidata/sheets.py index f9adf9828..0aef5a1c3 100644 --- a/visidata/sheets.py +++ b/visidata/sheets.py @@ -17,7 +17,6 @@ option('default_height', 10, 'default column height') option('textwrap_cells', True, 'wordwrap text for multiline rows') -option('cmd_after_edit', 'go-down', 'command longname to execute after successful edit') option('quitguard', False, 'confirm before quitting last sheet') option('debug', False, 'exit on error and display stacktrace') option('skip', 0, 'skip N rows before header', replay=True) @@ -1045,7 +1044,7 @@ def updateColNames(sheet, rows, cols, overwrite=False): Sheet.addCommand('!', 'key-col', 'toggleKeys([cursorCol])', 'toggle current column as a key column') Sheet.addCommand('z!', 'key-col-off', 'unsetKeys([cursorCol])', 'unset current column as a key column') -Sheet.addCommand('e', 'edit-cell', 'cursorCol.setValues([cursorRow], editCell(cursorVisibleColIndex)); options.cmd_after_edit and sheet.execCommand(options.cmd_after_edit)', 'edit contents of current cell') +Sheet.addCommand('e', 'edit-cell', 'cursorCol.setValues([cursorRow], editCell(cursorVisibleColIndex))', 'edit contents of current cell') Sheet.addCommand('ge', 'setcol-input', 'cursorCol.setValuesTyped(selectedRows, input("set selected to: ", value=cursorDisplay))', 'set contents of current column for selected rows to same input') Sheet.addCommand('"', 'dup-selected', 'vs=copy(sheet); vs.name += "_selectedref"; vs.reload=lambda vs=vs,rows=selectedRows: setattr(vs, "rows", list(rows)); vd.push(vs)', 'open duplicate sheet with only selected rows'), From 98ee7337b5d6d068afbe1d5c78c30ab21a600c17 Mon Sep 17 00:00:00 2001 From: Saul Pwanson Date: Wed, 18 Nov 2020 20:16:07 -0800 Subject: [PATCH 093/162] [fixed] fixed-width saver (uses col.width) --- dev/formats.jsonl | 2 +- visidata/loaders/fixed_width.py | 22 ++++++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/dev/formats.jsonl b/dev/formats.jsonl index 31f027178..a143874a8 100644 --- a/dev/formats.jsonl +++ b/dev/formats.jsonl @@ -6,7 +6,7 @@ {"filetype": "hdf5", "aliases": "h5", "requirements": "h5py", "format": "Hierarchical Data Format", "VisiData loader": "yes", "VisiData saver": "", "version_added": "0.28", "created": "199x", "creator": "NCSA", "description": "", "open format": "yes", "nestable": "", "plottable": "", "format_url": "https://support.hdfgroup.org/HDF5/"} {"filetype": "sqlite", "filetype_url": "#sqlite", "aliases": "db", "requirements": "", "format": "sqlite", "VisiData loader": "yes", "VisiData saver": "", "version_added": "0.42", "created": "2000", "creator": "D. Richard Hipp", "description": "full SQL relational database; the most used database engine in the world", "open format": "public domain", "nestable": "", "plottable": "", "format_url": "https://sqlite.org/"} {"filetype": "xls", "aliases": "", "requirements": "xlrd", "format": "Excel spreadsheets", "VisiData loader": "yes", "VisiData saver": "", "version_added": "0.42", "created": "1987", "creator": "Microsoft", "description": "", "open format": "no", "nestable": "", "plottable": "", "format_url": "https://msdn.microsoft.com/en-us/library/office/cc313154(v=office.12).aspx"} -{"filetype": "fixed", "filetype_url": "#fixed", "aliases": "", "requirements": "", "format": "fixed width text", "VisiData loader": "yes", "VisiData saver": "", "version_added": "0.97", "created": "", "creator": "", "description": "not recommended for new data", "open format": "prehistory", "nestable": "", "plottable": "", "format_url": "https://stackoverflow.com/questions/7666780/why-are-fixed-width-file-formats-still-in-use"} +{"filetype": "fixed", "filetype_url": "#fixed", "aliases": "", "requirements": "", "format": "fixed width text", "VisiData loader": "yes", "VisiData saver": "displayed text (as of 2.1)", "version_added": "0.97", "created": "", "creator": "", "description": "not recommended for new data", "open format": "prehistory", "nestable": "", "plottable": "", "format_url": "https://stackoverflow.com/questions/7666780/why-are-fixed-width-file-formats-still-in-use"} {"filetype": "postgres", "filetype_url": "#postgres", "aliases": "", "requirements": "", "format": "PostgreSQL database", "VisiData loader": "yes", "VisiData saver": "", "version_added": "0.97", "created": "1996", "creator": "", "description": "", "open format": "", "nestable": "", "plottable": "", "format_url": ""} {"filetype": "vd", "filetype_url": "#vd", "aliases": "vdj", "requirements": "", "format": "VisiData command log", "VisiData loader": "yes", "VisiData saver": "", "version_added": "0.97", "created": "2017", "creator": "VisiData", "description": "replayable", "open format": "yes", "nestable": "", "plottable": "", "format_url": "http://visidata.org/docs/save-restore/"} {"filetype": "mbtiles", "filetype_url": "#mbtiles", "aliases": "", "requirements": "mapbox-vector-tile", "format": "MapBox Tileset", "VisiData loader": "yes", "VisiData saver": "", "version_added": "0.98", "created": "2011", "creator": "MapBox", "description": "based on sqlite", "open format": "yes", "nestable": "", "plottable": "plottable", "format_url": "https://docs.mapbox.com/help/glossary/mbtiles/"} diff --git a/visidata/loaders/fixed_width.py b/visidata/loaders/fixed_width.py index 648e20a68..6830a0226 100644 --- a/visidata/loaders/fixed_width.py +++ b/visidata/loaders/fixed_width.py @@ -70,3 +70,25 @@ def iterload(self): def setCols(self, headerlines): self.headerlines = headerlines + + +@VisiData.api +def save_fixed(vd, p, *vsheets): + with p.open_text(mode='w') as fp: + for sheet in vsheets: + if len(vsheets) > 1: + fp.write('%s\n\n' % vs.name) + + # headers + for col in sheet.visibleCols: + fp.write('{0:{width}}'.format(col.name, width=col.width)) + fp.write('\n') + + # rows + with Progress(gerund='saving'): + for dispvals in sheet.iterdispvals(format=True): + for col, val in dispvals.items(): + fp.write('{0:{align}{width}}'.format(val, width=col.width, align='>' if isNumeric(col) else '<')) + fp.write('\n') + + vd.status('%s save finished' % p) From 0016ca73d13329c0a16630c151a27f633f5bf524 Mon Sep 17 00:00:00 2001 From: Thomas Sibley Date: Thu, 19 Nov 2020 15:38:39 -0800 Subject: [PATCH 094/162] [tests] Run tests with an empty config file and visidata dir Otherwise user-preferences and plugins affect test success/failure. For example, this bit me because I have a custom disp_date_fmt. --- dev/test.sh | 4 ++-- tests/.visidata/.gitignore | 1 + tests/.visidatarc | 0 3 files changed, 3 insertions(+), 2 deletions(-) create mode 100644 tests/.visidata/.gitignore create mode 100644 tests/.visidatarc diff --git a/dev/test.sh b/dev/test.sh index d24cdf79d..80079f691 100755 --- a/dev/test.sh +++ b/dev/test.sh @@ -20,10 +20,10 @@ for i in $TESTS ; do outbase=${i##tests/} if [ "${i%-nosave.vd}-nosave" == "${i%.vd}" ]; then - PYTHONPATH=. bin/vd --play "$i" --batch + PYTHONPATH=. bin/vd --play "$i" --batch --config tests/.visidatarc --visidata-dir tests/.visidata else for goldfn in tests/golden/${outbase%.vd}.*; do - PYTHONPATH=. bin/vd --confirm-overwrite=False --play "$i" --batch --output "$goldfn" + PYTHONPATH=. bin/vd --confirm-overwrite=False --play "$i" --batch --output "$goldfn" --config tests/.visidatarc --visidata-dir tests/.visidata echo "save: $goldfn" done fi diff --git a/tests/.visidata/.gitignore b/tests/.visidata/.gitignore new file mode 100644 index 000000000..0a2101fab --- /dev/null +++ b/tests/.visidata/.gitignore @@ -0,0 +1 @@ +/cache/ diff --git a/tests/.visidatarc b/tests/.visidatarc new file mode 100644 index 000000000..e69de29bb From 919c4e6b234f1440242c140edeb3c99c2decc605 Mon Sep 17 00:00:00 2001 From: anjakefala Date: Fri, 20 Nov 2020 00:58:36 -0500 Subject: [PATCH 095/162] [replay] move to replay context after getting sheet Closes #796 --- visidata/cmdlog.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/visidata/cmdlog.py b/visidata/cmdlog.py index 011618216..34d5e62d4 100644 --- a/visidata/cmdlog.py +++ b/visidata/cmdlog.py @@ -295,12 +295,13 @@ def replayOne(vd, r): vd.exceptionCaught(e) escaped = True else: - vd.moveToReplayContext(r, vs) if vs: vd.push(vs) else: vs = vd.sheets[0] # use top sheet by default + vd.moveToReplayContext(r, vs) + if r.comment: vd.status(r.comment) From e3a6d5d2125089d52a26b3953799e74b7d6bb4f2 Mon Sep 17 00:00:00 2001 From: Saul Pwanson Date: Sun, 22 Nov 2020 00:44:02 -0800 Subject: [PATCH 096/162] [expr] allow column attributes as variables #659 - add ExprColumn instance itself to LazyChainMap of contexts - includes docs/columns rework. --- docs/columns.md | 30 +++++++++++++++++++++++++++++- visidata/basesheet.py | 4 ++-- visidata/column.py | 2 +- visidata/expr.py | 4 ++-- visidata/sheets.py | 41 +++++++++++++++++++++++------------------ 5 files changed, 57 insertions(+), 24 deletions(-) diff --git a/docs/columns.md b/docs/columns.md index 057fbd7db..0ce6107e7 100644 --- a/docs/columns.md +++ b/docs/columns.md @@ -189,7 +189,35 @@ Note that by default the expansion logic will look for nested columns in **up to ## [How to create derivative columns](#derived) {#derived} -The `=` command takes a Python expression as input, evaluates the expression, and creates a new column from the result. Column names can be supplied as variables, in order to have the expression performed on the column cell-by-cell. VisiData supports `Tab` autocompletion of column names. +The `=` command takes a Python expression as input and creates a new column, where each cell evaluates the expression in the context of its row. + +These variables and functions are available in the scope of an expression: + +- **Column names** evaluate to the typed value of the cell in the named column for the same row. +- **`vd`** attributes and methods; use `Ctrl+X vd` to view the vd object, or [see the API](). +- **`Sheet`** attributes and methods; use `g Ctrl+Y` to view the sheet object (or see the API). +- **Global** functions and variables (add your own in your .visidatarc). +- **modules** that have been `import`ed in Python + - if you need a module that hasn't already been imported at runtime, use `g Ctrl+X import `. + +- **`sheet`**: the current sheet (a TableSheet object) +- **`col`**: the current column (as a Column object; use for Column metadata) +- **`row`**: the current row (a Python object of the internal rowtype) + +Additional attributes can be added to sheets and columns. + +`col` deliberately returns a Column object, but any other Column object is interpreted as the value within that column for the same row. + +For example, this customizes addcol-expr to set the `curcol` attribute on the new ExprColumn to a snapshot of the current cursor column (at the time the expression column is added): + +``` +Sheet.addCommand('=', 'addcol-expr', 'addColumnAtCursor(ExprColumn(inputExpr("new column expr="), curcol=cursorCol))', 'create + new column from Python expression, with column names as variables') +``` + +Then, an expression can use `curcol` as though it referred to the value in the saved column. + +`Tab` autocompletion when inputting an expression will cycle through valid column names only. The following examples use the file [sample.tsv](https://raw.githubusercontent.com/saulpw/visidata/stable/sample_data/sample.tsv). diff --git a/visidata/basesheet.py b/visidata/basesheet.py index 1cd360389..b89cc4af0 100644 --- a/visidata/basesheet.py +++ b/visidata/basesheet.py @@ -207,8 +207,8 @@ def checkCursorNoExceptions(self): except Exception as e: vd.exceptionCaught(e) - def evalExpr(self, expr, row=None): - 'Evaluate Python expression *expr* in the context of *row*.' + def evalExpr(self, expr, **kwargs): + 'Evaluate Python expression *expr* in the context of *kwargs* (may vary by sheet type).' return eval(expr, vd.getGlobals(), None) diff --git a/visidata/column.py b/visidata/column.py index e36dc5276..11a59cc92 100644 --- a/visidata/column.py +++ b/visidata/column.py @@ -496,7 +496,7 @@ def __init__(self, name, expr=None, **kwargs): def calcValue(self, row): t0 = time.perf_counter() - r = self.sheet.evalExpr(self.compiledExpr, row) + r = self.sheet.evalExpr(self.compiledExpr, row, col=self) t1 = time.perf_counter() self.ncalcs += 1 self.maxtime = max(self.maxtime, t1-t0) diff --git a/visidata/expr.py b/visidata/expr.py index 435d25b16..f4449cb4a 100644 --- a/visidata/expr.py +++ b/visidata/expr.py @@ -1,4 +1,4 @@ -from visidata import Progress, Sheet, Column, asyncthread, vd, ColumnExpr +from visidata import Progress, Sheet, Column, asyncthread, vd, ExprColumn class CompleteExpr: @@ -46,7 +46,7 @@ def inputExpr(self, prompt, *args, **kwargs): return vd.input(prompt, "expr", *args, completer=CompleteExpr(self), **kwargs) -Sheet.addCommand('=', 'addcol-expr', 'addColumnAtCursor(ColumnExpr(inputExpr("new column expr=")))', 'create new column from Python expression, with column names as variables') +Sheet.addCommand('=', 'addcol-expr', 'addColumnAtCursor(ExprColumn(inputExpr("new column expr="), curcol=cursorCol))', 'create new column from Python expression, with column names as variables') Sheet.addCommand('g=', 'setcol-expr', 'cursorCol.setValuesFromExpr(selectedRows, inputExpr("set selected="))', 'set current column for selected rows to result of Python expression') Sheet.addCommand('z=', 'setcell-expr', 'cursorCol.setValues([cursorRow], evalExpr(inputExpr("set expr="), cursorRow))', 'evaluate Python expression on current row and set current cell with result of Python expression') Sheet.addCommand('gz=', 'setcol-iter', 'cursorCol.setValues(selectedRows, *list(itertools.islice(eval(input("set column= ", "expr", completer=CompleteExpr())), len(selectedRows))))', 'set current column for selected rows to the items in result of Python sequence expression') diff --git a/visidata/sheets.py b/visidata/sheets.py index 0aef5a1c3..e6db7ca8d 100644 --- a/visidata/sheets.py +++ b/visidata/sheets.py @@ -95,11 +95,11 @@ class RecursiveExprException(Exception): class LazyComputeRow: 'Calculate column values as needed.' - def __init__(self, sheet, row): + def __init__(self, sheet, row, col=None): self.row = row self.sheet = sheet if not hasattr(self.sheet, '_lcm'): - self.sheet._lcm = LazyChainMap(sheet, vd) + self.sheet._lcm = LazyChainMap(sheet, vd, col) else: self.sheet._lcm.clear() # reset locals on lcm @@ -107,7 +107,7 @@ def __init__(self, sheet, row): self._keys = [c.name for c in self.sheet.columns] def keys(self): - return self._keys + self.sheet._lcm.keys() + ['row', 'sheet'] + return self._keys + self.sheet._lcm.keys() + ['row', 'sheet', 'col'] def __str__(self): return str(self.as_dict()) @@ -122,22 +122,26 @@ def __getitem__(self, colid): try: i = self._keys.index(colid) c = self.sheet.columns[i] - - if c in self._usedcols: - raise RecursiveExprException() - self._usedcols.add(c) - ret = c.getTypedValue(self.row) - self._usedcols.remove(c) - return ret except ValueError: try: - return self.sheet._lcm[colid] + c = self.sheet._lcm[colid] except (KeyError, AttributeError): - if colid == 'row': - return self.row - elif colid == 'sheet': - return self.sheet - raise KeyError(colid) + if colid == 'sheet': return self.sheet + elif colid == 'row': c = self.row + elif colid == 'col': c = self.col + else: + raise KeyError(colid) + + if not isinstance(c, Column): # columns calc in the context of the row of the cell being calc'ed + return c + + if c in self._usedcols: + raise RecursiveExprException() + + self._usedcols.add(c) + ret = c.getTypedValue(self.row) + self._usedcols.remove(c) + return ret class BasicRow(collections.defaultdict): def __init__(self): @@ -321,9 +325,10 @@ def __deepcopy__(self, memo): def __repr__(self): return self.name - def evalExpr(self, expr, row=None): + def evalExpr(self, expr, row=None, col=None): if row: - contexts = vd._evalcontexts.setdefault((self, self.rowid(row)), LazyComputeRow(self, row)) + # contexts are cached by sheet/rowid for duration of drawcycle + contexts = vd._evalcontexts.setdefault((self, self.rowid(row)), LazyComputeRow(self, row, col=col)) else: contexts = None From 572c6cc96fdc0d84037c213396c85dc7e2ffb47d Mon Sep 17 00:00:00 2001 From: Saul Pwanson Date: Mon, 23 Nov 2020 14:11:17 -0800 Subject: [PATCH 097/162] [status] use color_working for progress indicator #804 --- visidata/statusbar.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/visidata/statusbar.py b/visidata/statusbar.py index cc9f7a2ba..8a4a831fa 100644 --- a/visidata/statusbar.py +++ b/visidata/statusbar.py @@ -177,7 +177,7 @@ def drawRightStatus(vd, scr, vs): gerund = vs.progresses[0].gerund else: gerund = 'processing' - statcolors.insert(1, (' %s %s…' % (vs.progressPct, gerund), 'color_status')) + statcolors.insert(1, (' %s %s…' % (vs.progressPct, gerund), 'color_working')) if active and vd.currentReplay: statcolors.insert(0, (vd.replayStatus, 'color_status_replay')) From ec68c103834e69b2a17a2a27ffcf68cfa880ee4b Mon Sep 17 00:00:00 2001 From: anjakefala Date: Mon, 23 Nov 2020 19:40:24 -0800 Subject: [PATCH 098/162] [numeric_binning] if width of bins is 1, fallback to degenerate binning --- visidata/pivot.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/visidata/pivot.py b/visidata/pivot.py index b9070d898..edfa3a367 100644 --- a/visidata/pivot.py +++ b/visidata/pivot.py @@ -181,8 +181,8 @@ def groupRows(self, rowfunc=None): if width == 0: # only one value (and maybe errors) numericBins = [(minval, maxval)] - elif numericCols[0].type in (int, vlen) and nbins > (maxval - minval): - # more bins than int vals, just use the vals + elif (numericCols[0].type in (int, vlen) and nbins > (maxval - minval)) or (width == 1): + # (more bins than int vals) or (if bins are of width 1), just use the vals as bins degenerateBinning = True numericBins = [(minval+i, minval+i) for i in range(maxval-minval+1)] else: From f16a9fbf00ad02df573c9a3bcfa5ae7e22e43f53 Mon Sep 17 00:00:00 2001 From: anjakefala Date: Mon, 23 Nov 2020 19:41:29 -0800 Subject: [PATCH 099/162] [numeric_binning] degenerate binning should resemble non-numeric binning Closes #791 --- visidata/pivot.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/visidata/pivot.py b/visidata/pivot.py index edfa3a367..15081a11d 100644 --- a/visidata/pivot.py +++ b/visidata/pivot.py @@ -184,7 +184,8 @@ def groupRows(self, rowfunc=None): elif (numericCols[0].type in (int, vlen) and nbins > (maxval - minval)) or (width == 1): # (more bins than int vals) or (if bins are of width 1), just use the vals as bins degenerateBinning = True - numericBins = [(minval+i, minval+i) for i in range(maxval-minval+1)] + numericBins = [(val, val) for val in sorted(set(vals))] + nbins = len(numericBins) else: numericBins = [(minval+width*i, minval+width*(i+1)) for i in range(nbins)] @@ -214,7 +215,8 @@ def groupRows(self, rowfunc=None): if not width: binidx = 0 elif degenerateBinning: - binidx = val-minval + # in degenerate binning, each val has its own bin + binidx = numericBins.index((val, val)) else: binidx = int((val-minval)//width) groupRow = numericGroupRows[formatRange(numericCols[0], numericBins[min(binidx, nbins-1)])] From 28510022e0ded9d9417676b99e41603ecf400ad6 Mon Sep 17 00:00:00 2001 From: Thomas Sibley Date: Thu, 19 Nov 2020 14:52:56 -0800 Subject: [PATCH 100/162] [regex] Use capture names for column names, if available, in capture-col Allows for predetermining friendlier column names, saving a renaming step later. --- tests/capture-col-named.vd | 3 ++ tests/golden/capture-col-named.tsv | 50 ++++++++++++++++++++++++++++++ visidata/regex.py | 26 +++++++++++----- 3 files changed, 71 insertions(+), 8 deletions(-) create mode 100644 tests/capture-col-named.vd create mode 100644 tests/golden/capture-col-named.tsv diff --git a/tests/capture-col-named.vd b/tests/capture-col-named.vd new file mode 100644 index 000000000..9a55c89fa --- /dev/null +++ b/tests/capture-col-named.vd @@ -0,0 +1,3 @@ +sheet col row longname input keystrokes comment + open-file sample_data/benchmark.csv o +benchmark Date capture-col (?P\d{1,2})/(?P\d{1,2})/(?P\d{4}) (?P\d{1,2}):(?P\d{2})(?P[ap]) ; add new column from capture groups of regex; requires example row diff --git a/tests/golden/capture-col-named.tsv b/tests/golden/capture-col-named.tsv new file mode 100644 index 000000000..9f22b7dfd --- /dev/null +++ b/tests/golden/capture-col-named.tsv @@ -0,0 +1,50 @@ +Date Date_month Date_day Date_year Date_hour Date_min Date_meridiem Customer SKU Item Quantity Unit Paid +7/3/2018 1:47p 7 3 2018 1 47 p Robert Armstrong FOOD213 BFF Oh My Gravy! Beef & Salmon 2.8oz 4 $12.95 $51.8 +7/3/2018 3:32p 7 3 2018 3 32 p Kyle Kennedy FOOD121 Food, Adult Cat - 3.5 oz 1 $4.22 $4.22 +7/5/2018 4:15p 7 5 2018 4 15 p Douglas "Dougie" Powers FOOD121 Food, Adult Cat 3.5 oz 1 $4.22 $4.22 +7/6/2018 12:15p 7 6 2018 12 15 p 桜 高橋 (Sakura Takahashi) FOOD122 Food, Senior Wet Cat - 3 oz 12 $1.29 157¥ +7/10/2018 10:28a 7 10 2018 10 28 a David Attenborough NSCT201 Food, Salamander 30 $.05 $1.5 +7/10/2018 5:23p 7 10 2018 5 23 p Susan Ashworth CAT060 Cat, Korat (Felis catus) 1 $720.42 $720.42 +7/10/2018 5:23p 7 10 2018 5 23 p Susan Ashworth FOOD130 Food, Kitten 3kg 1 $14.94 $14.94 +7/13/2018 10:26a 7 13 2018 10 26 a Wil Wheaton NSCT523 Monster, Rust (Monstrus gygaxus) 1 $39.95 $39.95 +7/13/2018 3:49p 7 13 2018 3 49 p Robert Armstrong FOOD216 BFF Oh My Gravy! Chicken & Shrimp 2.8oz 4 $12.95 $51.8 +7/17/2018 9:01a 7 17 2018 9 01 a Robert Armstrong FOOD217 BFF Oh My Gravy! Duck & Tuna 2.8oz 4 $12.95 $51.8 +7/17/2018 11:30a 7 17 2018 11 30 a Helen Halestorm LAGO342 Rabbit (Oryctolagus cuniculus) 2 $32.94 $65.88 +7/18/2018 12:16p 7 18 2018 12 16 p 桜 高橋 (Sakura Takahashi) FOOD122 Food, Senior Wet Cat - 3 oz 6 $1.29 157¥ +7/19/2018 10:28a 7 19 2018 10 28 a Rubeus Hagrid FOOD170 Food, Dog - 5kg 5 $44.95 $224.75 +7/20/2018 2:13p 7 20 2018 2 13 p Jon Arbuckle FOOD167 Food, Premium Wet Cat - 3.5 oz 50 $3.95 $197.5 +7/23/2018 1:41p 7 23 2018 1 41 p Robert Armstrong FOOD215 BFF Oh My Gravy! Lamb & Tuna 2.8oz 4 $12.95 $51.8 +7/23/2018 4:23p 7 23 2018 4 23 p Douglas "Dougie" Powers TOY235 Laser Pointer 1 $16.12 $16.12 +7/24/2018 12:16p 7 24 2018 12 16 p 桜 高橋 (Sakura Takahashi) FOOD122 Food, Senior Wet Cat - 3 oz 3 $1.29 157¥ +7/26/2018 4:39p 7 26 2018 4 39 p Douglas "Dougie" Powers FOOD420 Food, Shark - 10 kg 1 $15.70 $15.7 +7/27/2018 12:16p 7 27 2018 12 16 p 桜 高橋 (Sakura Takahashi) FOOD122 Food, Senior Wet Cat - 3 oz 3 $1.29 157¥ +7/30/2018 12:17p 7 30 2018 12 17 p 桜 高橋 (Sakura Takahashi) RETURN Food, Senior Wet Cat - 3 oz 1 $1.29 157¥ +7/31/2018 5:42p 7 31 2018 5 42 p Rubeus Hagrid CAT060 Food, Dragon - 50kg 5 $720.42 $3602.1 +8/1/2018 2:44p 8 1 2018 2 44 p David Attenborough FOOD360 Food, Rhinocerous - 50kg 4 $5.72 $22.88 +8/2/2018 5:12p 8 2 2018 5 12 p Susan Ashworth CAT110 Cat, Maine Coon (Felix catus) 1 $1,309.68 $1309.68 +8/2/2018 5:12p 8 2 2018 5 12 p Susan Ashworth FOOD130 Food, Kitten 3kg 3 $14.94 $44.82 +8/6/2018 10:21a 8 6 2018 10 21 a Robert Armstrong FOOD212 BFF Oh My Gravy! Beef & Chicken 2.8oz 4 $12.95 $51.8 +8/7/2018 4:12p 8 7 2018 4 12 p Juan Johnson REPT082 Kingsnake, California (Lampropeltis getula) 1 $89.95 $89.95 +8/7/2018 4:12p 8 7 2018 4 12 p Juan Johnson RDNT443 Mouse, Pinky (Mus musculus) 1 $1.49 $1.49 +8/10/2018 4:31p 8 10 2018 4 31 p Robert Armstrong FOOD211 BFF Oh My Gravy! Chicken & Turkey 2.8oz 4 $12.95 $51.8 +8/13/2018 2:07p 8 13 2018 2 07 p Monica Johnson RDNT443 Mouse, Pinky (Mus musculus) 1 $1.49 $1.49 +8/13/2018 2:08p 8 13 2018 2 08 p María Fernández FOOD146 Forti Diet Prohealth Mouse/Rat 3lbs 2 $2.00 $4.0 +8/15/2018 11:57a 8 15 2018 11 57 a Mr. Praline RETURN Parrot, Norwegian Blue (Mopsitta tanta) 1 $2300.00 -$2300.0 +8/15/2018 3:48p 8 15 2018 3 48 p Kyle Kennedy FOOD121 Food, Adult Cat - 3.5 oz 2 $4.22 $8.44 +8/16/2018 11:50a 8 16 2018 11 50 a Helen Halestorm RETURN Rabbit (Oryctolagus cuniculus) 6 $0 $0.0 +8/16/2018 4:00p 8 16 2018 4 00 p Kyle Kennedy DOG010 Dog, Golden Retriever (Canis lupus familiaris) 1 $2,495.99 $2495.99 +8/16/2018 5:15p 8 16 2018 5 15 p Michael Smith BIRD160 Parakeet, Blue (Melopsittacus undulatus) 1 29.95 $31.85 +8/17/2018 9:26a 8 17 2018 9 26 a Rubeus Hagrid NSCT201 Food, Spider 5 $.05 $0.25 +8/20/2018 9:36a 8 20 2018 9 36 a Kyle Kennedy RETURN Dog, Golden Retriever (Canis lupus familiaris) 1 $1,247.99 -$1247.99 +8/20/2018 3:31p 8 20 2018 3 31 p Monica Johnson NSCT201 Crickets, Adult Live (Gryllus assimilis) 30 $.05 $1.5 +8/20/2018 5:12p 8 20 2018 5 12 p David Attenborough NSCT084 Food, Pangolin 30 $.17 $5.10 +8/21/2018 12:13p 8 21 2018 12 13 p Robert Armstrong FOOD214 BFF Oh My Gravy! Duck & Salmon 2.8oz 4 $12.95 $51.8 +8/22/2018 9:38a 8 22 2018 9 38 a David Attenborough BIRD160 Food, Quoll 1 29.95 $29.95 +8/22/2018 2:13p 8 22 2018 2 13 p Jon Arbuckle FOOD170 Food, Adult Dog - 5kg 1 $44.95 $44.95 +8/24/2018 11:42a 8 24 2018 11 42 a Robert Armstrong FOOD218 BFF Oh My Gravy! Chicken & Salmon 2.8oz 4 $12.95 $51.8 +8/27/2018 3:05p 8 27 2018 3 05 p Monica Johnson NSCT443 Mealworms, Large (Tenebrio molitor) 100ct 1 $1.99 $1.99 +8/28/2018 5:32p 8 28 2018 5 32 p Susan Ashworth CAT020 Cat, Scottish Fold (Felis catus) 1 $1,964.53 $1964.53 +8/28/2018 5:32p 8 28 2018 5 32 p Susan Ashworth FOOD130 Food, Kitten 3kg 2 $14.94 $29.88 +8/29/2018 10:07a 8 29 2018 10 07 a Robert Armstrong FOOD219 BFF Oh My Gravy! Chicken & Pumpkin 2.8oz 4 $12.95 $51.8 +8/31/2018 12:00a 8 31 2018 12 00 a Robert Armstrong FOOD219 BFF Oh My Gravy! Chicken & Pumpkin 2.8oz 144 $12.95 $1864.8 +8/31/2018 5:57p 8 31 2018 5 57 p Juan Johnson REPT217 Lizard, Spinytail (Uromastyx ornatus) 1 $99.95 $99.95 diff --git a/visidata/regex.py b/visidata/regex.py index 152a642dc..ecf76d83a 100644 --- a/visidata/regex.py +++ b/visidata/regex.py @@ -26,7 +26,7 @@ def makeRegexMatcher(regex, origcol): def _regexMatcher(row): m = regex.search(origcol.getDisplayValue(row)) if m: - return m.groups() + return m.groupdict() if m.groupdict() else m.groups() return _regexMatcher @asyncthread @@ -43,7 +43,7 @@ def addRegexColumns(regexMaker, vs, origcol, regexstr): else: exampleRows = vs.rows - cols = [] + cols = {} ncols = 0 # number of new columns added already for r in Progress(exampleRows + [vs.cursorRow]): try: @@ -53,12 +53,22 @@ def addRegexColumns(regexMaker, vs, origcol, regexstr): except Exception as e: vd.exceptionCaught(e) - for _ in range(len(m)-len(cols)): - cols.append(Column(origcol.name+'_re'+str(len(cols)), - getter=lambda col,row,i=len(cols),func=func: func(row)[i], - origCol=origcol)) - - vs.addColumnAtCursor(*cols) + if isinstance(m, dict): + for name in m: + if name in cols: + continue + cols[name] = Column(origcol.name+'_'+str(name), + getter=lambda col,row,name=name,func=func: func(row)[name], + origCol=origcol) + elif isinstance(m, (tuple, list)): + for _ in range(len(m)-len(cols)): + cols[len(cols)] = Column(origcol.name+'_re'+str(len(cols)), + getter=lambda col,row,i=len(cols),func=func: func(row)[i], + origCol=origcol) + else: + raise TypeError("addRegexColumns() expects a dict, list, or tuple from regexMaker, but got a "+type(m).__name__) + + vs.addColumnAtCursor(*cols.values()) def regexTransform(origcol, instr): From 052793914cf24b935af817f8e8a4215d20fd8fb1 Mon Sep 17 00:00:00 2001 From: anjakefala Date: Tue, 24 Nov 2020 21:40:19 -0800 Subject: [PATCH 101/162] [BasicRow-] match defaultdict's __init__ - needed bc super()'s copy() will instantiate a new BasicRow() and pass the original BasicRow as an argument --- visidata/sheets.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/visidata/sheets.py b/visidata/sheets.py index e6db7ca8d..ad2d23c2b 100644 --- a/visidata/sheets.py +++ b/visidata/sheets.py @@ -144,8 +144,8 @@ def __getitem__(self, colid): return ret class BasicRow(collections.defaultdict): - def __init__(self): - super().__init__(lambda: None) + def __init__(self, *args): + collections.defaultdict.__init__(self, lambda: None, *args) def __bool__(self): return True From 14175a9f08a6c3992c5bafa20120cff30d926fbf Mon Sep 17 00:00:00 2001 From: anjakefala Date: Tue, 24 Nov 2020 21:42:18 -0800 Subject: [PATCH 102/162] [edit-row] catch 'no rows' case --- visidata/sheets.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/visidata/sheets.py b/visidata/sheets.py index ad2d23c2b..6898fc134 100644 --- a/visidata/sheets.py +++ b/visidata/sheets.py @@ -1049,7 +1049,7 @@ def updateColNames(sheet, rows, cols, overwrite=False): Sheet.addCommand('!', 'key-col', 'toggleKeys([cursorCol])', 'toggle current column as a key column') Sheet.addCommand('z!', 'key-col-off', 'unsetKeys([cursorCol])', 'unset current column as a key column') -Sheet.addCommand('e', 'edit-cell', 'cursorCol.setValues([cursorRow], editCell(cursorVisibleColIndex))', 'edit contents of current cell') +Sheet.addCommand('e', 'edit-cell', 'cursorCol.setValues([cursorRow], editCell(cursorVisibleColIndex)) if cursorRow else fail("no rows to edit")', 'edit contents of current cell') Sheet.addCommand('ge', 'setcol-input', 'cursorCol.setValuesTyped(selectedRows, input("set selected to: ", value=cursorDisplay))', 'set contents of current column for selected rows to same input') Sheet.addCommand('"', 'dup-selected', 'vs=copy(sheet); vs.name += "_selectedref"; vs.reload=lambda vs=vs,rows=selectedRows: setattr(vs, "rows", list(rows)); vd.push(vs)', 'open duplicate sheet with only selected rows'), From f5a234cbea99090638e3cc400668bbb793a25938 Mon Sep 17 00:00:00 2001 From: Thomas Sibley Date: Wed, 25 Nov 2020 21:04:23 -0800 Subject: [PATCH 103/162] [docs] Note use of named captures with ; / capture-col Resolves #809. Should have been part of #808! --- docs/columns.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/columns.md b/docs/columns.md index 0ce6107e7..ae343d48e 100644 --- a/docs/columns.md +++ b/docs/columns.md @@ -136,7 +136,7 @@ uses the commands for column splitting and transformation with [xd/puzzles.tsv]( ### - `:` adds new columns derived from splitting the current column at positions defined by a *regex pattern*. The current row will be used to infer the number of columns that will be created. -- `;` adds new columns derived from pulling the contents of the current column which match the *regex within capture groups*. This command also requires an example row. +- `;` adds new columns derived from pulling the contents of the current column which match the *regex within capture groups*. The new columns are named using the capture group index, or if named capture groups are used, the capture group names. This command also requires an example row. - `*` followed by *regex*`/`*substring* replaces the text which matches the capture groups in *regex* with the contents of *substring*. *substring* may include backreferences (*\1* etc). ## [How do I substitute text in my column] From 38691f0859a5c3678d204e7d5d06c30c93412614 Mon Sep 17 00:00:00 2001 From: Saul Pwanson Date: Mon, 23 Nov 2020 15:34:58 -0800 Subject: [PATCH 104/162] [save] g^S is save-sheets-selected on IndexSheet - new command allows some or all sheets on an IndexSheet to be saved (and not the sheets on the sheet stack) - [sqlite] ensureLoaded when saving sheets to sqlite db --- visidata/loaders/sqlite.py | 4 ++++ visidata/save.py | 1 + 2 files changed, 5 insertions(+) diff --git a/visidata/loaders/sqlite.py b/visidata/loaders/sqlite.py index 388267f77..eb9cd0b72 100644 --- a/visidata/loaders/sqlite.py +++ b/visidata/loaders/sqlite.py @@ -143,6 +143,10 @@ def save_sqlite(vd, p, *vsheets): currency: 'REAL' } + for vs in vsheets: + vs.ensureLoaded() + vd.sync() + for vs in vsheets: tblname = clean_to_id(vs.name) sqlcols = [] diff --git a/visidata/save.py b/visidata/save.py index 06034b05b..c419f468f 100644 --- a/visidata/save.py +++ b/visidata/save.py @@ -136,5 +136,6 @@ def save_txt(vd, p, *vsheets): Sheet.addCommand('^S', 'save-sheet', 'vd.saveSheets(inputPath("save to: ", value=getDefaultSaveName()), sheet, confirm_overwrite=options.confirm_overwrite)', 'save current sheet to filename in format determined by extension (default .tsv)') BaseSheet.addCommand('g^S', 'save-all', 'vd.saveSheets(inputPath("save all sheets to: "), *vd.sheets, confirm_overwrite=options.confirm_overwrite)', 'save all sheets to given file or directory)') +IndexSheet.addCommand('g^S', 'save-sheets-selected', 'vd.saveSheets(inputPath("save all sheets to: "), *selectedRows, confirm_overwrite=options.confirm_overwrite)', 'save all sheets to given file or directory)') Sheet.addCommand('', 'save-col', 'save_cols([cursorCol])', 'save current column only to filename in format determined by extension (default .tsv)') Sheet.addCommand('', 'save-col-keys', 'save_cols(keyCols + [cursorCol])', 'save key columns and current column to filename in format determined by extension (default .tsv)') From e8e737b0601eb464753c1da548e0970cfa992582 Mon Sep 17 00:00:00 2001 From: Saul Pwanson Date: Mon, 23 Nov 2020 20:38:50 -0800 Subject: [PATCH 105/162] [dev] update loader checklist for dev/formats.jsonl --- dev/checklists/add-loader.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev/checklists/add-loader.md b/dev/checklists/add-loader.md index 8a5162b9f..8cc9092c1 100644 --- a/dev/checklists/add-loader.md +++ b/dev/checklists/add-loader.md @@ -21,6 +21,6 @@ For additional reading, see our detailed guide on [writing loaders for VisiData] ## Documenting the loader (to be done once the loader is shippable) -10) add a section on the loader to the "supported sources" of the manpage. +10) add or change an entry in dev/formats.jsonl for the filetype. - include its non-Python3 stdlib dependency requirements (if any) - - provide a short overview of how the loader allows users to engage with the data + - provide a short overview of additional features/commands offered for this data type (if any) From d54fe6980ae030b1f80c1eb3cf1004a9c8854758 Mon Sep 17 00:00:00 2001 From: Saul Pwanson Date: Mon, 23 Nov 2020 20:53:51 -0800 Subject: [PATCH 106/162] [pdf] options.pdf_tables to parse tables from pdf with tabula --- requirements.txt | 3 ++- visidata/loaders/pdf.py | 17 +++++++++++++++-- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/requirements.txt b/requirements.txt index cc0cad566..4e00f83d8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ python-dateutil # date type -# optional requirements +# optional dependencies pandas==1.0.3 # dta (Stata) requests # http lxml # html/xml @@ -21,6 +21,7 @@ dnslib # pcap namestand # graphviz datapackage # frictionless .json pdfminer.six # pdf +tabula # pdf tables vobject # vcf tabulate # tabulate saver wcwidth # tabulate saver with unicode diff --git a/visidata/loaders/pdf.py b/visidata/loaders/pdf.py index da39c19bf..4e3cafc55 100644 --- a/visidata/loaders/pdf.py +++ b/visidata/loaders/pdf.py @@ -2,10 +2,16 @@ from visidata import * +vd.option('pdf_tables', False, 'parse PDF for tables instead of pages of text', replay=True) + + def open_pdf(p): - return PdfSheet(p.name, source=p) + if vd.options.pdf_tables: + return TabulaSheet(p.name, source=p) + return PdfMinerSheet(p.name, source=p) + -class PdfSheet(TableSheet): +class PdfMinerSheet(TableSheet): rowtype='pages' # rowdef: [pdfminer.LTPage, pageid, text] columns=[ ColumnItem('pdfpage', 0, width=0), @@ -27,3 +33,10 @@ def iterload(self): interpreter = PDFPageInterpreter(newrsrcmgr, txtconv) interpreter.process_page(page) yield [page, page.pageid, output_string.getvalue()] + + +class TabulaSheet(IndexSheet): + def iterload(self): + import tabula + for i, t in enumerate(tabula.read_pdf(self.source, pages='all', multiple_tables=True)): + yield PandasSheet(self.source.name, i, source=t) From cd051f1f54044f8c4b076b345aee54aae01ff832 Mon Sep 17 00:00:00 2001 From: Saul Pwanson Date: Mon, 23 Nov 2020 21:33:54 -0800 Subject: [PATCH 107/162] [menu] text canvas with clickable labels wip - click doesn't work yet --- sample_data/hello.mnu | 2 ++ visidata/__init__.py | 1 + visidata/menu.py | 29 +++++++++++++++++++++++++++++ 3 files changed, 32 insertions(+) create mode 100644 sample_data/hello.mnu create mode 100644 visidata/menu.py diff --git a/sample_data/hello.mnu b/sample_data/hello.mnu new file mode 100644 index 000000000..57655b2a3 --- /dev/null +++ b/sample_data/hello.mnu @@ -0,0 +1,2 @@ +x y text color command input cond status +3 4 hello world 217 underline show-status Hello World! diff --git a/visidata/__init__.py b/visidata/__init__.py index f5d662486..21994fb9b 100644 --- a/visidata/__init__.py +++ b/visidata/__init__.py @@ -96,6 +96,7 @@ def getGlobals(): import visidata.customdate import visidata.misc from .macros import * +from .menu import * from .loaders.csv import * from .loaders.archive import * diff --git a/visidata/menu.py b/visidata/menu.py new file mode 100644 index 000000000..e6cd8f240 --- /dev/null +++ b/visidata/menu.py @@ -0,0 +1,29 @@ +from visidata import * + +def open_mnu(p): + return MenuSheet(p.name, source=p) + + +vd.save_mnu=vd.save_tsv + +class MenuSheet(VisiDataMetaSheet): + rowtype='labels' # { .x .y .text .color .command .input } + + +class MenuCanvas(BaseSheet): + rowtype='labels' + def click(self, r): + vd.replayOne(vd.cmdlog.newRow(sheet=self.name, col='', row='', longname=r.command, input=r.input)) + + def reload(self): + self.rows = self.source.rows + + def draw(self, scr): + vd.clearCaches() + for r in Progress(self.source.rows): + x, y = map(int, (r.x, r.y)) + clipdraw(scr, y, x, r.text, colors[r.color]) + vd.onMouse(scr, y, x, 1, len(r.text), BUTTON1_RELEASED=lambda y,x,key,r=r,sheet=self: sheet.click(r)) + + +MenuSheet.addCommand('z.', 'disp-menu', 'vd.push(MenuCanvas(name, "disp", source=sheet))', '') From 733ecda99c102d14ebb0b850e9f5d4ce865de4a1 Mon Sep 17 00:00:00 2001 From: Saul Pwanson Date: Mon, 23 Nov 2020 22:01:28 -0800 Subject: [PATCH 108/162] [dev] add history.jsonl --- dev/history.jsonl | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 dev/history.jsonl diff --git a/dev/history.jsonl b/dev/history.jsonl new file mode 100644 index 000000000..9eed77c46 --- /dev/null +++ b/dev/history.jsonl @@ -0,0 +1,6 @@ +{"date": "2020-02-29", "event": "\"VisiData is wonderful...\" - @jeremyphoward", "url": "https://twitter.com/jeremyphoward/status/1233956135226314753"} +{"date": "2018-03-04", "event": "[HN] VisiData Lightning Demo at PyCascades 2018", "url": "https://news.ycombinator.com/item?id=16515299"} +{"date": "2018-01-31", "event": "r/commandline post by @anjakefala", "url": "https://www.reddit.com/r/commandline/comments/7ua1hj/shell_show_and_tell_terminal_spreadsheet_tool/"} +{"date": "2017-06-29", "event": "[HN] Show HN: VisiData - vi for data", "url": "https://news.ycombinator.com/item?id=14662860"} +{"date": "2020-03-27", "event": "anja wins year-old bet, >140 users every weekday", "url": ""} +{"date": "2020-05-13", "event": "changelog podcast released", "url": "https://changelog.com/podcast/394"} From e334656be8bf9752a3d8f607412fe7c342fa278e Mon Sep 17 00:00:00 2001 From: Saul Pwanson Date: Mon, 23 Nov 2020 22:16:50 -0800 Subject: [PATCH 109/162] [dev] types table --- dev/types.jsonl | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 dev/types.jsonl diff --git a/dev/types.jsonl b/dev/types.jsonl new file mode 100644 index 000000000..3dfd78b30 --- /dev/null +++ b/dev/types.jsonl @@ -0,0 +1,7 @@ +{"type": "anytype", "description": "pass-through", "numeric": "", "command": "type-anytype", "keystrokes": "z~"} +{"type": "str", "description": "string", "numeric": "", "command": "type-str", "keystrokes": "~"} +{"type": "date", "description": "date/time", "numeric": "Y", "command": "type-date", "keystrokes": "@"} +{"type": "int", "description": "integer", "numeric": "Y", "command": "type-int", "keystrokes": "#"} +{"type": "float", "description": "decimal", "numeric": "Y", "command": "type-float", "keystrokes": "%"} +{"type": "currency", "description": "decimal with units", "numeric": "Y", "command": "type-currency", "keystrokes": "$"} +{"type": "vlen", "description": "container size", "numeric": "Y", "command": "type-vlen", "keystrokes": "z#"} From 9fa5d49d3f47a6331395a29a494bbc2ff7c44ee9 Mon Sep 17 00:00:00 2001 From: Saul Pwanson Date: Mon, 23 Nov 2020 22:45:41 -0800 Subject: [PATCH 110/162] [rec-] fix broken --- visidata/loaders/rec.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/visidata/loaders/rec.py b/visidata/loaders/rec.py index e90f5fcc9..73ad8c638 100644 --- a/visidata/loaders/rec.py +++ b/visidata/loaders/rec.py @@ -1,7 +1,7 @@ from visidata import * @VisiData.api -def open_rec(p): +def open_rec(vd, p): return RecIndexSheet(p.name, source=p) def decode_multiline(line, fp): From 438129aacb8fa258124999fd297d0f2a72e455ea Mon Sep 17 00:00:00 2001 From: Saul Pwanson Date: Wed, 25 Nov 2020 20:18:23 -0800 Subject: [PATCH 111/162] [tests] add test for replaying within cursor delete --- tests/graph-cursor-nosave.vd | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 tests/graph-cursor-nosave.vd diff --git a/tests/graph-cursor-nosave.vd b/tests/graph-cursor-nosave.vd new file mode 100644 index 000000000..fb076bfb9 --- /dev/null +++ b/tests/graph-cursor-nosave.vd @@ -0,0 +1,16 @@ +sheet col row longname input keystrokes comment + open-file http://visidata.org/usage.tsv o +usage daily_users type-int # +usage daily_users aggregate-col sum + +usage motd_fetches type-int # +usage motd_fetches aggregate-col sum + +usage date freq-col F +usage_date_freq date type-date @ +usage_date_freq exec-python import calendar g^X +usage_date_freq date addcol-expr date.isocalendar()[2] = +usage_date_freq date.isocalendar()[2] rename-col dotw ^ +usage_date_freq dotw addcol-expr 'Sat/Sun' if dotw >=6 else 'Mon-Fri' = +usage_date_freq 'Sat/Sun' if dotw >=6 else 'Mon-Fri' key-col ! +usage_date_freq date sort-desc ! +usage_date_freq daily_users_sum plot-column . +usage_date_freq_graph 2018-11-18 2019-03-26 51 140 delete-cursor d delete rows on source sheet contained within canvas cursor From 65ed1a024ecd5da8218ddb9a07458a23c3785f87 Mon Sep 17 00:00:00 2001 From: Saul Pwanson Date: Wed, 25 Nov 2020 20:21:44 -0800 Subject: [PATCH 112/162] [clickhouse] add clickhouse loader --- plugins/clickhouse.py | 72 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 plugins/clickhouse.py diff --git a/plugins/clickhouse.py b/plugins/clickhouse.py new file mode 100644 index 000000000..33bb6de0a --- /dev/null +++ b/plugins/clickhouse.py @@ -0,0 +1,72 @@ + +from visidata import vd, addGlobals, asyncthread, Sheet, ItemColumn +from visidata import * + +__all__ = ['openurl_clickhouse'] + +vd.option('clickhouse_host', '', '') +vd.option('clickhouse_port', 9000, '') + +def openurl_clickhouse(p, filetype=None): + url = urlparse(p.given) + options.clickhouse_host = url.hostname + options.clickhouse_port = url.port + return ClickhouseIndexSheet(p.name, source=p) + +BaseSheet.addCommand(ALT+'c', 'open-clickhouse', 'vd.push(vd.clickhouse_queries)') + +class ClickhouseQuerySheet(Sheet): + columns = [ + ColumnItem('name', 0), + ColumnItem('query', 1), + ColumnItem('nrows', 2), + ColumnItem('notes', 3), + ] + _rowtype=lambda: ['', '', 0, ''] + + def openRow(self, row): + return ClickhouseSheet(row[0], query=row[1]) + +@VisiData.lazy_property +def clickhouse_client(vd): + from clickhouse_driver import Client + return Client(**options.getall('clickhouse_')) + +@VisiData.lazy_property +def clickhouse_queries(self): + return ClickhouseQuerySheet("queries") + + +class ClickhouseSheet(Sheet): + @asyncthread + def reload(self): + if isinstance(self.source, ClickhouseSheet): + vd.clickhouse_client.execute('USE %s' % self.source.dbname) + + self.rows = [] + self.columns = [] + self.result = vd.clickhouse_client.execute(self.query, with_column_types=True) + result = self.result[0] + + for i, r in enumerate(self.result[1]): + self.addColumn(ItemColumn(r[0], i)) + + for r in result: + self.addRow(r) + +class ClickhouseIndexSheet(IndexSheet): + @asyncthread + def reload(self): + self.rows = [] + for r in vd.clickhouse_client.execute('SHOW DATABASES'): + self.addRow(ClickhouseDbSheet(r[0], source=self)) + +class ClickhouseDbSheet(IndexSheet): + @asyncthread + def reload(self): + vd.clickhouse_client.execute('USE %s' % self.name) + self.rows = [] + for r in vd.clickhouse_client.execute('SHOW TABLES'): + self.addRow(ClickhouseSheet(r[0], source=self, query='SELECT * FROM %s LIMIT 100' % r[0])) + +addGlobals({'openurl_clickhouse': openurl_clickhouse}) From 926146a2d45503e3e1b7b32842aa531bb853b4ef Mon Sep 17 00:00:00 2001 From: Saul Pwanson Date: Wed, 25 Nov 2020 20:25:56 -0800 Subject: [PATCH 113/162] [test] add test for option-unset --- tests/golden/issue733.tsv | 50 +++++++++++++++++++++++++++++++++++++++ tests/issue733.vd | 5 ++++ 2 files changed, 55 insertions(+) create mode 100644 tests/golden/issue733.tsv create mode 100644 tests/issue733.vd diff --git a/tests/golden/issue733.tsv b/tests/golden/issue733.tsv new file mode 100644 index 000000000..f20be3b9d --- /dev/null +++ b/tests/golden/issue733.tsv @@ -0,0 +1,50 @@ +Date Customer SKU Item Quantity Unit Paid +2018-07-03 13:47 Robert Armstrong FOOD213 BFF Oh My Gravy! Beef & Salmon 2.8oz 4 $12.95 $51.8 +2018-07-03 15:32 Kyle Kennedy FOOD121 Food, Adult Cat - 3.5 oz 1 $4.22 $4.22 +2018-07-05 16:15 Douglas "Dougie" Powers FOOD121 Food, Adult Cat 3.5 oz 1 $4.22 $4.22 +2018-07-06 12:15 桜 高橋 (Sakura Takahashi) FOOD122 Food, Senior Wet Cat - 3 oz 12 $1.29 157¥ +2018-07-10 10:28 David Attenborough NSCT201 Food, Salamander 30 $.05 $1.5 +2018-07-10 17:23 Susan Ashworth CAT060 Cat, Korat (Felis catus) 1 $720.42 $720.42 +2018-07-10 17:23 Susan Ashworth FOOD130 Food, Kitten 3kg 1 $14.94 $14.94 +2018-07-13 10:26 Wil Wheaton NSCT523 Monster, Rust (Monstrus gygaxus) 1 $39.95 $39.95 +2018-07-13 15:49 Robert Armstrong FOOD216 BFF Oh My Gravy! Chicken & Shrimp 2.8oz 4 $12.95 $51.8 +2018-07-17 09:01 Robert Armstrong FOOD217 BFF Oh My Gravy! Duck & Tuna 2.8oz 4 $12.95 $51.8 +2018-07-17 11:30 Helen Halestorm LAGO342 Rabbit (Oryctolagus cuniculus) 2 $32.94 $65.88 +2018-07-18 12:16 桜 高橋 (Sakura Takahashi) FOOD122 Food, Senior Wet Cat - 3 oz 6 $1.29 157¥ +2018-07-19 10:28 Rubeus Hagrid FOOD170 Food, Dog - 5kg 5 $44.95 $224.75 +2018-07-20 14:13 Jon Arbuckle FOOD167 Food, Premium Wet Cat - 3.5 oz 50 $3.95 $197.5 +2018-07-23 13:41 Robert Armstrong FOOD215 BFF Oh My Gravy! Lamb & Tuna 2.8oz 4 $12.95 $51.8 +2018-07-23 16:23 Douglas "Dougie" Powers TOY235 Laser Pointer 1 $16.12 $16.12 +2018-07-24 12:16 桜 高橋 (Sakura Takahashi) FOOD122 Food, Senior Wet Cat - 3 oz 3 $1.29 157¥ +2018-07-26 16:39 Douglas "Dougie" Powers FOOD420 Food, Shark - 10 kg 1 $15.70 $15.7 +2018-07-27 12:16 桜 高橋 (Sakura Takahashi) FOOD122 Food, Senior Wet Cat - 3 oz 3 $1.29 157¥ +2018-07-30 12:17 桜 高橋 (Sakura Takahashi) RETURN Food, Senior Wet Cat - 3 oz 1 $1.29 157¥ +2018-07-31 17:42 Rubeus Hagrid CAT060 Food, Dragon - 50kg 5 $720.42 $3602.1 +2018-08-01 14:44 David Attenborough FOOD360 Food, Rhinocerous - 50kg 4 $5.72 $22.88 +2018-08-02 17:12 Susan Ashworth CAT110 Cat, Maine Coon (Felix catus) 1 $1,309.68 $1309.68 +2018-08-02 17:12 Susan Ashworth FOOD130 Food, Kitten 3kg 3 $14.94 $44.82 +2018-08-06 10:21 Robert Armstrong FOOD212 BFF Oh My Gravy! Beef & Chicken 2.8oz 4 $12.95 $51.8 +2018-08-07 16:12 Juan Johnson REPT082 Kingsnake, California (Lampropeltis getula) 1 $89.95 $89.95 +2018-08-07 16:12 Juan Johnson RDNT443 Mouse, Pinky (Mus musculus) 1 $1.49 $1.49 +2018-08-10 16:31 Robert Armstrong FOOD211 BFF Oh My Gravy! Chicken & Turkey 2.8oz 4 $12.95 $51.8 +2018-08-13 14:07 Monica Johnson RDNT443 Mouse, Pinky (Mus musculus) 1 $1.49 $1.49 +2018-08-13 14:08 María Fernández FOOD146 Forti Diet Prohealth Mouse/Rat 3lbs 2 $2.00 $4.0 +2018-08-15 11:57 Mr. Praline RETURN Parrot, Norwegian Blue (Mopsitta tanta) 1 $2300.00 -$2300.0 +2018-08-15 15:48 Kyle Kennedy FOOD121 Food, Adult Cat - 3.5 oz 2 $4.22 $8.44 +2018-08-16 11:50 Helen Halestorm RETURN Rabbit (Oryctolagus cuniculus) 6 $0 $0.0 +2018-08-16 16:00 Kyle Kennedy DOG010 Dog, Golden Retriever (Canis lupus familiaris) 1 $2,495.99 $2495.99 +2018-08-16 17:15 Michael Smith BIRD160 Parakeet, Blue (Melopsittacus undulatus) 1 29.95 $31.85 +2018-08-17 09:26 Rubeus Hagrid NSCT201 Food, Spider 5 $.05 $0.25 +2018-08-20 09:36 Kyle Kennedy RETURN Dog, Golden Retriever (Canis lupus familiaris) 1 $1,247.99 -$1247.99 +2018-08-20 15:31 Monica Johnson NSCT201 Crickets, Adult Live (Gryllus assimilis) 30 $.05 $1.5 +2018-08-20 17:12 David Attenborough NSCT084 Food, Pangolin 30 $.17 $5.10 +2018-08-21 12:13 Robert Armstrong FOOD214 BFF Oh My Gravy! Duck & Salmon 2.8oz 4 $12.95 $51.8 +2018-08-22 09:38 David Attenborough BIRD160 Food, Quoll 1 29.95 $29.95 +2018-08-22 14:13 Jon Arbuckle FOOD170 Food, Adult Dog - 5kg 1 $44.95 $44.95 +2018-08-24 11:42 Robert Armstrong FOOD218 BFF Oh My Gravy! Chicken & Salmon 2.8oz 4 $12.95 $51.8 +2018-08-27 15:05 Monica Johnson NSCT443 Mealworms, Large (Tenebrio molitor) 100ct 1 $1.99 $1.99 +2018-08-28 17:32 Susan Ashworth CAT020 Cat, Scottish Fold (Felis catus) 1 $1,964.53 $1964.53 +2018-08-28 17:32 Susan Ashworth FOOD130 Food, Kitten 3kg 2 $14.94 $29.88 +2018-08-29 10:07 Robert Armstrong FOOD219 BFF Oh My Gravy! Chicken & Pumpkin 2.8oz 4 $12.95 $51.8 +2018-08-31 00:00 Robert Armstrong FOOD219 BFF Oh My Gravy! Chicken & Pumpkin 2.8oz 144 $12.95 $1864.8 +2018-08-31 17:57 Juan Johnson REPT217 Lizard, Spinytail (Uromastyx ornatus) 1 $99.95 $99.95 diff --git a/tests/issue733.vd b/tests/issue733.vd new file mode 100644 index 000000000..575c93ff3 --- /dev/null +++ b/tests/issue733.vd @@ -0,0 +1,5 @@ +sheet col row longname input keystrokes comment + open-file sample_data/benchmark.csv o +benchmark Date type-date @ set type of current column to date +benchmark disp_date_fmt set-option %Y +benchmark disp_date_fmt unset-option From f9b6fd7812df8158490d91324f3da84b9d410221 Mon Sep 17 00:00:00 2001 From: Saul Pwanson Date: Thu, 26 Nov 2020 00:38:15 -0800 Subject: [PATCH 114/162] [xlsx] only reload Workbook sheets to avoid error #797 --- visidata/loaders/xlsx.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/visidata/loaders/xlsx.py b/visidata/loaders/xlsx.py index 7b4987194..32dbe6e97 100644 --- a/visidata/loaders/xlsx.py +++ b/visidata/loaders/xlsx.py @@ -22,8 +22,10 @@ def iterload(self): import openpyxl self.workbook = openpyxl.load_workbook(str(self.source), data_only=True, read_only=True) for sheetname in self.workbook.sheetnames: - vs = XlsxSheet(self.name, sheetname, source=self.workbook[sheetname]) - vs.reload() + src = self.workbook[sheetname] + vs = XlsxSheet(self.name, sheetname, source=src) + if isinstance(src, openpyxl.Workbook): + vs.reload() yield vs From ecdf24525efa3a6606cc3635b18603a2ea5566ae Mon Sep 17 00:00:00 2001 From: Saul Pwanson Date: Thu, 26 Nov 2020 22:38:40 -0800 Subject: [PATCH 115/162] [replay] do not push replaying .vd on sheet stack #795 --- visidata/_input.py | 2 +- visidata/cmdlog.py | 9 +++++++-- visidata/main.py | 3 ++- visidata/mainloop.py | 8 ++++---- visidata/statusbar.py | 4 ++-- 5 files changed, 16 insertions(+), 10 deletions(-) diff --git a/visidata/_input.py b/visidata/_input.py index 522b03b8b..89f2aab1e 100644 --- a/visidata/_input.py +++ b/visidata/_input.py @@ -317,7 +317,7 @@ def input(self, prompt, type=None, defaultLast=False, history=[], **kwargs): else: history = type - sheet = self.sheets[0] + sheet = self.activeSheet rstatuslen = self.drawRightStatus(sheet._scr, sheet) attr = 0 promptlen = clipdraw(sheet._scr, sheet.windowHeight-1, 0, prompt, attr, w=sheet.windowWidth-rstatuslen-1) diff --git a/visidata/cmdlog.py b/visidata/cmdlog.py index 34d5e62d4..46e950bbc 100644 --- a/visidata/cmdlog.py +++ b/visidata/cmdlog.py @@ -271,6 +271,11 @@ def delay(vd, factor=1): acquired = vd.semaphore.acquire(timeout=options.replay_wait*factor if not vd.paused else None) return acquired or not vd.paused +@VisiData.property +def activeSheet(vd): + 'Return top sheet on sheets stack, or cmdlog sheets stack empty.' + return vd.sheets[0] if vd.sheets else vd.cmdlog + @VisiData.api def replayOne(vd, r): @@ -298,7 +303,7 @@ def replayOne(vd, r): if vs: vd.push(vs) else: - vs = vd.sheets[0] # use top sheet by default + vs = vd.activeSheet vd.moveToReplayContext(r, vs) @@ -341,7 +346,7 @@ def replay_sync(vd, cmdlog, live=False): cmdlog.cursorRowIndex += 1 prog.addProgress(1) - vd.sheets[0].ensureLoaded() + vd.activeSheet.ensureLoaded() vd.sync() while not vd.delay(): pass diff --git a/visidata/main.py b/visidata/main.py index 28a64d781..37be15d24 100755 --- a/visidata/main.py +++ b/visidata/main.py @@ -33,7 +33,7 @@ def eval_vd(logpath, *args, **kwargs): src = Path(logpath.given, fp=io.StringIO(log), filesize=len(log)) vs = vd.openSource(src, filetype=src.ext) vs.name += '_vd' - vd.push(vs) + vs.reload() vs.vd = vd return vs @@ -275,6 +275,7 @@ def main_vd(): if vd.replay_sync(vs): # error return 1 else: + vd.currentReplay = vs vd.replay(vs) run() diff --git a/visidata/mainloop.py b/visidata/mainloop.py index 66540781d..d0b13a6e4 100644 --- a/visidata/mainloop.py +++ b/visidata/mainloop.py @@ -74,6 +74,8 @@ def setWindows(vd, scr): @VisiData.api def draw_all(vd): 'Draw all sheets in all windows.' + if not vd.sheets: + return vd.draw_sheet(vd.win1, vd.sheets[0]) if vd.win2 and len(vd.sheets) > 1: vd.draw_sheet(vd.win2, vd.sheets[1]) @@ -104,11 +106,10 @@ def mainloop(self, scr): self.keystrokes = '' while True: - if not self.sheets: - # if no more sheets, exit + if not self.sheets and self.currentReplay is None: return - sheet = self.sheets[0] + sheet = self.activeSheet threading.current_thread().sheet = sheet vd.drawThread = threading.current_thread() @@ -201,7 +202,6 @@ def mainloop(self, scr): else: scr.timeout(curses_timeout) - def setupcolors(stdscr, f, *args): curses.raw() # get control keys instead of signals curses.meta(1) # allow "8-bit chars" diff --git a/visidata/statusbar.py b/visidata/statusbar.py index 8a4a831fa..3bc9c2422 100644 --- a/visidata/statusbar.py +++ b/visidata/statusbar.py @@ -97,7 +97,7 @@ def leftStatus(sheet): def drawLeftStatus(vd, scr, vs): 'Draw left side of status bar.' cattr = colors.get_color('color_status') - active = vs is vd.sheets[0] # active sheet + active = (vs is vd.sheets[0]) if vd.sheets else False # active sheet if active: cattr = update_attr(cattr, colors.color_active_status, 0) else: @@ -166,7 +166,7 @@ def drawRightStatus(vd, scr, vs): (vd.rightStatus(vs), 'color_status'), ] - active = vs is vd.sheets[0] # active sheet + active = vs is vd.activeSheet if active: statcolors.append((vd.keystrokes or '', 'color_keystrokes')) From d8cc2a265251f62541bf216ab99c23d77979d587 Mon Sep 17 00:00:00 2001 From: Saul Pwanson Date: Thu, 26 Nov 2020 22:47:01 -0800 Subject: [PATCH 116/162] [freq] change numeric_binning back to False by default --- visidata/freqtbl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/visidata/freqtbl.py b/visidata/freqtbl.py index e5a57297f..01bb45b88 100644 --- a/visidata/freqtbl.py +++ b/visidata/freqtbl.py @@ -7,7 +7,7 @@ theme('disp_histogram', '*', 'histogram element character') option('disp_histolen', 50, 'width of histogram column') option('histogram_bins', 0, 'number of bins for histogram of numeric columns') -option('numeric_binning', True, 'bin numeric columns into ranges', replay=True) +option('numeric_binning', False, 'bin numeric columns into ranges', replay=True) def valueNames(discrete_vals, numeric_vals): From 6baf0df290c8ee60f8077d1c8b5d75c48bf2e694 Mon Sep 17 00:00:00 2001 From: Saul Pwanson Date: Thu, 26 Nov 2020 23:24:10 -0800 Subject: [PATCH 117/162] [columns] add hidden keycol to ColumnsSheet #768 --- visidata/metasheets.py | 1 + 1 file changed, 1 insertion(+) diff --git a/visidata/metasheets.py b/visidata/metasheets.py index a51d320cd..d4a4af784 100644 --- a/visidata/metasheets.py +++ b/visidata/metasheets.py @@ -33,6 +33,7 @@ def setValue(self, srcCol, val): columns = [ ColumnAttr('sheet', type=str), ColumnAttr('name', width=options.default_width), + ColumnAttr('keycol', type=int, width=0), ColumnAttr('width', type=int), ColumnAttr('height', type=int), ColumnAttr('hoffset', type=int, width=0), From ba9b8a65e7400ffd69345b4d46abc0029216c841 Mon Sep 17 00:00:00 2001 From: Saul Pwanson Date: Fri, 27 Nov 2020 20:05:32 -0800 Subject: [PATCH 118/162] [tests] fix golden for issue733 --- tests/golden/issue733.tsv | 98 +++++++++++++++++++-------------------- 1 file changed, 49 insertions(+), 49 deletions(-) diff --git a/tests/golden/issue733.tsv b/tests/golden/issue733.tsv index f20be3b9d..05a8fbd97 100644 --- a/tests/golden/issue733.tsv +++ b/tests/golden/issue733.tsv @@ -1,50 +1,50 @@ Date Customer SKU Item Quantity Unit Paid -2018-07-03 13:47 Robert Armstrong FOOD213 BFF Oh My Gravy! Beef & Salmon 2.8oz 4 $12.95 $51.8 -2018-07-03 15:32 Kyle Kennedy FOOD121 Food, Adult Cat - 3.5 oz 1 $4.22 $4.22 -2018-07-05 16:15 Douglas "Dougie" Powers FOOD121 Food, Adult Cat 3.5 oz 1 $4.22 $4.22 -2018-07-06 12:15 桜 高橋 (Sakura Takahashi) FOOD122 Food, Senior Wet Cat - 3 oz 12 $1.29 157¥ -2018-07-10 10:28 David Attenborough NSCT201 Food, Salamander 30 $.05 $1.5 -2018-07-10 17:23 Susan Ashworth CAT060 Cat, Korat (Felis catus) 1 $720.42 $720.42 -2018-07-10 17:23 Susan Ashworth FOOD130 Food, Kitten 3kg 1 $14.94 $14.94 -2018-07-13 10:26 Wil Wheaton NSCT523 Monster, Rust (Monstrus gygaxus) 1 $39.95 $39.95 -2018-07-13 15:49 Robert Armstrong FOOD216 BFF Oh My Gravy! Chicken & Shrimp 2.8oz 4 $12.95 $51.8 -2018-07-17 09:01 Robert Armstrong FOOD217 BFF Oh My Gravy! Duck & Tuna 2.8oz 4 $12.95 $51.8 -2018-07-17 11:30 Helen Halestorm LAGO342 Rabbit (Oryctolagus cuniculus) 2 $32.94 $65.88 -2018-07-18 12:16 桜 高橋 (Sakura Takahashi) FOOD122 Food, Senior Wet Cat - 3 oz 6 $1.29 157¥ -2018-07-19 10:28 Rubeus Hagrid FOOD170 Food, Dog - 5kg 5 $44.95 $224.75 -2018-07-20 14:13 Jon Arbuckle FOOD167 Food, Premium Wet Cat - 3.5 oz 50 $3.95 $197.5 -2018-07-23 13:41 Robert Armstrong FOOD215 BFF Oh My Gravy! Lamb & Tuna 2.8oz 4 $12.95 $51.8 -2018-07-23 16:23 Douglas "Dougie" Powers TOY235 Laser Pointer 1 $16.12 $16.12 -2018-07-24 12:16 桜 高橋 (Sakura Takahashi) FOOD122 Food, Senior Wet Cat - 3 oz 3 $1.29 157¥ -2018-07-26 16:39 Douglas "Dougie" Powers FOOD420 Food, Shark - 10 kg 1 $15.70 $15.7 -2018-07-27 12:16 桜 高橋 (Sakura Takahashi) FOOD122 Food, Senior Wet Cat - 3 oz 3 $1.29 157¥ -2018-07-30 12:17 桜 高橋 (Sakura Takahashi) RETURN Food, Senior Wet Cat - 3 oz 1 $1.29 157¥ -2018-07-31 17:42 Rubeus Hagrid CAT060 Food, Dragon - 50kg 5 $720.42 $3602.1 -2018-08-01 14:44 David Attenborough FOOD360 Food, Rhinocerous - 50kg 4 $5.72 $22.88 -2018-08-02 17:12 Susan Ashworth CAT110 Cat, Maine Coon (Felix catus) 1 $1,309.68 $1309.68 -2018-08-02 17:12 Susan Ashworth FOOD130 Food, Kitten 3kg 3 $14.94 $44.82 -2018-08-06 10:21 Robert Armstrong FOOD212 BFF Oh My Gravy! Beef & Chicken 2.8oz 4 $12.95 $51.8 -2018-08-07 16:12 Juan Johnson REPT082 Kingsnake, California (Lampropeltis getula) 1 $89.95 $89.95 -2018-08-07 16:12 Juan Johnson RDNT443 Mouse, Pinky (Mus musculus) 1 $1.49 $1.49 -2018-08-10 16:31 Robert Armstrong FOOD211 BFF Oh My Gravy! Chicken & Turkey 2.8oz 4 $12.95 $51.8 -2018-08-13 14:07 Monica Johnson RDNT443 Mouse, Pinky (Mus musculus) 1 $1.49 $1.49 -2018-08-13 14:08 María Fernández FOOD146 Forti Diet Prohealth Mouse/Rat 3lbs 2 $2.00 $4.0 -2018-08-15 11:57 Mr. Praline RETURN Parrot, Norwegian Blue (Mopsitta tanta) 1 $2300.00 -$2300.0 -2018-08-15 15:48 Kyle Kennedy FOOD121 Food, Adult Cat - 3.5 oz 2 $4.22 $8.44 -2018-08-16 11:50 Helen Halestorm RETURN Rabbit (Oryctolagus cuniculus) 6 $0 $0.0 -2018-08-16 16:00 Kyle Kennedy DOG010 Dog, Golden Retriever (Canis lupus familiaris) 1 $2,495.99 $2495.99 -2018-08-16 17:15 Michael Smith BIRD160 Parakeet, Blue (Melopsittacus undulatus) 1 29.95 $31.85 -2018-08-17 09:26 Rubeus Hagrid NSCT201 Food, Spider 5 $.05 $0.25 -2018-08-20 09:36 Kyle Kennedy RETURN Dog, Golden Retriever (Canis lupus familiaris) 1 $1,247.99 -$1247.99 -2018-08-20 15:31 Monica Johnson NSCT201 Crickets, Adult Live (Gryllus assimilis) 30 $.05 $1.5 -2018-08-20 17:12 David Attenborough NSCT084 Food, Pangolin 30 $.17 $5.10 -2018-08-21 12:13 Robert Armstrong FOOD214 BFF Oh My Gravy! Duck & Salmon 2.8oz 4 $12.95 $51.8 -2018-08-22 09:38 David Attenborough BIRD160 Food, Quoll 1 29.95 $29.95 -2018-08-22 14:13 Jon Arbuckle FOOD170 Food, Adult Dog - 5kg 1 $44.95 $44.95 -2018-08-24 11:42 Robert Armstrong FOOD218 BFF Oh My Gravy! Chicken & Salmon 2.8oz 4 $12.95 $51.8 -2018-08-27 15:05 Monica Johnson NSCT443 Mealworms, Large (Tenebrio molitor) 100ct 1 $1.99 $1.99 -2018-08-28 17:32 Susan Ashworth CAT020 Cat, Scottish Fold (Felis catus) 1 $1,964.53 $1964.53 -2018-08-28 17:32 Susan Ashworth FOOD130 Food, Kitten 3kg 2 $14.94 $29.88 -2018-08-29 10:07 Robert Armstrong FOOD219 BFF Oh My Gravy! Chicken & Pumpkin 2.8oz 4 $12.95 $51.8 -2018-08-31 00:00 Robert Armstrong FOOD219 BFF Oh My Gravy! Chicken & Pumpkin 2.8oz 144 $12.95 $1864.8 -2018-08-31 17:57 Juan Johnson REPT217 Lizard, Spinytail (Uromastyx ornatus) 1 $99.95 $99.95 +2018-07-03 Robert Armstrong FOOD213 BFF Oh My Gravy! Beef & Salmon 2.8oz 4 $12.95 $51.8 +2018-07-03 Kyle Kennedy FOOD121 Food, Adult Cat - 3.5 oz 1 $4.22 $4.22 +2018-07-05 Douglas "Dougie" Powers FOOD121 Food, Adult Cat 3.5 oz 1 $4.22 $4.22 +2018-07-06 桜 高橋 (Sakura Takahashi) FOOD122 Food, Senior Wet Cat - 3 oz 12 $1.29 157¥ +2018-07-10 David Attenborough NSCT201 Food, Salamander 30 $.05 $1.5 +2018-07-10 Susan Ashworth CAT060 Cat, Korat (Felis catus) 1 $720.42 $720.42 +2018-07-10 Susan Ashworth FOOD130 Food, Kitten 3kg 1 $14.94 $14.94 +2018-07-13 Wil Wheaton NSCT523 Monster, Rust (Monstrus gygaxus) 1 $39.95 $39.95 +2018-07-13 Robert Armstrong FOOD216 BFF Oh My Gravy! Chicken & Shrimp 2.8oz 4 $12.95 $51.8 +2018-07-17 Robert Armstrong FOOD217 BFF Oh My Gravy! Duck & Tuna 2.8oz 4 $12.95 $51.8 +2018-07-17 Helen Halestorm LAGO342 Rabbit (Oryctolagus cuniculus) 2 $32.94 $65.88 +2018-07-18 桜 高橋 (Sakura Takahashi) FOOD122 Food, Senior Wet Cat - 3 oz 6 $1.29 157¥ +2018-07-19 Rubeus Hagrid FOOD170 Food, Dog - 5kg 5 $44.95 $224.75 +2018-07-20 Jon Arbuckle FOOD167 Food, Premium Wet Cat - 3.5 oz 50 $3.95 $197.5 +2018-07-23 Robert Armstrong FOOD215 BFF Oh My Gravy! Lamb & Tuna 2.8oz 4 $12.95 $51.8 +2018-07-23 Douglas "Dougie" Powers TOY235 Laser Pointer 1 $16.12 $16.12 +2018-07-24 桜 高橋 (Sakura Takahashi) FOOD122 Food, Senior Wet Cat - 3 oz 3 $1.29 157¥ +2018-07-26 Douglas "Dougie" Powers FOOD420 Food, Shark - 10 kg 1 $15.70 $15.7 +2018-07-27 桜 高橋 (Sakura Takahashi) FOOD122 Food, Senior Wet Cat - 3 oz 3 $1.29 157¥ +2018-07-30 桜 高橋 (Sakura Takahashi) RETURN Food, Senior Wet Cat - 3 oz 1 $1.29 157¥ +2018-07-31 Rubeus Hagrid CAT060 Food, Dragon - 50kg 5 $720.42 $3602.1 +2018-08-01 David Attenborough FOOD360 Food, Rhinocerous - 50kg 4 $5.72 $22.88 +2018-08-02 Susan Ashworth CAT110 Cat, Maine Coon (Felix catus) 1 $1,309.68 $1309.68 +2018-08-02 Susan Ashworth FOOD130 Food, Kitten 3kg 3 $14.94 $44.82 +2018-08-06 Robert Armstrong FOOD212 BFF Oh My Gravy! Beef & Chicken 2.8oz 4 $12.95 $51.8 +2018-08-07 Juan Johnson REPT082 Kingsnake, California (Lampropeltis getula) 1 $89.95 $89.95 +2018-08-07 Juan Johnson RDNT443 Mouse, Pinky (Mus musculus) 1 $1.49 $1.49 +2018-08-10 Robert Armstrong FOOD211 BFF Oh My Gravy! Chicken & Turkey 2.8oz 4 $12.95 $51.8 +2018-08-13 Monica Johnson RDNT443 Mouse, Pinky (Mus musculus) 1 $1.49 $1.49 +2018-08-13 María Fernández FOOD146 Forti Diet Prohealth Mouse/Rat 3lbs 2 $2.00 $4.0 +2018-08-15 Mr. Praline RETURN Parrot, Norwegian Blue (Mopsitta tanta) 1 $2300.00 -$2300.0 +2018-08-15 Kyle Kennedy FOOD121 Food, Adult Cat - 3.5 oz 2 $4.22 $8.44 +2018-08-16 Helen Halestorm RETURN Rabbit (Oryctolagus cuniculus) 6 $0 $0.0 +2018-08-16 Kyle Kennedy DOG010 Dog, Golden Retriever (Canis lupus familiaris) 1 $2,495.99 $2495.99 +2018-08-16 Michael Smith BIRD160 Parakeet, Blue (Melopsittacus undulatus) 1 29.95 $31.85 +2018-08-17 Rubeus Hagrid NSCT201 Food, Spider 5 $.05 $0.25 +2018-08-20 Kyle Kennedy RETURN Dog, Golden Retriever (Canis lupus familiaris) 1 $1,247.99 -$1247.99 +2018-08-20 Monica Johnson NSCT201 Crickets, Adult Live (Gryllus assimilis) 30 $.05 $1.5 +2018-08-20 David Attenborough NSCT084 Food, Pangolin 30 $.17 $5.10 +2018-08-21 Robert Armstrong FOOD214 BFF Oh My Gravy! Duck & Salmon 2.8oz 4 $12.95 $51.8 +2018-08-22 David Attenborough BIRD160 Food, Quoll 1 29.95 $29.95 +2018-08-22 Jon Arbuckle FOOD170 Food, Adult Dog - 5kg 1 $44.95 $44.95 +2018-08-24 Robert Armstrong FOOD218 BFF Oh My Gravy! Chicken & Salmon 2.8oz 4 $12.95 $51.8 +2018-08-27 Monica Johnson NSCT443 Mealworms, Large (Tenebrio molitor) 100ct 1 $1.99 $1.99 +2018-08-28 Susan Ashworth CAT020 Cat, Scottish Fold (Felis catus) 1 $1,964.53 $1964.53 +2018-08-28 Susan Ashworth FOOD130 Food, Kitten 3kg 2 $14.94 $29.88 +2018-08-29 Robert Armstrong FOOD219 BFF Oh My Gravy! Chicken & Pumpkin 2.8oz 4 $12.95 $51.8 +2018-08-31 Robert Armstrong FOOD219 BFF Oh My Gravy! Chicken & Pumpkin 2.8oz 144 $12.95 $1864.8 +2018-08-31 Juan Johnson REPT217 Lizard, Spinytail (Uromastyx ornatus) 1 $99.95 $99.95 From 8e9079f82dca489dcb39312ad79737a83493e451 Mon Sep 17 00:00:00 2001 From: anjakefala Date: Fri, 27 Nov 2020 20:49:15 -0800 Subject: [PATCH 119/162] [tests] update tests for d8cc2a26 and ecdf24525efa3a --- tests/aggregators-errors.vd | 1 + tests/exp-digits.vd | 2 +- tests/freq-error.vd | 1 - tests/freq-fmtstr.vd | 1 - tests/graph-sincos-nosave.vd | 2 +- tests/histogram.vd | 1 + tests/pivot-error.vd | 1 - tests/quantum-sum-nosave.vd | 2 +- 8 files changed, 5 insertions(+), 6 deletions(-) diff --git a/tests/aggregators-errors.vd b/tests/aggregators-errors.vd index ca6330c58..350210aee 100644 --- a/tests/aggregators-errors.vd +++ b/tests/aggregators-errors.vd @@ -1,5 +1,6 @@ sheet col row longname input keystrokes comment open-file sample_data/test.jsonl o + numeric_binning set-option True test key1 type-date @ test qty type-int # test amt type-float % diff --git a/tests/exp-digits.vd b/tests/exp-digits.vd index b035fb365..0399cbcb7 100644 --- a/tests/exp-digits.vd +++ b/tests/exp-digits.vd @@ -1,6 +1,6 @@ sheet col row longname input keystrokes comment incr_base set-option 0 -exp-digits_vd open-new 1 A open new blank sheet with number columns + open-new 1 A open new blank sheet with number columns unnamed 0 rename-col x ^ edit name of current column unnamed add-rows 100 ga add N blank rows unnamed select-rows gs diff --git a/tests/freq-error.vd b/tests/freq-error.vd index f6612b379..c14c3a2cb 100644 --- a/tests/freq-error.vd +++ b/tests/freq-error.vd @@ -1,6 +1,5 @@ sheet col row longname input keystrokes comment open-file sample_data/test.jsonl o -test numeric_binning set-option False test key1 type-date @ test qty type-int # test amt type-float % diff --git a/tests/freq-fmtstr.vd b/tests/freq-fmtstr.vd index a3c27ce4e..986fd247d 100644 --- a/tests/freq-fmtstr.vd +++ b/tests/freq-fmtstr.vd @@ -1,6 +1,5 @@ sheet col row longname input keystrokes comment open-file sample_data/benchmark.csv o - numeric_binning set-option False benchmark Date type-date @ benchmark columns-sheet C benchmark_columns fmtstr キDate edit-cell %Y-%m e diff --git a/tests/graph-sincos-nosave.vd b/tests/graph-sincos-nosave.vd index 5c356e91f..d892fdafe 100644 --- a/tests/graph-sincos-nosave.vd +++ b/tests/graph-sincos-nosave.vd @@ -1,5 +1,5 @@ sheet col row longname input keystrokes comment -graph-sincos-nosave_vd add-sheet 1 A open new blank sheet with number columns + add-sheet 1 A open new blank sheet with number columns unnamed add-rows 360 ga add N blank rows unnamed select-rows gs unnamed 0 setcol-iter range(360) gz= set selected rows in this column to the values in the given Python sequence expression diff --git a/tests/histogram.vd b/tests/histogram.vd index 634281ff9..340767bc4 100644 --- a/tests/histogram.vd +++ b/tests/histogram.vd @@ -1,5 +1,6 @@ sheet col row longname input keystrokes comment open-file sample_data/benchmark.csv o + numeric_binning set-option True benchmark Quantity type-int # benchmark Quantity addcol-expr Quantity > 1 = benchmark Quantity > 1 select-col-regex True | diff --git a/tests/pivot-error.vd b/tests/pivot-error.vd index cf45387f0..a487f7e75 100644 --- a/tests/pivot-error.vd +++ b/tests/pivot-error.vd @@ -1,6 +1,5 @@ sheet col row longname input keystrokes comment open-file sample_data/test.jsonl o -test numeric_binning set-option False test key1 type-date @ test qty type-int # test amt type-float % diff --git a/tests/quantum-sum-nosave.vd b/tests/quantum-sum-nosave.vd index 127abef04..677acdeef 100644 --- a/tests/quantum-sum-nosave.vd +++ b/tests/quantum-sum-nosave.vd @@ -1,5 +1,5 @@ sheet col row longname input keystrokes comment -quantum-sum-nosave_vd add-sheet 1 A open new blank sheet with number columns + add-sheet 1 A open new blank sheet with number columns unnamed add-rows 100000 ga add N blank rows unnamed 0 addcol-expr random.random() = create new column from Python expression, with column names as variables unnamed random.random() type-float % set type of current column to float From dd3aaa4a01e5116c8cd25a30a36bf39756a719b6 Mon Sep 17 00:00:00 2001 From: Saul Pwanson Date: Fri, 27 Nov 2020 23:45:29 -0800 Subject: [PATCH 120/162] [plugins rownum-] fix missing argument #811 --- plugins/rownum.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/rownum.py b/plugins/rownum.py index 69857ba8c..f673932dc 100644 --- a/plugins/rownum.py +++ b/plugins/rownum.py @@ -72,4 +72,4 @@ def addcol_rownum(sheet): return newcol Sheet.addCommand(None, 'addcol-rownum', 'addcol_rownum()', helpstr='add column with original row ordering') -Sheet.addCommand(None, 'addcol-delta', 'addcol_delta()', helpstr='add column with delta of current column') +Sheet.addCommand(None, 'addcol-delta', 'addcol_delta(cursorVisibleColIndex)', helpstr='add column with delta of current column') From 5921a9198f4ae32fd722d8084a30ca0de64126a6 Mon Sep 17 00:00:00 2001 From: anjakefala Date: Sat, 28 Nov 2020 18:09:00 -0800 Subject: [PATCH 121/162] [MissingAttrFormatter] handle missing attributes in status formatting Closes #764 --- visidata/statusbar.py | 4 ++-- visidata/utils.py | 17 ++++++++++++++++- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/visidata/statusbar.py b/visidata/statusbar.py index 3bc9c2422..d18219963 100644 --- a/visidata/statusbar.py +++ b/visidata/statusbar.py @@ -1,7 +1,7 @@ import collections import curses -from visidata import vd, VisiData, BaseSheet, Sheet, ColumnItem, Column, RowColorizer, options, colors, wrmap, clipdraw, ExpectedException, update_attr, theme +from visidata import vd, VisiData, BaseSheet, Sheet, ColumnItem, Column, RowColorizer, options, colors, wrmap, clipdraw, ExpectedException, update_attr, theme, MissingAttrFormatter __all__ = ['StatusSheet', 'status', 'error', 'fail', 'warning', 'debug'] @@ -153,7 +153,7 @@ def drawLeftStatus(vd, scr, vs): @VisiData.api def rightStatus(vd, sheet): 'Return right side of status bar. Overrideable.' - return options.disp_rstatus_fmt.format(sheet=sheet, vd=vd) + return MissingAttrFormatter().format(sheet.options.disp_rstatus_fmt, sheet=sheet, vd=vd) @VisiData.api diff --git a/visidata/utils.py b/visidata/utils.py index 9b2bcb2cf..44780befc 100644 --- a/visidata/utils.py +++ b/visidata/utils.py @@ -1,8 +1,9 @@ import operator +import string 'Various helper classes and functions.' -__all__ = ['AlwaysDict', 'AttrDict', 'moveListItem', 'namedlist', 'classproperty'] +__all__ = ['AlwaysDict', 'AttrDict', 'moveListItem', 'namedlist', 'classproperty', 'MissingAttrFormatter'] class AlwaysDict(dict): @@ -98,3 +99,17 @@ def __setattr__(self, k, v): super().__setattr__(k, v) return NamedListTemplate + +class MissingAttrFormatter(string.Formatter): + "formats {} fields with `''`, that would normally result in a raised KeyError or AttributeError; intended for user customisable format strings." + def get_field(self, field_name, *args, **kwargs): + try: + return super().get_field(field_name, *args, **kwargs) + except (KeyError, AttributeError): + return (None, field_name) + + def format_field(self, value, format_spec): + # value is missing + if not value: + return '' + return super().format_field(value, format_spec) From c2a9b3f96fc328708ae4a70f0464368138f30195 Mon Sep 17 00:00:00 2001 From: anjakefala Date: Sat, 28 Nov 2020 21:00:29 -0800 Subject: [PATCH 122/162] [macros] fixes for #787 - pass original macrosheet to append_tsv_row - vd.macrosheet returns a real_macrosheet IndexSheet - real_macrosheet is an IndexSheet with a slightly different structure than the actual file on disk recording the macros - that original macros file is the one we actually want to update - perform cmdlog.afterExecSheet after internal macro cmdlog is updated - internal cmdlog macro must be updated before the activeCommand is reset --- visidata/macros.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/visidata/macros.py b/visidata/macros.py index b4970ef51..819ff37c9 100644 --- a/visidata/macros.py +++ b/visidata/macros.py @@ -9,7 +9,7 @@ def macrosheet(vd): macrospath = Path(os.path.join(options.visidata_dir, 'macros.tsv')) macrosheet = vd.loadInternalSheet(TsvSheet, macrospath, columns=(ColumnItem('command', 0), ColumnItem('filename', 1))) or vd.error('error loading macros') - real_macrosheet = IndexSheet('user_macros', rows=[]) + real_macrosheet = IndexSheet('user_macros', rows=[], source=macrosheet) for ks, fn in macrosheet.rows: vs = vd.loadInternalSheet(CommandLog, Path(fn)) vd.status(f"setting {ks}") @@ -34,17 +34,20 @@ def saveMacro(self, rows, ks): macropath = Path(fnSuffix(options.visidata_dir+"macro")) vd.save_vd(macropath, vs) setMacro(ks, vs) - append_tsv_row(vd.macrosheet, (ks, macropath)) + append_tsv_row(vd.macrosheet.source, (ks, macropath)) @CommandLog.api @wraps(CommandLog.afterExecSheet) def afterExecSheet(cmdlog, sheet, escaped, err): - cmdlog.afterExecSheet.__wrapped__(cmdlog, sheet, escaped, err) if vd.macroMode and (vd.activeCommand is not None) and (vd.activeCommand is not UNLOADED): cmd = copy(vd.activeCommand) cmd.row = cmd.col = cmd.sheet = '' vd.macroMode.addRow(cmd) + # the following needs to happen at the end, bc + # once cmdlog.afterExecSheet.__wrapped__ runs, vd.activeCommand resets to None + cmdlog.afterExecSheet.__wrapped__(cmdlog, sheet, escaped, err) + @CommandLog.api def startMacro(cmdlog): if vd.macroMode: From 06f135e3e25a3f0cd7689c2bf7282d01d910a11f Mon Sep 17 00:00:00 2001 From: Saul Pwanson Date: Sat, 28 Nov 2020 00:10:39 -0800 Subject: [PATCH 123/162] [shell] empty stdin to avoid hanging processes #752 also warn if command has no $column in it --- visidata/shell.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/visidata/shell.py b/visidata/shell.py index 85d2024dc..4a59acc0f 100644 --- a/visidata/shell.py +++ b/visidata/shell.py @@ -65,7 +65,7 @@ def calcValue(self, row): else: args.append(arg) - p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + p = subprocess.Popen(args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) return p.communicate() except Exception as e: vd.exceptionCaught(e) @@ -203,9 +203,17 @@ def iterload(self): for fn in self.source.open_text(): yield Path(fn.rstrip()) + +@VisiData.api +def inputShell(vd): + cmd = vd.input("sh$ ", type="sh") + if '$' not in cmd: + vd.warning('no $column in command') + return cmd + globalCommand('', 'open-dir-current', 'vd.push(vd.currentDirSheet)', 'open Directory Sheet: browse properties of files in current directory') -Sheet.addCommand('z;', 'addcol-sh', 'cmd=input("sh$ ", type="sh"); addShellColumns(cmd, sheet)', 'create new column from bash expression, with $columnNames as variables') +Sheet.addCommand('z;', 'addcol-sh', 'cmd=inputShell(); addShellColumns(cmd, sheet)', 'create new column from bash expression, with $columnNames as variables') DirSheet.addCommand(ENTER, 'open-row', 'vd.push(openSource(cursorRow or fail("no row"), filetype="dir" if cursorRow.is_dir() else LazyComputeRow(sheet, cursorRow).ext))', 'open current file as a new sheet') DirSheet.addCommand('g'+ENTER, 'open-rows', 'for r in selectedRows: vd.push(openSource(r))', 'open selected files as new sheets') From 0d1fb38beb96f230db637b1895c26d18fbfdbafc Mon Sep 17 00:00:00 2001 From: anjakefala Date: Sun, 29 Nov 2020 00:22:20 -0800 Subject: [PATCH 124/162] [frozen] use putValue instead of setValue with frozen columns. - means frozen columns are not deferred Closes #786 --- visidata/freeze.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/visidata/freeze.py b/visidata/freeze.py index 2bba46b70..7fc794a4b 100644 --- a/visidata/freeze.py +++ b/visidata/freeze.py @@ -17,9 +17,9 @@ def calcRows_async(frozencol, rows, col): # no need to undo, addColumn undo is enough for r in Progress(rows, 'calculating'): try: - frozencol.setValue(r, col.getTypedValue(r)) + frozencol.putValue(r, col.getTypedValue(r)) except Exception as e: - frozencol.setValue(r, e) + frozencol.putValue(r, e) calcRows_async(frozencol, sheet.rows, col) return frozencol From 1b64f01aaae4234f339cc3c529182be69704ee39 Mon Sep 17 00:00:00 2001 From: AJ Kerrigan Date: Sun, 29 Nov 2020 22:25:53 -0500 Subject: [PATCH 125/162] [pandas] support loading Python objects directly If a data source isn't an existing DataFrame or a VisiData path-like object, try to pass it into `pd.DataFrame()`. This is most useful when using `visidata.view_pandas()` from a Python REPL. Closes #798 --- visidata/loaders/_pandas.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/visidata/loaders/_pandas.py b/visidata/loaders/_pandas.py index 340090b04..8cf2471ec 100644 --- a/visidata/loaders/_pandas.py +++ b/visidata/loaders/_pandas.py @@ -117,6 +117,11 @@ def reload(self): else: readfunc = getattr(pd, 'read_'+filetype) or vd.error('no pandas.read_'+filetype) df = readfunc(str(self.source), **options.getall('pandas_'+filetype+'_')) + else: + try: + df = pd.DataFrame(self.source) + except ValueError as err: + vd.fail('error building pandas DataFrame from source data: %s' % err) # reset the index here if type(df.index) is not pd.RangeIndex: From 7bb87d066569811cd209bfc18f2631773b3154a7 Mon Sep 17 00:00:00 2001 From: AJ Kerrigan Date: Sun, 29 Nov 2020 22:33:38 -0500 Subject: [PATCH 126/162] [pandas] Ensure that all column names are strings VisiData column names are defined as strings, and that assumption pops up in multiple places. It's not ideal to convert colum names to strings for a wide pandas DataFrame, but it _is_ better than throwing an error and dying altogether. Relates to #800 --- visidata/loaders/_pandas.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/visidata/loaders/_pandas.py b/visidata/loaders/_pandas.py index 8cf2471ec..3ab4f5561 100644 --- a/visidata/loaders/_pandas.py +++ b/visidata/loaders/_pandas.py @@ -127,6 +127,10 @@ def reload(self): if type(df.index) is not pd.RangeIndex: df = df.reset_index() + # VisiData assumes string column names but pandas does not. Forcing string + # columns at load-time avoids various errors later. + df.columns = df.columns.astype(str) + self.columns = [] for col in (c for c in df.columns if not c.startswith("__vd_")): self.addColumn(Column( From 349e7f3e7a3cad74890f385c5378b7d191230ec6 Mon Sep 17 00:00:00 2001 From: AJ Kerrigan Date: Sun, 29 Nov 2020 22:44:25 -0500 Subject: [PATCH 127/162] [pandas] Build freqtbl using a copy of the source Make a local shallow copy of the source DataFrame that the frequency table logic can use. This prevents pandas from getting worried about assigning values to copies of slices in some cases. Relates to #802 --- visidata/loaders/pandas_freqtbl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/visidata/loaders/pandas_freqtbl.py b/visidata/loaders/pandas_freqtbl.py index 0e3f3d6cd..5c3874045 100644 --- a/visidata/loaders/pandas_freqtbl.py +++ b/visidata/loaders/pandas_freqtbl.py @@ -97,7 +97,7 @@ def reload(self): # that operates similarly to pd.cut. super().initCols() - df = self.source.rows.df + df = self.source.df.copy() # Implementation (special case): for one row, this degenerates # to .value_counts(); however this does not order in a stable manner. From a156fe8f0cdadb8ba6f14307200a9be2d6f91f8d Mon Sep 17 00:00:00 2001 From: anjakefala Date: Sun, 29 Nov 2020 14:22:19 -0800 Subject: [PATCH 128/162] [select] rename someSelectedRows to onlySelectedRows --- docs/api/data.rst | 2 +- visidata/choose.py | 2 +- visidata/clipboard.py | 14 +++++++------- visidata/customdate.py | 2 +- visidata/loaders/archive.py | 4 ++-- visidata/metasheets.py | 22 +++++++++++----------- 6 files changed, 23 insertions(+), 23 deletions(-) diff --git a/docs/api/data.rst b/docs/api/data.rst index 40fd3de67..5205f1050 100644 --- a/docs/api/data.rst +++ b/docs/api/data.rst @@ -4,7 +4,7 @@ Selected Rows Each TableSheet has a set of *selected rows*, which is a strict subset of the rows on the sheet. .. autoattribute:: visidata.TableSheet.selectedRows -.. autoattribute:: visidata.TableSheet.someSelectedRows +.. autoattribute:: visidata.TableSheet.onlySelectedRows .. autoattribute:: visidata.TableSheet.nSelectedRows .. autofunction:: visidata.TableSheet.selectRow diff --git a/visidata/choose.py b/visidata/choose.py index 5eaf276c9..715d6cc8a 100644 --- a/visidata/choose.py +++ b/visidata/choose.py @@ -78,4 +78,4 @@ def throw_fancy(v, i): ChoiceSheet.addCommand(ENTER, 'choose-rows', 'makeChoice([cursorRow])') -ChoiceSheet.addCommand('g'+ENTER, 'choose-rows-selected', 'makeChoice(someSelectedRows)') +ChoiceSheet.addCommand('g'+ENTER, 'choose-rows-selected', 'makeChoice(onlySelectedRows)') diff --git a/visidata/clipboard.py b/visidata/clipboard.py index 6c0e9220a..515c99f92 100644 --- a/visidata/clipboard.py +++ b/visidata/clipboard.py @@ -181,26 +181,26 @@ def saveToClipboard(sheet, rows, filetype=None): Sheet.addCommand('p', 'paste-after', 'paste_after(cursorRowIndex)', 'paste clipboard rows after current row') Sheet.addCommand('P', 'paste-before', 'paste_before(cursorRowIndex)', 'paste clipboard rows before current row') -Sheet.addCommand('gd', 'delete-selected', 'copyRows(someSelectedRows); deleteSelected()', 'delete (cut) selected rows and move them to clipboard') -Sheet.addCommand('gy', 'copy-selected', 'copyRows(someSelectedRows)', 'yank (copy) selected rows to clipboard') +Sheet.addCommand('gd', 'delete-selected', 'copyRows(onlySelectedRows); deleteSelected()', 'delete (cut) selected rows and move them to clipboard') +Sheet.addCommand('gy', 'copy-selected', 'copyRows(onlySelectedRows)', 'yank (copy) selected rows to clipboard') Sheet.addCommand('zy', 'copy-cell', 'copyCells(cursorCol, [cursorRow])', 'yank (copy) current cell to clipboard') Sheet.addCommand('zp', 'paste-cell', 'cursorCol.setValuesTyped([cursorRow], vd.clipcells[0]) if vd.clipcells else warning("no cells to paste")', 'set contents of current cell to last clipboard value') Sheet.addCommand('zd', 'delete-cell', 'vd.clipcells = [cursorDisplay]; cursorCol.setValues([cursorRow], None)', 'delete (cut) current cell and move it to clipboard') -Sheet.addCommand('gzd', 'delete-cells', 'vd.clipcells = list(vd.sheet.cursorCol.getDisplayValue(r) for r in someSelectedRows); cursorCol.setValues(someSelectedRows, None)', 'delete (cut) contents of current column for selected rows and move them to clipboard') +Sheet.addCommand('gzd', 'delete-cells', 'vd.clipcells = list(vd.sheet.cursorCol.getDisplayValue(r) for r in onlySelectedRows); cursorCol.setValues(onlySelectedRows, None)', 'delete (cut) contents of current column for selected rows and move them to clipboard') Sheet.bindkey('BUTTON2_PRESSED', 'go-mouse') Sheet.addCommand('BUTTON2_RELEASED', 'syspaste-cells', 'pasteFromClipboard(visibleCols[cursorVisibleColIndex:], rows[cursorRowIndex:])', 'paste into VisiData from system clipboard') Sheet.bindkey('BUTTON2_CLICKED', 'go-mouse') -Sheet.addCommand('gzy', 'copy-cells', 'copyCells(cursorCol, someSelectedRows)', 'yank (copy) contents of current column for selected rows to clipboard') -Sheet.addCommand('gzp', 'setcol-clipboard', 'for r, v in zip(someSelectedRows, itertools.cycle(vd.clipcells)): cursorCol.setValuesTyped([r], v)', 'set cells of current column for selected rows to last clipboard value') +Sheet.addCommand('gzy', 'copy-cells', 'copyCells(cursorCol, onlySelectedRows)', 'yank (copy) contents of current column for selected rows to clipboard') +Sheet.addCommand('gzp', 'setcol-clipboard', 'for r, v in zip(onlySelectedRows, itertools.cycle(vd.clipcells)): cursorCol.setValuesTyped([r], v)', 'set cells of current column for selected rows to last clipboard value') Sheet.addCommand('Y', 'syscopy-row', 'syscopyRows([cursorRow])', 'yank (copy) current row to system clipboard (using options.clipboard_copy_cmd)') -Sheet.addCommand('gY', 'syscopy-selected', 'syscopyRows(someSelectedRows)', 'yank (copy) selected rows to system clipboard (using options.clipboard_copy_cmd)') +Sheet.addCommand('gY', 'syscopy-selected', 'syscopyRows(onlySelectedRows)', 'yank (copy) selected rows to system clipboard (using options.clipboard_copy_cmd)') Sheet.addCommand('zY', 'syscopy-cell', 'syscopyCells(cursorCol, [cursorRow])', 'yank (copy) current cell to system clipboard (using options.clipboard_copy_cmd)') -Sheet.addCommand('gzY', 'syscopy-cells', 'syscopyCells(cursorCol, someSelectedRows)', 'yank (copy) contents of current column from selected rows to system clipboard (using options.clipboard_copy_cmd') +Sheet.addCommand('gzY', 'syscopy-cells', 'syscopyCells(cursorCol, onlySelectedRows)', 'yank (copy) contents of current column from selected rows to system clipboard (using options.clipboard_copy_cmd') Sheet.bindkey('KEY_DC', 'delete-cell'), Sheet.bindkey('gKEY_DC', 'delete-cells'), diff --git a/visidata/customdate.py b/visidata/customdate.py index cfc0f647f..c21856d58 100644 --- a/visidata/customdate.py +++ b/visidata/customdate.py @@ -17,4 +17,4 @@ def __new__(cls, *args, **kwargs): Sheet.addCommand('z@', 'type-customdate', 'cursorCol.type=cursorCol.type=customdate(input("date format: ", type="fmtstr"))', 'set type of current column to custom date format') -ColumnsSheet.addCommand('gz@', 'type-customdate-selected', 'someSelectedRows.type=customdate(input("date format: ", type="fmtstr"))', 'set type of selected columns to date') +ColumnsSheet.addCommand('gz@', 'type-customdate-selected', 'onlySelectedRows.type=customdate(input("date format: ", type="fmtstr"))', 'set type of selected columns to date') diff --git a/visidata/loaders/archive.py b/visidata/loaders/archive.py index f1620ba88..7ac9720ea 100644 --- a/visidata/loaders/archive.py +++ b/visidata/loaders/archive.py @@ -82,6 +82,6 @@ def iterload(self): ZipSheet.addCommand('x', 'extract-file', 'extract(cursorRow)') -ZipSheet.addCommand('gx', 'extract-selected', 'extract(*someSelectedRows)') +ZipSheet.addCommand('gx', 'extract-selected', 'extract(*onlySelectedRows)') ZipSheet.addCommand('zx', 'extract-file-to', 'extract(cursorRow, path=inputPath("extract to: "))') -ZipSheet.addCommand('gzx', 'extract-selected-to', 'extract(*someSelectedRows, path=inputPath("extract %d files to: " % nSelected))') +ZipSheet.addCommand('gzx', 'extract-selected-to', 'extract(*onlySelectedRows, path=inputPath("extract %d files to: " % nSelected))') diff --git a/visidata/metasheets.py b/visidata/metasheets.py index d4a4af784..87ac1396e 100644 --- a/visidata/metasheets.py +++ b/visidata/metasheets.py @@ -164,7 +164,7 @@ def allColumnsSheet(vd): @ColumnsSheet.command('&', 'join-cols', 'add column from concatenating selected source columns') def join_cols(sheet): - cols = sheet.someSelectedRows + cols = sheet.onlySelectedRows destSheet = cols[0].sheet if len(set(c.sheet for c in cols)) > 1: @@ -187,19 +187,19 @@ def join_cols(sheet): Sheet.addCommand('C', 'columns-sheet', 'vd.push(ColumnsSheet(name+"_columns", source=[sheet]))', 'open Columns Sheet: edit column properties for current sheet') # used ColumnsSheet, affecting the 'row' (source column) -ColumnsSheet.addCommand('g!', 'key-selected', 'for c in someSelectedRows: c.sheet.setKeys([c])', 'toggle selected rows as key columns on source sheet') -ColumnsSheet.addCommand('gz!', 'key-off-selected', 'for c in someSelectedRows: c.sheet.unsetKeys([c])', 'unset selected rows as key columns on source sheet') +ColumnsSheet.addCommand('g!', 'key-selected', 'for c in onlySelectedRows: c.sheet.setKeys([c])', 'toggle selected rows as key columns on source sheet') +ColumnsSheet.addCommand('gz!', 'key-off-selected', 'for c in onlySelectedRows: c.sheet.unsetKeys([c])', 'unset selected rows as key columns on source sheet') -ColumnsSheet.addCommand('g-', 'hide-selected', 'someSelectedRows.hide()', 'hide selected columns on source sheet') +ColumnsSheet.addCommand('g-', 'hide-selected', 'onlySelectedRows.hide()', 'hide selected columns on source sheet') ColumnsSheet.addCommand(None, 'resize-source-rows-max', 'for c in selectedRows or [cursorRow]: c.setWidth(c.getMaxWidth(c.sheet.visibleRows))', 'adjust widths of selected source columns') -ColumnsSheet.addCommand('g%', 'type-float-selected', 'someSelectedRows.type=float', 'set type of selected columns to float') -ColumnsSheet.addCommand('g#', 'type-int-selected', 'someSelectedRows.type=int', 'set type of selected columns to int') -ColumnsSheet.addCommand('gz#', 'type-len-selected', 'someSelectedRows.type=vlen', 'set type of selected columns to len') -ColumnsSheet.addCommand('g@', 'type-date-selected', 'someSelectedRows.type=date', 'set type of selected columns to date') -ColumnsSheet.addCommand('g$', 'type-currency-selected', 'someSelectedRows.type=currency', 'set type of selected columns to currency') -ColumnsSheet.addCommand('g~', 'type-string-selected', 'someSelectedRows.type=str', 'set type of selected columns to str') -ColumnsSheet.addCommand('gz~', 'type-any-selected', 'someSelectedRows.type=anytype', 'set type of selected columns to anytype') +ColumnsSheet.addCommand('g%', 'type-float-selected', 'onlySelectedRows.type=float', 'set type of selected columns to float') +ColumnsSheet.addCommand('g#', 'type-int-selected', 'onlySelectedRows.type=int', 'set type of selected columns to int') +ColumnsSheet.addCommand('gz#', 'type-len-selected', 'onlySelectedRows.type=vlen', 'set type of selected columns to len') +ColumnsSheet.addCommand('g@', 'type-date-selected', 'onlySelectedRows.type=date', 'set type of selected columns to date') +ColumnsSheet.addCommand('g$', 'type-currency-selected', 'onlySelectedRows.type=currency', 'set type of selected columns to currency') +ColumnsSheet.addCommand('g~', 'type-string-selected', 'onlySelectedRows.type=str', 'set type of selected columns to str') +ColumnsSheet.addCommand('gz~', 'type-any-selected', 'onlySelectedRows.type=anytype', 'set type of selected columns to anytype') OptionsSheet.addCommand('d', 'unset-option', 'options.unset(cursorRow.name, str(source))', 'remove option override for this context') OptionsSheet.addCommand(None, 'edit-option', 'editOption(cursorRow)', 'edit option at current row') From 5b6cd8129481ccee12fe5d5a08e95dec4d5d6cfe Mon Sep 17 00:00:00 2001 From: anjakefala Date: Sun, 29 Nov 2020 14:23:00 -0800 Subject: [PATCH 129/162] [select] add new someSelectRows and options.selected_or_rows, if True, if no rows selected, returns all rows Closes #767 --- docs/api/data.rst | 1 + visidata/regex.py | 4 ++-- visidata/selection.py | 15 ++++++++++++++- 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/docs/api/data.rst b/docs/api/data.rst index 5205f1050..61ca4e179 100644 --- a/docs/api/data.rst +++ b/docs/api/data.rst @@ -4,6 +4,7 @@ Selected Rows Each TableSheet has a set of *selected rows*, which is a strict subset of the rows on the sheet. .. autoattribute:: visidata.TableSheet.selectedRows +.. autoattribute:: visidata.TableSheet.someSelectedRows .. autoattribute:: visidata.TableSheet.onlySelectedRows .. autoattribute:: visidata.TableSheet.nSelectedRows diff --git a/visidata/regex.py b/visidata/regex.py index ecf76d83a..e7bb0c615 100644 --- a/visidata/regex.py +++ b/visidata/regex.py @@ -113,5 +113,5 @@ def regex_flags(sheet): Sheet.addCommand(':', 'split-col', 'addRegexColumns(makeRegexSplitter, sheet, cursorCol, input("split regex: ", type="regex-split"))', 'add new columns from regex split; number of columns determined by example row at cursor') Sheet.addCommand(';', 'capture-col', 'addRegexColumns(makeRegexMatcher, sheet, cursorCol, input("match regex: ", type="regex-capture"))', 'add new column from capture groups of regex; requires example row') Sheet.addCommand('*', 'addcol-subst', 'addColumnAtCursor(Column(cursorCol.name + "_re", getter=regexTransform(cursorCol, input("transform column by regex: ", type="regex-subst"))))', 'add column derived from current column, replacing regex with subst (may include \1 backrefs)') -Sheet.addCommand('g*', 'setcol-subst', 'setSubst([cursorCol], selectedRows)', 'regex/subst - modify selected rows in current column, replacing regex with subst, (may include backreferences \\1 etc)') -Sheet.addCommand('gz*', 'setcol-subst-all', 'setSubst(visibleCols, selectedRows)', 'modify selected rows in all visible columns, replacing regex with subst (may include \\1 backrefs)') +Sheet.addCommand('g*', 'setcol-subst', 'setSubst([cursorCol], someSelectedRows)', 'regex/subst - modify selected rows in current column, replacing regex with subst, (may include backreferences \\1 etc)') +Sheet.addCommand('gz*', 'setcol-subst-all', 'setSubst(visibleCols, someSelectedRows)', 'modify selected rows in all visible columns, replacing regex with subst (may include \\1 backrefs)') diff --git a/visidata/selection.py b/visidata/selection.py index fc0ff8c73..0d0e07361 100644 --- a/visidata/selection.py +++ b/visidata/selection.py @@ -1,5 +1,6 @@ from visidata import vd, Sheet, Progress, option, asyncthread, options, rotateRange, Fanout, undoAttrCopyFunc, copy option('bulk_select_clear', False, 'clear selected rows before new bulk selections', replay=True) +option('some_selected_rows', False, 'if no rows selected, if True, someSelectedRows returns all rows; if False, fails') Sheet.init('_selectedRows', dict) # rowid(row) -> row @@ -94,12 +95,24 @@ def selectedRows(self): return Fanout((r for r in self.rows if self.rowid(r) in self._selectedRows)) @Sheet.property -def someSelectedRows(self): +def onlySelectedRows(self): 'List of selected rows in sheet order. Fail if no rows are selected.' if self.nSelectedRows == 0: vd.fail('no rows selected') return self.selectedRows +@Sheet.property +def someSelectedRows(self): + '''Return a list of rows: + (a) in batch mode, always return selectedRows + (b) in interactive mode, if options.some_selected_rows is True, return selectedRows or all rows if none selected + (c) in interactive mode, if options.some_selected_rows is False, return selectedRows or fail if none selected''' + if options.batch: + return self.selectedRows + if options.some_selected_rows: + return self.selectedRows or self.rows + return self.onlySelectedRows + @Sheet.property def nSelectedRows(self): 'Number of selected rows.' From d9c760f20263a90a9b75f27117bd6ad7cfe35ddb Mon Sep 17 00:00:00 2001 From: Saul Pwanson Date: Mon, 30 Nov 2020 00:30:50 -0800 Subject: [PATCH 130/162] [disp] show ellipsis on left side with non-zero hoffset #751 --- visidata/sheets.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/visidata/sheets.py b/visidata/sheets.py index 6898fc134..888cf74c6 100644 --- a/visidata/sheets.py +++ b/visidata/sheets.py @@ -679,6 +679,7 @@ def draw(self, scr): 'colsep': options.disp_column_sep, 'keysep': options.disp_keycol_sep, 'selectednote': options.disp_selected_note, + 'disp_truncator': options.disp_truncator, } self._rowLayout = {} # [rowidx] -> (y, height) @@ -723,9 +724,9 @@ def drawRow(self, scr, row, rowidx, ybase, rowcattr: ColorAttr, maxheight, endbotsep='', colsep='', keysep='', - selectednote='' + selectednote='', + disp_truncator='' ): - # sepattr is the attr between cell/columns sepcattr = update_attr(rowcattr, colors.color_column_sep, 1) @@ -833,7 +834,8 @@ def drawRow(self, scr, row, rowidx, ybase, rowcattr: ColorAttr, maxheight, else: sepchars = midsep - clipdraw(scr, y, x, (disp_column_fill if colwidth > 2 else '')+line[hoffset:], cattr.attr, w=colwidth-(1 if note else 0)) + pre = disp_truncator if hoffset != 0 else disp_column_fill + clipdraw(scr, y, x, (pre if colwidth > 2 else '')+line[hoffset:], cattr.attr, w=colwidth-(1 if note else 0)) vd.onMouse(scr, y, x, 1, colwidth, BUTTON3_RELEASED='edit-cell') if x+colwidth+len(sepchars) <= self.windowWidth: From 96548d1e0c4bc4ab4899ae6d0e17cac469eda779 Mon Sep 17 00:00:00 2001 From: Saul Pwanson Date: Mon, 30 Nov 2020 22:45:49 -0800 Subject: [PATCH 131/162] [movement] fix scroll-cells; add Column.visibleWidth #762 --- visidata/movement.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/visidata/movement.py b/visidata/movement.py index d035ab948..cb28547be 100644 --- a/visidata/movement.py +++ b/visidata/movement.py @@ -87,6 +87,13 @@ def nextColRegex(sheet, colregex): vd.fail('no column name matches /%s/' % colregex) + +@Column.property +def visibleWidth(self): + vcolidx = self.sheet.visibleCols.index(self) + return self.sheet._visibleColLayout[vcolidx][1] + + Sheet.addCommand(None, 'go-left', 'cursorRight(-1)', 'go left'), Sheet.addCommand(None, 'go-down', 'cursorDown(+1)', 'go down'), Sheet.addCommand(None, 'go-up', 'cursorDown(-1)', 'go up'), @@ -160,9 +167,9 @@ def go_mouse(sheet): Sheet.addCommand(None, 'scroll-leftmost', 'sheet.leftVisibleColIndex = cursorVisibleColIndex', 'scroll sheet to leftmost column') Sheet.addCommand(None, 'scroll-rightmost', 'tmp = cursorVisibleColIndex; pageLeft(); sheet.cursorVisibleColIndex = tmp', 'scroll sheet to rightmost column') -Sheet.addCommand('zl', 'scroll-cells-right', 'cursorCol.hoffset += cursorCol.width-2', 'scroll display of current column to the right') -Sheet.addCommand('zh', 'scroll-cells-left', 'cursorCol.hoffset -= cursorCol.width-2', 'scroll display of current column to the left') -Sheet.addCommand('gzl', 'scroll-cells-rightmost', 'cursorCol.hoffset = -cursorCol.width+1', 'scroll display of current column to the end') +Sheet.addCommand('zl', 'scroll-cells-right', 'cursorCol.hoffset += cursorCol.visibleWidth-2', 'scroll display of current column to the right') +Sheet.addCommand('zh', 'scroll-cells-left', 'cursorCol.hoffset -= cursorCol.visibleWidth-2', 'scroll display of current column to the left') +Sheet.addCommand('gzl', 'scroll-cells-rightmost', 'cursorCol.hoffset = -cursorCol.visibleWidth+2', 'scroll display of current column to the end') Sheet.addCommand('gzh', 'scroll-cells-leftmost', 'cursorCol.hoffset = 0', 'scroll display of current column to the beginning') Sheet.addCommand('zj', 'scroll-cells-down', 'cursorCol.voffset += 1', 'scroll display of current column down one line') From 37d6eddb3dcf0436997b7b342068f95199546374 Mon Sep 17 00:00:00 2001 From: Saul Pwanson Date: Tue, 1 Dec 2020 00:15:04 -0800 Subject: [PATCH 132/162] [expr-] fix with e3a6d5 #659 --- visidata/sheets.py | 1 + 1 file changed, 1 insertion(+) diff --git a/visidata/sheets.py b/visidata/sheets.py index 888cf74c6..38feb280d 100644 --- a/visidata/sheets.py +++ b/visidata/sheets.py @@ -97,6 +97,7 @@ class LazyComputeRow: 'Calculate column values as needed.' def __init__(self, sheet, row, col=None): self.row = row + self.col = col self.sheet = sheet if not hasattr(self.sheet, '_lcm'): self.sheet._lcm = LazyChainMap(sheet, vd, col) From f0ae2404be1b5fadb846fca2d25efa3f22eafde4 Mon Sep 17 00:00:00 2001 From: Saul Pwanson Date: Tue, 1 Dec 2020 00:15:20 -0800 Subject: [PATCH 133/162] [expr] never include computing column #756 - only checks for self-reference; 2+ cycles still raises RecursionException --- visidata/sheets.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/visidata/sheets.py b/visidata/sheets.py index 38feb280d..23836f9d9 100644 --- a/visidata/sheets.py +++ b/visidata/sheets.py @@ -123,6 +123,10 @@ def __getitem__(self, colid): try: i = self._keys.index(colid) c = self.sheet.columns[i] + if c is self.col: + j = self._keys[i+1:].index(colid) + c = self.sheet.columns[i+j+1] + except ValueError: try: c = self.sheet._lcm[colid] From 8a77d22818d7e0f994c5297b32dd021b042faf5a Mon Sep 17 00:00:00 2001 From: Saul Pwanson Date: Tue, 1 Dec 2020 00:35:26 -0800 Subject: [PATCH 134/162] [json] 50% loading speedup #765 - This makes JSON saving non-deterministic in Python 3.6, as the order of fields output is dependent on the order within the dict (this is the default behavior for dicts in Python3.7+). - Thanks to @lxcode for investigating and pointing this out! --- visidata/loaders/json.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/visidata/loaders/json.py b/visidata/loaders/json.py index f66bea967..6718b8ed6 100644 --- a/visidata/loaders/json.py +++ b/visidata/loaders/json.py @@ -28,7 +28,7 @@ def iterload(self): try: if L.startswith('#'): # skip commented lines continue - ret = json.loads(L, object_pairs_hook=OrderedDict) + ret = json.loads(L) if isinstance(ret, list): yield from Progress(ret) else: @@ -40,7 +40,7 @@ def iterload(self): yield TypedExceptionWrapper(json.loads, L, exception=e) # an error on one line else: with self.source.open_text() as fp: - ret = json.load(fp, object_pairs_hook=OrderedDict) + ret = json.load(fp) if isinstance(ret, list): yield from Progress(ret) else: From 36e7d7fb4910bdcbe6832a257fa0f4b4f9870570 Mon Sep 17 00:00:00 2001 From: Saul Pwanson Date: Tue, 1 Dec 2020 18:43:00 -0800 Subject: [PATCH 135/162] [frictionless] assume JSON if no format #803 From https://specs.frictionlessdata.io/data-resource/#data-location): > a consumer of resource object MAY assume if no format or mediatype property is provided that the data is JSON and attempt to process it as such. --- visidata/loaders/frictionless.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/visidata/loaders/frictionless.py b/visidata/loaders/frictionless.py index d38af9f9c..a1380a686 100644 --- a/visidata/loaders/frictionless.py +++ b/visidata/loaders/frictionless.py @@ -8,4 +8,4 @@ def iterload(self): import datapackage self.dp = datapackage.Package(self.source.open_text()) for r in Progress(self.dp.resources): - yield vd.openSource(self.source.with_name(r.descriptor['path']), filetype=r.descriptor['format']) + yield vd.openSource(self.source.with_name(r.descriptor['path']), filetype=r.descriptor.get('format', 'json')) From 03b3e347df96be1ffc797ba49af9f6c30ba102df Mon Sep 17 00:00:00 2001 From: anjakefala Date: Tue, 1 Dec 2020 18:58:43 -0800 Subject: [PATCH 136/162] [plugins] only reload plugins sheet, if not already loaded Closes #818 --- visidata/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/visidata/main.py b/visidata/main.py index 37be15d24..fd1ad8271 100755 --- a/visidata/main.py +++ b/visidata/main.py @@ -188,7 +188,7 @@ def main_vd(): options.set(k, v, obj='override') # fetch motd and plugins *after* options parsing/setting - visidata.PluginsSheet().reload() + vd.pluginsSheet.ensureLoaded() domotd() if args.batch: From 8ed0434f50c2bc5f61e8dbf16dcd53690599c8cc Mon Sep 17 00:00:00 2001 From: anjakefala Date: Tue, 1 Dec 2020 23:46:42 -0800 Subject: [PATCH 137/162] [docs] fix typo in top-level functions --- docs/api/index.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/api/index.rst b/docs/api/index.rst index 4add3d32b..d2c67b5ef 100644 --- a/docs/api/index.rst +++ b/docs/api/index.rst @@ -51,6 +51,6 @@ This extends VisiData so that when :kbd:`1` is pressed on any sheet, the command Toplevel functions =================== -.. autofunction visidata.vd.view -.. autofunction visidata.vd.view_pandas -.. autofunction visidata.vd.run +.. autofunction:: visidata.vd.view +.. autofunction:: visidata.vd.view_pandas +.. autofunction:: visidata.vd.run From 30e6797c159fa162d7aceb036b727a965ff6049a Mon Sep 17 00:00:00 2001 From: anjakefala Date: Wed, 2 Dec 2020 19:40:14 -0800 Subject: [PATCH 138/162] [edit-cell-] specify None; rows can be [] and {} which are false-y --- visidata/sheets.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/visidata/sheets.py b/visidata/sheets.py index 23836f9d9..f83a6f0d7 100644 --- a/visidata/sheets.py +++ b/visidata/sheets.py @@ -1056,7 +1056,7 @@ def updateColNames(sheet, rows, cols, overwrite=False): Sheet.addCommand('!', 'key-col', 'toggleKeys([cursorCol])', 'toggle current column as a key column') Sheet.addCommand('z!', 'key-col-off', 'unsetKeys([cursorCol])', 'unset current column as a key column') -Sheet.addCommand('e', 'edit-cell', 'cursorCol.setValues([cursorRow], editCell(cursorVisibleColIndex)) if cursorRow else fail("no rows to edit")', 'edit contents of current cell') +Sheet.addCommand('e', 'edit-cell', 'cursorCol.setValues([cursorRow], editCell(cursorVisibleColIndex)) if not (cursorRow is None) else fail("no rows to edit")', 'edit contents of current cell') Sheet.addCommand('ge', 'setcol-input', 'cursorCol.setValuesTyped(selectedRows, input("set selected to: ", value=cursorDisplay))', 'set contents of current column for selected rows to same input') Sheet.addCommand('"', 'dup-selected', 'vs=copy(sheet); vs.name += "_selectedref"; vs.reload=lambda vs=vs,rows=selectedRows: setattr(vs, "rows", list(rows)); vd.push(vs)', 'open duplicate sheet with only selected rows'), From 39b67d6b1f05eea4e6236176e2bb02b9228e2c3f Mon Sep 17 00:00:00 2001 From: anjakefala Date: Wed, 2 Dec 2020 22:20:01 -0800 Subject: [PATCH 139/162] [evalExpr] cache LazyComputeRow for each cell, instead of for each row necessary, bc LazyComputeRow is needed to keep track of the identity of the computing column f0ae2404be1b5fadb846fca2d25efa3f22eafde4 --- visidata/sheets.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/visidata/sheets.py b/visidata/sheets.py index f83a6f0d7..fc281e794 100644 --- a/visidata/sheets.py +++ b/visidata/sheets.py @@ -333,7 +333,7 @@ def __repr__(self): def evalExpr(self, expr, row=None, col=None): if row: # contexts are cached by sheet/rowid for duration of drawcycle - contexts = vd._evalcontexts.setdefault((self, self.rowid(row)), LazyComputeRow(self, row, col=col)) + contexts = vd._evalcontexts.setdefault((self, self.rowid(row), col), LazyComputeRow(self, row, col=col)) else: contexts = None From 4ef72846c31e076198b2b7551c3d0e4206eee9ff Mon Sep 17 00:00:00 2001 From: anjakefala Date: Wed, 2 Dec 2020 22:32:07 -0800 Subject: [PATCH 140/162] [expr] pass column identity to all evalExpr (g=, gz=) --- visidata/expr.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/visidata/expr.py b/visidata/expr.py index f4449cb4a..2e61009ff 100644 --- a/visidata/expr.py +++ b/visidata/expr.py @@ -36,7 +36,7 @@ def setValuesFromExpr(self, rows, expr): compiledExpr = compile(expr, '', 'eval') vd.addUndoSetValues([self], rows) for row in Progress(rows, 'setting'): - self.setValueSafe(row, self.sheet.evalExpr(compiledExpr, row)) + self.setValueSafe(row, self.sheet.evalExpr(compiledExpr, row, col=self)) self.recalc() vd.status('set %d values = %s' % (len(rows), expr)) @@ -48,7 +48,7 @@ def inputExpr(self, prompt, *args, **kwargs): Sheet.addCommand('=', 'addcol-expr', 'addColumnAtCursor(ExprColumn(inputExpr("new column expr="), curcol=cursorCol))', 'create new column from Python expression, with column names as variables') Sheet.addCommand('g=', 'setcol-expr', 'cursorCol.setValuesFromExpr(selectedRows, inputExpr("set selected="))', 'set current column for selected rows to result of Python expression') -Sheet.addCommand('z=', 'setcell-expr', 'cursorCol.setValues([cursorRow], evalExpr(inputExpr("set expr="), cursorRow))', 'evaluate Python expression on current row and set current cell with result of Python expression') +Sheet.addCommand('z=', 'setcell-expr', 'cursorCol.setValues([cursorRow], evalExpr(inputExpr("set expr="), cursorRow, col=cursorCol))', 'evaluate Python expression on current row and set current cell with result of Python expression') Sheet.addCommand('gz=', 'setcol-iter', 'cursorCol.setValues(selectedRows, *list(itertools.islice(eval(input("set column= ", "expr", completer=CompleteExpr())), len(selectedRows))))', 'set current column for selected rows to the items in result of Python sequence expression') Sheet.addCommand(None, 'show-expr', 'status(evalExpr(inputExpr("show expr="), cursorRow))', 'evaluate Python expression on current row and show result on status line') From 955b8cc8bbcf54e9e00320b47507d70f9e7aaae2 Mon Sep 17 00:00:00 2001 From: anjakefala Date: Wed, 2 Dec 2020 22:32:50 -0800 Subject: [PATCH 141/162] [expr] have setcol-expr and setcol-iter process someSelectedRows --- visidata/expr.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/visidata/expr.py b/visidata/expr.py index 2e61009ff..d49480462 100644 --- a/visidata/expr.py +++ b/visidata/expr.py @@ -47,8 +47,8 @@ def inputExpr(self, prompt, *args, **kwargs): Sheet.addCommand('=', 'addcol-expr', 'addColumnAtCursor(ExprColumn(inputExpr("new column expr="), curcol=cursorCol))', 'create new column from Python expression, with column names as variables') -Sheet.addCommand('g=', 'setcol-expr', 'cursorCol.setValuesFromExpr(selectedRows, inputExpr("set selected="))', 'set current column for selected rows to result of Python expression') +Sheet.addCommand('g=', 'setcol-expr', 'cursorCol.setValuesFromExpr(someSelectedRows, inputExpr("set selected="))', 'set current column for selected rows to result of Python expression') Sheet.addCommand('z=', 'setcell-expr', 'cursorCol.setValues([cursorRow], evalExpr(inputExpr("set expr="), cursorRow, col=cursorCol))', 'evaluate Python expression on current row and set current cell with result of Python expression') -Sheet.addCommand('gz=', 'setcol-iter', 'cursorCol.setValues(selectedRows, *list(itertools.islice(eval(input("set column= ", "expr", completer=CompleteExpr())), len(selectedRows))))', 'set current column for selected rows to the items in result of Python sequence expression') +Sheet.addCommand('gz=', 'setcol-iter', 'cursorCol.setValues(someSelectedRows, *list(itertools.islice(eval(input("set column= ", "expr", completer=CompleteExpr())), len(someSelectedRows))))', 'set current column for selected rows to the items in result of Python sequence expression') Sheet.addCommand(None, 'show-expr', 'status(evalExpr(inputExpr("show expr="), cursorRow))', 'evaluate Python expression on current row and show result on status line') From faf9beb8d3c83dde01fd1f1162e6d79061d1cf5d Mon Sep 17 00:00:00 2001 From: anjakefala Date: Thu, 3 Dec 2020 20:36:22 -0800 Subject: [PATCH 142/162] [plugins] remove livesearch --- plugins/plugins.jsonl | 1 - 1 file changed, 1 deletion(-) diff --git a/plugins/plugins.jsonl b/plugins/plugins.jsonl index 7eadc832c..80cc6ab46 100644 --- a/plugins/plugins.jsonl +++ b/plugins/plugins.jsonl @@ -2,7 +2,6 @@ {"name": "dedupe", "description": "adds commands for selection and removal of rows which are duplicates of prior rows", "maintainer": "Jeremy Singer-Vine @jsvine", "latest_release": "2020-10-11", "url": "https://raw.githubusercontent.com/jsvine/visidata-plugins/37cba82ed2cc49aedc1a377af984acfe1ef3d5cd/plugins/dedupe.py", "latest_ver": "0.1.0", "visidata_ver": "2.0", "pydeps": "", "vdplugindeps": "", "sha256": "2e077b8d62bc8ec2235d22d7d9711d99e7366c8c0ff20405f70647044cca67a1"} {"name": "normcol", "description": "normalizes column names in any given sheet so that the names are unique, valid Python identifiers, and only composed of lowercase letters, numbers, and underscores", "maintainer": "Jeremy Singer-Vine @jsvine", "latest_release": "2020-10-11", "url": "https://raw.githubusercontent.com/jsvine/visidata-plugins/37cba82ed2cc49aedc1a377af984acfe1ef3d5cd/plugins/normcol.py", "latest_ver": "0.1.0", "visidata_ver": "2.0", "pydeps": "", "vdplugindeps": "", "sha256": "ede6b9e508e1ce7842e00d308c372837c9b4d4906b88e0ba62efd5b481f816f6"} {"name": "fec", "description": "loader for .fec files from the Federal Election Commission", "maintainer": "Jeremy Singer-Vine @jsvine", "latest_release": "2019-04-21", "url": "https://raw.githubusercontent.com/jsvine/visidata-plugins/37cba82ed2cc49aedc1a377af984acfe1ef3d5cd/plugins/fec.py", "latest_ver": "0.0.0", "visidata_ver": "1.5.2", "pydeps": "fecfile", "vdplugindeps": "", "sha256": "e917170e5bb74ac6242f81113aa48ba7e36763ad32cb190420acb93c3f62765c"} -{"name": "livesearch", "description": "filter rows as you search", "maintainer": "Saul Pwanson @saulpw", "latest_release": "2019-06-09", "url": "https://raw.githubusercontent.com/saulpw/visidata/a185cb5f734a58d14478cf13477246c7773245f8/plugins/livesearch.py", "latest_ver": "0.9", "visidata_ver": "2.-1", "pydeps": "", "vdplugindeps": "", "sha256": "57ba2685252f0a4659d2582e05e7d9bab9e680314883328a202ed220dbd0bb6f"} {"name": "sparkline", "description": "add a sparkline column to visualise trends of numeric cells in a row", "maintainer": "Lucas Messenger @layertwo", "latest_release": "2020-09-13", "url": "https://raw.githubusercontent.com/saulpw/visidata/e4a373cd74b49b2b91876b17d7a1cd0cd0303848/plugins/sparkline.py", "latest_ver": "0.1", "visidata_ver": "2.0", "pydeps": "", "vdplugindeps": "", "sha256": "4256bea5535405a2478cb6827df4be86f28c69b3cb7f30cc08e2cd655e48b592"} {"name": "rownum", "description": "add column of original row ordering", "maintainer": "Saul Pwanson @saulpw", "latest_release": "2019-11-07", "url": "https://raw.githubusercontent.com/saulpw/visidata/a185cb5f734a58d14478cf13477246c7773245f8/plugins/rownum.py", "latest_ver": "0.9", "visidata_ver": "2.0", "pydeps": "", "vdplugindeps": "", "sha256": "caf896841639ecdef25c2f227f3370a4c4f0b0b31bdd72ef422dfe94ec5664f0"} {"name": "vmailcap", "description": "add mailcap-view(-selected) commands to DirSheet", "maintainer": "Saul Pwanson @saulpw", "latest_release": "2020-10-06", "url": "https://raw.githubusercontent.com/saulpw/visidata/0d8af5a226818aa2f99a2f74e08543152b22a36d/plugins/vmailcap.py", "latest_ver": "0.9", "visidata_ver": "2.0", "pydeps": "", "vdplugindeps": "", "sha256": "b8aa66821dfa38107ee45fe49a5262e766cc23a0ff362ce5aea5aa6e9b8e6fb0"} From 9eee3e6097a2594fc6c4d91d3b1c4c5843ea77e2 Mon Sep 17 00:00:00 2001 From: Saul Pwanson Date: Wed, 2 Dec 2020 00:32:26 -0800 Subject: [PATCH 143/162] [scroll] zj/zk do nothing in single-line mode - per @jsvine --- visidata/movement.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/visidata/movement.py b/visidata/movement.py index cb28547be..38b6101b8 100644 --- a/visidata/movement.py +++ b/visidata/movement.py @@ -172,8 +172,8 @@ def go_mouse(sheet): Sheet.addCommand('gzl', 'scroll-cells-rightmost', 'cursorCol.hoffset = -cursorCol.visibleWidth+2', 'scroll display of current column to the end') Sheet.addCommand('gzh', 'scroll-cells-leftmost', 'cursorCol.hoffset = 0', 'scroll display of current column to the beginning') -Sheet.addCommand('zj', 'scroll-cells-down', 'cursorCol.voffset += 1', 'scroll display of current column down one line') -Sheet.addCommand('zk', 'scroll-cells-up', 'cursorCol.voffset -= 1', 'scroll display of current column up one line') +Sheet.addCommand('zj', 'scroll-cells-down', 'cursorCol.voffset += 1 if cursorCol.height > 1 else fail("multiline column needed for scrolling")', 'scroll display of current column down one line') +Sheet.addCommand('zk', 'scroll-cells-up', 'cursorCol.voffset -= 1 if cursorCol.height > 1 else fail("multiline column needed for scrolling")', 'scroll display of current column up one line') Sheet.addCommand('gzj', 'scroll-cells-bottom', 'cursorCol.voffset = -1', 'scroll display of current column to the bottom') Sheet.addCommand('gzk', 'scroll-cells-top', 'cursorCol.voffset = 0', 'scroll display of current column to the top') From c9e8f5edc5044293ca4931de3d976ef10d8e2907 Mon Sep 17 00:00:00 2001 From: Saul Pwanson Date: Thu, 3 Dec 2020 21:23:09 -0800 Subject: [PATCH 144/162] [draw-] fix display for off-screen cursor with multiline rows --- visidata/movement.py | 4 +- visidata/sheets.py | 92 +++++++++++++++++++++++++------------------- 2 files changed, 54 insertions(+), 42 deletions(-) diff --git a/visidata/movement.py b/visidata/movement.py index 38b6101b8..04858a86c 100644 --- a/visidata/movement.py +++ b/visidata/movement.py @@ -98,8 +98,8 @@ def visibleWidth(self): Sheet.addCommand(None, 'go-down', 'cursorDown(+1)', 'go down'), Sheet.addCommand(None, 'go-up', 'cursorDown(-1)', 'go up'), Sheet.addCommand(None, 'go-right', 'cursorRight(+1)', 'go right'), -Sheet.addCommand(None, 'go-pagedown', 'cursorDown(nScreenRows); sheet.topRowIndex += nScreenRows', 'scroll one page forward'), -Sheet.addCommand(None, 'go-pageup', 'cursorDown(-nScreenRows); sheet.topRowIndex -= nScreenRows', 'scroll one page backward'), +Sheet.addCommand(None, 'go-pagedown', 'cursorDown(bottomRowIndex-topRowIndex); sheet.topRowIndex = bottomRowIndex', 'scroll one page forward'), +Sheet.addCommand(None, 'go-pageup', 'cursorDown(topRowIndex-bottomRowIndex); sheet.bottomRowIndex = topRowIndex', 'scroll one page backward'), Sheet.addCommand(None, 'go-leftmost', 'sheet.cursorVisibleColIndex = sheet.leftVisibleColIndex = 0', 'go all the way to the left of sheet'), Sheet.addCommand(None, 'go-top', 'sheet.cursorRowIndex = sheet.topRowIndex = 0', 'go all the way to the top of sheet'), diff --git a/visidata/sheets.py b/visidata/sheets.py index fc281e794..c578a4105 100644 --- a/visidata/sheets.py +++ b/visidata/sheets.py @@ -319,7 +319,19 @@ def __copy__(self): @property def bottomRowIndex(self): - return max(self._rowLayout.keys()) + return max(self._rowLayout.keys()) if self._rowLayout else self.topRowIndex+self.nScreenRows + + @bottomRowIndex.setter + def bottomRowIndex(self, newidx): + 'Set topRowIndex, by getting height of *newidx* row and going backwards until more than nScreenRows is allocated.' + nrows = 0 + i = 0 + while nrows < self.nScreenRows and newidx-i >= 0: + h = self.calc_height(self.rows[newidx-i]) + nrows += h + i += 1 + + self._topRowIndex = newidx-i+2 def __deepcopy__(self, memo): 'same as __copy__' @@ -535,14 +547,10 @@ def checkCursor(self): # check bounds, scroll if necessary if self.topRowIndex > self.cursorRowIndex: self.topRowIndex = self.cursorRowIndex - else: - if self.topRowIndex < self.cursorRowIndex-self.nScreenRows+1: - self.topRowIndex = self.cursorRowIndex-self.nScreenRows+1 - elif self._rowLayout: # only check this if topRowIndex has not been set (which clears _rowLayout) - bottomRowIndex = self.bottomRowIndex - y, h = self._rowLayout[bottomRowIndex] - if self.cursorRowIndex > bottomRowIndex and y+h > self.nScreenRows: - self._topRowIndex += bottomRowIndex-self.cursorRowIndex+2 + elif self.bottomRowIndex < self.cursorRowIndex: + self.bottomRowIndex = self.cursorRowIndex + elif self.bottomRowIndex == self.cursorRowIndex and self._rowLayout and self._rowLayout[self.bottomRowIndex][1] > 1: + self.bottomRowIndex = self.cursorRowIndex if self.cursorCol and self.cursorCol.keycol: return @@ -708,41 +716,16 @@ def draw(self, scr): rowcattr = self._colorize(None, row) - y += self.drawRow(scr, row, self.topRowIndex+rowidx, y, rowcattr, maxheight=self.windowHeight-y, **drawparams) + y += self.drawRow(scr, row, self.topRowIndex+rowidx, y, rowcattr, maxheight=self.windowHeight-y-1, **drawparams) if vcolidx+1 < self.nVisibleCols: scr.addstr(headerRow, self.windowWidth-2, options.disp_more_right, colors.color_column_sep) scr.refresh() - def drawRow(self, scr, row, rowidx, ybase, rowcattr: ColorAttr, maxheight, - isNull='', - topsep='', - midsep='', - botsep='', - endsep='', - keytopsep='', - keymidsep='', - keybotsep='', - endtopsep='', - endmidsep='', - endbotsep='', - colsep='', - keysep='', - selectednote='', - disp_truncator='' - ): - # sepattr is the attr between cell/columns - sepcattr = update_attr(rowcattr, colors.color_column_sep, 1) - - # apply current row here instead of in a colorizer, because it needs to know dispRowIndex - if rowidx == self.cursorRowIndex: - color_current_row = colors.get_color('color_current_row', 5) - basecellcattr = sepcattr = update_attr(rowcattr, color_current_row) - else: - basecellcattr = rowcattr - - displines = {} # [vcolidx] -> list of lines in that cell + def calc_height(self, row, displines=None, isNull=None): + if displines is None: + displines = {} # [vcolidx] -> list of lines in that cell for vcolidx, (x, colwidth) in sorted(self._visibleColLayout.items()): if x < self.windowWidth: # only draw inside window @@ -755,7 +738,7 @@ def drawRow(self, scr, row, rowidx, ybase, rowcattr: ColorAttr, maxheight, cellval.display = cellval.display.rjust(colwidth-2) try: - if isNull(cellval.value): + if isNull and isNull(cellval.value): cellval.note = options.disp_note_none cellval.notecolor = 'color_note_type' except (TypeError, ValueError): @@ -772,8 +755,37 @@ def drawRow(self, scr, row, rowidx, ybase, rowcattr: ColorAttr, maxheight, h = len(lines) # of this cell heights.append(min(col.height, h)) - height = min(max(heights), maxheight) or 1 # display even empty rows + return max(heights) + def drawRow(self, scr, row, rowidx, ybase, rowcattr: ColorAttr, maxheight, + isNull='', + topsep='', + midsep='', + botsep='', + endsep='', + keytopsep='', + keymidsep='', + keybotsep='', + endtopsep='', + endmidsep='', + endbotsep='', + colsep='', + keysep='', + selectednote='', + disp_truncator='' + ): + # sepattr is the attr between cell/columns + sepcattr = update_attr(rowcattr, colors.color_column_sep, 1) + + # apply current row here instead of in a colorizer, because it needs to know dispRowIndex + if rowidx == self.cursorRowIndex: + color_current_row = colors.get_color('color_current_row', 5) + basecellcattr = sepcattr = update_attr(rowcattr, color_current_row) + else: + basecellcattr = rowcattr + + displines = {} # [vcolidx] -> list of lines in that cell + height = min(self.calc_height(row, displines), maxheight) or 1 # display even empty rows self._rowLayout[rowidx] = (ybase, height) for vcolidx, (col, cellval, lines) in displines.items(): From 34894514a63543d536ba5d28567fe3ad955df3a5 Mon Sep 17 00:00:00 2001 From: Saul Pwanson Date: Thu, 3 Dec 2020 23:09:19 -0800 Subject: [PATCH 145/162] [types] floatsi parser #661 - floatsi type now parses SI strings (like 2.3M) - use z% (still) to set column type to floatsi - @anjakefala sponsored this feature --- visidata/_types.py | 35 ++++++++++++++++++++++++----------- 1 file changed, 24 insertions(+), 11 deletions(-) diff --git a/visidata/_types.py b/visidata/_types.py index b826a9da0..97419b5ba 100644 --- a/visidata/_types.py +++ b/visidata/_types.py @@ -47,17 +47,33 @@ def numericFormatter(fmtstr, typedval): return str(typedval) -vd.si_prefixes='p n u m . k M G T P Q'.split() +vd.si_prefixes='p n u m . kK M G T P Q'.split() + +def floatsi(*args): + if not args: + return 0.0 + if not isinstance(args[0], str): + return args[0] + + s=args[0].strip() + for i, p in enumerate(vd.si_prefixes): + if s[-1] in p: + return float(s[:-1]) * (1000 ** (i-4)) + + return float(s) + + def SIFormatter(fmtstr, val): level = 4 - while abs(val) > 1000: - val /= 1000 - level += 1 - while abs(val) < 0.001: - val *= 1000 - level -= 1 + if val != 0: + while abs(val) > 1000: + val /= 1000 + level += 1 + while abs(val) < 0.001: + val *= 1000 + level -= 1 - return numericFormatter(fmtstr, val) + (vd.si_prefixes[level] if level != 4 else '') + return numericFormatter(fmtstr, val) + (vd.si_prefixes[level][0] if level != 4 else '') class VisiDataType: @@ -106,9 +122,6 @@ def isNumeric(col): ## -def floatsi(*args): - return float(*args) - floatchars='+-0123456789.' def currency(*args): 'dirty float (strip non-numeric characters)' From 41130329d423e1c3f784f88c18cfc530e9070c5e Mon Sep 17 00:00:00 2001 From: Saul Pwanson Date: Fri, 4 Dec 2020 00:38:07 -0800 Subject: [PATCH 146/162] [modify] add colname input to addcol-new --- visidata/modify.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/visidata/modify.py b/visidata/modify.py index 2bf8475f5..366467019 100644 --- a/visidata/modify.py +++ b/visidata/modify.py @@ -259,7 +259,7 @@ def commit(sheet, *rows): Sheet.addCommand('a', 'add-row', 'addNewRows(1, cursorRowIndex); cursorDown(1)', 'append a blank row') Sheet.addCommand('ga', 'add-rows', 'addNewRows(int(input("add rows: ", value=1)), cursorRowIndex); cursorDown(1)', 'append N blank rows') -Sheet.addCommand('za', 'addcol-new', 'addColumnAtCursor(SettableColumn()); cursorRight(1)', 'append an empty column') +Sheet.addCommand('za', 'addcol-new', 'addColumnAtCursor(SettableColumn(input("column name: "))); cursorRight(1)', 'append an empty column') Sheet.addCommand('gza', 'addcol-bulk', 'addColumnAtCursor(*(SettableColumn() for c in range(int(input("add columns: "))))); cursorRight(1)', 'append N empty columns') Sheet.addCommand('z^S', 'commit-sheet', 'commit()') From ab33e4a46c8c500067e889580474b64ebf75724e Mon Sep 17 00:00:00 2001 From: Saul Pwanson Date: Fri, 4 Dec 2020 00:54:52 -0800 Subject: [PATCH 147/162] [cell] DisplayWrapper __bool__ should always return bool --- visidata/column.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/visidata/column.py b/visidata/column.py index 11a59cc92..d331f7b50 100644 --- a/visidata/column.py +++ b/visidata/column.py @@ -52,7 +52,7 @@ def __init__(self, value=None, *, display=None, note=None, notecolor=None, error self.error = error # list of strings for stacktrace def __bool__(self): - return self.value + return bool(self.value) def __eq__(self, other): return self.value == other From df01039a5a6ccb6b7dcf0db6ddf2df5df91637e7 Mon Sep 17 00:00:00 2001 From: Saul Pwanson Date: Fri, 4 Dec 2020 00:56:59 -0800 Subject: [PATCH 148/162] [plugins] colorize plugin rows by status - color_working for up-to-date plugins - color_warning for installed but not latest_ver --- visidata/plugins.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/visidata/plugins.py b/visidata/plugins.py index 4c82f15d2..8748a3530 100644 --- a/visidata/plugins.py +++ b/visidata/plugins.py @@ -46,9 +46,18 @@ def _checkHash(data, sha): import hashlib return hashlib.sha256(data.strip().encode('utf-8')).hexdigest() == sha +def _pluginColorizer(s,c,r,v): + if not r: return None + ver = _loadedVersion(r) + if not ver: return None + if ver != r.latest_ver: return 'color_warning' + return 'color_working' class PluginsSheet(JsonLinesSheet): - rowtype = "plugins" + rowtype = "plugins" # rowdef: AttrDict of json dict + colorizers = [ + CellColorizer(3, None, _pluginColorizer) + ] def iterload(self): for r in JsonLinesSheet.iterload(self): From 6b42d2bdb5c6ed84f6360befdd7db7568403b084 Mon Sep 17 00:00:00 2001 From: anjakefala Date: Fri, 4 Dec 2020 21:13:59 -0800 Subject: [PATCH 149/162] [dev] update CHANGELOG --- CHANGELOG.md | 104 ++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 99 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index acac57892..863f1f5b8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,32 +1,126 @@ # VisiData version history -# v2.?.? (????) +# v2.1 (????) + - [add] add bulk rows and cols leave cursor on first added (like add singles) + - [add] add colname input to `addcol-new` + - [aggregators] add mode and stdev to aggregator options (thanks @jsvine for PR #754) - [api] add options.unset() + - [columns] add hidden 'keycol' to **ColumnsSheet** (thanks @geekscrapy for feature request #768) + - [cli] support running as `python -m visidata` (thanks @abitrolly for PR #785) + - [cli] add `#!vd -p` as first line of `.vdj` for executable vd script + - [cli] allow `=` in `.vd` replay parameters - [clipboard] clipboard commands now require some selected rows #681 - [commands] add unset-option command bound to `d` on OptionsSheet #733 + - [config] `--config=''` now ignores visidatarc (thanks @rswgnu for feature request #777) + - [defer] commit changes, even if no deferred changes + - [display] add sort indication #582 + - [display] show ellipsis on left side with non-zero hoffset (thanks @frosencrantz for feature request #751) + - [expr] allow column attributes as variables (thanks @frosencrantz for feature request #659) + - [freq] change `numeric_binning` back to False by default + - [input] Shift+Arrow within `edit-cell` to move cursor and re-enter edit mode + - [loaders clickhouse] add clickhouse loader - [loaders http] have automatic API pagination (thanks @geekscrapy for feature request #480) + - [loaders json] improve loading speedup 50% (thanks @lxcode for investigating and pointing this out #765) + - this makes JSON saving non-deterministic in Python 3.6, as the order of fields output is dependent on the order within the dict + - (this is the default behaviour for dicts in Python 3.7+) + - [loaders json] try loading as jsonl before json (inverted) + - jsonl is a streamable format, so this way it doesn't have to wait for the entire contents to be loaded + before failing to parse as json and then trying to parse as jsonl + - fixes api loading with http so that contents of each response are added as they happen + - unfurl toplevel lists + - functionally now jsonl and json are identical + - [loaders json] try parsing `options.json_indent` as int (thanks @frosencrantz for the bug report #753) + this means json output can't be indented with a number. this seems like an uncommon use case + - [loaders json] skip lines starting with `#` + - [loaders pdf] `options.pdf_tables` to parse tables from pdf with tabular + - [loaders sqlite] use rowid to update and delete rows + - note that this will not work with WITHOUT ROWID sqlite tables - [loaders xlsx] add active column (thanks @kbd for feature request #726) + - [loaders zip] add extract-file, extract-selected, extract-file-to, extract-selected-to commands + - [macros] add improved macro system (thanks @bob-u for feature request #755) + - `m` (`macro-record`) begins recording macro; `m` prompts for keystroke, and completes recording + - macro can then be executed everytime provided keystroke is used, will override existing keybinding + - `gm` opens an index of all existing macros, can be directly viewed with `Enter` and then modified with `Ctrl+S` + - macros will run command on current row, column sheet + - remove deprecated `z Ctrl+D` older iteration of macro system + - [regex] use capture names for column names, if available, in `capture-col` (thanks @tsibley for PR #808) + - allows for pre-determining friendlier column names, saving a renaming step later + - [save] `g Ctrl+S` is `save-sheets-selected` on **IndexSheet** + - new command allows some or all sheets on an **IndexSheet** to be saved (and not the sheets on the sheet stack) + - [saver] add fixed-width saver (uses col.width) + - [saver sqlite] ensureLoaded when saving sheets to sqlite db - [search] `search-next` and `searchr-next` are now bound to n and N (was `next-search` and `search-prev`) - [select] differentiate select-equal- and select-exact- (thanks @geekscrapy for feature request #734) - previous select-equal- matched type value - now select-equal- matches display value - add `z,` and `gz,` bindings for select-exact-cell/-row - - [display] sort indication #582 + - [sheets] sorting on **SheetsSheet** now does not sort **SheetsSheet** itself. (thanks @klartext and @geekscrapy for bug reports #761 #518) + - [status] use `color_working` for progress indicator (thanks @geekscrapy for feature request #804) + - [types] add floatsi parser (sponsored feature by @anjakefala #661) + - floatsi type now parses SI strings (like 2.3M) + - use `z%` to set column type to floatsi ## Bugfixes - [api] expose visidata.view (thanks @alekibango for bug report #732) + - [color] use `color_column_sep` for sep chars (thanks @geekscrapy for bug report) + - [defer] frozen columns should not be deferred (thanks @frosencrantz for bug report #786) + - [dir] fix commit-sheet and delete-row on DirSheet + - [draw] fix display for off-screen cursor with multiline rows + - [expr] remove duplicate tabbing suggestions (thanks @geekscrapy for bug report #747) + - [expr] never include computing column (thanks @geekscrapy for bug report #756) + - only checks for self-reference; 2+ cycles still raises RecursionException + - caches are now for each cell, instead of for each row + - [freeze] freeze-sheet with errors should replace with null + - [loaders frictionless] assume JSON if no format (thanks scls19fr for bug report #803) + - from https://specs.frictionlessdata.io/data-resource/#data-location): + - a consumer of resource object MAY assume if no format or mediatype property is provided that the data is JSON and attempt to process it as such. + - [loaders hdf5] misc bugfixes to hdf5 dataset loading (thanks @amotl for PR #728) + - [loaders jsonl] fix copy-rows + - [loaders pandas] support loading Python objects directly (thanks @ajkerrigan for PR #816 and scls19fr for bug report #798) + - [loaders pandas] ensure all column names are strings (thanks @ajkerrigan for PR #816 and scls19fr for bug report #800) + - [loaders pandas] build frequency table using a copy of the source (thanks @ajkerrigan for PR #816 and scls19fr for bug report #802) + - [loaders sqlite] fix commit-sheet + - [loaders sqlite] fix commit deletes + - [loaders xlsx] only reload Workbook sheets to avoid error (thanks @aborruso for bug report #797) + - [loaders vdj] fix add-row - [man] fix warnings with manpage (thanks @jsvine for the bug report #718) + - [movement] fix scroll-cells (thanks @jsvine for bug report #762) - [numeric binning] perform degenerate binning when number of bins greater than number of values - (instead of when greater than width of bins) + - [numeric binning] if width of bins is 1, fallback to degenerate binning + - [numeric binning] degenerate binning should resemble non-numeric binning (thanks @setop for bug report #791) + - [options] fix `confirm_overwrite` in batch mode + - fix `-y` to set `confirm_overwrite` to False (means, no confirmation necessary for overwrite) + - make `confirm()` always fail in batch mode + - make `confirm_overwrite` a sheet-specific option + - [plugins] only reload **Plugins Sheet** if not already loaded + - [replay] move to replay context after getting sheet (thanks @rswgnu for bug report #796) + - [replay] do not push replaying .vd on sheet stack (thanks @rswgnu for bug report #795) + - [scroll] zj/zk do nothing in single-line mode (thanks @jsvine for suggestion) + - [shell] empty stdin to avoid hanging process (thanks @frosencrantz for bug report #752) + - [status] handle missing attributes in `disp_rstatus_fmt` (thanks @geekscrapy for bug report #764) - [tabulate] fix savers to save in their own format (thanks @frosencrantz for bug report #723) - - [color] use `color_column_sep` for sep chars (thanks @geekscrapy for bug report) + - [typing] fix indefinite hang for typing (thanks @lxcode for issue #794) - [windows] add Ctrl+M as alias for Ctrl+J #741 (thanks @bob-u for bug report #741) - [windows man] package man/vd.txt as a fallback for when manis not available on os (thanks @bob-u for bug report #745) - - [loaders jsonl] fix copy-rows - - [loaders vdj] fix add-row + +## Plugins +- add conll loader to **PluginsSheet** (thanks @polm) +- remove livesearch + +## Commands +- if `options.selected_or_rows` is True, `setcol-expr`, `setcol-iter`, `setcol-subst`, `setcol-subst`, `setcol-subst-all` will return all rows, if none selected + +## API +- [columns[ add Column.visibleWidth +- [open] additionally search for `open_filetype` within the vd scope +- [select] rename `someSelectedRows` to `onlySelectedRows` +- [select] add new `someSelectedRows` and `options.selected_or_rows` (thanks maufdez for feature request #767) + - if options is True, and no rows are selected, `someSelectedRows` will return all rows +- [status] allow non-hashable status msgs by deduping based on stringified contents # v2.0.1 (2020-10-13) From a133cd26a6b7e02a9e8290f12768a18c14dcfd4f Mon Sep 17 00:00:00 2001 From: anjakefala Date: Sat, 5 Dec 2020 16:30:34 -0800 Subject: [PATCH 150/162] [expr-] revert 4ef728 "pass column identity to all evalExpr" expressions that are only calculated once, do not need to pass column identity they can reference their "previous selves" once without causing a recursive problem thanks aac on #visidata for the bug report --- CHANGELOG.md | 4 ++-- visidata/expr.py | 6 ++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 863f1f5b8..c88c1df8d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -112,13 +112,13 @@ - remove livesearch ## Commands -- if `options.selected_or_rows` is True, `setcol-expr`, `setcol-iter`, `setcol-subst`, `setcol-subst`, `setcol-subst-all` will return all rows, if none selected +- if `options.some_selected_rows` is True, `setcol-expr`, `setcol-iter`, `setcol-subst`, `setcol-subst`, `setcol-subst-all` will return all rows, if none selected ## API - [columns[ add Column.visibleWidth - [open] additionally search for `open_filetype` within the vd scope - [select] rename `someSelectedRows` to `onlySelectedRows` -- [select] add new `someSelectedRows` and `options.selected_or_rows` (thanks maufdez for feature request #767) +- [select] add new `someSelectedRows` and `options.some_selected_rows` (thanks maufdez for feature request #767) - if options is True, and no rows are selected, `someSelectedRows` will return all rows - [status] allow non-hashable status msgs by deduping based on stringified contents diff --git a/visidata/expr.py b/visidata/expr.py index d49480462..44a872d15 100644 --- a/visidata/expr.py +++ b/visidata/expr.py @@ -36,7 +36,9 @@ def setValuesFromExpr(self, rows, expr): compiledExpr = compile(expr, '', 'eval') vd.addUndoSetValues([self], rows) for row in Progress(rows, 'setting'): - self.setValueSafe(row, self.sheet.evalExpr(compiledExpr, row, col=self)) + # Note: expressions that are only calculated once, do not need to pass column identity + # they can reference their "previous selves" once without causing a recursive problem + self.setValueSafe(row, self.sheet.evalExpr(compiledExpr, row)) self.recalc() vd.status('set %d values = %s' % (len(rows), expr)) @@ -48,7 +50,7 @@ def inputExpr(self, prompt, *args, **kwargs): Sheet.addCommand('=', 'addcol-expr', 'addColumnAtCursor(ExprColumn(inputExpr("new column expr="), curcol=cursorCol))', 'create new column from Python expression, with column names as variables') Sheet.addCommand('g=', 'setcol-expr', 'cursorCol.setValuesFromExpr(someSelectedRows, inputExpr("set selected="))', 'set current column for selected rows to result of Python expression') -Sheet.addCommand('z=', 'setcell-expr', 'cursorCol.setValues([cursorRow], evalExpr(inputExpr("set expr="), cursorRow, col=cursorCol))', 'evaluate Python expression on current row and set current cell with result of Python expression') +Sheet.addCommand('z=', 'setcell-expr', 'cursorCol.setValues([cursorRow], evalExpr(inputExpr("set expr="), cursorRow,))', 'evaluate Python expression on current row and set current cell with result of Python expression') Sheet.addCommand('gz=', 'setcol-iter', 'cursorCol.setValues(someSelectedRows, *list(itertools.islice(eval(input("set column= ", "expr", completer=CompleteExpr())), len(someSelectedRows))))', 'set current column for selected rows to the items in result of Python sequence expression') Sheet.addCommand(None, 'show-expr', 'status(evalExpr(inputExpr("show expr="), cursorRow))', 'evaluate Python expression on current row and show result on status line') From e2a395eaa26941f2cf3c598523a87fdf0825707b Mon Sep 17 00:00:00 2001 From: anjakefala Date: Sat, 5 Dec 2020 20:04:10 -0800 Subject: [PATCH 151/162] [tests] add test for bulk renaming columns, using previous column names as reference. --- tests/bulk-rename-cols.vd | 4 ++++ tests/golden/bulk-rename-cols.tsv | 8 ++++++++ 2 files changed, 12 insertions(+) create mode 100644 tests/bulk-rename-cols.vd create mode 100644 tests/golden/bulk-rename-cols.tsv diff --git a/tests/bulk-rename-cols.vd b/tests/bulk-rename-cols.vd new file mode 100644 index 000000000..9e9f317b8 --- /dev/null +++ b/tests/bulk-rename-cols.vd @@ -0,0 +1,4 @@ +sheet col row longname input keystrokes comment + open-file sample_data/sample.tsv o +sample columns-sheet C open Columns Sheet: edit column properties for current sheet +sample_columns name setcol-expr f"feature_{name}" g= set current column for selected rows to result of Python expression diff --git a/tests/golden/bulk-rename-cols.tsv b/tests/golden/bulk-rename-cols.tsv new file mode 100644 index 000000000..6fb25357e --- /dev/null +++ b/tests/golden/bulk-rename-cols.tsv @@ -0,0 +1,8 @@ +name width height type fmtstr value expr aggregators +feature_OrderDate 1 2016-01-06 0 +feature_Region 1 East 1 +feature_Rep 1 Jones 2 +feature_Item 1 Pencil 3 +feature_Units 1 95 4 +feature_Unit_Cost 1 1.99 5 +feature_Total 1 189.05 6 From dc37f70c83499232e4dab8a3befca812e158afdb Mon Sep 17 00:00:00 2001 From: anjakefala Date: Sat, 5 Dec 2020 20:22:50 -0800 Subject: [PATCH 152/162] [dev] update manpage for 2.1 --- CHANGELOG.md | 6 +++--- visidata/man/vd.1 | 34 ++++++++++++++++++++++++++-------- visidata/man/vd.inc | 22 +++++++++++++++++----- visidata/man/vd.txt | 40 ++++++++++++++++++++++++++++++---------- 4 files changed, 76 insertions(+), 26 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c88c1df8d..5d5cf788a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ - [commands] add unset-option command bound to `d` on OptionsSheet #733 - [config] `--config=''` now ignores visidatarc (thanks @rswgnu for feature request #777) - [defer] commit changes, even if no deferred changes + - [deprecated] add traceback warnings for deprecated calls (thanks @ajkerrigan for PR #724) - [display] add sort indication #582 - [display] show ellipsis on left side with non-zero hoffset (thanks @frosencrantz for feature request #751) - [expr] allow column attributes as variables (thanks @frosencrantz for feature request #659) @@ -25,8 +26,7 @@ - this makes JSON saving non-deterministic in Python 3.6, as the order of fields output is dependent on the order within the dict - (this is the default behaviour for dicts in Python 3.7+) - [loaders json] try loading as jsonl before json (inverted) - - jsonl is a streamable format, so this way it doesn't have to wait for the entire contents to be loaded - before failing to parse as json and then trying to parse as jsonl + - jsonl is a streamable format, so this way it doesn't have to wait for the entire contents to be loaded before failing to parse as json and then trying to parse as jsonl - fixes api loading with http so that contents of each response are added as they happen - unfurl toplevel lists - functionally now jsonl and json are identical @@ -105,7 +105,7 @@ - [tabulate] fix savers to save in their own format (thanks @frosencrantz for bug report #723) - [typing] fix indefinite hang for typing (thanks @lxcode for issue #794) - [windows] add Ctrl+M as alias for Ctrl+J #741 (thanks @bob-u for bug report #741) - - [windows man] package man/vd.txt as a fallback for when manis not available on os (thanks @bob-u for bug report #745) + - [windows man] package man/vd.txt as a fallback for when man is not available on os (thanks @bob-u for bug report #745) ## Plugins - add conll loader to **PluginsSheet** (thanks @polm) diff --git a/visidata/man/vd.1 b/visidata/man/vd.1 index d5ad85025..87d054d29 100644 --- a/visidata/man/vd.1 +++ b/visidata/man/vd.1 @@ -188,7 +188,7 @@ set name of current column to combined contents of current cell in selected rows set name of all visible columns to combined contents of current column for selected rows (or current row) .Pp .It Ic " =" Ar expr -.No create new column from Python Ar expr Ns , with column names as variables +.No create new column from Python Ar expr Ns , with column names, and attributes, as variables .It Ic " g=" Ar expr .No set current column for selected rows to result of Python Ar expr .It Ic "gz=" Ar expr @@ -253,9 +253,11 @@ select/toggle/unselect all rows from cursor to bottom .It Ic "z| z\e\ " Ns Ar expr .No select/unselect rows matching Python Ar expr No in any visible column .It Ic " \&," Ns " (comma)" -select rows matching current cell in current column +select rows matching typed value of current cell in current column .It Ic "g\&," -select rows matching current row in all visible columns +select rows matching typed value of current row in all visible columns +.It Ic "z\&, gz\&," +select rows matching display value of current cell/row in current column/all visible columns . .El . @@ -343,6 +345,8 @@ toggle insert mode set contents to previous/next in history .It Ic "Tab Shift+Tab" autocomplete input (when available) +.It Ic "Shift+Arrow" +.No move cursor in direction of Sy Arrow No and re-enter edit mode . .El . @@ -466,6 +470,10 @@ show cursor position and bounds of current sheet on status line show version and copyright information on status line .It Ic " ^P" .No open Sy Status History +.It "m" Ar keystroke +.No first, begin recording macro; second, prompt for Ar keystroke No, and complete recording. Macro can then be executed everytime provided keystroke is used. Will override existing keybinding. Macros will run on current row, column, sheet. +.It "gm" +.No open an index of all existing macros. Can be directly viewed with Sy Enter Ns , and then modified with Sy ^S Ns . . .El .Pp @@ -591,8 +599,8 @@ toggle/unset selected columns as key columns on source sheet add Ar aggregator No to selected source columns .It Ic "g-" No (hyphen) hide selected columns on source sheet -.It Ic "g~ g# g% g$ g@ gz#" -set type of selected columns on source sheet to str/int/float/currency/date/len +.It Ic "g~ g# g% g$ g@ gz# z%" +set type of selected columns on source sheet to str/int/float/currency/date/len/floatsi .It Ic " Enter" .No open a Sy Frequency Table No sheet grouped by column referenced in current row .El @@ -624,6 +632,8 @@ add row to reference a new blank sheet .No reload all selected sheets .It Ic "z^C gz^C" abort async threads for current/selected sheets(s) +.It Ic "g^S" +save selected or all sheets .It Ic " &" Ar jointype .No merge selected sheets with visible columns from all, keeping rows according to Ar jointype Ns : .El @@ -662,6 +672,8 @@ abort async threads for current/selected sheets(s) .Bl -tag -width XXXXXXXXXXXXXXX -compact -offset XXX .It Ic "Enter e" edit option at current row +.It Ic "d" +remove option override for this context .El . .Ss CommandLog (Shift+D) @@ -802,8 +814,6 @@ default column width default column height .It Sy --textwrap-cells Ns = Ns Ar "bool " No "True" wordwrap text for multiline rows -.It Sy --cmd-after-edit Ns = Ns Ar "str " No "go-down" -command longname to execute after successful edit .It Sy --quitguard No " False" confirm before quitting last sheet .It Sy --debug No " False" @@ -844,6 +854,8 @@ encoding passed to codecs.open encoding_errors passed to codecs.open .It Sy --bulk-select-clear No " False" clear selected rows before new bulk selections +.It Sy --some-selected-rows No " False" +if no rows selected, if True, someSelectedRows returns all rows; if False, fails .It Sy --delimiter Ns = Ns Ar "str " No " " field delimiter to use for tsv/usv filetype .It Sy --row-delimiter Ns = Ns Ar "str " No " @@ -879,7 +891,7 @@ a nicer selection interface for aggregators and jointype numeric aggregators to calculate on Describe sheet .It Sy --histogram-bins Ns = Ns Ar "int " No "0" number of bins for histogram of numeric columns -.It Sy --numeric-binning Ns = Ns Ar "bool " No "True" +.It Sy --numeric-binning No " False" bin numeric columns into ranges .It Sy --replay-wait Ns = Ns Ar "float " No "0.0" time to wait between replayed commands, in seconds @@ -950,6 +962,8 @@ table header when saving to html (y/s/n) if save_dot includes all internet hosts separately (y), combined (s), or does not include the internet (n) .It Sy --graphviz-edge-labels Ns = Ns Ar "bool " No "True" whether to include edge labels on graphviz diagrams +.It Sy --pdf-tables No " False" +parse PDF for tables instead of pages of text .It Sy --plugins-url Ns = Ns Ar "str " No "https://visidata.org/plugins/plugins.jsonl" source of plugins sheet .El @@ -1017,6 +1031,10 @@ separator between key columns and rest of columns .It Sy "disp_selected_note " No "\[u2022]" +.It Sy "disp_sort_asc " No "\[u2191]\[u219F]\[u21DE]\[u21E1]\[u21E7]\[u21D1]" +characters for ascending sort +.It Sy "disp_sort_desc " No "\[u2193]\[u21A1]\[u21DF]\[u21E3]\[u21E9]\[u21D3]" +characters for descending sort .It Sy "color_default " No "normal" the default color .It Sy "color_default_hdr " No "bold" diff --git a/visidata/man/vd.inc b/visidata/man/vd.inc index 027699b9d..a2450fb9c 100644 --- a/visidata/man/vd.inc +++ b/visidata/man/vd.inc @@ -188,7 +188,7 @@ set name of current column to combined contents of current cell in selected rows set name of all visible columns to combined contents of current column for selected rows (or current row) .Pp .It Ic " =" Ar expr -.No create new column from Python Ar expr Ns , with column names as variables +.No create new column from Python Ar expr Ns , with column names, and attributes, as variables .It Ic " g=" Ar expr .No set current column for selected rows to result of Python Ar expr .It Ic "gz=" Ar expr @@ -253,9 +253,11 @@ select/toggle/unselect all rows from cursor to bottom .It Ic "z| z\e\ " Ns Ar expr .No select/unselect rows matching Python Ar expr No in any visible column .It Ic " \&," Ns " (comma)" -select rows matching current cell in current column +select rows matching typed value of current cell in current column .It Ic "g\&," -select rows matching current row in all visible columns +select rows matching typed value of current row in all visible columns +.It Ic "z\&, gz\&," +select rows matching display value of current cell/row in current column/all visible columns . .El . @@ -343,6 +345,8 @@ toggle insert mode set contents to previous/next in history .It Ic "Tab Shift+Tab" autocomplete input (when available) +.It Ic "Shift+Arrow" +.No move cursor in direction of Sy Arrow No and re-enter edit mode . .El . @@ -466,6 +470,10 @@ show cursor position and bounds of current sheet on status line show version and copyright information on status line .It Ic " ^P" .No open Sy Status History +.It "m" Ar keystroke +.No first, begin recording macro; second, prompt for Ar keystroke No, and complete recording. Macro can then be executed everytime provided keystroke is used. Will override existing keybinding. Macros will run on current row, column, sheet. +.It "gm" +.No open an index of all existing macros. Can be directly viewed with Sy Enter Ns , and then modified with Sy ^S Ns . . .El .Pp @@ -591,8 +599,8 @@ toggle/unset selected columns as key columns on source sheet add Ar aggregator No to selected source columns .It Ic "g-" No (hyphen) hide selected columns on source sheet -.It Ic "g~ g# g% g$ g@ gz#" -set type of selected columns on source sheet to str/int/float/currency/date/len +.It Ic "g~ g# g% g$ g@ gz# z%" +set type of selected columns on source sheet to str/int/float/currency/date/len/floatsi .It Ic " Enter" .No open a Sy Frequency Table No sheet grouped by column referenced in current row .El @@ -624,6 +632,8 @@ add row to reference a new blank sheet .No reload all selected sheets .It Ic "z^C gz^C" abort async threads for current/selected sheets(s) +.It Ic "g^S" +save selected or all sheets .It Ic " &" Ar jointype .No merge selected sheets with visible columns from all, keeping rows according to Ar jointype Ns : .El @@ -662,6 +672,8 @@ abort async threads for current/selected sheets(s) .Bl -tag -width XXXXXXXXXXXXXXX -compact -offset XXX .It Ic "Enter e" edit option at current row +.It Ic "d" +remove option override for this context .El . .Ss CommandLog (Shift+D) diff --git a/visidata/man/vd.txt b/visidata/man/vd.txt index 995e141a5..2dd91990f 100644 --- a/visidata/man/vd.txt +++ b/visidata/man/vd.txt @@ -108,8 +108,8 @@ DESCRIPTION gz^ set name of all visible columns to combined contents of current column for selected rows (or current row) - = expr create new column from Python expr, with column names as - variables + = expr create new column from Python expr, with column names, + and attributes, as variables g= expr set current column for selected rows to result of Python expr gz= expr set current column for selected rows to the items in @@ -160,8 +160,12 @@ DESCRIPTION column z| z\ expr select/unselect rows matching Python expr in any visible column - , (comma) select rows matching current cell in current column - g, select rows matching current row in all visible columns + , (comma) select rows matching typed value of current cell in cur‐ + rent column + g, select rows matching typed value of current row in all + visible columns + z, gz, select rows matching display value of current cell/row + in current column/all visible columns Row Sorting/Filtering [ ] sort ascending/descending by current column; replace any @@ -215,6 +219,8 @@ DESCRIPTION Insert toggle insert mode Up Down set contents to previous/next in history Tab Shift+Tab autocomplete input (when available) + Shift+Arrow move cursor in direction of Arrow and re-enter edit + mode Data Toolkit o input open input in VisiData @@ -296,6 +302,13 @@ DESCRIPTION tus line ^V show version and copyright information on status line ^P open Status History + m keystroke first, begin recording macro; second, prompt for + keystroke No, and complete recording. Macro can then be + executed everytime provided keystroke is used. Will + override existing keybinding. Macros will run on current + row, column, sheet. + gm open an index of all existing macros. Can be directly + viewed with Enter, and then modified with ^S. ^Y z^Y g^Y open current row/cell/sheet as Python object ^X expr evaluate Python expr and opens result as Python object @@ -370,9 +383,9 @@ DESCRIPTION source sheet g+ aggregator add Ar aggregator No to selected source columns g- (hyphen) hide selected columns on source sheet - g~ g# g% g$ g@ gz# + g~ g# g% g$ g@ gz# z% set type of selected columns on source sheet to - str/int/float/currency/date/len + str/int/float/currency/date/len/floatsi Enter open a Frequency Table sheet grouped by column referenced in current row @@ -391,6 +404,7 @@ DESCRIPTION columns from selected sheets g^R reload all selected sheets z^C gz^C abort async threads for current/selected sheets(s) + g^S save selected or all sheets & jointype merge selected sheets with visible columns from all, keeping rows according to jointype: . inner keep only rows which match keys on all @@ -414,6 +428,7 @@ DESCRIPTION gO open options.config as TextSheet (sheet-specific commands) Enter e edit option at current row + d remove option override for this context CommandLog (Shift+D) (global commands) @@ -507,9 +522,6 @@ COMMANDLINE OPTIONS --default-height=int 10 default column height --textwrap-cells=bool True wordwrap text for multi‐ line rows - --cmd-after-edit=str go-down command longname to exe‐ - cute after successful - edit --quitguard False confirm before quitting last sheet --debug False exit on error and display @@ -551,6 +563,10 @@ COMMANDLINE OPTIONS codecs.open --bulk-select-clear False clear selected rows be‐ fore new bulk selections + --some-selected-rows False if no rows selected, if + True, someSelectedRows + returns all rows; if + False, fails --delimiter=str field delimiter to use for tsv/usv filetype --row-delimiter=str " row delimiter to use @@ -589,7 +605,7 @@ COMMANDLINE OPTIONS sheet --histogram-bins=int 0 number of bins for his‐ togram of numeric columns - --numeric-binning=bool True bin numeric columns into + --numeric-binning False bin numeric columns into ranges --replay-wait=float 0.0 time to wait between re‐ played commands, in sec‐ @@ -672,6 +688,8 @@ COMMANDLINE OPTIONS --graphviz-edge-labels=bool True whether to include edge labels on graphviz dia‐ grams + --pdf-tables False parse PDF for tables in‐ + stead of pages of text --plugins-url=str https://visidata.org/plugins/plugins.jsonl source of plugins sheet @@ -722,6 +740,8 @@ COMMANDLINE OPTIONS disp_endmid_sep ║ disp_endbot_sep ║ disp_selected_note • + disp_sort_asc ↑↟⇞⇡⇧⇑ characters for ascending sort + disp_sort_desc ↓↡⇟⇣⇩⇓ characters for descending sort color_default normal the default color color_default_hdr bold color of the column headers color_bottom_hdr underline color of the bottom header row From bf9d4fd06e391399df2be74c895821ac1c75b149 Mon Sep 17 00:00:00 2001 From: anjakefala Date: Sat, 5 Dec 2020 20:23:35 -0800 Subject: [PATCH 153/162] [dev] update date on manpage --- visidata/man/vd.1 | 2 +- visidata/man/vd.inc | 2 +- visidata/man/vd.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/visidata/man/vd.1 b/visidata/man/vd.1 index 87d054d29..c12fbca65 100644 --- a/visidata/man/vd.1 +++ b/visidata/man/vd.1 @@ -1,4 +1,4 @@ -.Dd Oct 13, 2020 +.Dd Dec 5, 2020 .Dt vd \&1 "Quick Reference Guide" .Os Linux/MacOS . diff --git a/visidata/man/vd.inc b/visidata/man/vd.inc index a2450fb9c..40ba6543a 100644 --- a/visidata/man/vd.inc +++ b/visidata/man/vd.inc @@ -1,4 +1,4 @@ -.Dd Oct 13, 2020 +.Dd Dec 5, 2020 .Dt vd \&1 "Quick Reference Guide" .Os Linux/MacOS . diff --git a/visidata/man/vd.txt b/visidata/man/vd.txt index 2dd91990f..03c2ee0e2 100644 --- a/visidata/man/vd.txt +++ b/visidata/man/vd.txt @@ -895,4 +895,4 @@ SUPPORTED OUTPUT FORMATS AUTHOR VisiData was made by Saul Pwanson . -Linux/MacOS Oct 13, 2020 Linux/MacOS +Linux/MacOS Dec 5, 2020 Linux/MacOS From ac5810f09738b051edbe228ac019e9f48af9f88f Mon Sep 17 00:00:00 2001 From: anjakefala Date: Sat, 5 Dec 2020 20:25:26 -0800 Subject: [PATCH 154/162] [tests] add gs to bulk-select-rows.vd --- tests/bulk-rename-cols.vd | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/bulk-rename-cols.vd b/tests/bulk-rename-cols.vd index 9e9f317b8..ae326b906 100644 --- a/tests/bulk-rename-cols.vd +++ b/tests/bulk-rename-cols.vd @@ -1,4 +1,5 @@ sheet col row longname input keystrokes comment open-file sample_data/sample.tsv o sample columns-sheet C open Columns Sheet: edit column properties for current sheet +sample_columns select-rows gs sample_columns name setcol-expr f"feature_{name}" g= set current column for selected rows to result of Python expression From 503050af507680f26f9b6bc99f659c56772d9a63 Mon Sep 17 00:00:00 2001 From: Stuart Henderson Date: Sun, 6 Dec 2020 14:03:07 +0000 Subject: [PATCH 155/162] use correct loader for .xls 5d0d6c3f933 converted from vd.filetype to open_ext, but missed the 'x' in the XlsxIndexSheet --- visidata/loaders/xlsx.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/visidata/loaders/xlsx.py b/visidata/loaders/xlsx.py index 32dbe6e97..c7dbfbe68 100644 --- a/visidata/loaders/xlsx.py +++ b/visidata/loaders/xlsx.py @@ -1,11 +1,12 @@ from visidata import * +def open_xls(p): + return XlsIndexSheet(p.name, source=p) + def open_xlsx(p): return XlsxIndexSheet(p.name, source=p) -open_xls = open_xlsx - class XlsxIndexSheet(IndexSheet): 'Load XLSX file (in Excel Open XML format).' rowtype = 'sheets' # rowdef: xlsxSheet From b87521784cf0582fc5d58eb0e4fe7312ae778ded Mon Sep 17 00:00:00 2001 From: anjakefala Date: Sun, 6 Dec 2020 15:47:04 -0800 Subject: [PATCH 156/162] [isNumeric] move isNumeric to vdobj api --- docs/api/columns.rst | 2 +- visidata/_types.py | 3 ++- visidata/column.py | 1 - visidata/deprecated.py | 6 +++++- visidata/describe.py | 2 +- visidata/freqtbl.py | 2 +- visidata/graph.py | 9 ++++----- visidata/loaders/fixed_width.py | 2 +- visidata/loaders/graphviz.py | 6 +++--- visidata/loaders/markdown.py | 2 +- visidata/loaders/xlsx.py | 2 +- visidata/pivot.py | 2 +- visidata/sheets.py | 4 ++-- 13 files changed, 23 insertions(+), 20 deletions(-) diff --git a/docs/api/columns.rst b/docs/api/columns.rst index 605f82117..6ecf56986 100644 --- a/docs/api/columns.rst +++ b/docs/api/columns.rst @@ -156,7 +156,7 @@ Types API ~~~~~~~~~~~ .. autoclass:: visidata.vd.addType -.. autofunction:: visidata.isNumeric +.. autofunction:: visidata.vd.isNumeric Examples ~~~~~~~~~ diff --git a/visidata/_types.py b/visidata/_types.py index 97419b5ba..990ca0420 100644 --- a/visidata/_types.py +++ b/visidata/_types.py @@ -117,7 +117,8 @@ def getType(vd, typetype): vdtype(dict, '') vdtype(list, '') -def isNumeric(col): +@VisiData.api +def isNumeric(vd, col): return col.type in (int,vlen,float,currency,date,floatsi) ## diff --git a/visidata/column.py b/visidata/column.py index d331f7b50..4a8c20d2e 100644 --- a/visidata/column.py +++ b/visidata/column.py @@ -7,7 +7,6 @@ import time from visidata import option, options, anytype, stacktrace, vd -from visidata import isNumeric from visidata import asyncthread, dispwidth from visidata import wrapply, TypedWrapper, TypedExceptionWrapper from visidata import Extensible, AttrDict, undoAttrFunc diff --git a/visidata/deprecated.py b/visidata/deprecated.py index 7bf90ce08..5776bb37f 100644 --- a/visidata/deprecated.py +++ b/visidata/deprecated.py @@ -84,7 +84,11 @@ def push_pyobj(name, pyobj): else: vd.error("cannot push '%s' as pyobj" % type(pyobj).__name__) -visidata.addGlobals({'load_pyobj': load_pyobj}) +@deprecated('2.1', 'vd.isNumeric instead') +def isNumeric(col): + return vd.isNumeric(col) + +visidata.addGlobals({'load_pyobj': load_pyobj, 'isNumeric': isNumeric}) # The longnames on the left are deprecated for 2.0 diff --git a/visidata/describe.py b/visidata/describe.py index 3822adfa3..f1f98e205 100644 --- a/visidata/describe.py +++ b/visidata/describe.py @@ -87,7 +87,7 @@ def reloadColumn(self, srccol): d['errors'].append(sr) d['mode'] = self.calcStatistic(d, mode, vals) - if isNumeric(srccol): + if vd.isNumeric(srccol): for func in [min, max, sum, median]: # use type d[func.__name__] = self.calcStatistic(d, func, vals) for aggrname in options.describe_aggrs.split(): diff --git a/visidata/freqtbl.py b/visidata/freqtbl.py index 01bb45b88..d23a4911f 100644 --- a/visidata/freqtbl.py +++ b/visidata/freqtbl.py @@ -59,7 +59,7 @@ def reload(self): self.column('percent').hide() self.column('histogram').hide() - if not [c for c in self.groupByCols if isNumeric(c)]: + if not [c for c in self.groupByCols if vd.isNumeric(c)]: self.orderBy(self.column('count'), reverse=True) def openRow(self, row): diff --git a/visidata/graph.py b/visidata/graph.py index 0197c6258..14e9a7024 100644 --- a/visidata/graph.py +++ b/visidata/graph.py @@ -5,8 +5,7 @@ def numericCols(cols): - # isNumeric from describe.py - return [c for c in cols if isNumeric(c)] + return [c for c in cols if vd.isNumeric(c)] class InvertedCanvas(Canvas): @@ -49,7 +48,7 @@ def reload(self): self.reset() vd.status('loading data points') - catcols = [c for c in self.xcols if not isNumeric(c)] + catcols = [c for c in self.xcols if not vd.isNumeric(c)] numcols = numericCols(self.xcols) for ycol in self.ycols: for rownum, row in enumerate(Progress(self.sourceRows, 'plotting')): # rows being plotted from source @@ -92,7 +91,7 @@ def moveToCol(self, colstr): return True def formatX(self, amt): - return ','.join(xcol.format(xcol.type(amt)) for xcol in self.xcols if isNumeric(xcol)) + return ','.join(xcol.format(xcol.type(amt)) for xcol in self.xcols if vd.isNumeric(xcol)) def formatY(self, amt): srccol = self.ycols[0] @@ -143,7 +142,7 @@ def createLabels(self): # TODO: if 0 line is within visible bounds, explicitly draw the axis # TODO: grid lines corresponding to axis labels - xname = ','.join(xcol.name for xcol in self.xcols if isNumeric(xcol)) or 'row#' + xname = ','.join(xcol.name for xcol in self.xcols if vd.isNumeric(xcol)) or 'row#' xname, _ = clipstr(xname, self.leftMarginPixels//2-2) self.plotlabel(0, self.plotviewBox.ymax+4, xname+'»', colors.color_graph_axis) diff --git a/visidata/loaders/fixed_width.py b/visidata/loaders/fixed_width.py index 6830a0226..2b5d4a648 100644 --- a/visidata/loaders/fixed_width.py +++ b/visidata/loaders/fixed_width.py @@ -88,7 +88,7 @@ def save_fixed(vd, p, *vsheets): with Progress(gerund='saving'): for dispvals in sheet.iterdispvals(format=True): for col, val in dispvals.items(): - fp.write('{0:{align}{width}}'.format(val, width=col.width, align='>' if isNumeric(col) else '<')) + fp.write('{0:{align}{width}}'.format(val, width=col.width, align='>' if vd.isNumeric(col) else '<')) fp.write('\n') vd.status('%s save finished' % p) diff --git a/visidata/loaders/graphviz.py b/visidata/loaders/graphviz.py index 517ed6e37..1b57238b9 100644 --- a/visidata/loaders/graphviz.py +++ b/visidata/loaders/graphviz.py @@ -1,5 +1,5 @@ from visidata import vd, options, option, TypedWrapper, asyncthread, Progress -from visidata import wrapply, clean_to_id, isNumeric, VisiData, SIFormatter +from visidata import wrapply, clean_to_id, VisiData, SIFormatter option('graphviz_edge_labels', True, 'whether to include edge labels on graphviz diagrams') @@ -29,7 +29,7 @@ def save_dot(vd, p, vs): downsrc = clean_to_id(str(src)) or src downdst = clean_to_id(str(dst)) or dst - edgenotes = [c.getTypedValue(row) for c in vs.nonKeyVisibleCols if not isNumeric(c)] + edgenotes = [c.getTypedValue(row) for c in vs.nonKeyVisibleCols if not vd.isNumeric(c)] edgetype = '-'.join(str(x) for x in edgenotes if is_valid(x)) color = assignedColors.get(edgetype, None) if not color: @@ -37,7 +37,7 @@ def save_dot(vd, p, vs): assignedColors[edgetype] = color if options.graphviz_edge_labels: - nodelabels = [wrapply(SIFormatter, '%0.1f', c.getTypedValue(row)) for c in vs.nonKeyVisibleCols if isNumeric(c)] + nodelabels = [wrapply(SIFormatter, '%0.1f', c.getTypedValue(row)) for c in vs.nonKeyVisibleCols if vd.isNumeric(c)] label = '/'.join(str(x) for x in nodelabels if is_valid(x)) else: label = '' diff --git a/visidata/loaders/markdown.py b/visidata/loaders/markdown.py index 01d74262d..457756ed7 100644 --- a/visidata/loaders/markdown.py +++ b/visidata/loaders/markdown.py @@ -13,7 +13,7 @@ def markdown_escape(s, style='orgmode'): return ret def markdown_colhdr(col): - if isNumeric(col): + if vd.isNumeric(col): return ('-' * (col.width-1)) + ':' else: return '-' * (col.width or options.default_width) diff --git a/visidata/loaders/xlsx.py b/visidata/loaders/xlsx.py index c7dbfbe68..b4218f947 100644 --- a/visidata/loaders/xlsx.py +++ b/visidata/loaders/xlsx.py @@ -89,7 +89,7 @@ def save_xlsx(vd, p, *sheets): for col, v in dispvals.items(): if col.type == date: v = datetime.datetime.fromtimestamp(int(v.timestamp())) - elif not isNumeric(col): + elif not vd.isNumeric(col): v = str(v) row.append(v) diff --git a/visidata/pivot.py b/visidata/pivot.py index 15081a11d..234730d27 100644 --- a/visidata/pivot.py +++ b/visidata/pivot.py @@ -45,7 +45,7 @@ def __init__(self, name, groupByCols, pivotCols, **kwargs): self.groupByCols = groupByCols # whose values become rows def isNumericRange(self, col): - return isNumeric(col) and self.source.options.numeric_binning + return vd.isNumeric(col) and self.source.options.numeric_binning def initCols(self): self.columns = [] diff --git a/visidata/sheets.py b/visidata/sheets.py index c578a4105..9769c41ee 100644 --- a/visidata/sheets.py +++ b/visidata/sheets.py @@ -4,7 +4,7 @@ import textwrap from visidata import VisiData, Extensible, globalCommand, ColumnAttr, ColumnItem, vd, ENTER, EscapeException, drawcache, drawcache_property, LazyChainMap, asyncthread, ExpectedException -from visidata import (options, theme, isNumeric, Column, option, namedlist, SettableColumn, +from visidata import (options, theme, Column, option, namedlist, SettableColumn, TypedExceptionWrapper, BaseSheet, UNLOADED, vd, clipdraw, ColorAttr, update_attr, colors, undoAttrFunc) import visidata @@ -734,7 +734,7 @@ def calc_height(self, row, displines=None, isNull=None): continue col = vcols[vcolidx] cellval = col.getCell(row) - if colwidth > 1 and isNumeric(col): + if colwidth > 1 and vd.isNumeric(col): cellval.display = cellval.display.rjust(colwidth-2) try: From f5eeef5c7c9876a1d066e68772cfccb5d45d841c Mon Sep 17 00:00:00 2001 From: anjakefala Date: Sun, 6 Dec 2020 17:35:40 -0800 Subject: [PATCH 157/162] [dev] update history --- dev/history.jsonl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dev/history.jsonl b/dev/history.jsonl index 9eed77c46..fe5080072 100644 --- a/dev/history.jsonl +++ b/dev/history.jsonl @@ -4,3 +4,5 @@ {"date": "2017-06-29", "event": "[HN] Show HN: VisiData - vi for data", "url": "https://news.ycombinator.com/item?id=14662860"} {"date": "2020-03-27", "event": "anja wins year-old bet, >140 users every weekday", "url": ""} {"date": "2020-05-13", "event": "changelog podcast released", "url": "https://changelog.com/podcast/394"} +{"date": "2020-12-05", "event": "anja's tweet about visidata being a tool everyone should gets >100 likes", "url": "https://twitter.com/nevoitbien/status/1335398803604586498"} +{"date": "2020-12-06", "event": "[HN] VisiData in 60 seconds", "url": "https://news.ycombinator.com/item?id=25322091"} From f3c12c2a70ea4dce1f044d896032596ef41ee0a0 Mon Sep 17 00:00:00 2001 From: anjakefala Date: Sun, 6 Dec 2020 18:21:39 -0800 Subject: [PATCH 158/162] [docs] update api docs with 2.1 changes --- CHANGELOG.md | 3 ++- docs/api/columns.rst | 7 +++++++ docs/api/data.rst | 2 ++ visidata/movement.py | 1 + 4 files changed, 12 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5d5cf788a..73aa07186 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -115,12 +115,13 @@ - if `options.some_selected_rows` is True, `setcol-expr`, `setcol-iter`, `setcol-subst`, `setcol-subst`, `setcol-subst-all` will return all rows, if none selected ## API -- [columns[ add Column.visibleWidth +- [columns] add Column.visibleWidth - [open] additionally search for `open_filetype` within the vd scope - [select] rename `someSelectedRows` to `onlySelectedRows` - [select] add new `someSelectedRows` and `options.some_selected_rows` (thanks maufdez for feature request #767) - if options is True, and no rows are selected, `someSelectedRows` will return all rows - [status] allow non-hashable status msgs by deduping based on stringified contents +- [isNumeric] isNumeric is part of vdobj # v2.0.1 (2020-10-13) diff --git a/docs/api/columns.rst b/docs/api/columns.rst index 6ecf56986..af934ad66 100644 --- a/docs/api/columns.rst +++ b/docs/api/columns.rst @@ -17,6 +17,9 @@ Instead, apps and plugins should call ``getValue`` and ``setValue``, which provi .. autoattribute:: visidata.Column.name .. autoattribute:: visidata.Column.type .. autoattribute:: visidata.Column.width +.. autoattribute:: visidata.Column.visibleWidth +.. versionadded:: 2.1 + .. autoattribute:: visidata.Column.fmtstr .. autoattribute:: visidata.Column.hidden @@ -157,6 +160,10 @@ Types API .. autoclass:: visidata.vd.addType .. autofunction:: visidata.vd.isNumeric +.. versionadded:: 2.1 +.. autofunction:: visidata.isNumeric +.. deprecated:: 2.1 + Use ``vd.isNumeric`` instead. Examples ~~~~~~~~~ diff --git a/docs/api/data.rst b/docs/api/data.rst index 61ca4e179..9d7f945ab 100644 --- a/docs/api/data.rst +++ b/docs/api/data.rst @@ -5,7 +5,9 @@ Each TableSheet has a set of *selected rows*, which is a strict subset of the ro .. autoattribute:: visidata.TableSheet.selectedRows .. autoattribute:: visidata.TableSheet.someSelectedRows +.. versionchanged:: 2.1 .. autoattribute:: visidata.TableSheet.onlySelectedRows +.. versionchanged:: 2.1 .. autoattribute:: visidata.TableSheet.nSelectedRows .. autofunction:: visidata.TableSheet.selectRow diff --git a/visidata/movement.py b/visidata/movement.py index 04858a86c..eddb8fee3 100644 --- a/visidata/movement.py +++ b/visidata/movement.py @@ -90,6 +90,7 @@ def nextColRegex(sheet, colregex): @Column.property def visibleWidth(self): + 'Width of column as is displayed in terminal' vcolidx = self.sheet.visibleCols.index(self) return self.sheet._visibleColLayout[vcolidx][1] From 9b84f3f51afbf79b0fdcf96b0b2908f90c810e46 Mon Sep 17 00:00:00 2001 From: anjakefala Date: Sun, 6 Dec 2020 19:04:41 -0800 Subject: [PATCH 159/162] [docs] move api change modules to helpstr --- docs/api/columns.rst | 2 -- docs/api/data.rst | 2 -- visidata/_types.py | 1 + visidata/deprecated.py | 1 + visidata/movement.py | 4 +++- visidata/selection.py | 6 ++++-- 6 files changed, 9 insertions(+), 7 deletions(-) diff --git a/docs/api/columns.rst b/docs/api/columns.rst index af934ad66..d8e526386 100644 --- a/docs/api/columns.rst +++ b/docs/api/columns.rst @@ -18,7 +18,6 @@ Instead, apps and plugins should call ``getValue`` and ``setValue``, which provi .. autoattribute:: visidata.Column.type .. autoattribute:: visidata.Column.width .. autoattribute:: visidata.Column.visibleWidth -.. versionadded:: 2.1 .. autoattribute:: visidata.Column.fmtstr @@ -160,7 +159,6 @@ Types API .. autoclass:: visidata.vd.addType .. autofunction:: visidata.vd.isNumeric -.. versionadded:: 2.1 .. autofunction:: visidata.isNumeric .. deprecated:: 2.1 Use ``vd.isNumeric`` instead. diff --git a/docs/api/data.rst b/docs/api/data.rst index 9d7f945ab..61ca4e179 100644 --- a/docs/api/data.rst +++ b/docs/api/data.rst @@ -5,9 +5,7 @@ Each TableSheet has a set of *selected rows*, which is a strict subset of the ro .. autoattribute:: visidata.TableSheet.selectedRows .. autoattribute:: visidata.TableSheet.someSelectedRows -.. versionchanged:: 2.1 .. autoattribute:: visidata.TableSheet.onlySelectedRows -.. versionchanged:: 2.1 .. autoattribute:: visidata.TableSheet.nSelectedRows .. autofunction:: visidata.TableSheet.selectRow diff --git a/visidata/_types.py b/visidata/_types.py index 990ca0420..d0ac58ca1 100644 --- a/visidata/_types.py +++ b/visidata/_types.py @@ -119,6 +119,7 @@ def getType(vd, typetype): @VisiData.api def isNumeric(vd, col): + '.. versionadded:: 2.1' return col.type in (int,vlen,float,currency,date,floatsi) ## diff --git a/visidata/deprecated.py b/visidata/deprecated.py index 5776bb37f..7f8252276 100644 --- a/visidata/deprecated.py +++ b/visidata/deprecated.py @@ -86,6 +86,7 @@ def push_pyobj(name, pyobj): @deprecated('2.1', 'vd.isNumeric instead') def isNumeric(col): + '.. deprecated:: 2.1' return vd.isNumeric(col) visidata.addGlobals({'load_pyobj': load_pyobj, 'isNumeric': isNumeric}) diff --git a/visidata/movement.py b/visidata/movement.py index eddb8fee3..fef5f2de5 100644 --- a/visidata/movement.py +++ b/visidata/movement.py @@ -90,7 +90,9 @@ def nextColRegex(sheet, colregex): @Column.property def visibleWidth(self): - 'Width of column as is displayed in terminal' + '''Width of column as is displayed in terminal + + .. versionadded:: 2.1''' vcolidx = self.sheet.visibleCols.index(self) return self.sheet._visibleColLayout[vcolidx][1] diff --git a/visidata/selection.py b/visidata/selection.py index 0d0e07361..f946cd264 100644 --- a/visidata/selection.py +++ b/visidata/selection.py @@ -96,7 +96,8 @@ def selectedRows(self): @Sheet.property def onlySelectedRows(self): - 'List of selected rows in sheet order. Fail if no rows are selected.' + '''List of selected rows in sheet order. Fail if no rows are selected. + .. versionchanged:: 2.1''' if self.nSelectedRows == 0: vd.fail('no rows selected') return self.selectedRows @@ -106,7 +107,8 @@ def someSelectedRows(self): '''Return a list of rows: (a) in batch mode, always return selectedRows (b) in interactive mode, if options.some_selected_rows is True, return selectedRows or all rows if none selected - (c) in interactive mode, if options.some_selected_rows is False, return selectedRows or fail if none selected''' + (c) in interactive mode, if options.some_selected_rows is False, return selectedRows or fail if none selected + .. versionchanged:: 2.1''' if options.batch: return self.selectedRows if options.some_selected_rows: From b4bb7d9532eeb62775481aea9cf7e5e285306d44 Mon Sep 17 00:00:00 2001 From: anjakefala Date: Sun, 6 Dec 2020 20:37:26 -0800 Subject: [PATCH 160/162] Revert "[docs] move api change modules to helpstr" This reverts commit 9b84f3f51afbf79b0fdcf96b0b2908f90c810e46. For now, we are keeping the versionadded in the docs, instead of in the helpstr. --- docs/api/columns.rst | 2 ++ docs/api/data.rst | 2 ++ visidata/_types.py | 1 - visidata/deprecated.py | 1 - visidata/movement.py | 4 +--- visidata/selection.py | 6 ++---- 6 files changed, 7 insertions(+), 9 deletions(-) diff --git a/docs/api/columns.rst b/docs/api/columns.rst index d8e526386..af934ad66 100644 --- a/docs/api/columns.rst +++ b/docs/api/columns.rst @@ -18,6 +18,7 @@ Instead, apps and plugins should call ``getValue`` and ``setValue``, which provi .. autoattribute:: visidata.Column.type .. autoattribute:: visidata.Column.width .. autoattribute:: visidata.Column.visibleWidth +.. versionadded:: 2.1 .. autoattribute:: visidata.Column.fmtstr @@ -159,6 +160,7 @@ Types API .. autoclass:: visidata.vd.addType .. autofunction:: visidata.vd.isNumeric +.. versionadded:: 2.1 .. autofunction:: visidata.isNumeric .. deprecated:: 2.1 Use ``vd.isNumeric`` instead. diff --git a/docs/api/data.rst b/docs/api/data.rst index 61ca4e179..9d7f945ab 100644 --- a/docs/api/data.rst +++ b/docs/api/data.rst @@ -5,7 +5,9 @@ Each TableSheet has a set of *selected rows*, which is a strict subset of the ro .. autoattribute:: visidata.TableSheet.selectedRows .. autoattribute:: visidata.TableSheet.someSelectedRows +.. versionchanged:: 2.1 .. autoattribute:: visidata.TableSheet.onlySelectedRows +.. versionchanged:: 2.1 .. autoattribute:: visidata.TableSheet.nSelectedRows .. autofunction:: visidata.TableSheet.selectRow diff --git a/visidata/_types.py b/visidata/_types.py index d0ac58ca1..990ca0420 100644 --- a/visidata/_types.py +++ b/visidata/_types.py @@ -119,7 +119,6 @@ def getType(vd, typetype): @VisiData.api def isNumeric(vd, col): - '.. versionadded:: 2.1' return col.type in (int,vlen,float,currency,date,floatsi) ## diff --git a/visidata/deprecated.py b/visidata/deprecated.py index 7f8252276..5776bb37f 100644 --- a/visidata/deprecated.py +++ b/visidata/deprecated.py @@ -86,7 +86,6 @@ def push_pyobj(name, pyobj): @deprecated('2.1', 'vd.isNumeric instead') def isNumeric(col): - '.. deprecated:: 2.1' return vd.isNumeric(col) visidata.addGlobals({'load_pyobj': load_pyobj, 'isNumeric': isNumeric}) diff --git a/visidata/movement.py b/visidata/movement.py index fef5f2de5..eddb8fee3 100644 --- a/visidata/movement.py +++ b/visidata/movement.py @@ -90,9 +90,7 @@ def nextColRegex(sheet, colregex): @Column.property def visibleWidth(self): - '''Width of column as is displayed in terminal - - .. versionadded:: 2.1''' + 'Width of column as is displayed in terminal' vcolidx = self.sheet.visibleCols.index(self) return self.sheet._visibleColLayout[vcolidx][1] diff --git a/visidata/selection.py b/visidata/selection.py index f946cd264..0d0e07361 100644 --- a/visidata/selection.py +++ b/visidata/selection.py @@ -96,8 +96,7 @@ def selectedRows(self): @Sheet.property def onlySelectedRows(self): - '''List of selected rows in sheet order. Fail if no rows are selected. - .. versionchanged:: 2.1''' + 'List of selected rows in sheet order. Fail if no rows are selected.' if self.nSelectedRows == 0: vd.fail('no rows selected') return self.selectedRows @@ -107,8 +106,7 @@ def someSelectedRows(self): '''Return a list of rows: (a) in batch mode, always return selectedRows (b) in interactive mode, if options.some_selected_rows is True, return selectedRows or all rows if none selected - (c) in interactive mode, if options.some_selected_rows is False, return selectedRows or fail if none selected - .. versionchanged:: 2.1''' + (c) in interactive mode, if options.some_selected_rows is False, return selectedRows or fail if none selected''' if options.batch: return self.selectedRows if options.some_selected_rows: From 07cf36a05165747ef286be07868379aa8da1b489 Mon Sep 17 00:00:00 2001 From: anjakefala Date: Sun, 6 Dec 2020 21:34:27 -0800 Subject: [PATCH 161/162] [docs] update docs --- CHANGELOG.md | 2 +- docs/api/options.rst | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 73aa07186..726bcb09a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,7 +20,6 @@ - [expr] allow column attributes as variables (thanks @frosencrantz for feature request #659) - [freq] change `numeric_binning` back to False by default - [input] Shift+Arrow within `edit-cell` to move cursor and re-enter edit mode - - [loaders clickhouse] add clickhouse loader - [loaders http] have automatic API pagination (thanks @geekscrapy for feature request #480) - [loaders json] improve loading speedup 50% (thanks @lxcode for investigating and pointing this out #765) - this makes JSON saving non-deterministic in Python 3.6, as the order of fields output is dependent on the order within the dict @@ -110,6 +109,7 @@ ## Plugins - add conll loader to **PluginsSheet** (thanks @polm) - remove livesearch +- add clickhouse loader ## Commands - if `options.some_selected_rows` is True, `setcol-expr`, `setcol-iter`, `setcol-subst`, `setcol-subst`, `setcol-subst-all` will return all rows, if none selected diff --git a/docs/api/options.rst b/docs/api/options.rst index 03c9c4964..50d165927 100644 --- a/docs/api/options.rst +++ b/docs/api/options.rst @@ -67,6 +67,7 @@ Options API .. autofunction:: visidata.vd.options.get .. autofunction:: visidata.vd.options.set .. autofunction:: visidata.vd.options.unset +.. versionadded:: 2.1 .. autofunction:: visidata.vd.options.getall The dict returned by ``options.getall('foo_')`` is designed to be used as kwargs to other loaders, so that their options can be passed through VisiData transparently. From e69959dab133c981a465a4e20d9241d7118c98bd Mon Sep 17 00:00:00 2001 From: anjakefala Date: Sun, 6 Dec 2020 21:36:42 -0800 Subject: [PATCH 162/162] [dev] push version to 2.1 --- CHANGELOG.md | 2 +- README.md | 2 +- setup.py | 2 +- visidata/__init__.py | 2 +- visidata/main.py | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 726bcb09a..f4a67b905 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # VisiData version history -# v2.1 (????) +# v2.1 (2020-12-06) - [add] add bulk rows and cols leave cursor on first added (like add singles) - [add] add colname input to `addcol-new` diff --git a/README.md b/README.md index 91f218e7e..1f42e892a 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ -# VisiData v2.0.1 [![twitter @VisiData][1.1]][1] [![CircleCI](https://circleci.com/gh/saulpw/visidata/tree/develop.svg?style=svg)](https://circleci.com/gh/saulpw/visidata/tree/develop) [![Gitpod ready-to-code](https://img.shields.io/badge/Gitpod-ready--to--code-blue?logo=gitpod)](https://gitpod.io/#https://github.com/saulpw/visidata) +# VisiData v2.1 [![twitter @VisiData][1.1]][1] [![CircleCI](https://circleci.com/gh/saulpw/visidata/tree/develop.svg?style=svg)](https://circleci.com/gh/saulpw/visidata/tree/develop) [![Gitpod ready-to-code](https://img.shields.io/badge/Gitpod-ready--to--code-blue?logo=gitpod)](https://gitpod.io/#https://github.com/saulpw/visidata) A terminal interface for exploring and arranging tabular data. diff --git a/setup.py b/setup.py index 3d5557072..dfdbe23c6 100755 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ from setuptools import setup # tox can't actually run python3 setup.py: https://github.com/tox-dev/tox/issues/96 #from visidata import __version__ -__version__ = '2.0+' +__version__ = '2.1' setup(name='visidata', version=__version__, diff --git a/visidata/__init__.py b/visidata/__init__.py index 21994fb9b..6d5bd7d00 100644 --- a/visidata/__init__.py +++ b/visidata/__init__.py @@ -1,6 +1,6 @@ 'VisiData: a curses interface for exploring and arranging tabular data' -__version__ = '2.0+' +__version__ = '2.1' __version_info__ = 'VisiData v' + __version__ __author__ = 'Saul Pwanson ' __status__ = 'Production/Stable' diff --git a/visidata/main.py b/visidata/main.py index fd1ad8271..9b3f67aef 100755 --- a/visidata/main.py +++ b/visidata/main.py @@ -2,7 +2,7 @@ # Usage: $0 [] [ ...] # $0 [] --play [--batch] [-w ] [-o ] [field=value ...] -__version__ = '2.0+' +__version__ = '2.1' __version_info__ = 'saul.pw/VisiData v' + __version__ from copy import copy