From 4bab6e39d78bd98975cbc9c292bce8ef3a175c8e Mon Sep 17 00:00:00 2001 From: molley Date: Sun, 6 Dec 2020 15:33:24 +0400 Subject: [PATCH 01/27] More accurate return of int(0), float(0.0) If values evaluate to False, `""` (empty string) is returned. This is the same case with 0. This patch evaluates the string to return the str value of the value, so even 0 gets returned --- visidata/utils.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/visidata/utils.py b/visidata/utils.py index 44780befc..939e5696a 100644 --- a/visidata/utils.py +++ b/visidata/utils.py @@ -110,6 +110,8 @@ def get_field(self, field_name, *args, **kwargs): def format_field(self, value, format_spec): # value is missing - if not value: + if value is None: return '' + elif not value: + return str(value) return super().format_field(value, format_spec) From e7857f39e1216adaed4518467b5c29ec14770bcb Mon Sep 17 00:00:00 2001 From: Paul O'Leary McCann Date: Mon, 7 Dec 2020 15:47:29 +0900 Subject: [PATCH 02/27] Update conll The previous version used the Visidata decorator but didn't take visidata arguments, so it actually doesn't work. --- plugins/plugins.jsonl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/plugins.jsonl b/plugins/plugins.jsonl index 80cc6ab46..cbec3b101 100644 --- a/plugins/plugins.jsonl +++ b/plugins/plugins.jsonl @@ -10,4 +10,4 @@ {"name": "genericSQL", "description": "add loaders for SQL databases (Oracle, MySQL)", "maintainer": "Andrew Swanson @aswan89", "latest_release": "2020-04-15", "url": "https://raw.githubusercontent.com/aswan89/visidata_plugin_genericSQL/1.0.0/generic_sql.py", "latest_ver": "1.0.0", "visidata_ver": "2.-3", "pydeps": "sqlalchemy cx_oracle mysqlclient pyodbc", "vdplugindeps": "", "sha256": "71bbaabaa4ffb973ef7745dc3eadca108b34a7e0f0bf6cf6631eca3fa4532513"} {"name": "diff", "description": "adds command to create diff sheets", "maintainer": "Anja Kefala @anjakefala", "latest_release": "2020-10-09", "url": "https://raw.githubusercontent.com/saulpw/visidata/492d558ff4aba55a192965a27ae6499f8d219072/plugins/diff.py", "latest_ver": "0.9", "visidata_ver": "2.0", "sha256": "0085bf8afdc9abf74b7dd52c251f7b7f7befc507aae262bbb656c2ab7379ddd1"} {"name": "marks", "description": "adds commands for marking selected rows, and selecting + viewing marked rows", "maintainer": "Saul Pwanson @saulpw", "latest_release": "2020-10-09", "url": "https://raw.githubusercontent.com/saulpw/visidata/54e49feee97a607cce5355bf0e3a79b2466c3d8a/plugins/marks.py", "latest_ver": "0.1", "visidata_ver": "2.0", "sha256": "f15977f327ccc387be208922b66afe4b65666912e8066d02352c53b1aaff941f"} -{"name": "conll", "description": "CoNLL data loader", "maintainer": "Paul McCann ", "latest_release": "2020-11-09", "url": "https://raw.githubusercontent.com/polm/visidata-conll/83579a939813b3a3bca638bbb15bb9e8cf4e08ac/conll.py", "latest_ver": "0.1.0", "visidata_ver": "2.0", "pydeps": "pyconll", "sha256": "ff0df4c817121e57780ddd3e16ae67703fa69129c9e33e83fe40dd5220ad93e2"} +{"name":"conll", "description":"CoNLL data loader", "maintainer": "Paul McCann ", "latest_release": "2020-12-07", "latest_ver": "v0.1.1", "url": "https://raw.githubusercontent.com/polm/visidata-conll/1bc43a825b329c833ff3f9eb8377d850d58ec4cd/conll.py", "visidata_ver": "v2.1", "pydeps": "pyconll","sha256":"ef24974de79c4f8f2a0fdf48dea3d694ef5d6f4dc66fb0c9977c33a236da984c"} From 9ffe805504e4c19ea4714a4bf06b60ff0eb21c9a Mon Sep 17 00:00:00 2001 From: anjakefala Date: Tue, 8 Dec 2020 22:39:42 -0800 Subject: [PATCH 03/27] [layout] do not calc topRowIndex offset with single-line rows Offset required for paging with multiline-rows. Closes #832 --- visidata/sheets.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/visidata/sheets.py b/visidata/sheets.py index 9769c41ee..36ba13b6f 100644 --- a/visidata/sheets.py +++ b/visidata/sheets.py @@ -331,7 +331,7 @@ def bottomRowIndex(self, newidx): nrows += h i += 1 - self._topRowIndex = newidx-i+2 + self._topRowIndex = newidx-i+2 if nrows == self.nScreenRows-1 else newidx-self.nScreenRows+1 def __deepcopy__(self, memo): 'same as __copy__' From 6c1f3188174d7f85564592fa68952865e737bed0 Mon Sep 17 00:00:00 2001 From: anjakefala Date: Tue, 8 Dec 2020 22:41:12 -0800 Subject: [PATCH 04/27] [layout-] fix bottomRowIndex calculation topRowIndex is already counted among nScreenRows, so subtract 1 --- visidata/sheets.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/visidata/sheets.py b/visidata/sheets.py index 36ba13b6f..8109d2b26 100644 --- a/visidata/sheets.py +++ b/visidata/sheets.py @@ -319,7 +319,7 @@ def __copy__(self): @property def bottomRowIndex(self): - return max(self._rowLayout.keys()) if self._rowLayout else self.topRowIndex+self.nScreenRows + return max(self._rowLayout.keys()) if self._rowLayout else self.topRowIndex+self.nScreenRows-1 @bottomRowIndex.setter def bottomRowIndex(self, newidx): From 2a7434b778e173c71575a503e6b74ba4508ef995 Mon Sep 17 00:00:00 2001 From: Saul Pwanson Date: Tue, 8 Dec 2020 22:15:37 -0800 Subject: [PATCH 05/27] [wrapper] add TypedWrapper.__len__ thanks @geekscrapy --- visidata/wrappers.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/visidata/wrappers.py b/visidata/wrappers.py index 879dd8b5b..f8afee69a 100644 --- a/visidata/wrappers.py +++ b/visidata/wrappers.py @@ -28,6 +28,9 @@ def __init__(self, func, *args): def __bool__(self): return False + def __len__(self): + return 0 + def __str__(self): return '%s(%s)' % (self.type.__name__, ','.join(str(x) for x in self.args)) From 4b60224bb1464b4655a5c40d08dec4dd931e5b54 Mon Sep 17 00:00:00 2001 From: Saul Pwanson Date: Thu, 10 Dec 2020 19:21:36 -0800 Subject: [PATCH 06/27] [save] better default save filename for url sheets #824 - previously would offer to save as the url itself - @geekscrapy suggests the leaf of the url path (name+ext), which seems reasonable --- visidata/save.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/visidata/save.py b/visidata/save.py index c419f468f..aaf1c3bfb 100644 --- a/visidata/save.py +++ b/visidata/save.py @@ -68,6 +68,8 @@ def itervals(sheet, *cols, format=False): @Sheet.api def getDefaultSaveName(sheet): src = getattr(sheet, 'source', None) + if src.scheme: + return src.name + src.suffix if isinstance(src, Path): return str(src) else: From a1dd30c777c8f05442b69013a93a95991faf6f2b Mon Sep 17 00:00:00 2001 From: anjakefala Date: Thu, 17 Dec 2020 23:38:37 -0800 Subject: [PATCH 07/27] [plugins] fix sha256 for conll --- plugins/plugins.jsonl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/plugins.jsonl b/plugins/plugins.jsonl index cbec3b101..77f372d42 100644 --- a/plugins/plugins.jsonl +++ b/plugins/plugins.jsonl @@ -10,4 +10,4 @@ {"name": "genericSQL", "description": "add loaders for SQL databases (Oracle, MySQL)", "maintainer": "Andrew Swanson @aswan89", "latest_release": "2020-04-15", "url": "https://raw.githubusercontent.com/aswan89/visidata_plugin_genericSQL/1.0.0/generic_sql.py", "latest_ver": "1.0.0", "visidata_ver": "2.-3", "pydeps": "sqlalchemy cx_oracle mysqlclient pyodbc", "vdplugindeps": "", "sha256": "71bbaabaa4ffb973ef7745dc3eadca108b34a7e0f0bf6cf6631eca3fa4532513"} {"name": "diff", "description": "adds command to create diff sheets", "maintainer": "Anja Kefala @anjakefala", "latest_release": "2020-10-09", "url": "https://raw.githubusercontent.com/saulpw/visidata/492d558ff4aba55a192965a27ae6499f8d219072/plugins/diff.py", "latest_ver": "0.9", "visidata_ver": "2.0", "sha256": "0085bf8afdc9abf74b7dd52c251f7b7f7befc507aae262bbb656c2ab7379ddd1"} {"name": "marks", "description": "adds commands for marking selected rows, and selecting + viewing marked rows", "maintainer": "Saul Pwanson @saulpw", "latest_release": "2020-10-09", "url": "https://raw.githubusercontent.com/saulpw/visidata/54e49feee97a607cce5355bf0e3a79b2466c3d8a/plugins/marks.py", "latest_ver": "0.1", "visidata_ver": "2.0", "sha256": "f15977f327ccc387be208922b66afe4b65666912e8066d02352c53b1aaff941f"} -{"name":"conll", "description":"CoNLL data loader", "maintainer": "Paul McCann ", "latest_release": "2020-12-07", "latest_ver": "v0.1.1", "url": "https://raw.githubusercontent.com/polm/visidata-conll/1bc43a825b329c833ff3f9eb8377d850d58ec4cd/conll.py", "visidata_ver": "v2.1", "pydeps": "pyconll","sha256":"ef24974de79c4f8f2a0fdf48dea3d694ef5d6f4dc66fb0c9977c33a236da984c"} +{"name": "conll", "description": "CoNLL data loader", "maintainer": "Paul McCann ", "latest_release": "2020-12-07", "url": "https://raw.githubusercontent.com/polm/visidata-conll/1bc43a825b329c833ff3f9eb8377d850d58ec4cd/conll.py", "latest_ver": "v0.1.1", "visidata_ver": "v2.1", "pydeps": "pyconll", "sha256": "7a89537e1ad173d9cf8ec934cb81836be2e4ddcdc06c7632af92e6754594285c"} From 8fe24d5c748614b4b2023195ca42ae68e0390279 Mon Sep 17 00:00:00 2001 From: anjakefala Date: Fri, 18 Dec 2020 00:14:54 -0800 Subject: [PATCH 08/27] [pyobj-] SheetList should construct based on contents of list Closes #843 --- visidata/pyobj.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/visidata/pyobj.py b/visidata/pyobj.py index e135a7d9c..477c712cc 100644 --- a/visidata/pyobj.py +++ b/visidata/pyobj.py @@ -152,10 +152,10 @@ def SheetList(*names, **kwargs): vd.status('no content in %s' % names) return - if isinstance(src, dict): + if isinstance(src[0], dict): return ListOfDictSheet(*names, **kwargs) - elif isinstance(src, tuple): - if getattr(src, '_fields', None): # looks like a namedtuple + elif isinstance(src[0], tuple): + if getattr(src[0], '_fields', None): # looks like a namedtuple return ListOfNamedTupleSheet(*names, **kwargs) # simple list From 115cb8e0d1b6b0d7c108166f038ce92fc0c42442 Mon Sep 17 00:00:00 2001 From: anjakefala Date: Fri, 18 Dec 2020 00:22:02 -0800 Subject: [PATCH 09/27] [save-] only check scheme if attr exists --- visidata/save.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/visidata/save.py b/visidata/save.py index aaf1c3bfb..df5bff4fe 100644 --- a/visidata/save.py +++ b/visidata/save.py @@ -68,7 +68,7 @@ def itervals(sheet, *cols, format=False): @Sheet.api def getDefaultSaveName(sheet): src = getattr(sheet, 'source', None) - if src.scheme: + if hasattr(src, 'scheme') and src.scheme: return src.name + src.suffix if isinstance(src, Path): return str(src) From 1f6a959ca1021e85cf4d7f86509c3b8d25aa1705 Mon Sep 17 00:00:00 2001 From: anjakefala Date: Fri, 18 Dec 2020 00:22:31 -0800 Subject: [PATCH 10/27] [tests] add test for #843 --- tests/golden/listofdictobj.tsv | 3 +++ tests/listofdictobj.vd | 4 ++++ 2 files changed, 7 insertions(+) create mode 100644 tests/golden/listofdictobj.tsv create mode 100644 tests/listofdictobj.vd diff --git a/tests/golden/listofdictobj.tsv b/tests/golden/listofdictobj.tsv new file mode 100644 index 000000000..8ce4ae959 --- /dev/null +++ b/tests/golden/listofdictobj.tsv @@ -0,0 +1,3 @@ +imag real +0.00 6.08 +0.00 50.77 diff --git a/tests/listofdictobj.vd b/tests/listofdictobj.vd new file mode 100644 index 000000000..83aa030c0 --- /dev/null +++ b/tests/listofdictobj.vd @@ -0,0 +1,4 @@ +sheet col row longname input keystrokes comment + open-file sample_data/y77d-th95.json.gz o +y77d-th95 geolocation expand-col-depth 1 z( expand current column of containers to given depth (0=fully) + open-cell z^J open sheet with copies of rows referenced in current cell From de8cbb9a720946be7094574e0e5e07c192b3af81 Mon Sep 17 00:00:00 2001 From: anjakefala Date: Fri, 18 Dec 2020 00:30:53 -0800 Subject: [PATCH 11/27] [test-] specify column and row in open-cell --- tests/listofdictobj.vd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/listofdictobj.vd b/tests/listofdictobj.vd index 83aa030c0..e322ea3f7 100644 --- a/tests/listofdictobj.vd +++ b/tests/listofdictobj.vd @@ -1,4 +1,4 @@ sheet col row longname input keystrokes comment open-file sample_data/y77d-th95.json.gz o y77d-th95 geolocation expand-col-depth 1 z( expand current column of containers to given depth (0=fully) - open-cell z^J open sheet with copies of rows referenced in current cell + geolocation.coordinates 0 open-cell z^J open sheet with copies of rows referenced in current cell From d9892b1671eb541d7c5037f18bd4488f42a72960 Mon Sep 17 00:00:00 2001 From: anjakefala Date: Fri, 18 Dec 2020 00:34:56 -0800 Subject: [PATCH 12/27] [cmdlog-] record column, sheet and row info for open-cell --- visidata/cmdlog.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/visidata/cmdlog.py b/visidata/cmdlog.py index 46e950bbc..6d250b985 100644 --- a/visidata/cmdlog.py +++ b/visidata/cmdlog.py @@ -147,7 +147,7 @@ def beforeExecHook(self, sheet, cmd, args, keystrokes): self.afterExecSheet(sheet, False, '') colname, rowname, sheetname = '', '', None - if sheet and not (cmd.longname.startswith('open-') and cmd.longname != 'open-row'): + if sheet and not (cmd.longname.startswith('open-') and not cmd.longname in ('open-row', 'open-cell')): sheetname = sheet contains = lambda s, *substrs: any((a in s) for a in substrs) From 67c7c7c1c47537faebe4c1f717ca5f339b86b54a Mon Sep 17 00:00:00 2001 From: anjakefala Date: Fri, 18 Dec 2020 20:52:56 -0800 Subject: [PATCH 13/27] [history] add recent events --- dev/history.jsonl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dev/history.jsonl b/dev/history.jsonl index fe5080072..9be9ce048 100644 --- a/dev/history.jsonl +++ b/dev/history.jsonl @@ -6,3 +6,5 @@ {"date": "2020-05-13", "event": "changelog podcast released", "url": "https://changelog.com/podcast/394"} {"date": "2020-12-05", "event": "anja's tweet about visidata being a tool everyone should gets >100 likes", "url": "https://twitter.com/nevoitbien/status/1335398803604586498"} {"date": "2020-12-06", "event": "[HN] VisiData in 60 seconds", "url": "https://news.ycombinator.com/item?id=25322091"} +{"date": "2020-12-05", "event": "[reddit] VisiData - a cool vim-like tool for examining and manipulating data", "url": "https://www.reddit.com/r/vim/comments/kf4wgb/visidata_a_cool_vimlike_tool_for_examining_and/"} +{"date": "2020-12-06", "event": "RealPython tweets about VisiData", "url": "https://twitter.com/realpython/status/1339669258448547842"} From b154a7993bca3b9805297245eb836c2338947f55 Mon Sep 17 00:00:00 2001 From: anjakefala Date: Sat, 19 Dec 2020 14:17:02 -0800 Subject: [PATCH 14/27] [api] add vd.isLongname, which returns True if ks is longname --- visidata/basesheet.py | 5 +++++ visidata/help.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/visidata/basesheet.py b/visidata/basesheet.py index b89cc4af0..b06e8c99b 100644 --- a/visidata/basesheet.py +++ b/visidata/basesheet.py @@ -226,6 +226,11 @@ def sheet(self): 'the top sheet on the stack' return self.sheets[0] if self.sheets else None +@VisiData.api +def isLongname(self, ks): + 'Return True if *ks* is a longname.' + return ('-' in ks) and (ks[-1] != '-') + @VisiData.api def getSheet(vd, sheetname): diff --git a/visidata/help.py b/visidata/help.py index fded3ac51..92ada1076 100644 --- a/visidata/help.py +++ b/visidata/help.py @@ -36,7 +36,7 @@ def iterload(self): self.revbinds = {} # [longname] -> keystrokes itbindings = vd.bindkeys.iterall() for (keystrokes, _), longname in itbindings: - if (keystrokes not in self.revbinds) and ('-' not in keystrokes or keystrokes[-1] == '-'): + if (keystrokes not in self.revbinds) and not vd.isLongname(keystrokes): self.revbinds[longname] = keystrokes From f7e489234e88b49ffbd5b19f67c4ceef3b511d54 Mon Sep 17 00:00:00 2001 From: anjakefala Date: Sat, 19 Dec 2020 14:18:50 -0800 Subject: [PATCH 15/27] [macros] allow macro interfaces to be longnames Closes #787 --- visidata/macros.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/visidata/macros.py b/visidata/macros.py index 819ff37c9..6c4805c95 100644 --- a/visidata/macros.py +++ b/visidata/macros.py @@ -24,7 +24,10 @@ def runMacro(vd, macro): def setMacro(ks, vs): vd.macrobindings[ks] = vs - BaseSheet.addCommand(ks, vs.name, 'runMacro(vd.macrobindings[keystrokes])') + if vd.isLongname(ks): + BaseSheet.addCommand('', ks, 'runMacro(vd.macrobindings[longname])') + else: + BaseSheet.addCommand(ks, vs.name, 'runMacro(vd.macrobindings[keystrokes])') @CommandLog.api From 10a393da671c4d503a138c5e631183b209a38018 Mon Sep 17 00:00:00 2001 From: anjakefala Date: Tue, 22 Dec 2020 23:59:55 -0800 Subject: [PATCH 16/27] [LazyComputeRow] associate the LazyChainMap with current column, or sheet, if no column Used to be associated with the current sheet, always. Ran into a bug where curcol would be cached for an entire sheet, because it was referenced in a sheet's LazyChainMap. Closes #659. Co-authored-by: saulpw --- docs/columns.md | 18 ++++++++++++++++- tests/curcol.vd | 6 ++++++ tests/golden/curcol.tsv | 44 +++++++++++++++++++++++++++++++++++++++++ visidata/sheets.py | 18 ++++++++++------- 4 files changed, 78 insertions(+), 8 deletions(-) create mode 100644 tests/curcol.vd create mode 100644 tests/golden/curcol.tsv diff --git a/docs/columns.md b/docs/columns.md index ae343d48e..a1a018e96 100644 --- a/docs/columns.md +++ b/docs/columns.md @@ -203,10 +203,13 @@ These variables and functions are available in the scope of an expression: - **`sheet`**: the current sheet (a TableSheet object) - **`col`**: the current column (as a Column object; use for Column metadata) - **`row`**: the current row (a Python object of the internal rowtype) +- **curcol**: evaluate to the typed value of this row in the column that the cursor was on at the time that the expression column was added. +- **cursorCol**: evaluate to the typed value of this row for the column the cursor is on. Changes as the cursor moves for `=`. Uses the column from the time the calculation was made for `g=`, `gz=`, and `z=`. Additional attributes can be added to sheets and columns. -`col` deliberately returns a Column object, but any other Column object is interpreted as the value within that column for the same row. +`col` deliberately returns a Column object, but any other Column object is interpreted as the value within that column for the same row. For example, both `curcol` and `cursorcol` return values, not the object itself. + For example, this customizes addcol-expr to set the `curcol` attribute on the new ExprColumn to a snapshot of the current cursor column (at the time the expression column is added): @@ -235,6 +238,19 @@ The following examples use the file [sample.tsv](https://raw.githubusercontent.c 2. Set the type of the new derived column by pressing `@` (date). 3. Type `^` followed by `Date` to rename the column to **Date**. +**Question** I have a dataset with **Date** column that is missing a prefix of '2020-'. How do I add it to the **Date** column? + +When using `=`, and wanting to reference the current column, we recommend using `curcol`. When using `g=`, `gz=`, and `z=`, we recommend cursorCol. `=`, unlike the others, is dynamic and changes with adjustment of underlying values, which means it will change along with the movement of the cursor (tracked by `cursorCol`). `curcol` is a special attribute of a new **ExprColumn**, which remembers the cursorCol at the time of creation. + +1. Move the cursor to **Date**. +2. Type `g=` followed by *f"2020-{cursorCol}"*. + +**Question** I have a dataset with **file names**. How do I create a new column with the **file names** lower cased? + +1. Move the cursor to **file names** column. +2. Type `=` followed by **curcol.casefold()**. +3. Move to the newly created column, and rename it with `^`, followed by the desired name. + --- ## How to configure multiple columns diff --git a/tests/curcol.vd b/tests/curcol.vd new file mode 100644 index 000000000..1836f0dc8 --- /dev/null +++ b/tests/curcol.vd @@ -0,0 +1,6 @@ +sheet col row longname input keystrokes comment + open-file sample_data/sample.tsv o +sample Region addcol-expr curcol.casefold() = create new column from Python expression, with column names as variables +sample Item addcol-expr curcol.casefold() = create new column from Python expression, with column names as variables +sample curcol.casefold() rename-col Region_col ^ edit name of current column +sample curcol.casefold() rename-col Item_col ^ edit name of current column diff --git a/tests/golden/curcol.tsv b/tests/golden/curcol.tsv new file mode 100644 index 000000000..cdd46faff --- /dev/null +++ b/tests/golden/curcol.tsv @@ -0,0 +1,44 @@ +OrderDate Region Region_col Rep Item Item_col Units Unit_Cost Total +2016-01-06 East east Jones Pencil pencil 95 1.99 189.05 +2016-01-23 Central central Kivell Binder binder 50 19.99 999.50 +2016-02-09 Central central Jardine Pencil pencil 36 4.99 179.64 +2016-02-26 Central central Gill Pen pen 27 19.99 539.73 +2016-03-15 West west Sorvino Pencil pencil 56 2.99 167.44 +2016-04-01 East east Jones Binder binder 60 4.99 299.40 +2016-04-18 Central central Andrews Pencil pencil 75 1.99 149.25 +2016-05-05 Central central Jardine Pencil pencil 90 4.99 449.10 +2016-05-22 West west Thompson Pencil pencil 32 1.99 63.68 +2016-06-08 East east Jones Binder binder 60 8.99 539.40 +2016-06-25 Central central Morgan Pencil pencil 90 4.99 449.10 +2016-07-12 East east Howard Binder binder 29 1.99 57.71 +2016-07-29 East east Parent Binder binder 81 19.99 1619.19 +2016-08-15 East east Jones Pencil pencil 35 4.99 174.65 +2016-09-01 Central central Smith Desk desk 2 125.00 250.00 +2016-09-18 East east Jones Pen Set pen set 16 15.99 255.84 +2016-10-05 Central central Morgan Binder binder 28 8.99 251.72 +2016-10-22 East east Jones Pen pen 64 8.99 575.36 +2016-11-08 East east Parent Pen pen 15 19.99 299.85 +2016-11-25 Central central Kivell Pen Set pen set 96 4.99 479.04 +2016-12-12 Central central Smith Pencil pencil 67 1.29 86.43 +2016-12-29 East east Parent Pen Set pen set 74 15.99 1183.26 +2017-01-15 Central central Gill Binder binder 46 8.99 413.54 +2017-02-01 Central central Smith Binder binder 87 15.00 1305.00 +2017-02-18 East east Jones Binder binder 4 4.99 19.96 +2017-03-07 West west Sorvino Binder binder 7 19.99 139.93 +2017-03-24 Central central Jardine Pen Set pen set 50 4.99 249.50 +2017-04-10 Central central Andrews Pencil pencil 66 1.99 131.34 +2017-04-27 East east Howard Pen pen 96 4.99 479.04 +2017-05-14 Central central Gill Pencil pencil 53 1.29 68.37 +2017-05-31 Central central Gill Binder binder 80 8.99 719.20 +2017-06-17 Central central Kivell Desk desk 5 125.00 625.00 +2017-07-04 East east Jones Pen Set pen set 62 4.99 309.38 +2017-07-21 Central central Morgan Pen Set pen set 55 12.49 686.95 +2017-08-07 Central central Kivell Pen Set pen set 42 23.95 1005.90 +2017-08-24 West west Sorvino Desk desk 3 275.00 825.00 +2017-09-10 Central central Gill Pencil pencil 7 1.29 9.03 +2017-09-27 West west Sorvino Pen pen 76 1.99 151.24 +2017-10-14 West west Thompson Binder binder 57 19.99 1139.43 +2017-10-31 Central central Andrews Pencil pencil 14 1.29 18.06 +2017-11-17 Central central Jardine Binder binder 11 4.99 54.89 +2017-12-04 Central central Jardine Binder binder 94 19.99 1879.06 +2017-12-21 Central central Andrews Binder binder 28 4.99 139.72 diff --git a/visidata/sheets.py b/visidata/sheets.py index 8109d2b26..cdb83c5e6 100644 --- a/visidata/sheets.py +++ b/visidata/sheets.py @@ -99,16 +99,20 @@ def __init__(self, sheet, row, col=None): self.row = row self.col = col self.sheet = sheet - if not hasattr(self.sheet, '_lcm'): - self.sheet._lcm = LazyChainMap(sheet, vd, col) - else: - self.sheet._lcm.clear() # reset locals on lcm - self._usedcols = set() self._keys = [c.name for c in self.sheet.columns] + self._lcm.clear() # reset locals on lcm + + @property + def _lcm(self): + lcmobj = self.col or self.sheet + if not hasattr(lcmobj, '_lcm'): + lcmobj._lcm = LazyChainMap(self.sheet, vd, self.col) + return lcmobj._lcm + def keys(self): - return self._keys + self.sheet._lcm.keys() + ['row', 'sheet', 'col'] + return self._keys + self._lcm.keys() + ['row', 'sheet', 'col'] def __str__(self): return str(self.as_dict()) @@ -129,7 +133,7 @@ def __getitem__(self, colid): except ValueError: try: - c = self.sheet._lcm[colid] + c = self._lcm[colid] except (KeyError, AttributeError): if colid == 'sheet': return self.sheet elif colid == 'row': c = self.row From bfd24ee960b8bc1c42a0f7349a3c331f948f71ec Mon Sep 17 00:00:00 2001 From: anjakefala Date: Sat, 26 Dec 2020 11:29:43 -0800 Subject: [PATCH 17/27] [docs] fix typo in manpage --- visidata/man/vd.inc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/visidata/man/vd.inc b/visidata/man/vd.inc index 40ba6543a..079d9e421 100644 --- a/visidata/man/vd.inc +++ b/visidata/man/vd.inc @@ -890,7 +890,7 @@ Core VisiData includes these sources: .It Sy sqlite .Bl -inset -compact -offset xxx .It May include multiple tables. The initial sheet is the table directory; -.Sy Enter No loaders the entrie table into memory. Sy z^S No saves modifications to source. +.Sy Enter No loads the entire table into memory. Sy z^S No saves modifications to source. .El .El .Pp From 5722fb93ce035c803fe36683e6c73c76f4860f84 Mon Sep 17 00:00:00 2001 From: anjakefala Date: Sun, 27 Dec 2020 21:03:55 -0800 Subject: [PATCH 18/27] [isLongname] refine check to include longer than 3 chars + lowercase --- visidata/basesheet.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/visidata/basesheet.py b/visidata/basesheet.py index b06e8c99b..90640f418 100644 --- a/visidata/basesheet.py +++ b/visidata/basesheet.py @@ -229,7 +229,7 @@ def sheet(self): @VisiData.api def isLongname(self, ks): 'Return True if *ks* is a longname.' - return ('-' in ks) and (ks[-1] != '-') + return ('-' in ks) and (ks[-1] != '-') or (len(ks) > 3 and ks.islower()) @VisiData.api From e80f57d07997d7930a695b7d33ce1b6c8d04824f Mon Sep 17 00:00:00 2001 From: anjakefala Date: Mon, 28 Dec 2020 14:39:33 -0800 Subject: [PATCH 19/27] [docs] add docs for selecting rows based on null-ness. --- docs/index.md | 2 ++ docs/rows.md | 17 +++++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/docs/index.md b/docs/index.md index b3db83cb8..3c7b1303a 100644 --- a/docs/index.md +++ b/docs/index.md @@ -37,6 +37,8 @@ * How to perform operations on a subset of rows * How to filter rows * How to filter a random subset of rows + * How to select rows where the current column is not null + * How to select rows where the current column is null * How to move, copy and remove rows * How to sort rows * [Columns](/docs/columns) diff --git a/docs/rows.md b/docs/rows.md index 30dea6f52..1e4636a43 100644 --- a/docs/rows.md +++ b/docs/rows.md @@ -65,6 +65,23 @@ The following example uses the file [sample.tsv](https://raw.githubusercontent.c --- +## How to select rows where the current column is not null? + +'Null' cells, by default, are cells which contain `None`. This can be changed with `options.null_value`. Null cells can be set with `zd` (set current cell to `None`) or `gzd` (set selected rows in current column to `None`). Null cells are distinguished with a yellow ∅' symbol on the right hand corner. They are distinct from empty cells (which are `''` in columns of string type.) + +1. Type `|` followed by *.* to select all rows with empty and null cells in the current column. + +--- + +## How to select rows where the current column is null? + +There are several different options: + +- Move to an empty or null cell in the column of interest and press `,` to select all empty cells in that column. +- Open a **DescribeSheet** for the current sheet with `Shift+I`. Move to the **nulls** column, and then move to the row which references the source column of interest. Type `zd` to select all null rows for that column. +- For non-numerical columns `z|` followed by **not ColumnName**, will select all empty cells for that column. For numerical columns it will also select cells with `0`. +--- + ## How to move, copy and remove rows Command(s) Operation From 216629428b88d215f367832f85e4db0f3eeb3172 Mon Sep 17 00:00:00 2001 From: Jeremy Dormitzer Date: Sun, 27 Dec 2020 10:53:46 -0500 Subject: [PATCH 20/27] Account for postgres_schema when rendering Postgres tables --- visidata/loaders/postgres.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/visidata/loaders/postgres.py b/visidata/loaders/postgres.py index 5dc995271..e600c067a 100644 --- a/visidata/loaders/postgres.py +++ b/visidata/loaders/postgres.py @@ -112,7 +112,11 @@ def openRow(self, row): class PgTable(Sheet): @asyncthread def reload(self): - with self.sql.cur("SELECT * FROM " + self.source) as cur: + if self.options.postgres_schema: + source = f"{self.options.postgres_schema}.{self.source}" + else: + source = self.source + with self.sql.cur(f"SELECT * FROM {source}") as cur: self.rows = [] r = cur.fetchone() if r: From 92979c984ae9d328fe23f5795a8f02c4ea0ad717 Mon Sep 17 00:00:00 2001 From: anjakefala Date: Wed, 30 Dec 2020 16:12:14 -0800 Subject: [PATCH 21/27] [sort-] increment progress once per row, instead of once per column * row - Fixes bug where total progress is 100 * sorted column --- visidata/sort.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/visidata/sort.py b/visidata/sort.py index b5972239e..0a305e2b0 100644 --- a/visidata/sort.py +++ b/visidata/sort.py @@ -47,8 +47,9 @@ def sortkey(self, r, prog=None): val = col.getTypedValue(r) ret.append(Reversor(val) if reverse else val) - if prog: - prog.addProgress(1) + if prog: + prog.addProgress(1) + return ret @Sheet.api From 0fe78b4fe95b418082359d2bd8e10d11f4735e3d Mon Sep 17 00:00:00 2001 From: anjakefala Date: Fri, 1 Jan 2021 22:11:33 -0800 Subject: [PATCH 22/27] [stdout] handle broken pipe gracefully Closes #851 --- visidata/main.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/visidata/main.py b/visidata/main.py index 9b3f67aef..e07f02b67 100755 --- a/visidata/main.py +++ b/visidata/main.py @@ -297,6 +297,8 @@ def vd_cli(): rc = -1 try: rc = main_vd() + except BrokenPipeError: + os.dup2(os.open(os.devnull, os.O_WRONLY), sys.stdout.fileno()) # handle broken pipe gracefully except visidata.ExpectedException as e: print('Error: ' + str(e)) except FileNotFoundError as e: From 7de88d5dd9720bd17301058da20e0dc629a36213 Mon Sep 17 00:00:00 2001 From: molley Date: Fri, 1 Jan 2021 22:13:31 -0800 Subject: [PATCH 23/27] [open_url] fail unknown URL scheme Currently if visidata is given a URI handler it doesn't understand, like `abc://` it will fail in the console like the following: ``` Traceback (most recent call last): File "bin/vd", line 6, in visidata.main.vd_cli() File "/Users/geekscrapy/vd_dev/visidata_upstream/visidata/main.py", line 299, in vd_cli rc = main_vd() File "/Users/geekscrapy/vd_dev/visidata_upstream/visidata/main.py", line 213, in main_vd vs = vd.openSource(p, **opts) File "/Users/geekscrapy/vd_dev/visidata_upstream/visidata/_open.py", line 78, in openSource vs = vd.openPath(Path(p), filetype=filetype) # convert to Path and recurse File "/Users/geekscrapy/vd_dev/visidata_upstream/visidata/_open.py", line 45, in openPath return vd.getGlobals()[openfunc](p, filetype=filetype) KeyError: 'openurl_abc' ``` My suggestion is to catch this KeyError and returns the following instead: `Error: no loader for url schema: abc` --- visidata/_open.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/visidata/_open.py b/visidata/_open.py index 6cf029621..04265d14e 100644 --- a/visidata/_open.py +++ b/visidata/_open.py @@ -42,7 +42,10 @@ def openPath(vd, p, filetype=None): 'Call ``open_(p)`` or ``openurl_(p, filetype)``. Return constructed but unloaded sheet of appropriate type.' if p.scheme and not p.fp: # isinstance(p, UrlPath): openfunc = 'openurl_' + p.scheme - return vd.getGlobals()[openfunc](p, filetype=filetype) + try: + return vd.getGlobals()[openfunc](p, filetype=filetype) + except KeyError: + vd.fail(f'no loader for url scheme: {p.scheme}') if not filetype: if p.is_dir(): From 562628d4b0f0775c80d7f484e8029ced421afd0a Mon Sep 17 00:00:00 2001 From: anjakefala Date: Sat, 2 Jan 2021 22:52:52 -0800 Subject: [PATCH 24/27] [v2.1.1] update changelog --- CHANGELOG.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f4a67b905..eaeb4fda5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,24 @@ # VisiData version history +# v2.1.1 + + - [macros] allow macro interfaces to be longnames (thanks @frosencrantz #787) + - [save] better default save filename for url sheets (thanks @geekscrapy #824) + +## Bugfixes + - [cmdlog] record column, sheet, and row info for open-cell + - [expr-col] `curcol` now works for multiple invocations (thanks @geekscrapy #659) + - [loaders postgres] account for postgres_schema when rendering Postgres tables (thanks @jdormit for PR #852) + - [loaders url] fail unknown URL scheme (thanks @geekscrapy for PR #84) + - [pyobj] fix Pyobj Sheets for lists (thanks @brookskindle #843) + - [pipe] handle broken pipes gracefully (thanks @robdmc #851) + - [scroll] fix issue with jagged scrolling down (thanks @uoee #832) + - [sort] fix bug where total progress in sorting is (100 * # of columns to sort) (thanks @cwarden) + +## api + - format_field formats int(0) and float(0.0) as "0" (thanks @geekscrapy for PR #821) + - add TypedWrapper.__len__ (thanks @geekscrapy) + # v2.1 (2020-12-06) - [add] add bulk rows and cols leave cursor on first added (like add singles) From 7acc6155377a7a31711720304e36aa877147ea69 Mon Sep 17 00:00:00 2001 From: anjakefala Date: Sat, 2 Jan 2021 22:57:15 -0800 Subject: [PATCH 25/27] [v2.1.1] bump version to 2.1.1 --- CHANGELOG.md | 2 +- README.md | 2 +- setup.py | 2 +- visidata/__init__.py | 2 +- visidata/main.py | 2 +- visidata/man/vd.1 | 2 +- visidata/man/vd.txt | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index eaeb4fda5..7b71734ab 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # VisiData version history -# v2.1.1 +# v2.1.1 (2021-01-03) - [macros] allow macro interfaces to be longnames (thanks @frosencrantz #787) - [save] better default save filename for url sheets (thanks @geekscrapy #824) diff --git a/README.md b/README.md index 1f42e892a..5ad968cc1 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ -# VisiData v2.1 [![twitter @VisiData][1.1]][1] [![CircleCI](https://circleci.com/gh/saulpw/visidata/tree/develop.svg?style=svg)](https://circleci.com/gh/saulpw/visidata/tree/develop) [![Gitpod ready-to-code](https://img.shields.io/badge/Gitpod-ready--to--code-blue?logo=gitpod)](https://gitpod.io/#https://github.com/saulpw/visidata) +# VisiData v2.1.1 [![twitter @VisiData][1.1]][1] [![CircleCI](https://circleci.com/gh/saulpw/visidata/tree/develop.svg?style=svg)](https://circleci.com/gh/saulpw/visidata/tree/develop) [![Gitpod ready-to-code](https://img.shields.io/badge/Gitpod-ready--to--code-blue?logo=gitpod)](https://gitpod.io/#https://github.com/saulpw/visidata) A terminal interface for exploring and arranging tabular data. diff --git a/setup.py b/setup.py index dfdbe23c6..72e8cbf21 100755 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ from setuptools import setup # tox can't actually run python3 setup.py: https://github.com/tox-dev/tox/issues/96 #from visidata import __version__ -__version__ = '2.1' +__version__ = '2.1.1' setup(name='visidata', version=__version__, diff --git a/visidata/__init__.py b/visidata/__init__.py index 6d5bd7d00..1ae2ad575 100644 --- a/visidata/__init__.py +++ b/visidata/__init__.py @@ -1,6 +1,6 @@ 'VisiData: a curses interface for exploring and arranging tabular data' -__version__ = '2.1' +__version__ = '2.1.1' __version_info__ = 'VisiData v' + __version__ __author__ = 'Saul Pwanson ' __status__ = 'Production/Stable' diff --git a/visidata/main.py b/visidata/main.py index e07f02b67..4ac08d958 100755 --- a/visidata/main.py +++ b/visidata/main.py @@ -2,7 +2,7 @@ # Usage: $0 [] [ ...] # $0 [] --play [--batch] [-w ] [-o ] [field=value ...] -__version__ = '2.1' +__version__ = '2.1.1' __version_info__ = 'saul.pw/VisiData v' + __version__ from copy import copy diff --git a/visidata/man/vd.1 b/visidata/man/vd.1 index c12fbca65..2276a6cce 100644 --- a/visidata/man/vd.1 +++ b/visidata/man/vd.1 @@ -1198,7 +1198,7 @@ Core VisiData includes these sources: .It Sy sqlite .Bl -inset -compact -offset xxx .It May include multiple tables. The initial sheet is the table directory; -.Sy Enter No loaders the entrie table into memory. Sy z^S No saves modifications to source. +.Sy Enter No loads the entire table into memory. Sy z^S No saves modifications to source. .El .El .Pp diff --git a/visidata/man/vd.txt b/visidata/man/vd.txt index 03c2ee0e2..6608ab1be 100644 --- a/visidata/man/vd.txt +++ b/visidata/man/vd.txt @@ -868,7 +868,7 @@ SUPPORTED SOURCES sqlite May include multiple tables. The initial sheet is the table - directory; Enter loaders the entrie table into memory. z^S saves + directory; Enter loads the entire table into memory. z^S saves modifications to source. URL schemes are also supported: From df3f2b0911bc02ad323feec520a2e6bd145545f0 Mon Sep 17 00:00:00 2001 From: anjakefala Date: Sun, 3 Jan 2021 17:31:28 -0800 Subject: [PATCH 26/27] [cmdlog] catch case of 'override' sheet for set-option --- visidata/cmdlog.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/visidata/cmdlog.py b/visidata/cmdlog.py index 6d250b985..d393f9665 100644 --- a/visidata/cmdlog.py +++ b/visidata/cmdlog.py @@ -281,12 +281,13 @@ def activeSheet(vd): def replayOne(vd, r): 'Replay the command in one given row.' vd.currentReplayRow = r - if r.sheet: + longname = getattr(r, 'longname', None) + + if r.sheet and not (r.sheet == 'override' and longname in ['set-option', 'unset-option']): vs = vd.getSheet(r.sheet) or vd.error('no sheet named %s' % r.sheet) else: vs = None - longname = getattr(r, 'longname', None) if longname in ['set-option', 'unset-option']: try: context = vs if r.sheet and vs else vd From 0caf62f95824aeeb4b8202bf49863becfca6c932 Mon Sep 17 00:00:00 2001 From: anjakefala Date: Sun, 3 Jan 2021 22:09:24 -0800 Subject: [PATCH 27/27] [2.1.1] update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7b71734ab..2e08adc07 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ ## Bugfixes - [cmdlog] record column, sheet, and row info for open-cell + - [cmdlog] catch case of 'override' sheet for set-option - [expr-col] `curcol` now works for multiple invocations (thanks @geekscrapy #659) - [loaders postgres] account for postgres_schema when rendering Postgres tables (thanks @jdormit for PR #852) - [loaders url] fail unknown URL scheme (thanks @geekscrapy for PR #84)