Merge pull request #1525 from saulpw/develop

v2.10.1

CHANGELOG.md

@@ -1,5 +1,37 @@
 # VisiData version history
 
+# v2.10.1 (2022-09-12)
+
+## Improvements
+
+- [docs] document `-d` option (thanks @abitrolly for PR #1515)
+- [freq] disable histogram if `disp_histlen` or `disp_histogram` set to 0 or empty string
+- [guard] add `guard-sheet-off` which unsets `options.quitguard` on current sheet (thanks @hanfried for PR #1517)
+- [menu] add `BUTTON1_CLICKED` (same as `BUTTON1_PRESSED`)
+- [open] add `zo` to open file or url from path in current cell
+
+## Bugfixes
+
+- fix Guix build problems (reported by @ryanprior #1499)
+- add support for sheet names with multiple `.` (periods) in the name (requested by @geekscrapy #1494)
+- [cmdlog] add more portable shebang in vdj
+- [date] fix custom date greater than or equal to comparison
+- [macros] fix `macro-record` (#1513)
+- [macros] refresh `macro-sheet` upon macro addition
+- [macros] ensure macros are set upon startup
+- [plugins] update usd plugin api (thanks @hanfried for PR #1510)
+- [repeat] fix `repeat-` (#1513)
+- [status] reduce priority of active colouring (reported by @geekscrapy #804)
+
+## API
+
+- add `ExpandedColumn` to globals
+- add `Extensible.before` and `Extensible.after`
+  - `def foo` decorated with `@VisiData.before` will run it before `vd.foo()`
+  - `def foo` decorated with `@VisiData.after` will run it immediately after
+
+
+
 # v2.10 (2022-08-28)
 
 - [plugins] load all entry points in `visidata.plugins` group before config load

MANIFEST.in

@@ -4,3 +4,4 @@ include visidata/man/vd.1
 include visidata/man/vd.txt
 include visidata/man/visidata.1
 include visidata/ddw/input.ddw
+include visidata/tests/sample.tsv

plugins/usd.py

@@ -7,6 +7,7 @@
 import json
 
 vd.option('fixer_key', '', 'API Key for fixer.io')
+vd.option('fixer_currency_cache_days', 1, 'Cache days for currency conversions')
 
 currency_symbols = {
     '$': 'USD',
@@ -19,9 +20,25 @@
     '₫': 'VND',
 }
 
-def currency_rates_json(date='latest'):
-    url = 'http://data.fixer.io/api/%s?access_key=%s' % (date, vd.options.fixer_key)
-    return vd.urlcache(url).read_text()
+def currency_rates_json(date='latest', base='USD'):
+    url = 'https://api.apilayer.com/fixer/%s?base=%s' % (date, base)
+    return vd.urlcache(
+        url, 
+        days=vd.options.fixer_currency_cache_days, 
+        headers={
+            # First need to set some additional headers as otherwise apilayers will block it with a 403
+            # See also https://stackoverflow.com/questions/13303449/urllib2-httperror-http-error-403-forbidden
+            'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11',
+            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+            'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.3',
+            'Accept-Encoding': 'none',
+            'Accept-Language': 'en-US,en;q=0.8',
+            'Connection': 'keep-alive',
+
+            # Finally set Apikey
+            'apikey': vd.options.fixer_key
+        } 
+    ).read_text()
 
 @functools.lru_cache()
 def currency_rates():
@@ -37,6 +54,7 @@ def currency_multiplier(src_currency, dest_currency):
     usd_mult = eur_usd_mult/eur_src_mult
     if dest_currency == 'USD':
         return usd_mult
+
     return usd_mult/currency_rates()[dest_currency]
 
 def USD(s):

setup.py

@@ -3,7 +3,7 @@
 from setuptools import setup
 # tox can't actually run python3 setup.py: https://github.com/tox-dev/tox/issues/96
 #from visidata import __version__
-__version__ = '2.10'
+__version__ = '2.11dev'
 
 setup(name='visidata',
       version=__version__,
@@ -29,7 +29,7 @@
       packages=['visidata',  'visidata.loaders', 'visidata.vendor', 'visidata.tests'],
       include_package_data=True,
       data_files = [('share/man/man1', ['visidata/man/vd.1', 'visidata/man/visidata.1'])],
-      package_data={'visidata': ['man/vd.1', 'man/vd.txt', 'ddw/input.ddw']},
+      package_data={'visidata': ['man/vd.1', 'man/vd.txt', 'ddw/input.ddw', 'tests/sample.tsv']},
       license='GPLv3',
       classifiers=[
           'Development Status :: 5 - Production/Stable',

tests/diff-join.vd

@@ -3,7 +3,7 @@ sheet	col	row	longname	input	keystrokes	comment
 			open-file	tests/data2.tsv	o	
 data1	Key		key-col		!	
 data1			sheets-stack		S	
-sheets		キdata2	open-row		^J	
+sheets		キdata2	open-row-pyobj		^J	
 data2	Key		key-col		!	
 data2			sheets-stack		S	
 sheets		キdata2	select-row		s	

tests/edit-joinkey-2.vd

@@ -16,4 +16,4 @@ data2+data1	A		key-col		!
 data2+data1	Key	キ2	edit-cell	4	e	
 data2+data1			reload-sheet		^R	
 data2			sheets-stack		S	
-sheets		キdata1	open-row		^J	
+sheets		キdata1	open-row-pyobj		^J	

tests/edit-joinregular-2.vd

@@ -15,4 +15,4 @@ data2+data1			sort-keys-asc		g[
 data2+data1	A		key-col		!	
 data2+data1	C	キ2	edit-cell	a3	e	
 data2+data1			sheets-stack		S	
-sheets		キdata2	open-row		^J	
+sheets		キdata2	open-row-pyobj		^J	

tests/invalid_unicode_sqlite.vd

@@ -2,4 +2,4 @@ sheet	col	row	longname	input	keystrokes	comment
 global		encoding	set-option	latin-1		
 invalid_unicode		encoding	set-option	latin-1		
 			open-file	tests/invalid_unicode.sqlite	o	
-invalid_unicode		キTest	open-row		Enter	open sheet with copies of rows referenced in current row
+invalid_unicode		キTest	open-row-pyobj		Enter	open sheet with copies of rows referenced in current row

tests/join-cols-single-sheet.vd

@@ -6,4 +6,4 @@ sample_columns		キRegion	select-row		s	select current row
 sample_columns		キRep	select-row		s	select current row
 sample_columns			join-cols		&	add column from concatenating selected source columns
 sample_columns			sheets-stack		S	open Sheets Stack: join or jump between the active sheets on the current stack
-sheets		キsample	open-row		^J	open sheet with copies of rows referenced in current row
+sheets		キsample	open-row-pyobj		^J	open sheet with copies of rows referenced in current row

tests/join-different-types.vd

@@ -1,16 +1,16 @@
 sheet	col	row	longname	input	keystrokes	comment
 			open-file	tests/joining_error.xlsx	o	
 			open-file	tests/joining_error_interesting_records.csv	o	
-joining_error		キrecords	open-row		^J	open sheet with copies of rows referenced in current row
+joining_error		キrecords	open-row-pyobj		^J	open sheet with copies of rows referenced in current row
 joining_error_records			sheets-stack		S	open Sheets Stack: join or jump between the active sheets on the current stack
 sheets		キjoining_error_interesting_records	slide-up		K	slide current row up
 sheets		キjoining_error_interesting_records	slide-up		K	slide current row up
 sheets		キjoining_error_interesting_records	select-row		s	select current row
 sheets		キjoining_error_records	select-row		s	select current row
-sheets		キjoining_error_interesting_records	open-row		^J	open sheet referenced in current row
+sheets		キjoining_error_interesting_records	open-row-pyobj		^J	open sheet referenced in current row
 joining_error_interesting_records	Row		key-col		!	toggle current column as a key column
 joining_error_interesting_records			sheets-stack		S	open Sheets Stack: join or jump between the active sheets on the current stack
-sheets		キjoining_error_records	open-row		^J	open sheet referenced in current row
+sheets		キjoining_error_records	open-row-pyobj		^J	open sheet referenced in current row
 joining_error_records	Row		key-col		!	toggle current column as a key column
 joining_error_records			sheets-stack		S	open Sheets Stack: join or jump between the active sheets on the current stack
 sheets		キjoining_error_interesting_records	slide-up		K	slide current row up

tests/join-non-unique-cols.vd

@@ -14,4 +14,4 @@ sheets			join-sheets	outer	&
 benchmark+sample			columns-sheet		C	
 benchmark+sample_columns	name		sort-desc		]	
 benchmark+sample_columns			sheets-stack		S	
-sheets		キbenchmark+sample	open-row		^J	
+sheets		キbenchmark+sample	open-row-pyobj		^J	

tests/listofdictobj.vd

@@ -1,4 +1,4 @@
 sheet	col	row	longname	input	keystrokes	comment
 			open-file	sample_data/y77d-th95.json.gz	o	
 y77d-th95	geolocation		expand-col-depth	1	z(	expand current column of containers to given depth (0=fully)
-	geolocation.coordinates	0	open-cell		z^J	open sheet with copies of rows referenced in current cell
+	geolocation.coordinates	0	open-cell-pyobj		z^J	open sheet with copies of rows referenced in current cell

tests/load-ods.vd

@@ -1,3 +1,3 @@
 sheet	col	row	longname	input	keystrokes	comment
 			open-file	sample_data/benchmark.ods	o	
-benchmark		キbenchmark	open-row		Enter	open sheet with copies of rows referenced in current row
+benchmark		キbenchmark	open-row-pyobj		Enter	open sheet with copies of rows referenced in current row

tests/load-sqlite-view.vd

@@ -1,3 +1,3 @@
 sheet	col	row	longname	input	keystrokes	comment
 			open-file	sample_data/employees.sqlite	o	
-employees		キemp_view	open-row		Enter	open sheet with copies of rows referenced in current row
+employees		キemp_view	open-row-pyobj		Enter	open sheet with copies of rows referenced in current row

tests/load-zip.vd

@@ -1,3 +1,3 @@
-sheet	col	row	longname	input	keystrokes	comment	keystrokes	input	longname	sheet	col	row	comment	undofuncs
-			open-file	sample_data/benchmark.zip	o		o	sample_data/benchmark.zip	open-file					
-benchmark		キsample_data/,benchmark.csv	open-row		Enter	open sheet with copies of rows referenced in current row	Enter		open-row	benchmark		キsample_data/,benchmark.csv	open sheet with copies of rows referenced in current row	[29] [3] <built-in function setattr>; [3] benchmark; _names; [1] benchmark; {0}; [3] <built-in function setattr>; [3] benchmark; _names; [1] benchmark; {0}; [3] <function undoAttrFunc.<locals>._undofunc at 0x7fb5c11b3f40>; [0] ; {0}; [3] <built-in function setattr>; [3] benchmark_cmdlog; _names; [1] benchmark_cmdlog; {0}; [3] <built-in function setattr>; [3] benchmark_cmdlog; _names; [1] benchmark_cmdlog; {0}; [3] <function undoAttrFunc.<locals>._undofunc at 0x7fb5c11b3eb0>; [0] ; {0}; [3] <built-in function setattr>; [3] benchmark_cmdlog; _names; [1] benchmark_cmdlog; {0}; [3] <built-in function setattr>; [3] benchmark_cmdlog; _names; [1] benchmark_cmdlog; {0}; [3] <function undoAttrFunc.<locals>._undofunc at 0x7fb5c11b3d90>; [0] ; {0}; [3] <built-in method remove of list object at 0x7fb5c1118740>; [1] <visidata.column.ItemColumn object at 0x7fb5c11e8850>; {0}; [3] <built-in method remove of list object at 0x7fb5c1118740>; [1] <visidata.column.ItemColumn object at 0x7fb5c11af880>; {0}; [3] <built-in method remove of list object at 0x7fb5c1118740>; [1] <visidata.column.ItemColumn object at 0x7fb5c11e8be0>; {0}; [3] <built-in method remove of list object at 0x7fb5c1118740>; [1] <visidata.column.ItemColumn object at 0x7fb5c11eabf0>; {0}; [3] <built-in method remove of list object at 0x7fb5c1118740>; [1] <visidata.column.ItemColumn object at 0x7fb5c0528c40>; {0}; [3] <built-in method remove of list object at 0x7fb5c0516e40>; [1] <visidata.column.ItemColumn object at 0x7fb5c05287c0>; {0}; [3] <built-in method remove of list object at 0x7fb5c0516e40>; [1] <visidata.column.ItemColumn object at 0x7fb5c0528ca0>; {0}; [3] <built-in method remove of list object at 0x7fb5c0516e40>; [1] <visidata.column.ItemColumn object at 0x7fb5c0528cd0>; {0}; [3] <built-in method remove of list object at 0x7fb5c0516e40>; [1] <visidata.column.ItemColumn object at 0x7fb5c0528d00>; {0}; [3] <built-in method remove of list object at 0x7fb5c0516e40>; [1] <visidata.column.ItemColumn object at 0x7fb5c0528d30>; {0}; [3] <built-in method remove of list object at 0x7fb5c0516e40>; [1] <visidata.column.ItemColumn object at 0x7fb5c0528d60>; {0}; [3] <built-in method remove of list object at 0x7fb5c0516e40>; [1] <visidata.column.ItemColumn object at 0x7fb5c0528d90>; {0}; [3] <built-in method remove of list object at 0x7fb5c0516e40>; [1] <visidata.column.ItemColumn object at 0x7fb5c0528dc0>; {0}; [3] <built-in method remove of list object at 0x7fb5c11fdcc0>; [1] <visidata.column.ItemColumn object at 0x7fb5c0529000>; {0}; [3] <built-in method remove of list object at 0x7fb5c11fdcc0>; [1] <visidata.column.ItemColumn object at 0x7fb5c11de0b0>; {0}; [3] <built-in method remove of list object at 0x7fb5c11fdcc0>; [1] <visidata.column.ItemColumn object at 0x7fb5c0529030>; {0}; [3] <built-in method remove of list object at 0x7fb5c11fdcc0>; [1] <visidata.column.ItemColumn object at 0x7fb5c0529090>; {0}; [3] <built-in method remove of list object at 0x7fb5c11fdcc0>; [1] <visidata.column.ItemColumn object at 0x7fb5c05290c0>; {0}; [3] <built-in method remove of list object at 0x7fb5c11fdcc0>; [1] <visidata.column.ItemColumn object at 0x7fb5c0529150>; {0}; [3] <built-in method remove of list object at 0x7fb5c11fdcc0>; [1] <visidata.column.ItemColumn object at 0x7fb5c0529180>; {0}
+sheet	col	row	longname	input	keystrokes	comment
+			open-file	sample_data/benchmark.zip	o	
+benchmark		キsample_data/,benchmark.csv	open-row-pyobj		Enter	open sheet with copies of rows referenced in current row

tests/sqlite_withoutrowid.vd

@@ -1,3 +1,3 @@
 sheet	col	row	longname	input	keystrokes	comment
 			open-file	tests/without_rowid.db	o	
-without_rowid		キwithoutrowid	open-row		Enter	open sheet with copies of rows referenced in current row
+without_rowid		キwithoutrowid	open-row-pyobj		Enter	open sheet with copies of rows referenced in current row

tests/xlsx-color-cells.vd

@@ -1,4 +1,4 @@
 sheet	col	row	longname	input	keystrokes	comment
 global		xlsx_meta_columns	set-option	True		
 			open-file	sample_data/color-merged-cells.xlsx	o	
-color-merged-cells		キSheet1	open-row		Enter	open sheet with copies of rows referenced in current row
+color-merged-cells		キSheet1	open-row-pyobj		Enter	open sheet with copies of rows referenced in current row

tests/xlsx-empty-cell.vd

@@ -1,3 +1,3 @@
 sheet	col	row	longname	input	keystrokes	comment
 			open-file	sample_data/empty-cell.xlsx	o	
-empty-cell		キSheet1	open-row		Enter	open sheet with copies of rows referenced in current row
+empty-cell		キSheet1	open-row-pyobj		Enter	open sheet with copies of rows referenced in current row

tests/xlsx-merged-cells.vd

@@ -1,3 +1,3 @@
 sheet	col	row	longname	input	keystrokes	comment
 			open-file	sample_data/color-merged-cells.xlsx	o	
-color-merged-cells		キSheet1	open-row		Enter	open sheet with copies of rows referenced in current row
+color-merged-cells		キSheet1	open-row-pyobj		Enter	open sheet with copies of rows referenced in current row

visidata/__init__.py

@@ -1,6 +1,6 @@
 'VisiData: a curses interface for exploring and arranging tabular data'
 
-__version__ = '2.10'
+__version__ = '2.11dev'
 __version_info__ = 'VisiData v' + __version__
 __author__ = 'Saul Pwanson <vd@saul.pw>'
 __status__ = 'Production/Stable'

visidata/_open.py

@@ -132,3 +132,4 @@ def loadInternalSheet(vd, cls, p, **kwargs):
 
 
 BaseSheet.addCommand('o', 'open-file', 'vd.push(openSource(inputFilename("open: "), create=True))', 'Open file or URL')
+TableSheet.addCommand('zo', 'open-cell', 'vd.push(openSource(cursorDisplay))', 'Open file or URL from path in current cell')

visidata/_types.py

@@ -192,7 +192,7 @@ def __le__(self, b):
         return NotImplemented
 
     def __ge__(self, b):
-        if isinstance(b, datetime.datetime): return datetime.datetime.__le__(self, b)
+        if isinstance(b, datetime.datetime): return datetime.datetime.__ge__(self, b)
         elif isinstance(b, datetime.date):   return self.date().__ge__(b)
         return NotImplemented
 

visidata/cmdlog.py

@@ -35,7 +35,7 @@ def open_vdj(vd, p):
 @VisiData.api
 def save_vdj(vd, p, *vsheets):
     with p.open_text(mode='w', encoding=vsheets[0].options.encoding) as fp:
-        fp.write("#!vd -p\n")
+        fp.write("#!/usr/bin/env vd -p\n")
         for vs in vsheets:
             vs.write_jsonl(fp)
 
@@ -178,7 +178,7 @@ def beforeExecHook(self, sheet, cmd, args, keystrokes):
             self.afterExecSheet(sheet, False, '')
 
         colname, rowname, sheetname = '', '', None
-        if sheet and not (cmd.longname.startswith('open-') and not cmd.longname in ('open-row', 'open-cell')):
+        if sheet and not (cmd.longname.startswith('open-') and not cmd.longname in ('open-row-pyobj', 'open-cell-pyobj')):
             sheetname = sheet.name
 
             colname, rowname = sheet.commandCursor(cmd.execstr)
@@ -464,14 +464,14 @@ def modifyCommand(vd):
     return vd.cmdlog.rows[-1]
 
 
-@CommandLog.api
+@CommandLogJsonl.api
 @asyncthread
 def repeat_for_n(cmdlog, r, n=1):
     r.sheet = r.row = r.col = ""
     for i in range(n):
         vd.replayOne(r)
 
-@CommandLog.api
+@CommandLogJsonl.api
 @asyncthread
 def repeat_for_selected(cmdlog, r):
     r.sheet = r.row = r.col = ""

visidata/deprecated.py

@@ -115,8 +115,9 @@ def isNumeric(col):
 alias('next-null', 'go-next-null')
 alias('page-right', 'go-right-page')
 alias('page-left', 'go-left-page')
-alias('dive-cell', 'open-cell')
-alias('dive-row', 'open-row')
+alias('dive-cell', 'open-cell-pyobj')
+alias('dive-row', 'open-row-pyobj')
+alias('open-row', 'open-row-pyobj')
 alias('add-sheet', 'open-new')
 alias('save-sheets-selected', 'save-selected')
 alias('join-sheets', 'join-selected')

visidata/extensible.py

@@ -42,6 +42,37 @@ def api(cls, func):
         setattr(cls, func.__name__, func)
         return func
 
+    @classmethod
+    def before(cls, beforefunc):
+        funcname = beforefunc.__name__
+        oldfunc = getattr(cls, funcname, None)
+        if not oldfunc:
+            vd.fail('@before on non-existing func {cls.__name__}.{funcname}')
+
+        @wraps(oldfunc)
+        def wrappedfunc(*args, **kwargs):
+            beforefunc(*args, **kwargs)
+            return oldfunc(*args, **kwargs)
+
+        setattr(cls,  funcname, wrappedfunc)
+        return wrappedfunc
+
+    @classmethod
+    def after(cls, beforefunc):
+        funcname = beforefunc.__name__
+        oldfunc = getattr(cls, funcname, None)
+        if not oldfunc:
+            vd.fail('@after on non-existing func {cls.__name__}.{funcname}')
+
+        @wraps(oldfunc)
+        def wrappedfunc(*args, **kwargs):
+            r = oldfunc(*args, **kwargs)
+            beforefunc(*args, **kwargs)
+            return r
+
+        setattr(cls,  funcname, wrappedfunc)
+        return wrappedfunc
+
     @classmethod
     def class_api(cls, func):
         name = func.__get__(None, dict).__func__.__name__

visidata/freqtbl.py

@@ -1,8 +1,7 @@
-import math
-import collections
+from copy import copy
 
-from visidata import *
-from visidata.pivot import PivotSheet
+from visidata import vd, asyncthread, vlen, VisiData, Column, AttrColumn, Sheet, ColumnsSheet, ENTER
+from visidata.pivot import PivotSheet, PivotGroupRow
 
 
 vd.option('disp_histogram', '*', 'histogram element character')
@@ -47,10 +46,18 @@ def reload(self):
 
         # add default bonus columns
         for c in [
-                    ColumnAttr('count', 'sourcerows', type=vlen),
-                    Column('percent', type=float, getter=lambda col,row: len(row.sourcerows)*100/col.sheet.source.nRows),
-                    Column('histogram', type=str, getter=lambda col,row: options.disp_histogram*(options.disp_histolen*len(row.sourcerows)//col.sheet.largest), width=options.disp_histolen+2),
-                    ]:
+            AttrColumn('count', 'sourcerows', type=vlen),
+            Column('percent', type=float, getter=lambda col,row: len(row.sourcerows)*100/col.sheet.source.nRows),
+        ]:
+            self.addColumn(c)
+
+        if self.options.disp_histolen and self.options.disp_histogram:
+            def histogram(col, row):
+                histogram = col.sheet.options.disp_histogram
+                histolen = col.sheet.options.disp_histolen
+                return histogram*(histolen*len(row.sourcerows)//col.sheet.largest)
+
+            c = Column('histogram', type=str, getter=histogram, width=self.options.disp_histolen+2)
             self.addColumn(c)
 
         # two more threads