diff --git a/.gitignore b/.gitignore index ae450c1..ff25d76 100644 --- a/.gitignore +++ b/.gitignore @@ -223,3 +223,10 @@ coverage.xml # CCLS files .ccls-cache/* + +# HDB Test files +HDB/** +multiColSplay/** +singleColSplay/** +sym +symsEnumsSplay/** diff --git a/conda-recipe/meta.yaml b/conda-recipe/meta.yaml index fab0bcf..bcdb2bc 100644 --- a/conda-recipe/meta.yaml +++ b/conda-recipe/meta.yaml @@ -27,7 +27,7 @@ requirements: run: - python - numpy>=1.22 - - pandas>=1.2,<2.0 + - pandas>=1.2 - pytz>=2022.1 - toml>=0.10.2 diff --git a/docs/examples/server/archive.zip b/docs/examples/server/archive.zip index 9be7a8c..cee51d2 100644 Binary files a/docs/examples/server/archive.zip and b/docs/examples/server/archive.zip differ diff --git a/docs/examples/server/server.md b/docs/examples/server/server.md index b9849f6..989e63c 100644 --- a/docs/examples/server/server.md +++ b/docs/examples/server/server.md @@ -7,11 +7,14 @@ To follow along with this example please feel free to download this =1.2` - `numpy~=1.22` -- `pytz~=2022.1` +- `pytz>=2022.1` - `toml~=0.10.2` They are installed automatically by `pip` when PyKX is installed. -PyKX also has an optional Python dependency of `pyarrow>=3.0.0`, which can be included by installing the `pyarrow` extra, e.g. `pip install pykx[pyarrow]` +### Optional Python Dependencies + +- `pyarrow>=3.0.0`, which can be included by installing the `pyarrow` extra, e.g. `pip install pykx[pyarrow]`. +- `find-libpython~=0.2`, which can be included by installing the `debug` extra, e.g. `pip install pykx[debug]`. This dependency can be used to help find `libpython` in the scenario that `pykx.q` fails to find it. !!! Warning diff --git a/docs/getting-started/q_magic_command.ipynb b/docs/getting-started/q_magic_command.ipynb index f820b18..9411f67 100644 --- a/docs/getting-started/q_magic_command.ipynb +++ b/docs/getting-started/q_magic_command.ipynb @@ -46,7 +46,7 @@ "metadata": {}, "source": [ "#### Executing against Embedded q\n", - "A cell begining with `%%q` will execute q within `PyKX`'s `EmbeddedQ` module." + "A cell beginning with `%%q` will execute q within `PyKX`'s `EmbeddedQ` module." ] }, { @@ -60,6 +60,33 @@ "til 10" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "`print` is the default method called on returned objects. To use `display` you can instead pass `--display`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%%q\n", + "([] a: 1 2 3)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%%q --display\n", + "([] a: 1 2 3)" + ] + }, { "cell_type": "markdown", "id": "89ec26e4", diff --git a/docs/getting-started/quickstart.md b/docs/getting-started/quickstart.md index 279eaac..233b9f4 100644 --- a/docs/getting-started/quickstart.md +++ b/docs/getting-started/quickstart.md @@ -130,6 +130,16 @@ PyKX objects can be interacted with in a variety of ways, through indexing using pykx.FloatVector(pykx.q('0.08123546 0.9367503 0.2782122')) ``` +* Assign objects to PyKX lists + + ```python + >>> qarray = kx.random.random(3, 10.0, seed=10) + pykx.FloatVector(pykx.q('0.891041 8.345194 3.621949')) + >>> qarray[1] = 0.1 + >>> qarray + pykx.FloatVector(pykx.q('0.891041 0.1 3.621949')) + ``` + * Create a PyKX table and manipulate using Pythonic syntax ```python diff --git a/docs/index.md b/docs/index.md index f7050ad..7a59acc 100644 --- a/docs/index.md +++ b/docs/index.md @@ -8,46 +8,26 @@ For Python developers, PyKX unlocks the speed and power of kdb+ for data process For q developers, PyKX brings together Python's data science ecosystem and the power of kdb+'s vector and time-series analytics. This makes them available in both q and Python environments. You can use it to run q code within a Python environment or embed Python analytics within your q session. -To begin your journey with PyKX follow the sections in the grid below. +To begin your journey with PyKX follow the sections below. -
+## Documentation Breakdown -- :material-run-fast:{ .lg .middle } __Getting Started__ +### [Getting Started](getting-started/what_is_pykx.md) - --- +Documentation for users new to PyKX! Contains installation instructions alongside quickstart guides and sample getting started notebooks. - Documentation for users new to PyKX! Contains installation instructions - alongside quickstart guides and sample getting started notebooks. - +### [User Guide](user-guide/index.md) - [:octicons-arrow-right-24: Getting started](getting-started/what_is_pykx.md) +Useful information allowing users to understand the key concepts behind PyKX. Including how the library is intended to be used and examples of this functionality. -- :material-book-open-variant:{ .lg .middle } __User Guide__ +### [API](api/pykx-execution/q.md) - --- +Detailed descriptions of the functions, modules and objects managed by PyKX. Using the API reference assumes you have an understanding of how PyKX is intended to be used through the getting started and user guide. - Useful information allowing users to understand the key concepts that behind PyKX. - Including how the library is intended to be used and examples of this fucntionality. - +### [Release Notes](release-notes/changelog.md) - [:octicons-arrow-right-24: User Guide](user-guide/index.md) +The latest additions and fixes for PyKX alongside historical changes. -- :material-api:{ .lg .middle } __API Reference__ +### [Roadmap](roadmap.md) - --- - - Detailed descriptions of the functions, modules and objects managed by PyKX. Using - the API reference assumes you have an understanding of how PyKX is intended to be used - through the getting started and user guide. - - [:octicons-arrow-right-24: API reference](api/pykx-execution/q.md) - -- :material-alert-decagram:{ .lg .middle } __Release Notes__ - - --- - - The latest additions and fixes for PyKX alongside historical changes. - - [:octicons-arrow-right-24: Release notes](release-notes/changelog.md) - -
+What to look out for in the next weeks, months and years from the PyKX team. diff --git a/docs/release-notes/changelog.md b/docs/release-notes/changelog.md index 00f021d..53073e8 100644 --- a/docs/release-notes/changelog.md +++ b/docs/release-notes/changelog.md @@ -4,6 +4,348 @@ The changelog presented here outlines changes to PyKX when operating within a Python environment specifically, if you require changelogs associated with PyKX operating under a q environment see [here](./underq-changelog.md). +## PyKX 2.2.0 + +#### Release Date + +2023-11-09 + +!!! Warning + + PyKX 2.2.0 presently does not include a Python 3.11 release for MacOS x86 and Linux x86 architectures, this will be rectified in an upcoming patch release. + +### Additions + +- Addition of `agg` method for application of aggregation functions on `pykx.Table` and `pykx.GroupbyTable` objects + + ```python + >>> import pykx as kx + >>> import numpy as np + >>> import statistics + >>> def mode(x): + ... return statistics.mode(x) + >>> tab = kx.Table(data={ + ... 'x': kx.random.random(1000, 10), + ... 'x1': kx.random.random(1000, 10.0)}) + >>> tab.agg(mode) + pykx.Dictionary(pykx.q(' + x | 6 + x1| 2.294631 + ')) + >>> tab.agg(['min', 'mean']) + pykx.KeyedTable(pykx.q(' + function| x x1 + --------| ----------------- + min | 0 0.009771725 + mean | 4.588 5.152194 + ')) + >>> + >>> group_tab = kx.Table(data={ + ... 'x': kx.random.random(1000, ['a', 'b']), + ... 'y': kx.random.random(1000, 10.0)}) + >>> group_tab.groupby('x').agg('mean') + pykx.KeyedTable(pykx.q(' + x| y + -| -------- + a| 5.239048 + b| 4.885599 + ')) + >>> group_tab.groupby('x').agg(mode) + pykx.KeyedTable(pykx.q(' + x| y + -| -------- + a| 1.870281 + b| 4.46898 + ')) + ``` + +- Addition of the ability for users to run `min`, `max`, `mean`, `median`, `sum` and `mode` methods on vector objects within PyKX. + + ```python + >>> import pykx as kx + >>> random_vec = kx.random.random(5, 3, seed=20) + pykx.LongVector(pykx.q('0 1 0 1 1')) + >>> random_vec.mode() + pykx.LongVector(pykx.q(',1')) + >>> random_vec.mean() + pykx.FloatAtom(pykx.q('0.6')) + ``` + +- Addition of the ability for users to assign objects to `pykx.*Vector` and `pykx.List` objects + + ```python + >>> import pykx as kx + >>> qvec = kx.q.til(10) + >>> qvec + pykx.LongVector(pykx.q('0 1 2 3 4 5 6 7 8 9')) + >>> qvec[3] = 45 + >>> qvec + pykx.LongVector(pykx.q('0 1 2 45 4 5 6 7 8 9')) + >>> qvec[-1] = 20 + >>> qvec + pykx.LongVector(pykx.q('0 1 2 45 4 5 6 7 8 20')) + ``` + +- Users can now assign/update keys of a `pykx.Dictionary` object using an in-built `__setitem__` method as follows + + ```python + >>> import pykx as kx + >>> pykx_dict = kx.toq({'x': 1}) + >>> pykx_dict + pykx.Dictionary(pykx.q('x| 1')) + >>> pykx_dict['x1'] = 2 + >>> pykx_dict + pykx.Dictionary(pykx.q(' + x | 1 + x1| 2 + ')) + >>> for i in range(3): + ... pykx_dict['x']+=i + ... + >>> pykx_dict + pykx.Dictionary(pykx.q(' + x | 4 + x1| 2 + ')) + ``` + +- Addition of `null` and `inf` properties for `pykx.Atom` objects allowing for Pythonic retrieval of nulls and infinities + + ```python + >>> import pykx as kx + >>> kx.FloatAtom.null + pykx.FloatAtom(pykx.q('0n')) + >>> kx.GUIDAtom.null + pykx.GUIDAtom(pykx.q('00000000-0000-0000-0000-000000000000')) + >>> kx.IntAtom.inf + pykx.IntAtom(pykx.q('0Wi')) + >>> -kx.IntAtom.inf + pykx.IntAtom(pykx.q('-0Wi')) + ``` + +- Users can now use the environment variables `PYKX_UNLICENSED="true"` or `PYKX_LICENSED="true"` set this as part of configuration within their `.pykx-config` file to allow `unlicensed` or `licensed` mode to be the default behaviour on initialisation for example: + + ```python + >>> import os + >>> os.environ['PYKX_UNLICESED'] = "true" + >>> import pykx as kx + >>> kx.toq([1, 2, 3]) + pykx.List._from_addr(0x7fee46000a00) + ``` + +- Addition of `append` and `extend` methods to `pykx.*Vector` and `pykx.List` objects + + ```python + >>> import pykx as kx + >>> qvec = kx.q.til(5) + >>> qvec.append(100) + >>> qvec + pykx.LongVector(pykx.q('0 1 2 3 4 100')) + >>> qvec.extend([1, 2, 3]) + >>> qvec + pykx.LongVector(pykx.q('0 1 2 3 4 100 1 2 3')) + ``` + +- Addition of `debug` keyword argument to the `__call__` method on `EmbeddedQ` and `QConnection` objects to provide backtraces on `q` code. + + ```python + >>> import pykx as kx + >>> kx.q('{[x] a: 5; b: til a; c: til x; b,c}', b'foo', debug=True) + backtrace: + [3] (.q.til) + + [2] {[x] a: 5; b: til a; c: til x; b,c} + ^ + [1] (.Q.trp) + + [0] {[pykxquery] .Q.trp[value; pykxquery; {2@"backtrace: + ^ + ",.Q.sbt y;'x}]} + Traceback (most recent call last): + File "", line 1, in + File "...\site-packages\pykx\embedded_q.py", line 226, in __call__ + return factory(result, False) + File "pykx\\_wrappers.pyx", line 504, in pykx._wrappers._factory + File "pykx\\_wrappers.pyx", line 497, in pykx._wrappers.factory + pykx.exceptions.QError: type + ``` + +- Added feature to extract individual elements of both `TimestampAtom` and `TimestampVector` in a pythonic way including: + + * `date` - DateAtom / DateVector + * `time` - TimeAtom / TimeVector + * `year` - IntAtom / IntVector + * `month` - IntAtom / IntVector + * `day` - IntAtom / IntVector + * `hour` - IntAtom / IntVector + * `minute` - IntAtom / IntVector + * `second` - IntAtom / IntVector + + ```python + >>> timestamp_atom = kx.q('2023.10.25D16:42:01.292070013') + + >>> timestamp_atom.time + pykx.TimeAtom(pykx.q('16:42:01.292')) + >>> timestamp_atom.date + pykx.DateAtom(pykx.q('2023.10.25')) + >>> timestamp_atom.minute + pykx.IntAtom(pykx.q('42i')) + + >>> timestamp_atom_2 = kx.q('2018.11.09D12:21:08.456123789') + >>> timestamp_vector = kx.q('enlist', timestamp_atom, timestamp_atom_2) + + >>> timestamp_vector.time + pykx.TimeVector(pykx.q('16:42:01.292 12:21:08.456')) + >>> timestamp_vector.date + pykx.DateVector(pykx.q('2023.10.25 2018.11.09')) + >>> timestamp_vector.hour + pykx.IntVector(pykx.q('16 12i')) + ``` + +- Addition of `poll_recv_async` to `RawQConnection` objects to support asynchronous polling. + +### Fixes and Improvements + +- Fix to allow users to use Python functions when operating on a `pykx.GroupbyTable` with an `apply` function + + ```python + >>> import pykx as kx + >>> import statistics + >>> def mode(x): + ... return statistics.mode(x) + >>> tab = kx.q('([]sym:`a`b`a`a;1 1 0 0)') + >>> tab.groupby('sym').apply(mode) + pykx.KeyedTable(pykx.q(' + sym| x + ---| - + a | 0 + b | 1 + ')) + ``` + +- Added debug dependency for `find-libpython` that can be installed using `pip install "pykx[debug]"`. This dependency can be used to help find `libpython` in the scenario that `pykx.q` fails to find it. +- Usage of the `QARGS` to enable/disable various elements of kdb Insights functionality has been formalised, outlined [here](../user-guide/configuration.md). For example users can now use `QARGS="--no-objstor"` to disable object storage capabilities. + +- Failure to initialise PyKX with `exp` or `embedq` license errors will now prompt users to ask if they wish to download an appropriate license following expiry or use of an invalid license + + === "'exp' License Prompt" + + ```python + Your PyKX license has now expired. + + Captured output from initialization attempt: + '2023.10.18T13:27:59.719 licence error: exp + + Would you like to renew your license? [Y/n]: + ``` + + === "'embedq' License Prompt" + + ```python + You appear to be using a non kdb Insights license. + + Captured output from initialization attempt: + '2023.10.18T13:27:59.719 licence error: embedq + + Running PyKX in the absence of a kdb Insights license has reduced functionality. + Would you like to install a kdb Insights personal license? [Y/n]: + ``` + + === "'upd' License Prompt" + + ```python + Your installed license is out of date for this version of PyKX and must be updated. + + Captured output from initialization attempt: + '2023.10.18T13:27:59.719 licence error: upd + + Would you like to install an updated kdb Insights personal license? [Y/n]: + ``` + +- PyKX sets `PYKX_EXECUTABLE` to use when loading embedded q to prevent errors if launched using a different Python executable than that which will be found in `PATH` + +- Jupyter Notebook: + - Removal of `FutureWarning` when displaying tables and dictionaries. + - Revert issue causing results to be displayed as pointer references rather than Python objects in unlicensed mode. + - `%%q` magic now suppresses displaying of `::`. + - `%%q` magic addition of `--display` option to have `display` be called on returned items in place of the default `print`. + +- `PyKXReimport` now additionally unsets/resets: `PYKX_SKIP_UNDERQ`, `PYKX_EXECUTABLE`, `PYKX_DIR` +- When attempting to deserialize unsupported byte representations `pykx.deserialize` would result in a segmentation fault, this has been updated such that an error message is now raised. + + ```python + >>> import pykx as kx + >>> kx.deserialize(b'invalid byte string') + Traceback (most recent call last): + File "", line 1, in + File "/usr/local/anaconda3/lib/python3.8/site-packages/pykx/serialize.py", line 123, in deserialize + return _deserialize(data) + File "pykx/_wrappers.pyx", line 131, in pykx._wrappers.deserialize + File "pykx/_wrappers.pyx", line 135, in pykx._wrappers.deserialize + pykx.exceptions.QError: Failed to deserialize supplied non PyKX IPC serialized format object + ``` + +- Fixed an issue when using multiple asynchronous `QConnection` connected to multiple servers. +- Users can now access the length of and index into `pykx.CharAtom` objects to align with Pythonic equivalent data + + ```python + >>> qatom = kx.CharAtom('a') + >>> len(qatom) + 1 + >>> qatom[0] + pykx.CharAtom(pykx.q('"a"')) + ``` + +## PyKX 2.1.2 + +#### Release Date + +2023-10-24 + +### Fixes and Improvements + +- Fix to issue where functions retrieved using the Context Interface with names `update/delete/select/exec` would result in an `AttributeError` + + === "Behavior prior to change" + + ```python + >>> import pykx as kx + >>> kx.q.test + + >>> kx.q.test.ctx.update(1) + Traceback (most recent call last): + File "", line 1, in + File "/usr/local/anaconda3/lib/python3.8/site-packages/pykx/ctx.py", line 121, in __getattr__ + raise AttributeError(f'{key}: {self._unsupported_keys_with_msg[key]}') + AttributeError: update: Usage of 'update' function directly via 'q' context not supported, please consider using 'pykx.q.qsql.update' + ``` + + === "Behavior post change" + + ```python + >>> import pykx as kx + >>> kx.q.test + + >>> kx.q.test.ctx.update(1) + pykx.LongAtom(pykx.q('2')) + ``` + +## PyKX 2.1.1 + +#### Release Date + +2023-10-10 + +### Fixes and Improvements + +- Fix to regression in PyKX 2.1.0 where execution of `from pykx import *` would result in the following behaviour + + ``` + >>> from pykx import * + ... + AttributeError: module 'pykx' has no attribute 'PyKXSerialized' + ``` + ## PyKX 2.1.0 #### Release Date @@ -27,16 +369,16 @@ the following reads a CSV file and specifies the types of the three columns name >>> df = pd.DataFrame.from_dict({'x': [1, 2], 'y': ['a', 'b']}) >>> kx.toq(df).dtypes pykx.Table(pykx.q(' - columns type + columns type ----------------------- - x "kx.LongAtom" + x "kx.LongAtom" y "kx.SymbolAtom" ')) >>> kx.toq(df, ktype={'x': kx.FloatAtom}).dtypes pykx.Table(pykx.q(' - columns type + columns type ----------------------- - x "kx.FloatAtom" + x "kx.FloatAtom" y "kx.SymbolAtom" ')) ``` @@ -98,19 +440,19 @@ the following reads a CSV file and specifies the types of the three columns name pykx.Table(pykx.q(' c1 -- - + ')) >>> tab[-4] pykx.Table(pykx.q(' c1 -- - + ')) >>> tab[3] pykx.Table(pykx.q(' c1 -- - + ')) ``` @@ -123,7 +465,7 @@ the following reads a CSV file and specifies the types of the three columns name pykx.Table(pykx.q(' c1 -- - 2 + 2 ')) >>> tab[-4] ... @@ -183,7 +525,7 @@ the following reads a CSV file and specifies the types of the three columns name ... }) >>> tab.groupby('sym').apply(kx.q.sum) pykx.KeyedTable(pykx.q(' - sym| price + sym| price ---| -------- a | 166759.4 b | 166963.6 @@ -192,7 +534,7 @@ the following reads a CSV file and specifies the types of the three columns name ``` - Addition of a new module `pykx.random` which provides functionality for the generation of random data and setting of random seeds. For more information see [here](../api/random.md) - + ```python >>> import pykx as kx >>> kx.random.random(5, 1.0, seed=123) diff --git a/docs/release-notes/underq-changelog.md b/docs/release-notes/underq-changelog.md index 7e9f226..c30b87f 100644 --- a/docs/release-notes/underq-changelog.md +++ b/docs/release-notes/underq-changelog.md @@ -6,11 +6,66 @@ This changelog provides updates from PyKX 2.0.0 and above, for information relat The changelog presented here outlines changes to PyKX when operating within a q environment specifically, if you require changelogs associated with PyKX operating within a Python environment see [here](./changelog.md). +## PyKX 2.2.0 + +### Additions + +- Addition of `PYKX_EXECUTABLE` environment/configuration variable to allow control of which Python executable is used under q. + +### Fixes and Improvements + +- Failure to access and load PyKX resulting in an `os` error now returns Python backtrace outlining the underlying Python error allowing for easier debugging + + === "Behavior prior to change" + + ```q + q)\l pykx.q + 'os + [4] \python3 -c "import pykx; print(pykx.config.pykx_dir)" 2>/dev/null + ^ + ``` + + === "Behavior post change" + + ```q + q)\l pykx.q + Traceback (most recent call last): + File "", line 1, in + File "/usr/local/anaconda3/lib/python3.8/site-packages/pykx/__init__.py", line 27, in + from . import core + File "pykx/core.pyx", line 6, in init pykx.core + from .util import num_available_cores + File "/usr/local/anaconda3/lib/python3.8/site-packages/pykx/util.py", line 8, in + import pandas as pd + File "/usr/local/anaconda3/lib/python3.8/site-packages/pandas/__init__.py", line 16, in + raise ImportError( + ImportError: Unable to import required dependencies: + numpy: cannot import name 'SystemRandom' from 'random' (/Users/projects/pykx/src/pykx/random.py) + Traceback (most recent call last): + File "", line 1, in + File "/usr/local/anaconda3/lib/python3.8/site-packages/pykx/__init__.py", line 27, in + from . import core + File "pykx/core.pyx", line 6, in init pykx.core + from .util import num_available_cores + File "/usr/local/anaconda3/lib/python3.8/site-packages/pykx/util.py", line 8, in + import pandas as pd + File "/usr/local/anaconda3/lib/python3.8/site-packages/pandas/__init__.py", line 16, in + raise ImportError( + ImportError: Unable to import required dependencies: + numpy: cannot import name 'SystemRandom' from 'random' (/Users/projects/pykx/src/pykx/random.py) + 'os + [4] \python3 -c "import pykx; print('PYKX_DIR: ' + str(pykx.config.pykx_dir))" + ``` + +- Fixed `type` error if converting dictionaries or keyed tables with conversion set to `default` +- On load now sets `PYKX_SKIP_UNDERQ` rather than deprecated `SKIP_UNDERQ` +- `safeReimport` now additionally unsets/resets: `PYKX_DEFAULT_CONVERSION`, `PYKX_SKIP_UNDERQ`, `PYKX_EXECUTABLE`, `PYKX_DIR` + ## PyKX 2.1.0 ### Fixes and Improvements -- Update to default conversion logic for q objects passed to PyKX functions to more closely match embedPy based conversion expectations.For version <=2.0 conversions of KX lists would produce N Dimensional Numpy arrays of singular type. This results in issues when applying to many analytic libraries which rely on lists of lists rather than singular N Dimensional arrays. Additionally q tables and keyed tables would be converted to numpy recarrays, these are now converted to Pandas DataFrames. To maintain previous behaviour please set the following environment variable `PYKX_DEFAULT_CONVERSION="np"`. +- Update to default conversion logic for q objects passed to PyKX functions to more closely match embedPy based conversion expectations.For version <=2.0 conversions of KX lists would produce N Dimensional Numpy arrays of singular type. This results in issues when applying to many analytic libraries which rely on lists of lists rather than singular N Dimensional arrays. Additionally q tables and keyed tables would be converted to Numpy recarrays, these are now converted to Pandas DataFrames. To maintain previous behavior please set the following environment variable `PYKX_DEFAULT_CONVERSION="np"`. === "Behaviour prior to change" diff --git a/docs/roadmap.md b/docs/roadmap.md new file mode 100644 index 0000000..65e3496 --- /dev/null +++ b/docs/roadmap.md @@ -0,0 +1,29 @@ +# PyKX Roadmap + +This page outlines areas of development focus for the PyKX team to provide you with an understanding of the development direction of the library. This is not an exhaustive list of all features/areas of focus but should give you a view on what to expect from the team over the coming months. Additionally this list is subject to change based on the complexity of the features and any customer feature requests raised following the publishing of this list. + +If you need a feature that's not included in this list please let us know by raising a [Github issue](https://github.com/KxSystems/pykx/issues)! + +## Nov 2023 - Jan 2024 + +- Support Python 3.12 +- Tighter integration with [Streamlit](https://streamlit.io/) allowing streamlit applications to interact with kdb+ servers and on-disk databases +- User defined Python functions to be supported when operating with local qsql.select functionality +- [JupyterQ](https://github.com/KxSystems/jupyterq) and [ML-Toolkit](https://github.com/KxSystems/ml) updates to allow optional PyKX backend replacing embedPy +- Pythonic data sorting for PyKX Tables + +## Feb - Apr 2024 + +- Database management functionality allowing for Pythonic persistence and management of on-disk kdb+ Databases (Beta) +- Improvements to multi-threaded PyKX efficiency, reducing per-call overhead for running PyKX on separate threads +- Configurable initialisation logic in the absense of a license. Thus allowing users who have their own workflows for license access to modify the instructions for their users. +- Addition of `cast` keyword when inserting/upserting data into a table reducing mismatch issues + +## Future + +- Tighter integration between PyKX/q objects and PyArrow arrays/Tables +- Expansion of supported datatypes for translation to/from PyKX +- Continued additions of Pandas-like functionality on PyKX Table objects +- Performance improvements through enhanced usage of Cython +- Real-time/Streaming functionality utilities +- Data pre-processing and statitics modules for operation on PyKX tables and vector objects diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 1044461..c7118c7 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -2,18 +2,38 @@ ## License issues -The following section outlines issues and potential solutions when dealing with failures to initialise PyKX with a `license error` +The following section outlines practical information useful when dealing with getting access to and managing licenses for PyKX. -### Failing to start PyKX with an error after following email based installation guide +### Accessing a license valid for PyKX -If on initially completing the installation guide for PyKX [here](getting-started/installing.md) you receive the following error +A number of trial and enterprise type licenses exist for q/kdb+. Not all licenses for q/kdb+ however are valid for PyKX. In particular users require access to a license which contains the feature flags `pykx` and `embedq` which provide access to the PyKX functionality. The following locations can be used for the retrieval of evaluation/personal licenses +- For non-commercial personal users you can access a 12 month kdb+ license with PyKX enabled [here](https://kx.com/kdb-insights-personal-edition-license-download). +- For commercial evaluation you can download a 30 day PyKX license [here](https://kx.com/kdb-insights-commercial-evaluation-license-download/). + +For non-personal or non-commercial usage please contact sales@kx.com. + +Once you have access to your license you can install the license following the walkthrough provided [here](getting-started/installing.md) or through installation using the function `kx.license.install` as follows + +```python +>>> import pykx as kx +>>> kx.license.install('/path/to/downloaded/kc.lic') ``` + +### Initialization failing with a 'embedq' error + +Failure to initialize PyKX while raising an error `embedq` indicates that the license you are attempting to use for PyKX in [licensed modality](modes.md) does not have the sufficient feature flags necessary to run PyKX. To access a license which does allow for running PyKX in this modality please following the instructions [here](#accessing-a-license-valid-for-pykx) to get a new license with appropriate feature flags. + +### Initialization failing with a 'kc.lic' error + +If after initially completing the installation guide for PyKX [here](getting-started/installing.md) you receive the following error: + +```python pykx.exceptions.PyKXException: Failed to initialize embedded q. Captured output from initialization attempt: '2023.09.02T21:28:45.699 licence error: kc.lic ``` -It is usually indicates that your license was not correctly written to disk, to check that the installed license matches the license you expect. To do this please have to hand the email you received on sign up for PyKX. +It usually indicates that your license was not correctly written to disk or a license could not be found, to check that the installed license matches the license you expect. === "License file based checking" diff --git a/docs/user-guide/advanced/Pandas_API.ipynb b/docs/user-guide/advanced/Pandas_API.ipynb index cb98590..239c4c8 100644 --- a/docs/user-guide/advanced/Pandas_API.ipynb +++ b/docs/user-guide/advanced/Pandas_API.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "markdown", - "id": "12a1e6cd", + "id": "2d0c8656", "metadata": {}, "source": [ "# Pandas API\n", @@ -23,7 +23,7 @@ { "cell_type": "code", "execution_count": null, - "id": "2f4437ca", + "id": "17f28b87", "metadata": {}, "outputs": [], "source": [ @@ -38,7 +38,7 @@ }, { "cell_type": "markdown", - "id": "13e03cb2", + "id": "774122a0", "metadata": {}, "source": [ "## Constructing Tables" @@ -46,7 +46,7 @@ }, { "cell_type": "markdown", - "id": "e76d1fc5", + "id": "0fd8910c", "metadata": {}, "source": [ "### Table\n", @@ -75,7 +75,7 @@ { "cell_type": "code", "execution_count": null, - "id": "7b5f69f2", + "id": "9a748c27", "metadata": {}, "outputs": [], "source": [ @@ -84,7 +84,7 @@ }, { "cell_type": "markdown", - "id": "673d9e9a", + "id": "231a5e28", "metadata": {}, "source": [ "Create a Table from an array like object." @@ -93,7 +93,7 @@ { "cell_type": "code", "execution_count": null, - "id": "c701c1b9", + "id": "7e43d716", "metadata": {}, "outputs": [], "source": [ @@ -102,7 +102,7 @@ }, { "cell_type": "markdown", - "id": "c8eb1622", + "id": "1e426cda", "metadata": {}, "source": [ "Create a Table from an array like object and provide names for the columns to use." @@ -111,7 +111,7 @@ { "cell_type": "code", "execution_count": null, - "id": "103f866d", + "id": "2b3c2edf", "metadata": {}, "outputs": [], "source": [ @@ -120,7 +120,7 @@ }, { "cell_type": "markdown", - "id": "bfd1da8e", + "id": "be094191", "metadata": {}, "source": [ "### Keyed Table\n", @@ -150,7 +150,7 @@ { "cell_type": "code", "execution_count": null, - "id": "765be7de", + "id": "d93e73d3", "metadata": {}, "outputs": [], "source": [ @@ -159,7 +159,7 @@ }, { "cell_type": "markdown", - "id": "f8082a5a", + "id": "119c2e1f", "metadata": {}, "source": [ "Create a keyed table from a list of rows." @@ -168,7 +168,7 @@ { "cell_type": "code", "execution_count": null, - "id": "ecd10819", + "id": "959fcd3d", "metadata": {}, "outputs": [], "source": [ @@ -177,7 +177,7 @@ }, { "cell_type": "markdown", - "id": "7be93c23", + "id": "9d83854e", "metadata": {}, "source": [ "Create a keyed table from a list of rows and provide names for the resulting columns." @@ -186,7 +186,7 @@ { "cell_type": "code", "execution_count": null, - "id": "51d94e4d", + "id": "4b2c6989", "metadata": {}, "outputs": [], "source": [ @@ -195,7 +195,7 @@ }, { "cell_type": "markdown", - "id": "8157961c", + "id": "356b29d8", "metadata": {}, "source": [ "Create a keyed table with a specified index column." @@ -204,7 +204,7 @@ { "cell_type": "code", "execution_count": null, - "id": "2405a759", + "id": "acbe339c", "metadata": {}, "outputs": [], "source": [ @@ -213,7 +213,7 @@ }, { "cell_type": "markdown", - "id": "e9ff8aa6", + "id": "95a04686", "metadata": {}, "source": [ "## Metadata" @@ -222,7 +222,7 @@ { "cell_type": "code", "execution_count": null, - "id": "fed3a938", + "id": "a52fdc82", "metadata": {}, "outputs": [], "source": [ @@ -233,7 +233,7 @@ }, { "cell_type": "markdown", - "id": "3e5de382", + "id": "280baf05", "metadata": {}, "source": [ "### Table.columns\n", @@ -244,7 +244,7 @@ { "cell_type": "code", "execution_count": null, - "id": "a355d654", + "id": "a2ee3fad", "metadata": {}, "outputs": [], "source": [ @@ -253,7 +253,7 @@ }, { "cell_type": "markdown", - "id": "9baab247", + "id": "40da029e", "metadata": {}, "source": [ "### Table.dtypes\n", @@ -264,7 +264,7 @@ { "cell_type": "code", "execution_count": null, - "id": "f72c7071", + "id": "70bd32d2", "metadata": {}, "outputs": [], "source": [ @@ -273,7 +273,7 @@ }, { "cell_type": "markdown", - "id": "5393cbb5", + "id": "00e49e84", "metadata": {}, "source": [ "### Table.empty\n", @@ -284,7 +284,7 @@ { "cell_type": "code", "execution_count": null, - "id": "d62ce629", + "id": "9dc49e08", "metadata": {}, "outputs": [], "source": [ @@ -293,7 +293,7 @@ }, { "cell_type": "markdown", - "id": "de0a60d6", + "id": "c00e46ef", "metadata": {}, "source": [ "### Table.ndim\n", @@ -304,7 +304,7 @@ { "cell_type": "code", "execution_count": null, - "id": "27aa4a92", + "id": "db113636", "metadata": {}, "outputs": [], "source": [ @@ -313,7 +313,7 @@ }, { "cell_type": "markdown", - "id": "d8b6533c", + "id": "5ea4b315", "metadata": {}, "source": [ "### Table.shape\n", @@ -324,7 +324,7 @@ { "cell_type": "code", "execution_count": null, - "id": "3bd69cae", + "id": "78125654", "metadata": {}, "outputs": [], "source": [ @@ -333,7 +333,7 @@ }, { "cell_type": "markdown", - "id": "50f7c03c", + "id": "1e3f85a5", "metadata": {}, "source": [ "### Table.size\n", @@ -344,7 +344,7 @@ { "cell_type": "code", "execution_count": null, - "id": "072269ec", + "id": "c77c5bc7", "metadata": { "scrolled": false }, @@ -355,7 +355,7 @@ }, { "cell_type": "markdown", - "id": "394b9b9d", + "id": "2be2ece3", "metadata": {}, "source": [ "### Table.mean()\n", @@ -382,7 +382,7 @@ }, { "cell_type": "markdown", - "id": "7d0ae9ce", + "id": "cb8c5ef8", "metadata": {}, "source": [ "**Examples:**\n", @@ -393,7 +393,7 @@ { "cell_type": "code", "execution_count": null, - "id": "0a3883ab", + "id": "0c3e5d76", "metadata": {}, "outputs": [], "source": [ @@ -411,7 +411,7 @@ { "cell_type": "code", "execution_count": null, - "id": "65f9d7ed", + "id": "9986a550", "metadata": {}, "outputs": [], "source": [ @@ -420,7 +420,7 @@ }, { "cell_type": "markdown", - "id": "94f312a5", + "id": "24ac0b99", "metadata": {}, "source": [ "Calculate the mean across the rows of a table" @@ -429,7 +429,7 @@ { "cell_type": "code", "execution_count": null, - "id": "3d96dc12", + "id": "41f6f669", "metadata": {}, "outputs": [], "source": [ @@ -438,7 +438,7 @@ }, { "cell_type": "markdown", - "id": "d102ec1b", + "id": "7bf853c5", "metadata": {}, "source": [ "### Table.median()\n", @@ -465,7 +465,7 @@ }, { "cell_type": "markdown", - "id": "e2341a7c", + "id": "98da458a", "metadata": {}, "source": [ "**Examples:**\n", @@ -476,7 +476,7 @@ { "cell_type": "code", "execution_count": null, - "id": "d9a17a3b", + "id": "bff5ac07", "metadata": {}, "outputs": [], "source": [ @@ -494,7 +494,7 @@ { "cell_type": "code", "execution_count": null, - "id": "d93621d4", + "id": "579c8b33", "metadata": {}, "outputs": [], "source": [ @@ -503,7 +503,7 @@ }, { "cell_type": "markdown", - "id": "b9822d60", + "id": "f6698350", "metadata": {}, "source": [ "Calculate the median across the rows of a table" @@ -512,7 +512,7 @@ { "cell_type": "code", "execution_count": null, - "id": "ab0b5159", + "id": "5664bd93", "metadata": { "scrolled": false }, @@ -523,7 +523,7 @@ }, { "cell_type": "markdown", - "id": "7041b59d", + "id": "33af56bb", "metadata": {}, "source": [ "### Table.mode()\n", @@ -551,7 +551,7 @@ }, { "cell_type": "markdown", - "id": "100d30fa", + "id": "4201c9af", "metadata": {}, "source": [ "**Examples:**\n", @@ -562,7 +562,7 @@ { "cell_type": "code", "execution_count": null, - "id": "806786d9", + "id": "b4bfe36c", "metadata": {}, "outputs": [], "source": [ @@ -580,7 +580,7 @@ { "cell_type": "code", "execution_count": null, - "id": "7ccc77a8", + "id": "e1a7eeb1", "metadata": { "scrolled": true }, @@ -591,7 +591,7 @@ }, { "cell_type": "markdown", - "id": "3bf74453", + "id": "6a47af49", "metadata": {}, "source": [ "Calculate the median across the rows of a table" @@ -600,7 +600,7 @@ { "cell_type": "code", "execution_count": null, - "id": "d3c86b05", + "id": "130081ce", "metadata": { "scrolled": false }, @@ -611,7 +611,7 @@ }, { "cell_type": "markdown", - "id": "ab19909d", + "id": "29dffe0d", "metadata": {}, "source": [ "Calculate the mode across columns and keep null values." @@ -620,7 +620,7 @@ { "cell_type": "code", "execution_count": null, - "id": "cf30480e", + "id": "53a8251a", "metadata": { "scrolled": true }, @@ -639,7 +639,7 @@ { "cell_type": "code", "execution_count": null, - "id": "f5e48708", + "id": "f8558148", "metadata": {}, "outputs": [], "source": [ @@ -648,7 +648,7 @@ }, { "cell_type": "markdown", - "id": "24cf11d3", + "id": "7e2813b4", "metadata": {}, "source": [ "## Indexing" @@ -657,7 +657,7 @@ { "cell_type": "code", "execution_count": null, - "id": "6fb377dc", + "id": "77ab64ab", "metadata": {}, "outputs": [], "source": [ @@ -669,7 +669,7 @@ }, { "cell_type": "markdown", - "id": "c1c04832", + "id": "69313988", "metadata": {}, "source": [ "### Table.head()\n", @@ -695,7 +695,7 @@ }, { "cell_type": "markdown", - "id": "3a1376fd", + "id": "edf33458", "metadata": {}, "source": [ "**Examples:**\n", @@ -706,7 +706,7 @@ { "cell_type": "code", "execution_count": null, - "id": "f0bf8f86", + "id": "916fcf4d", "metadata": { "scrolled": false }, @@ -717,7 +717,7 @@ }, { "cell_type": "markdown", - "id": "93e184ff", + "id": "cb58279a", "metadata": {}, "source": [ "Return the first 10 rows of the table." @@ -726,7 +726,7 @@ { "cell_type": "code", "execution_count": null, - "id": "9e3e5b67", + "id": "bf32db40", "metadata": {}, "outputs": [], "source": [ @@ -735,7 +735,7 @@ }, { "cell_type": "markdown", - "id": "76e7a8fe", + "id": "a5c4a5e9", "metadata": {}, "source": [ "### Table.tail()\n", @@ -761,7 +761,7 @@ }, { "cell_type": "markdown", - "id": "bc99337e", + "id": "4e3fee46", "metadata": {}, "source": [ "**Examples:**\n", @@ -772,7 +772,7 @@ { "cell_type": "code", "execution_count": null, - "id": "3b261b98", + "id": "a0d34e0b", "metadata": {}, "outputs": [], "source": [ @@ -781,7 +781,7 @@ }, { "cell_type": "markdown", - "id": "9871118a", + "id": "e223e705", "metadata": {}, "source": [ "Return the last 10 rows of the table." @@ -790,7 +790,7 @@ { "cell_type": "code", "execution_count": null, - "id": "dd3970b3", + "id": "4edae0c3", "metadata": {}, "outputs": [], "source": [ @@ -799,7 +799,7 @@ }, { "cell_type": "markdown", - "id": "507b8049", + "id": "c87325f8", "metadata": {}, "source": [ "### Table.get()\n", @@ -826,7 +826,7 @@ }, { "cell_type": "markdown", - "id": "ec0f77c7", + "id": "7c96cd34", "metadata": {}, "source": [ "**Examples:**\n", @@ -837,7 +837,7 @@ { "cell_type": "code", "execution_count": null, - "id": "5a5a6d38", + "id": "7f64d914", "metadata": { "scrolled": true }, @@ -848,7 +848,7 @@ }, { "cell_type": "markdown", - "id": "528ef898", + "id": "88ee5698", "metadata": {}, "source": [ "Get the `y` and `z` columns from the table." @@ -857,7 +857,7 @@ { "cell_type": "code", "execution_count": null, - "id": "50dc3d41", + "id": "daef6ce6", "metadata": { "scrolled": true }, @@ -868,7 +868,7 @@ }, { "cell_type": "markdown", - "id": "5671306b", + "id": "26a53f6d", "metadata": {}, "source": [ "Attempt to get the `q` column from the table and recieve none as that column does not exist." @@ -877,7 +877,7 @@ { "cell_type": "code", "execution_count": null, - "id": "f4e793d7", + "id": "3856084d", "metadata": {}, "outputs": [], "source": [ @@ -886,7 +886,7 @@ }, { "cell_type": "markdown", - "id": "4ae7804e", + "id": "91932d32", "metadata": {}, "source": [ "Attempt to get the `q` column from the table and recieve the default value `not found` as that column does not exist." @@ -895,7 +895,7 @@ { "cell_type": "code", "execution_count": null, - "id": "4fb7dafd", + "id": "7d2a2bcf", "metadata": {}, "outputs": [], "source": [ @@ -904,7 +904,7 @@ }, { "cell_type": "markdown", - "id": "b9bffb97", + "id": "9e831e14", "metadata": {}, "source": [ "### Table.at[]\n", @@ -922,7 +922,7 @@ }, { "cell_type": "markdown", - "id": "631f538b", + "id": "97519657", "metadata": {}, "source": [ "**Examples:**\n", @@ -933,7 +933,7 @@ { "cell_type": "code", "execution_count": null, - "id": "e88eb0b9", + "id": "9cd275bf", "metadata": {}, "outputs": [], "source": [ @@ -942,7 +942,7 @@ }, { "cell_type": "markdown", - "id": "3b2a6f91", + "id": "1fd39083", "metadata": {}, "source": [ "Reassign the value of the `z` column in the 997th row to `3.14159`." @@ -951,7 +951,7 @@ { "cell_type": "code", "execution_count": null, - "id": "514ceeb0", + "id": "814fa8e0", "metadata": {}, "outputs": [], "source": [ @@ -961,7 +961,7 @@ }, { "cell_type": "markdown", - "id": "9d807946", + "id": "7815e8c3", "metadata": {}, "source": [ "### Table.loc[]\n", @@ -997,7 +997,7 @@ }, { "cell_type": "markdown", - "id": "fc696884", + "id": "5ee06186", "metadata": {}, "source": [ "**Examples:**\n", @@ -1008,7 +1008,7 @@ { "cell_type": "code", "execution_count": null, - "id": "076e08a4", + "id": "12fc6807", "metadata": { "scrolled": true }, @@ -1019,7 +1019,7 @@ }, { "cell_type": "markdown", - "id": "dd6c4a2f", + "id": "97206dd7", "metadata": {}, "source": [ "Get all rows of the table where the value in the `z` column is greater than `250.0`" @@ -1028,7 +1028,7 @@ { "cell_type": "code", "execution_count": null, - "id": "eec5938e", + "id": "a6c9add0", "metadata": {}, "outputs": [], "source": [ @@ -1037,7 +1037,7 @@ }, { "cell_type": "markdown", - "id": "8ce7195e", + "id": "a32aca6b", "metadata": {}, "source": [ "Replace all null values in the column `v` with the value `-100`." @@ -1046,7 +1046,7 @@ { "cell_type": "code", "execution_count": null, - "id": "4fec8625", + "id": "c1ad3a23", "metadata": { "scrolled": true }, @@ -1058,7 +1058,7 @@ }, { "cell_type": "markdown", - "id": "81343ea4", + "id": "447b9fd2", "metadata": {}, "source": [ "Replace all locations in column `v` where the value is `-100` with a null." @@ -1067,7 +1067,7 @@ { "cell_type": "code", "execution_count": null, - "id": "d49ba7ff", + "id": "31ea02c9", "metadata": {}, "outputs": [], "source": [ @@ -1077,7 +1077,7 @@ }, { "cell_type": "markdown", - "id": "dirty-deviation", + "id": "ac4c5e4b", "metadata": {}, "source": [ "Usage of the `loc` functionality under the hood additionally allows users to set columns within a table for single or multiple columns. Data passed for this can be q/Python." @@ -1086,7 +1086,7 @@ { "cell_type": "code", "execution_count": null, - "id": "economic-administration", + "id": "f378ba4a", "metadata": {}, "outputs": [], "source": [ @@ -1096,7 +1096,7 @@ { "cell_type": "code", "execution_count": null, - "id": "parliamentary-simon", + "id": "0f2936b9", "metadata": {}, "outputs": [], "source": [ @@ -1105,7 +1105,7 @@ }, { "cell_type": "markdown", - "id": "8aeb5b10", + "id": "a3368987", "metadata": {}, "source": [ "### Table.iloc[]\n", @@ -1135,7 +1135,7 @@ }, { "cell_type": "markdown", - "id": "a6e24ecf", + "id": "0ef4d8cf", "metadata": {}, "source": [ "**Examples:**\n", @@ -1146,7 +1146,7 @@ { "cell_type": "code", "execution_count": null, - "id": "a3460c85", + "id": "683ab48b", "metadata": { "scrolled": true }, @@ -1157,7 +1157,7 @@ }, { "cell_type": "markdown", - "id": "2bdb5d71", + "id": "e71bebdb", "metadata": {}, "source": [ "Get the first 5 rows from a table." @@ -1166,7 +1166,7 @@ { "cell_type": "code", "execution_count": null, - "id": "4ef3767c", + "id": "a13730fd", "metadata": { "scrolled": false }, @@ -1177,7 +1177,7 @@ }, { "cell_type": "markdown", - "id": "f869425e", + "id": "60f892e0", "metadata": {}, "source": [ "Get all rows of the table where the `y` column is equal to `AAPL`." @@ -1186,7 +1186,7 @@ { "cell_type": "code", "execution_count": null, - "id": "bd3d1613", + "id": "d7afdf65", "metadata": { "scrolled": true }, @@ -1197,7 +1197,7 @@ }, { "cell_type": "markdown", - "id": "bcc638af", + "id": "8b3b9279", "metadata": {}, "source": [ "Get all rows of the table where the `y` column is equal to `AAPL`, and only return the `y`, `z` and `w` columns." @@ -1206,7 +1206,7 @@ { "cell_type": "code", "execution_count": null, - "id": "19491b1a", + "id": "a0d9f08d", "metadata": {}, "outputs": [], "source": [ @@ -1215,7 +1215,7 @@ }, { "cell_type": "markdown", - "id": "7a7bcdd8", + "id": "045bc156", "metadata": {}, "source": [ "Replace all null values in the column `v` with the value `-100`." @@ -1224,7 +1224,7 @@ { "cell_type": "code", "execution_count": null, - "id": "8dbd832b", + "id": "7e21c163", "metadata": {}, "outputs": [], "source": [ @@ -1234,7 +1234,7 @@ }, { "cell_type": "markdown", - "id": "37ad1ee6", + "id": "76021266", "metadata": {}, "source": [ "### Table.pop()\n", @@ -1260,7 +1260,7 @@ }, { "cell_type": "markdown", - "id": "a7e8dc98", + "id": "e5fdfbd3", "metadata": {}, "source": [ "**Examples:**\n", @@ -1271,7 +1271,7 @@ { "cell_type": "code", "execution_count": null, - "id": "cc748ad8", + "id": "7a960191", "metadata": { "scrolled": true }, @@ -1286,7 +1286,7 @@ }, { "cell_type": "markdown", - "id": "231ebfbb", + "id": "35062560", "metadata": {}, "source": [ "Remove the `z` and `w` columns from the table and return them." @@ -1295,7 +1295,7 @@ { "cell_type": "code", "execution_count": null, - "id": "2aea8b3e", + "id": "a46189b2", "metadata": { "scrolled": false }, @@ -1310,7 +1310,7 @@ }, { "cell_type": "markdown", - "id": "56ce1b1d", + "id": "f71b6917", "metadata": {}, "source": [ "## Reindexing" @@ -1319,7 +1319,7 @@ { "cell_type": "code", "execution_count": null, - "id": "a4c1d74b", + "id": "a2b1a198", "metadata": { "scrolled": true }, @@ -1333,7 +1333,7 @@ }, { "cell_type": "markdown", - "id": "e47a4340", + "id": "f5a7ac0e", "metadata": {}, "source": [ "### Table.drop()\n", @@ -1360,7 +1360,7 @@ }, { "cell_type": "markdown", - "id": "a0417a0f", + "id": "008a2e74", "metadata": {}, "source": [ "**Examples:**\n", @@ -1371,7 +1371,7 @@ { "cell_type": "code", "execution_count": null, - "id": "2932fb5f", + "id": "0f74d3f2", "metadata": {}, "outputs": [], "source": [ @@ -1380,7 +1380,7 @@ }, { "cell_type": "markdown", - "id": "5368f9f1", + "id": "cb4e82aa", "metadata": {}, "source": [ "Drop columns from a table." @@ -1389,7 +1389,7 @@ { "cell_type": "code", "execution_count": null, - "id": "02c1221f", + "id": "57ad6a64", "metadata": {}, "outputs": [], "source": [ @@ -1398,7 +1398,7 @@ }, { "cell_type": "markdown", - "id": "a88ea856", + "id": "90db87b0", "metadata": {}, "source": [ "### Table.drop_duplicates()\n", @@ -1418,7 +1418,7 @@ }, { "cell_type": "markdown", - "id": "90493dae", + "id": "3af33f03", "metadata": {}, "source": [ "**Examples:**\n", @@ -1429,7 +1429,7 @@ { "cell_type": "code", "execution_count": null, - "id": "baccc6bd", + "id": "af182307", "metadata": {}, "outputs": [], "source": [ @@ -1439,7 +1439,7 @@ }, { "cell_type": "markdown", - "id": "cd94c2b6", + "id": "48143d51", "metadata": {}, "source": [ "Drop all duplicate rows from the table." @@ -1448,7 +1448,7 @@ { "cell_type": "code", "execution_count": null, - "id": "c6dfca99", + "id": "eeff16e7", "metadata": {}, "outputs": [], "source": [ @@ -1457,7 +1457,7 @@ }, { "cell_type": "markdown", - "id": "ece21d55", + "id": "6d71c8c0", "metadata": {}, "source": [ "### Table.rename()\n", @@ -1483,7 +1483,7 @@ }, { "cell_type": "markdown", - "id": "d49a17ce", + "id": "73260da1", "metadata": {}, "source": [ "**Examples:**\n", @@ -1494,7 +1494,7 @@ { "cell_type": "code", "execution_count": null, - "id": "585ea2e9", + "id": "3cc68fa6", "metadata": {}, "outputs": [], "source": [ @@ -1503,7 +1503,7 @@ }, { "cell_type": "markdown", - "id": "b88b46fd", + "id": "eef94948", "metadata": {}, "source": [ "Rename column `y` to `symbol` and `z` to `price`." @@ -1512,7 +1512,7 @@ { "cell_type": "code", "execution_count": null, - "id": "ed9c511f", + "id": "d5e76248", "metadata": {}, "outputs": [], "source": [ @@ -1521,7 +1521,7 @@ }, { "cell_type": "markdown", - "id": "10582eaa", + "id": "05124590", "metadata": {}, "source": [ "### Table.sample()\n", @@ -1553,7 +1553,7 @@ }, { "cell_type": "markdown", - "id": "0271484d", + "id": "e8f78917", "metadata": {}, "source": [ "**Examples:**\n", @@ -1564,7 +1564,7 @@ { "cell_type": "code", "execution_count": null, - "id": "187059eb", + "id": "d88ab348", "metadata": {}, "outputs": [], "source": [ @@ -1573,7 +1573,7 @@ }, { "cell_type": "markdown", - "id": "d5d52b8b", + "id": "78e03554", "metadata": {}, "source": [ "Sample 10% of the rows." @@ -1582,7 +1582,7 @@ { "cell_type": "code", "execution_count": null, - "id": "f6aa2289", + "id": "8585d62e", "metadata": {}, "outputs": [], "source": [ @@ -1591,7 +1591,7 @@ }, { "cell_type": "markdown", - "id": "a9d80fe9", + "id": "c77712d3", "metadata": {}, "source": [ "Sample 10% of the rows and allow the same row to be sampled twice." @@ -1600,7 +1600,7 @@ { "cell_type": "code", "execution_count": null, - "id": "17096534", + "id": "b138f770", "metadata": {}, "outputs": [], "source": [ @@ -1609,7 +1609,7 @@ }, { "cell_type": "markdown", - "id": "32794d29", + "id": "6f6f5672", "metadata": {}, "source": [ "### Table.select_dtypes()\n", @@ -1638,7 +1638,7 @@ }, { "cell_type": "markdown", - "id": "a94cc6e5", + "id": "6a703c57", "metadata": {}, "source": [ "**Examples:**\n", @@ -1649,7 +1649,7 @@ { "cell_type": "code", "execution_count": null, - "id": "8fcaa4fb", + "id": "5e9734f7", "metadata": {}, "outputs": [], "source": [ @@ -1658,7 +1658,7 @@ }, { "cell_type": "markdown", - "id": "3dc6ef75", + "id": "42d9ffa6", "metadata": {}, "source": [ "Exclude columns contatining symbols" @@ -1667,7 +1667,7 @@ { "cell_type": "code", "execution_count": null, - "id": "7d1a3e61", + "id": "3d934cf0", "metadata": {}, "outputs": [], "source": [ @@ -1676,7 +1676,7 @@ }, { "cell_type": "markdown", - "id": "7009cb76", + "id": "e4302f7d", "metadata": {}, "source": [ "Include a list of column types" @@ -1685,7 +1685,7 @@ { "cell_type": "code", "execution_count": null, - "id": "b71e87fa", + "id": "f698f5f0", "metadata": {}, "outputs": [], "source": [ @@ -1694,7 +1694,7 @@ }, { "cell_type": "markdown", - "id": "54417754", + "id": "5590d1ca", "metadata": {}, "source": [ "### Table.astype()\n", @@ -1723,7 +1723,7 @@ }, { "cell_type": "markdown", - "id": "20546f87", + "id": "f9ca98d2", "metadata": {}, "source": [ "**Examples:**\n", @@ -1734,7 +1734,7 @@ { "cell_type": "code", "execution_count": null, - "id": "33e3cf56", + "id": "831836c8", "metadata": {}, "outputs": [], "source": [ @@ -1743,7 +1743,7 @@ }, { "cell_type": "markdown", - "id": "16b2ee25", + "id": "0bf0d78f", "metadata": {}, "source": [ "Cast all columns to dtype LongVector" @@ -1752,7 +1752,7 @@ { "cell_type": "code", "execution_count": null, - "id": "ac977798", + "id": "6833400a", "metadata": {}, "outputs": [], "source": [ @@ -1761,7 +1761,7 @@ }, { "cell_type": "markdown", - "id": "51850e87", + "id": "7a2bfcd3", "metadata": {}, "source": [ "Casting as specified in the dcitionary supplied with given dtype per column" @@ -1770,7 +1770,7 @@ { "cell_type": "code", "execution_count": null, - "id": "d8c2f1f9", + "id": "872db9aa", "metadata": {}, "outputs": [], "source": [ @@ -1779,7 +1779,7 @@ }, { "cell_type": "markdown", - "id": "ada1bfd4", + "id": "ef3b4225", "metadata": {}, "source": [ "The next example will use this table" @@ -1788,7 +1788,7 @@ { "cell_type": "code", "execution_count": null, - "id": "ab8261b8", + "id": "6a20abdd", "metadata": {}, "outputs": [], "source": [ @@ -1797,7 +1797,7 @@ }, { "cell_type": "markdown", - "id": "a2972dc0", + "id": "908fa4ea", "metadata": {}, "source": [ "Casting char and string columns to symbol columns" @@ -1806,7 +1806,7 @@ { "cell_type": "code", "execution_count": null, - "id": "c2c019bc", + "id": "5ea7fe9e", "metadata": {}, "outputs": [], "source": [ @@ -1815,7 +1815,7 @@ }, { "cell_type": "markdown", - "id": "e0b57863", + "id": "718584f8", "metadata": {}, "source": [ "## Merging" @@ -1823,7 +1823,7 @@ }, { "cell_type": "markdown", - "id": "8b1c0dc5", + "id": "ef401426", "metadata": {}, "source": [ "### Table.merge()\n", @@ -1875,7 +1875,7 @@ }, { "cell_type": "markdown", - "id": "9542857c", + "id": "9e613e3c", "metadata": {}, "source": [ "**Examples:**\n", @@ -1886,7 +1886,7 @@ { "cell_type": "code", "execution_count": null, - "id": "847941a9", + "id": "a3b0ec9f", "metadata": { "scrolled": true }, @@ -1899,7 +1899,7 @@ }, { "cell_type": "markdown", - "id": "4b6793c9", + "id": "6e32596c", "metadata": {}, "source": [ "Merge tab1 and tab2 on the lkey and rkey columns using a native q inner join. The value columns have the default suffixes, \\_x and \\_y, appended." @@ -1908,7 +1908,7 @@ { "cell_type": "code", "execution_count": null, - "id": "023b555b", + "id": "8ea253c9", "metadata": {}, "outputs": [], "source": [ @@ -1917,7 +1917,7 @@ }, { "cell_type": "markdown", - "id": "9449f90b", + "id": "2d9240b3", "metadata": {}, "source": [ "Merge tab1 and tab2 with specified left and right suffixes appended to any overlapping columns." @@ -1926,7 +1926,7 @@ { "cell_type": "code", "execution_count": null, - "id": "87ad643d", + "id": "64425a1d", "metadata": {}, "outputs": [], "source": [ @@ -1935,7 +1935,7 @@ }, { "cell_type": "markdown", - "id": "49deadfd", + "id": "e749c7e0", "metadata": {}, "source": [ "Merge tab1 and tab2 but raise an exception if the Tables have any overlapping columns." @@ -1944,7 +1944,7 @@ { "cell_type": "code", "execution_count": null, - "id": "a3d45ec4", + "id": "a938230d", "metadata": { "scrolled": true }, @@ -1959,7 +1959,7 @@ { "cell_type": "code", "execution_count": null, - "id": "ee3ef9a8", + "id": "b1d99a31", "metadata": {}, "outputs": [], "source": [ @@ -1969,7 +1969,7 @@ }, { "cell_type": "markdown", - "id": "b62f2e3b", + "id": "385c0465", "metadata": {}, "source": [ "Merge tab1 and tab2 on the `a` column using an inner join." @@ -1978,7 +1978,7 @@ { "cell_type": "code", "execution_count": null, - "id": "8f17070f", + "id": "7431a148", "metadata": { "scrolled": true }, @@ -1989,7 +1989,7 @@ }, { "cell_type": "markdown", - "id": "2aaf6b4b", + "id": "230a7666", "metadata": {}, "source": [ "Merge tab1 and tab2 on the `a` column using a left join." @@ -1998,7 +1998,7 @@ { "cell_type": "code", "execution_count": null, - "id": "64cfb314", + "id": "04b96b08", "metadata": {}, "outputs": [], "source": [ @@ -2007,7 +2007,7 @@ }, { "cell_type": "markdown", - "id": "374c2905", + "id": "d991656c", "metadata": {}, "source": [ "Merge tab1 and tab2 using a cross join." @@ -2016,7 +2016,7 @@ { "cell_type": "code", "execution_count": null, - "id": "51f94109", + "id": "09886503", "metadata": { "scrolled": true }, @@ -2029,7 +2029,7 @@ }, { "cell_type": "markdown", - "id": "d4293b88", + "id": "b2f4aff1", "metadata": {}, "source": [ "### Table.merge_asof()\n", @@ -2086,7 +2086,7 @@ }, { "cell_type": "markdown", - "id": "bf6bc139", + "id": "fc696ccf", "metadata": {}, "source": [ "**Examples:**\n", @@ -2097,7 +2097,7 @@ { "cell_type": "code", "execution_count": null, - "id": "338f553c", + "id": "6cb634e0", "metadata": {}, "outputs": [], "source": [ @@ -2109,7 +2109,7 @@ { "cell_type": "code", "execution_count": null, - "id": "2d7b0d9b", + "id": "81b10932", "metadata": {}, "outputs": [], "source": [ @@ -2119,7 +2119,7 @@ { "cell_type": "code", "execution_count": null, - "id": "5949e8b3", + "id": "411d19d2", "metadata": {}, "outputs": [], "source": [ @@ -2128,7 +2128,7 @@ }, { "cell_type": "markdown", - "id": "6976cfc0", + "id": "324d24ec", "metadata": {}, "source": [ "Perform a asof join on two tables but first merge them on the by column." @@ -2137,7 +2137,7 @@ { "cell_type": "code", "execution_count": null, - "id": "7e9c7ee0", + "id": "d805fa5c", "metadata": {}, "outputs": [], "source": [ @@ -2183,7 +2183,7 @@ { "cell_type": "code", "execution_count": null, - "id": "444c426a", + "id": "665d0e74", "metadata": {}, "outputs": [], "source": [ @@ -2193,7 +2193,7 @@ { "cell_type": "code", "execution_count": null, - "id": "d2e16d4a", + "id": "9398ab6a", "metadata": {}, "outputs": [], "source": [ @@ -2202,7 +2202,7 @@ }, { "cell_type": "markdown", - "id": "ca5c9a5f", + "id": "acca5289", "metadata": {}, "source": [ "## Computations" @@ -2211,7 +2211,7 @@ { "cell_type": "code", "execution_count": null, - "id": "674cb468", + "id": "852b5f34", "metadata": {}, "outputs": [], "source": [ @@ -2223,7 +2223,7 @@ }, { "cell_type": "markdown", - "id": "92136283", + "id": "93a50ee2", "metadata": {}, "source": [ "### Table.abs()\n", @@ -2250,7 +2250,7 @@ { "cell_type": "code", "execution_count": null, - "id": "7b08d857", + "id": "7da5d72b", "metadata": { "scrolled": true }, @@ -2261,7 +2261,7 @@ }, { "cell_type": "markdown", - "id": "ad57d9cf", + "id": "cbcdf84e", "metadata": {}, "source": [ "### Table.all()\n", @@ -2290,7 +2290,7 @@ { "cell_type": "code", "execution_count": null, - "id": "b37d2e15", + "id": "7d0b0cd3", "metadata": {}, "outputs": [], "source": [ @@ -2299,7 +2299,7 @@ }, { "cell_type": "markdown", - "id": "fd14012f", + "id": "aa02cf1c", "metadata": {}, "source": [ "### Table.any()\n", @@ -2328,7 +2328,7 @@ { "cell_type": "code", "execution_count": null, - "id": "581cf133", + "id": "a4806993", "metadata": {}, "outputs": [], "source": [ @@ -2337,7 +2337,7 @@ }, { "cell_type": "markdown", - "id": "c42f7ec0", + "id": "a3c3fccd", "metadata": {}, "source": [ "### Table.max()\n", @@ -2366,7 +2366,7 @@ { "cell_type": "code", "execution_count": null, - "id": "dd8a84e1", + "id": "8e9abf02", "metadata": {}, "outputs": [], "source": [ @@ -2375,7 +2375,7 @@ }, { "cell_type": "markdown", - "id": "fb28288f", + "id": "301ab2c2", "metadata": {}, "source": [ "### Table.min()\n", @@ -2404,7 +2404,7 @@ { "cell_type": "code", "execution_count": null, - "id": "0fe40c65", + "id": "c1255ac7", "metadata": {}, "outputs": [], "source": [ @@ -2413,7 +2413,7 @@ }, { "cell_type": "markdown", - "id": "af783468", + "id": "a389f7aa", "metadata": {}, "source": [ "### Table.sum()\n", @@ -2443,7 +2443,7 @@ { "cell_type": "code", "execution_count": null, - "id": "32519605", + "id": "af638f53", "metadata": {}, "outputs": [], "source": [ @@ -2452,7 +2452,7 @@ }, { "cell_type": "markdown", - "id": "621766f6", + "id": "9bf62b1a", "metadata": {}, "source": [ "### Table.prod()\n", @@ -2482,7 +2482,7 @@ { "cell_type": "code", "execution_count": null, - "id": "97c7c26b", + "id": "0ddad367", "metadata": { "scrolled": true }, @@ -2500,7 +2500,7 @@ { "cell_type": "code", "execution_count": null, - "id": "9222c8ba", + "id": "151411e2", "metadata": {}, "outputs": [], "source": [ @@ -2509,7 +2509,7 @@ }, { "cell_type": "markdown", - "id": "655c3ad2", + "id": "499025cb", "metadata": {}, "source": [ "## Setting Indexes" @@ -2517,7 +2517,7 @@ }, { "cell_type": "markdown", - "id": "6ad74ce0", + "id": "4dc576e8", "metadata": {}, "source": [ "### Table.set_index()\n", @@ -2558,7 +2558,7 @@ { "cell_type": "code", "execution_count": null, - "id": "bdd21889", + "id": "42a288f7", "metadata": {}, "outputs": [], "source": [ @@ -2569,7 +2569,7 @@ { "cell_type": "code", "execution_count": null, - "id": "cf782f61", + "id": "f744959e", "metadata": {}, "outputs": [], "source": [ @@ -2580,7 +2580,7 @@ { "cell_type": "code", "execution_count": null, - "id": "0c54f760", + "id": "00c31275", "metadata": {}, "outputs": [], "source": [ @@ -2591,7 +2591,7 @@ { "cell_type": "code", "execution_count": null, - "id": "825afd87", + "id": "858bbeb2", "metadata": {}, "outputs": [], "source": [ @@ -2602,7 +2602,7 @@ }, { "cell_type": "markdown", - "id": "72efa53c", + "id": "450c30ee", "metadata": {}, "source": [ "Appending:" @@ -2611,7 +2611,7 @@ { "cell_type": "code", "execution_count": null, - "id": "d2b9b266", + "id": "b475c811", "metadata": {}, "outputs": [], "source": [ @@ -2622,7 +2622,7 @@ { "cell_type": "code", "execution_count": null, - "id": "af569818", + "id": "0fb2c59c", "metadata": {}, "outputs": [], "source": [ @@ -2632,7 +2632,7 @@ }, { "cell_type": "markdown", - "id": "3224889a", + "id": "887ffb99", "metadata": {}, "source": [ "Verify Integrity:" @@ -2641,7 +2641,7 @@ { "cell_type": "code", "execution_count": null, - "id": "74347bef", + "id": "49367c46", "metadata": {}, "outputs": [], "source": [ @@ -2652,7 +2652,7 @@ { "cell_type": "code", "execution_count": null, - "id": "edca3507", + "id": "7bb2aaf0", "metadata": {}, "outputs": [], "source": [ @@ -2665,7 +2665,7 @@ }, { "cell_type": "markdown", - "id": "e9d74bb5", + "id": "7e415861", "metadata": {}, "source": [ "## Group By" @@ -2673,7 +2673,7 @@ }, { "cell_type": "markdown", - "id": "ae3ec2eb", + "id": "8b2d72fb", "metadata": {}, "source": [ "### Table.groupby()\n", @@ -2724,7 +2724,7 @@ { "cell_type": "code", "execution_count": null, - "id": "189eabbe", + "id": "0789d3f4", "metadata": { "scrolled": true }, @@ -2741,7 +2741,7 @@ }, { "cell_type": "markdown", - "id": "d805052d", + "id": "8baae3c9", "metadata": {}, "source": [ "Group on the `Animal` column and calculate the mean of the resulting `Max Speed` and `Max Altitude` columns." @@ -2750,7 +2750,7 @@ { "cell_type": "code", "execution_count": null, - "id": "00cc7660", + "id": "734cb6ff", "metadata": { "scrolled": true }, @@ -2761,7 +2761,7 @@ }, { "cell_type": "markdown", - "id": "c7ef160d", + "id": "b3b759af", "metadata": {}, "source": [ "Example table with multiple columns to group on." @@ -2770,7 +2770,7 @@ { "cell_type": "code", "execution_count": null, - "id": "3204bd59", + "id": "7966c28c", "metadata": {}, "outputs": [], "source": [ @@ -2786,7 +2786,7 @@ }, { "cell_type": "markdown", - "id": "77008f71", + "id": "e3ab5b1f", "metadata": {}, "source": [ "Group on multiple columns using thier indexes." @@ -2795,7 +2795,7 @@ { "cell_type": "code", "execution_count": null, - "id": "cfd22d01", + "id": "c01d3cc9", "metadata": {}, "outputs": [], "source": [ @@ -2804,7 +2804,7 @@ }, { "cell_type": "markdown", - "id": "58e77d29", + "id": "d46304f0", "metadata": {}, "source": [ "Example table with Nulls." @@ -2813,7 +2813,7 @@ { "cell_type": "code", "execution_count": null, - "id": "96bb6e4d", + "id": "dc222240", "metadata": {}, "outputs": [], "source": [ @@ -2831,7 +2831,7 @@ }, { "cell_type": "markdown", - "id": "a13c11f4", + "id": "4c38e902", "metadata": {}, "source": [ "Group on column `a` and keep null groups." @@ -2840,7 +2840,7 @@ { "cell_type": "code", "execution_count": null, - "id": "95d7734a", + "id": "833e4a92", "metadata": { "scrolled": true }, @@ -2851,7 +2851,7 @@ }, { "cell_type": "markdown", - "id": "1645ae2b", + "id": "c26a98ff", "metadata": {}, "source": [ "Group on column `a` keeping null groups and not using the groups as an index column." @@ -2860,7 +2860,7 @@ { "cell_type": "code", "execution_count": null, - "id": "bf8dc14c", + "id": "bb5d1bac", "metadata": {}, "outputs": [], "source": [ @@ -2869,7 +2869,7 @@ }, { "cell_type": "markdown", - "id": "undefined-bruce", + "id": "af8fad39", "metadata": {}, "source": [ "## Apply\n", @@ -2917,7 +2917,7 @@ { "cell_type": "code", "execution_count": null, - "id": "cooperative-construction", + "id": "02f41281", "metadata": {}, "outputs": [], "source": [ @@ -2928,7 +2928,7 @@ }, { "cell_type": "markdown", - "id": "micro-dodge", + "id": "cf555661", "metadata": {}, "source": [ "Apply square root on each item within a column" @@ -2937,7 +2937,7 @@ { "cell_type": "code", "execution_count": null, - "id": "handmade-bridal", + "id": "173acc13", "metadata": {}, "outputs": [], "source": [ @@ -2946,7 +2946,7 @@ }, { "cell_type": "markdown", - "id": "accepted-planning", + "id": "a00dda0c", "metadata": {}, "source": [ "Apply a reducing function sum on either axis" @@ -2955,7 +2955,7 @@ { "cell_type": "code", "execution_count": null, - "id": "acquired-wholesale", + "id": "4936ea30", "metadata": {}, "outputs": [], "source": [ @@ -2965,17 +2965,60 @@ { "cell_type": "code", "execution_count": null, - "id": "informal-algebra", + "id": "5df4a3ac", "metadata": {}, "outputs": [], "source": [ "tab.apply(lambda x: sum(x), axis=1)" ] + }, + { + "cell_type": "markdown", + "id": "8da6da7c", + "metadata": {}, + "source": [ + "## Aggregate\n", + "\n", + "### Table.agg()\n", + "\n", + "```\n", + "Table.agg(\n", + " func,\n", + " axis=0,\n", + " *args,\n", + " **kwargs\n", + ")\n", + "```\n", + "\n", + "Aggregate data using one or more operations over a specified axis\n", + "\n", + "Objects passed to a function are passed as kx vector/list objects.\n", + "\n", + "**Parameters:**\n", + "\n", + "| Name | Type | Description | Default |\n", + "| :--------------: | :---------------------------------: | :-------------------------------------------------------------------------- | :------: |\n", + "| func | function, str, list or dict | Function to use for aggregating the data. If a function this must either work when passed a `Table` or when passed to `Table.apply`

Accepted combinations are:
  • function
  • string function name
  • list of functions and/or function names, e.g. `[kx.q.sum, 'mean']`
  • dict of axis labels -> functions or function names
      • | |\n", + "| `*args` | any | Positional arguments to pass to `func` in addition to the kx list. | |\n", + "| axis | int | The axis along which the function is applied, `0` applies function to each column, at present row based application is not supported. | 0 | \n", + "| `**kwargs` | dict | Additional keyword arguments to pass as keywords to `func`, this argument is not implemented in the case `func` is a kx callable function. | None | \n", + "\n", + "\n", + "**Returns:**\n", + "\n", + "| Type | Description |\n", + "| :-----------------------: | :---------------------------------------------- |\n", + "| List, Dictionary or Table | Result of applying `func` along the giveen axis of the `kx.Table`. |\n", + "\n", + "**Examples:**\n", + "\n", + "Example Table." + ] } ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, diff --git a/docs/user-guide/advanced/context_interface.md b/docs/user-guide/advanced/context_interface.md index 347a8d9..4b9927e 100644 --- a/docs/user-guide/advanced/context_interface.md +++ b/docs/user-guide/advanced/context_interface.md @@ -224,6 +224,22 @@ in order until one is found to exist, or they have all been checked: AttributeError: 'pykx.ctx.QContext' object has no attribute 'test_extension' ``` +### Adding context interface search locations + +In addition to the default search locations users can add additional locations to be searched through appending of additional search paths to the `kx.q.paths` list which underpins the search. The following shows a practical example of this accessing a file `my_context.q` at a new location `/tmp/files`, in this example we attempt initially to access the namespace without this location set for search + +```python +>>> import pykx as kx +>>> from pathlib import Path +>>> kx.q.my_context +Traceback (most recent call last): + File "/usr/local/anaconda3/lib/python3.8/site-packages/pykx/__init__.py", line 132, in __getattr__ +.. +>>> kx.q.paths.append(Path('/tmp/files')) +>>> kx.q.my_context + +``` + ## Best Practices To take full advantage of the automatic script loading one should ensure that every q/k script diff --git a/docs/user-guide/advanced/modes.md b/docs/user-guide/advanced/modes.md index 13210b4..bd2eb7d 100644 --- a/docs/user-guide/advanced/modes.md +++ b/docs/user-guide/advanced/modes.md @@ -136,15 +136,15 @@ The following are the differences provided through operation with a valid KX Lic Users can choose to initialise PyKX under one of these modalities explicitly through the use of the `QARGS` environment variable as follows: - | Modality argument| Description| |------------------|----------| | `--unlicensed` | Starts PyKX in unlicensed mode. No license check will be performed, and no warning will be emitted at start-up if embedded q initialization fails. | | `--licensed` | Raise a `PyKXException` (as opposed to emitting a `PyKXWarning`) if embedded q initialization fails. - In addition to the PyKX specific start-up arguments `QARGS` also can be used to set the standard [q command-line arguments](https://code.kx.com/q/basics/cmdline/). +Alternatively for users who wish to make use of PyKX in unlicensed mode they can set the environment variable `PYKX_UNLICENSED="true"` or define this in their `.pykx-config` file as outlined [here](../configuration.md). + ## PyKX within a q session diff --git a/docs/user-guide/configuration.md b/docs/user-guide/configuration.md index 06ac6a4..5a59770 100644 --- a/docs/user-guide/configuration.md +++ b/docs/user-guide/configuration.md @@ -70,18 +70,20 @@ The following variables can be used to enable or disable advanced features of Py | Option | Default | Values | Description | Status | |---------------------------------|---------|-----------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------| | `PYKX_IGNORE_QHOME` | `False` | `1` or `true` | When loading PyKX on a machine with an existing q installation (and the environment variable `QHOME` set to the installation folder), PyKX will look within this directory for q scripts their dependencies. It will then symlink these files to make them available to load under PyKX. This variable instructs PyKX to not perform this symlinking. | | -| `PYKX_KEEP_LOCAL_TIMES` | `False` | `1` or `true` | When converting a Python datetime object to q, PyKX will translate the Python datetime into UTC before the conversion. This variable instructs PyKX to convert the Python datetime using the local time zone. | | +| `PYKX_KEEP_LOCAL_TIMES` | `False` | `1` or `true` | When converting a Python datetime object to q, PyKX will translate the Python datetime into UTC before the conversion. This variable instructs PyKX to convert the Python datetime using the local time zone. | | | `PYKX_ALLOCATOR` | `False` | `1` or `true` | When converting a Numpy array to q, PyKX implements a full data copy in order to translate the Numpy array to q representation in memory. When this is set PyKX implements [NEP-49](https://numpy.org/neps/nep-0049.html) which allows q to handle memory allocation of all Numpy arrays so they can be converted more efficiently to q. This avoids the need to resort to a copy where possible. | | | `PYKX_GC` | `False` | `1` or `true` | When PYKX_ALLOCATOR is enabled, PyKX can trigger q garbage collector when Numpy arrays allocated by PyKX are deallocated. This variable enables this behavior which will release q memory to the OS following deallocation of the Numpy array at the cost of a small overhead. | | | `PYKX_LOAD_PYARROW_UNSAFE` | `False` | `1` or `true` | By default, PyKX uses a subprocess to import pyarrow as it can result in a crash when the version of pyarrow is incompatible. This variable will trigger a normal import of pyarrow and importing PyKX should be slightly faster. | | | `PYKX_MAX_ERROR_LENGTH` | `256` | size in characters | By default, PyKX reports IPC connection errors with a message buffer of size 256 characters. This allows the length of these error messages to be modified reducing the chance of excessive error messages polluting logs. | | | `PYKX_NOQCE` | `False` | `1` or `true` | On Linux, PyKX comes with q Cloud Edition features from Insights Core (https://code.kx.com/insights/1.2/core/). This variable allows a user to skip the loading of q Cloud Edition functionality, saving some time when importing PyKX but removing access to possibly supported additional functionality. | | -| `PYKX_Q_LIB_LOCATION` | `UNSET` | Path to a directory containing q libraries necessary for loading PyKX | See [here](../release-notes/changelog.md#pykx-131) for detailed information. This allows a user to centralise the q libraries, `q.k`, `read.q`, `libq.so` etc to a managed location within their environment which is decentralised from the Python installation. This is required for some enterprise use-cases. | | +| `PYKX_Q_LIB_LOCATION` | `UNSET` | Path to a directory containing q libraries necessary for loading PyKX | See [here](../release-notes/changelog.md#pykx-131) for detailed information. This allows a user to centralise the q libraries, `q.k`, `read.q`, `libq.so` etc to a managed location within their environment which is decentralised from the Python installation. This is required for some enterprise use-cases. | | | `PYKX_RELEASE_GIL` | `False` | `1` or `true` | When PYKX_RELEASE_GIL is enabled the Python Global Interpreter Lock will not be held when calling into q. | | -| `PYKX_Q_LOCK` | `False` | `1` or `true` | When PYKX_Q_LOCK is enabled a re-entrant lock is added around calls into q, this lock will stop multiple threads from calling into q at the same time. This allows embedded q to be thread safe even when using PYKX_RELEASE_GIL. | | +| `PYKX_Q_LOCK` | `False` | `1` or `true` | When PYKX_Q_LOCK is enabled a re-entrant lock is added around calls into q, this lock will stop multiple threads from calling into q at the same time. This allows embedded q to be thread safe even when using PYKX_RELEASE_GIL. | | | `PYKX_DEBUG_INSIGHTS_LIBRARIES` | `False` | `1` or `true` | If the insights libraries failed to load this variable can be used to print out the full error output for debugging purposes. | | +| `PYKX_UNLICENSED` | `False` | `1` or `true` | Set PyKX to make use of the library in `unlicensed` mode at all times. | | +| `PYKX_LICENSED` | `False` | `1` or `true` | Set PyKX to make use of the library in `licensed` mode at all times. | | | `IGNORE_QHOME` | `True` | `1` or `true` | When loading PyKX on a machine with an existing q installation (and the environment variable `QHOME` set to the installation folder), PyKX will look within this directory for q scripts their dependencies. It will then symlink these files to make them available to load under PyKX. This variable instructs PyKX to not perform this symlinking. | `DEPRECATED`, please use `PYKX_IGNORE_QHOME` | -| `KEEP_LOCAL_TIMES` | `False` | `1` or `true` | When converting a Python datetime object to q, PyKX will translate the Python datetime into UTC before the conversion. This variable instructs PyKX to convert the Python datetime using the local time zone. | `DEPRECATED`, please use `PYKX_KEEP_LOCAL_TIMES` | +| `KEEP_LOCAL_TIMES` | `False` | `1` or `true` | When converting a Python datetime object to q, PyKX will translate the Python datetime into UTC before the conversion. This variable instructs PyKX to convert the Python datetime using the local time zone. | `DEPRECATED`, please use `PYKX_KEEP_LOCAL_TIMES` | The variables below can be used to set the environment for q (embedded in PyKX, in licensed mode): @@ -91,7 +93,19 @@ The variables below can be used to set the environment for q (embedded in PyKX, | `QARGS` | See link | Command-line flags to pass to q, see [here](https://code.kx.com/q/basics/cmdline/) for more information. | | `QHOME` | Path to the users q installation folder | See [here](https://code.kx.com/q/learn/install/#step-5-edit-your-profile) for more information. | | `QLIC` | Path to the folder where the q license should be found | See [here](https://code.kx.com/q/learn/install/#step-5-edit-your-profile) for more information. | +| `QINIT` | Path to an additional `*.q` file loaded after `PyKX` has initialized | See [here](https://code.kx.com/q4m3/14_Introduction_to_Kdb%2B/#1481-the-environment-variables) for more information. | +#### PyKX QARGS Supported Additions + +When using PyKX users can use the following values when defining `QARGS` to modify the behaviour of PyKX at initialisation when running within a Linux environment. + +| Input | Description | +|----------------|---------------------------------------------------------------------------------| +| `--no-qce` | Ensure that no kdb Insights libraries are loaded at initialisation of PyKX. | +| `--no-kurl` | Ensure that the kdb Insights `kurl` library is not loaded at initialisation. | +| `--no-objstor` | Ensure that the kdb Insights `objstor` library is not loaded at initialisation. | +| `--no-qlog` | Ensure that the kdb Insights `qlog` library is not loaded at initialisation. | +| `--no-sql` | Ensure that the kdb Insights `sql` library is not loaded at initialisation. | ### PyKX under q @@ -100,12 +114,13 @@ PyKX can be loaded and used from a q session (see [here](../pykx-under-q/intro.m | Variable | Values | Description | Status | |---------------------------|-------------------------------|-------------|--------| | `PYKX_DEFAULT_CONVERSION` | `py`, `np`, `pd`, `pa` or `k` | Default conversion to apply when passing q objects to Python. Converting to Numpy (`np`) by default. | | -| `PYKX_SKIP_UNDERQ` | `1` or `true` | When importing PyKX from Python, PyKX will also load `pykx.q` under its embedded q. This variable skip this step. | | +| `PYKX_SKIP_UNDERQ` | `1` or `true` | When importing PyKX from Python, PyKX will also load `pykx.q` under its embedded q. This variable skips this step. | | | `PYKX_UNSET_GLOBALS` | `1` or `true` | By default "PyKX under q" will load some utility functions into the global namespace (eg. `print`). This variable prevents this. | | +| `PYKX_EXECUTABLE` | File path | The path to use for the Python executable | | | `PYKX_PYTHON_LIB_PATH` | File path | The path to use for loading libpython. | | | `PYKX_PYTHON_BASE_PATH` | File path | The path to use for the base directory of your Python installation. | | | `PYKX_PYTHON_HOME_PATH` | File path | The path to use for the base Python home directory (used to find site packages). | | -| `SKIP_UNDERQ` | `1` or `true` | When importing PyKX from Python, PyKX will also load `pykx.q` under its embedded q. This variable skip this step. | `DEPRECATED`, please use `PYKX_SKIP_UNDERQ` | +| `SKIP_UNDERQ` | `1` or `true` | When importing PyKX from Python, PyKX will also load `pykx.q` under its embedded q. This variable skips this step. | `DEPRECATED`, please use `PYKX_SKIP_UNDERQ` | | `UNSET_PYKX_GLOBALS` | `1` or `true` | By default "PyKX under q" will load some utility functions into the global namespace (eg. `print`). This variable prevents this. | `DEPRECATED`, please use `PYKX_UNSET_GLOBALS` | diff --git a/docs/user-guide/fundamentals/creating.md b/docs/user-guide/fundamentals/creating.md index 88239d2..e7bf8bb 100644 --- a/docs/user-guide/fundamentals/creating.md +++ b/docs/user-guide/fundamentals/creating.md @@ -167,6 +167,15 @@ Create a vector of random floating point precision values pykx.FloatVector(pykx.q('9.030751 7.750292 3.869818')) ``` +Additionally, users when generating random data can use PyKX null/infinite data to create data across larger data ranges as follows + +```python +>>> kx.random.random(2, kx.GUIDAtom.null) +pykx.GUIDVector(pykx.q('8c6b8b64-6815-6084-0a3e-178401251b68 5ae7962d-49f2-404d-5aec-f7c8abbae288')) +>>> kx.random.random(3, kx.IntAtom.inf) +pykx.IntVector(pykx.q('986388794 824432196 2022020141i')) +``` + Create a two-dimensional list of random symbol values ```python diff --git a/docs/user-guide/fundamentals/indexing.md b/docs/user-guide/fundamentals/indexing.md index 59abe02..ccb0910 100644 --- a/docs/user-guide/fundamentals/indexing.md +++ b/docs/user-guide/fundamentals/indexing.md @@ -13,7 +13,7 @@ Indexes used on K objects in PyKX are converted to equivalent K objects in q usi The following provides some examples of applying indexing to various q objects: -## Basic Array Indexing +## Basic Vectors Indexing Indexing in PyKX spans elements `0` to element `N-1` where `N` is the length of the object being indexed. @@ -27,6 +27,11 @@ Single element indexing works similarly to any other standard Python sequence. S pykx.LongAtom(pykx.q('2')) >>> x[-2] pykx.LongAtom(pykx.q('8')) +>>> y = kx.CharVector('abcdefg') +>>> y[0] +pykx.CharAtom(pykx.q('"a"')) +>>> y[-2] +pykx.CharAtom(pykx.q('"f"')) ``` Similar to Numpy indexing an array out of bounds will result in an `IndexError` being raised. @@ -43,7 +48,7 @@ Traceback (most recent call last): IndexError: index out of range ``` -N Dimensional list arrays can also be manipulated using single element indexing as follows +N Dimensional list vectors can also be manipulated using single element indexing as follows ```python >>> x = kx.q('4 4#16?1f') @@ -60,7 +65,7 @@ pykx.FloatAtom(pykx.q('0.6919531')) ### Slicing -Slicing arrays in PyKX is more simplistic than the functionality provided by Numpy. Arrays of N dimensions are indexed using `obj[start:stop:step]` semantics. This slice syntax operates where `start` is the starting index, `stop` is the stopping index and `step` is the number of steps between the elements where `step` is non zero +Slicing vectors in PyKX is more simplistic than the functionality provided by Numpy. Vectors of N dimensions are indexed using `obj[start:stop:step]` semantics. This slice syntax operates where `start` is the starting index, `stop` is the stopping index and `step` is the number of steps between the elements where `step` is non zero ```python >>> x = kx.q.til(10) @@ -77,9 +82,118 @@ pykx.List(pykx.q(' 0.1477547 0.274227 0.5635053 0.883823 0.2439194 0.6718125 0.8639591 0.8439807 ')) + +>>> y = kx.CharVector('abcdefg') +>>> y[2:4] +pykx.CharVector(pykx.q('"cd"')) +>>> y[3:] +pykx.CharVector(pykx.q('"defg"')) +>>> y[:6:2] +pykx.CharVector(pykx.q('"ace"')) +``` + +## Assigning and Adding Values to Vectors/Lists + +Vector assignment in PyKX operates similarly to that provided by Numpy and operations supported on basic Python lists. As with the previous sections this functionality supports both individual element assignment and slice assignment as follows: + +```python +>>> vec = kx.q.til(10) +>>> vec +pykx.LongVector(pykx.q('0 1 2 3 4 5 6 7 8 9')) +>>> vec[1] = 15 +>>> vec +pykx.LongVector(pykx.q('0 15 2 3 4 5 6 7 8 9')) +>>> vec[-1] = 10 +>>> vec +pykx.LongVector(pykx.q('0 15 2 3 4 5 6 7 8 10')) +>>> vec[:5] = 0 +>>> vec +pykx.LongVector(pykx.q('0 0 0 0 0 5 6 7 8 10')) ``` -## Indexing Non Array Objects +??? Note "N-Dimensional vector element assignment not supported" + + ```python + >>> x = kx.q('4 4#16?1f') + >>> x + pykx.List(pykx.q(' + 0.3927524 0.5170911 0.5159796 0.4066642 + 0.1780839 0.3017723 0.785033 0.5347096 + 0.7111716 0.411597 0.4931835 0.5785203 + 0.08388858 0.1959907 0.375638 0.6137452 + ')) + >>> x[0][2] = 3.0 + >>> x + pykx.List(pykx.q(' + 0.3927524 0.5170911 0.5159796 0.4066642 + 0.1780839 0.3017723 0.785033 0.5347096 + 0.7111716 0.411597 0.4931835 0.5785203 + 0.08388858 0.1959907 0.375638 0.6137452 + ')) + ``` + +In addition to positional assignment users can make use of the `append` and `extend` methods for `pykx.*Vector` and `pykx.List` objects. When appending objects to a list this can be achieved for single item assignments, while extend will look to add multiple elements to a Vector or List object. The following tabbed section shows the use of append and extend operations including failing cases. + +=== "pykx.*Vector" + + ```python + >>> import pykx as kx + >>> qvec = kx.random.random(3, 1.0, seed = 42) + >>> qvec + pykx.FloatVector(pykx.q('0.7742128 0.7049724 0.5212126')) + >>> qvec.append(1.1) + >>> qvec + pykx.FloatVector(pykx.q('0.7742128 0.7049724 0.5212126 1.1')) + >>> + >>> qvec.append([1.2, 1.3, 1.4]) + Traceback (most recent call last): + File "", line 1, in + File "/usr/local/anaconda3/lib/python3.8/site-packages/pykx/wrappers.py", line 1262, in append + raise QError(f'Appending data of type: {type(data)} ' + pykx.exceptions.QError: Appending data of type: to vector of type: not supported + >>> + >>> qvec.extend([1.2, 1.3, 1.4]) + pykx.FloatVector(pykx.q('0.7742128 0.7049724 0.5212126 1.1 1.2 1.3 1.4')) + >>> + >>> qvec.extend([1, 2, 3]) + Traceback (most recent call last): + File "", line 1, in + File "/usr/local/anaconda3/lib/python3.8/site-packages/pykx/wrappers.py", line 1271, in extend + raise QError(f'Extending data of type: {type(data)} ' + pykx.exceptions.QError: Extending data of type: to vector of type: not supported + ``` + +=== "pykx.List" + + ```python + >>> qlist = kx.toq([1, 2, 1.3]) + >>> qlist + pykx.List(pykx.q(' + 1 + 2 + 1.3 + ')) + >>> qlist.append({'x': 1}) + >>> qlist + pykx.List(pykx.q(' + 1 + 2 + 1.3 + (,`x)!,1 + ')) + >>> qlist.extend([1, 2]) + >>> qlist + pykx.List(pykx.q(' + 1 + 2 + 1.3 + (,`x)!,1 + 1 + 2 + ')) + ``` + +## Indexing Non Vector Objects In addition to being able to index and slice PyKX vector and list objects it is also possible to apply index and slicing semantics on PyKX Table objects. Application of slice/index semantics on tabular objects will return table like objects diff --git a/docs/user-guide/fundamentals/nulls_and_infinities.md b/docs/user-guide/fundamentals/nulls_and_infinities.md index 7f2e291..f74fd48 100644 --- a/docs/user-guide/fundamentals/nulls_and_infinities.md +++ b/docs/user-guide/fundamentals/nulls_and_infinities.md @@ -10,6 +10,65 @@ For example, the q null short integer `0Nh` is stored as the value `-32768` (i.e Due to the design of nulls and infinites in q, there are some technical considerations - detailed on this page - regarding converting nulls and infinities between Python and q in either direction. +## Generation of null and infinite values + +To facilitate the generation of null and infinite values there are a number of properties for `pykx.Atom` objects which allow this to be completed Pythonically. In all cases this requires access to [licensed mode](../advanced/modes.md). The following examples show the generation of various null and infinite values. + +### Null generation + +Where possible null values can be returned to you as follows: + +```python +>>> import pykx as kx +>>> kx.LongAtom.null +pykx.LongAtom(pykx.q('0N')) +>>> kx.TimespanAtom.null +pykx.TimespanAtom(pykx.q('0Nn')) +>>> kx.GUIDAtom.null +pykx.GUIDAtom(pykx.q('00000000-0000-0000-0000-000000000000')) +>>> kx.SymbolAtom.null +pykx.SymbolAtom(pykx.q('`')) +``` + +Unsupported values will return a `NotImplemetedError` as follows: + +```python +>>> import pykx as kx +>>> kx.ByteAtom.null +Traceback (most recent call last): + File "", line 1, in + File "/usr/local/anaconda3/lib/python3.8/site-packages/pykx/wrappers.py", line 250, in null + raise NotImplementedError('Retrieval of null values not supported for this type') +NotImplementedError: Retrieval of null values not supported for this type +``` + +### Infinite generation + +Where possible positive and negative infinite values can be returned to you as follows: + +```python +>>> import pykx as kx +>>> kx.TimeAtom.inf +pykx.TimeAtom(pykx.q('0Wt')) +>>> -kx.TimeAtom.inf +pykx.TimeAtom(pykx.q('-0Wt')) +>>> kx.IntAtom.inf +pykx.IntAtom(pykx.q('0Wi')) +>>> -kx.IntAtom.inf +pykx.IntAtom(pykx.q('-0Wi')) +``` + +Unsupported values will return a `NotImplementedError` as follows: + +```python +>>> kx.SymbolAtom.inf +Traceback (most recent call last): + File "", line 1, in + File "/usr/local/anaconda3/lib/python3.8/site-packages/pykx/wrappers.py", line 274, in inf + raise NotImplementedError('Retrieval of infinite values not supported for this type') +NotImplementedError: Retrieval of infinite values not supported for this type +``` + ## Checking for nulls and infinities [The q function named null](https://code.kx.com/q/ref/null/) can be applied to most PyKX objects, and will return if the object is null by returning `1b`, or if it contains nulls by returning a collection of booleans whose shape matches the object. Like with any function from the `.q` namespace, it can be accessed via the [context interface](../../api/pykx-execution/ctx.md): [`q.null`](../../api/pykx-execution/q.md#null)). diff --git a/mkdocs.yml b/mkdocs.yml index bc0afa1..b88ea41 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -139,15 +139,8 @@ theme: - content.tabs.link # Insiders - header.autohide - navigation.tabs - - navigation.tabs.sticky - - navigation.path - - navigation.top - - navigation.tracking - content.code.annotate - content.action.edit - - search.suggest - - search.highlight - - search.share palette: - media: "(prefers-color-scheme: light)" scheme: kx-light @@ -235,6 +228,8 @@ nav: - Release notes: - PyKX: release-notes/changelog.md - PyKX under q: release-notes/underq-changelog.md + - Roadmap: roadmap.md + - Troubleshooting: troubleshooting.md - Frequently Asked Questions (FAQ): faq.md - Support: support.md - License: license.md diff --git a/pyproject.toml b/pyproject.toml index 1329f28..e321ca0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -61,19 +61,22 @@ doc = [ "black==22.1.0", # Used by mkdocstrings to format function signatures "griffe==0.14.0", # Force-install a newer version of griffe (mkdocstrings backend/dep) to get a required bugfix "jupyter==1.0.0", - "mkdocs==1.4.3", + "mkdocs~=1.5.0", "markdown==3.3.7", "markdown-grid-tables==0.0.1", "mkdocs-autorefs==0.4.1", "mkdocs-click==0.5.0", "mkdocs-exclude==1.0.2", - "mkdocs-jupyter==0.20.0", - "mkdocs-material==8.1.10", + "mkdocs-jupyter~=0.24", + "mkdocs-material~=9.4.5", "mkdocs-render-swagger-plugin==0.0.3", "mkdocs-spellcheck==0.2.0", "mkdocstrings[python]==0.18.0", - "pygments==2.11", - "pymdown-extensions==9.3", + "pygments~=2.12", + "pymdown-extensions>=9.3", +] +debug = [ + "find-libpython~=0.2", ] lint = [ "flake8==4.0.1", diff --git a/src/pykx/.gitignore b/src/pykx/.gitignore index 078a728..9d85b4c 100644 --- a/src/pykx/.gitignore +++ b/src/pykx/.gitignore @@ -1,5 +1,6 @@ # Auto-generated files *.c +!_pykx_helpers.c !_numpy.c !python.c !pykx.c diff --git a/src/pykx/__init__.py b/src/pykx/__init__.py index 6445cd9..8ffdbb7 100644 --- a/src/pykx/__init__.py +++ b/src/pykx/__init__.py @@ -410,9 +410,9 @@ def deactivate_numpy_allocator(): __all__ = sorted([ 'AsyncQConnection', + 'deserialize', 'EmbeddedQ', 'EmbeddedQFuture', - 'PyKXSerialized', 'Q', 'qargs', 'QConnection', @@ -424,6 +424,7 @@ def deactivate_numpy_allocator(): 'random', 'QWriter', 'qlic', + 'serialize', 'SyncQConnection', 'RawQConnection', 'activate_numpy_allocator', diff --git a/src/pykx/_pykx_helpers.c b/src/pykx/_pykx_helpers.c new file mode 100644 index 0000000..5da9bf1 --- /dev/null +++ b/src/pykx/_pykx_helpers.c @@ -0,0 +1,35 @@ +#define PY_SSIZE_T_CLEAN +#include +#include +#include +#include + + +static PyObject* clear_all_errors(PyObject* self, PyObject* args) { + PyErr_Clear(); + Py_RETURN_NONE; +} + + +static PyMethodDef _HelperMethods[] = { + {"clean_errors", clear_all_errors, METH_NOARGS, "Clean the python error state."}, + {NULL, NULL, 0, NULL} // Sentinel +}; + + +static struct PyModuleDef _helpermodule = { + PyModuleDef_HEAD_INIT, + "_pykx_helpers",// Module name. + NULL, // Module documentation; may be NULL. + -1, // Size of per-interpreter state of the module, or -1 if the module keeps state in global variables. + _HelperMethods, // Module methods. + NULL, // Module slots. + NULL, // A traversal function to call during GC traversal of the module object. + NULL, // A clear function to call during GC clearing of the module object. + NULL // A function to call during deallocation of the module object. +}; + + +PyMODINIT_FUNC PyInit__pykx_helpers(void) { + return PyModule_Create(&_helpermodule); +} diff --git a/src/pykx/_wrappers.pyx b/src/pykx/_wrappers.pyx index 6803ed0..310be65 100644 --- a/src/pykx/_wrappers.pyx +++ b/src/pykx/_wrappers.pyx @@ -130,6 +130,9 @@ def k_str(self): cpdef inline deserialize(x): cdef core.K buff = core.kpn(x, len(x)) + if 0 == core.okx(buff): + core.r0(buff) + raise QError('Failed to deserialize supplied non PyKX IPC serialized format object') cdef core.K kx = core.ee(core.d9(buff)) core.r0(buff) return factory(kx, False) diff --git a/src/pykx/config.py b/src/pykx/config.py index ce6e96d..9dd7414 100644 --- a/src/pykx/config.py +++ b/src/pykx/config.py @@ -72,6 +72,7 @@ def _is_set(envvar): pykx_dir = Path(__file__).parent.resolve(strict=True) os.environ['PYKX_DIR'] = str(pykx_dir) +os.environ['PYKX_EXECUTABLE'] = sys.executable pykx_lib_dir = Path(_get_config_value('PYKX_Q_LIB_LOCATION', pykx_dir/'lib')) pykx_platlib_dir = pykx_lib_dir/q_lib_dir_name lib_prefix = '' if system == 'Windows' else 'lib' @@ -111,26 +112,26 @@ def _is_set(envvar): if not license_located: qlic = Path(qhome) -qargs_tmp = tuple(shlex.split(_get_config_value('QARGS', ''))) +qargs = tuple(shlex.split(_get_config_value('QARGS', ''))) -arglist = ['--unlicensed', '--licensed'] -if any(i in qargs_tmp for i in arglist) or not hasattr(sys, 'ps1'): # noqa: C901 - pass -elif not license_located: + +def _license_install(intro=None, return_value=False): # noqa: modes_url = "https://code.kx.com/pykx/user-guide/advanced/modes.html" lic_url = "https://kx.com/kdb-insights-personal-edition-license-download" unlicensed_message = '\nPyKX unlicensed mode enabled. To set this as your default behavior '\ - "please set the following environment variable 'QARGS=--unlicensed'\n\n"\ - 'For more information on PyKX modes of operation, please visit '\ + "please set the following environment variable 'PYKX_UNLICENSED='true'"\ + '\n\nFor more information on PyKX modes of operation, please visit '\ f'{modes_url}.\nTo apply for a PyKX license please visit {lic_url}' - continue_license = input('\nThank you for installing PyKX!\n\n' - 'We have been unable to locate your license for PyKX. ' - 'Running PyKX in unlicensed mode has reduced functionality.\n' - 'Would you like to continue with license installation? [Y/n]: ') - + first_user = '\nThank you for installing PyKX!\n\n'\ + 'We have been unable to locate your license for PyKX. '\ + 'Running PyKX in unlicensed mode has reduced functionality.\n'\ + 'Would you like to continue with license installation? [Y/n]: ' + continue_license = input(first_user if intro is None else intro) if continue_license in ('n', 'N'): - os.environ['QARGS']='--unlicensed' + os.environ['PYKX_UNLICENSED']='true' print(unlicensed_message) + if return_value: + return False elif continue_license in ('y', 'Y', ''): redirect = input(f'\nTo apply for a PyKX license, please visit {lic_url}.\n' @@ -181,10 +182,22 @@ def _is_set(envvar): binary_file.write(lic) print('PyKX license successfully installed!\n') + elif install_type == '3': + if return_value: + return False else: raise Exception('Invalid input provided please try again') + if return_value: + return True + + +_arglist = ['--unlicensed', '--licensed'] +_licenvset = _is_enabled('PYKX_LICENSED', '--licensed') or _is_enabled('PYKX_UNLICENSED', '--unlicensed') # noqa: E501 +if any(i in qargs for i in _arglist) or _licenvset or not hasattr(sys, 'ps1'): # noqa: C901 + pass +elif not license_located: + _license_install() -qargs = tuple(shlex.split(_get_config_value('QARGS', ''))) licensed = False under_q = _is_enabled('PYKX_UNDER_Q') diff --git a/src/pykx/core.pyx b/src/pykx/core.pyx index 0b0d786..0ddff16 100644 --- a/src/pykx/core.pyx +++ b/src/pykx/core.pyx @@ -2,9 +2,11 @@ import os, platform from pathlib import Path from threading import RLock from typing import List, Tuple +import re +import sys from .util import num_available_cores -from .config import _is_enabled +from .config import _is_enabled, _license_install def _normalize_qargs(user_args: List[str]) -> Tuple[bytes]: @@ -93,7 +95,10 @@ from .exceptions import PyKXException, PyKXWarning if '--licensed' in qargs and '--unlicensed' in qargs: raise PyKXException("$QARGS includes mutually exclusive flags '--licensed' and '--unlicensed'") - +elif ('--unlicensed' in qargs or _is_enabled('PYKX_UNLICENSED', '--unlicensed')) & \ + ('--licensed' in qargs or _is_enabled('PYKX_LICENSED', '--licensed')): + raise PyKXException("User specified options for setting 'licensed' and 'unlicensed' behaviour " + "resulting in conflicts") class QLock: def __init__(self, timeout): @@ -241,7 +246,7 @@ if under_q: # nocov licensed = True # nocov else: # To make Cython happy, we indirectly assign Python values to `_libq_path` - if '--unlicensed' in qargs: + if '--unlicensed' in qargs or _is_enabled('PYKX_UNLICENSED', '--unlicensed'): _libq_path_py = bytes(find_core_lib('e')) _libq_path = _libq_path_py _q_handle = dlopen(_libq_path, RTLD_NOW | RTLD_GLOBAL) @@ -251,7 +256,7 @@ else: from ctypes.util import find_library # nocov if find_library("msvcr100.dll") is None: # nocov msvcrMSG = "Needed dependency msvcr100.dll missing. See: https://code.kx.com/pykx/getting-started/installing.html" # nocov - if '--licensed' in qargs: # nocov + if '--licensed' in qargs or _is_enabled('PYKX_LICENSED', --licensed): # nocov raise PyKXException(msvcrMSG) # nocov else: # nocov warn(msvcrMSG, PyKXWarning) # nocov @@ -275,12 +280,34 @@ else: } ) _qinit_output = ' ' + ' '.join(_qinit_check_proc.stdout.strip().splitlines(True)) + _license_message = False if _qinit_check_proc.returncode: # Fallback to unlicensed mode if _qinit_output != ' ': - _capout_msg = f' Captured output from initialization attempt:\n{_qinit_output}' + _capout_msg = f'Captured output from initialization attempt:\n{_qinit_output}' else: _capout_msg = '' # nocov - this can only occur under extremely weird circumstances. - if '--licensed' in qargs: + if hasattr(sys, 'ps1'): + if re.compile('exp').search(_capout_msg): + _exp_license = 'Your PyKX license has now expired.\n\n'\ + f'{_capout_msg}\n\n'\ + 'Would you like to renew your license? [Y/n]: ' + _license_message = _license_install(_exp_license, True) + elif re.compile('embedq').search(_capout_msg): + _ce_license = 'You appear to be using a non kdb Insights license.\n\n'\ + f'{_capout_msg}\n\n'\ + 'Running PyKX in the absence of a kdb Insights license '\ + 'has reduced functionality.\nWould you like to install '\ + 'a kdb Insights personal license? [Y/n]: ' + _license_message = _license_install(_ce_license, True) + elif re.compile('upd').search(_capout_msg): + _upd_license = 'Your installed license is out of date for this version'\ + ' of PyKX and must be updated.\n\n'\ + f'{_capout_msg}\n\n'\ + 'Would you like to install an updated kdb '\ + 'Insights personal license? [Y/n]: ' + _license_message = _license_install(_upd_license, True) + if (not _license_message) and _qinit_check_proc.returncode: + if '--licensed' in qargs or _is_enabled('PYKX_LICENSED', '--licensed'): raise PyKXException(f'Failed to initialize embedded q.{_capout_msg}') else: warn(f'Failed to initialize PyKX successfully with the following error: {_capout_msg}', PyKXWarning) diff --git a/src/pykx/ctx.py b/src/pykx/ctx.py index 86fa2e4..f2c134e 100644 --- a/src/pykx/ctx.py +++ b/src/pykx/ctx.py @@ -117,7 +117,7 @@ def __getattr__(self, key): # noqa raise AttributeError if key == 'z' and self._fqn == '': return ZContext(proxy(self)) - elif key in self._unsupported_keys_with_msg: + elif self._fqn in {'', '.q'} and key in self._unsupported_keys_with_msg: raise AttributeError(f'{key}: {self._unsupported_keys_with_msg[key]}') if self._fqn in {'', '.q'} and key in self._q.reserved_words: # Reserved words aren't actually part of the `.q` context dict diff --git a/src/pykx/embedded_q.py b/src/pykx/embedded_q.py index a7d915e..9303dec 100644 --- a/src/pykx/embedded_q.py +++ b/src/pykx/embedded_q.py @@ -11,7 +11,7 @@ from . import toq from . import wrappers from . import schema -from .config import find_core_lib, licensed, no_qce, pykx_dir, skip_under_q +from .config import find_core_lib, licensed, no_qce, pykx_dir, qargs, skip_under_q from .core import keval as _keval from .exceptions import FutureCancelled, LicenseException, NoResults, PyKXException, PyKXWarning, QError # noqa from ._wrappers import _factory as factory @@ -132,9 +132,9 @@ def __init__(self): # noqa code += f'2:[`$"{pykx_qlib_path}q";(`k_pykx_init; 1)][`$"{find_core_lib("q").as_posix()}"];' # noqa: E501 code += f'`.pykx.modpow set {{((`$"{pykx_qlib_path}q") 2: (`k_modpow; 3))["j"$x;"j"$y;$[z~(::);(::);"j"$z]]}};' # noqa: E501 code += '@[get;`.pykx.i.kxic.loadfailed;{()!()}]' - kxic_loadfailed = self._call(code).py() - if not no_qce: - sql = self._call('$["insights.lib.sql" in " " vs .z.l 4; @[system; "l s.k_";{x}];::]').py() # noqa: E501 + kxic_loadfailed = self._call(code, debug=False).py() + if (not no_qce) and ('--no-sql' not in qargs): + sql = self._call('$[("insights.lib.sql" in " " vs .z.l 4)¬ `s in key`; @[system; "l s.k_";{x}];::]', debug=False).py() # noqa: E501 if sql is not None: kxic_loadfailed['s.k'] = sql for lib, msg in kxic_loadfailed.items(): @@ -147,9 +147,9 @@ def __init__(self): # noqa and os.getenv('PYKX_UNDER_Q') is None ): os.environ['PYKX_Q_LOADED_MARKER'] = 'loaded' - self._call('setenv[`PYKX_Q_LOADED_MARKER; "loaded"]') + self._call('setenv[`PYKX_Q_LOADED_MARKER; "loaded"]', debug=False) try: - self._call('.Q.ld') + self._call('.Q.ld', debug=False) except QError as err: if '.Q.ld' in str(err): # .Q.ld is not defined on the server so we define it here @@ -157,12 +157,12 @@ def __init__(self): # noqa lines = f.readlines() for line in lines: if 'pykxld:' in line: - self._call('k).Q.' + line) + self._call('k).Q.' + line, debug=False) break else: raise err pykx_qini_path = (Path(__file__).parent.absolute()/'pykx_init.q_') - self._call(f'\l {pykx_qini_path}') # noqa + self._call(f'\l {pykx_qini_path}', debug=False) # noqa pykx_q_path = (Path(__file__).parent.absolute()/'pykx.q') with open(pykx_q_path, 'r') as f: code = f.read() @@ -170,9 +170,9 @@ def __init__(self): # noqa self._call( "{[code;file] value (@';last file;enlist[file],/:.Q.pykxld code)}", code, - b'pykx.q' + b'pykx.q', debug=False ) - self._call('.pykx.setdefault[enlist"k"]') + self._call('.pykx.setdefault[enlist"k"]', debug=False) super().__init__() def __repr__(self): @@ -182,7 +182,8 @@ def __call__(self, query: Union[str, bytes, wrappers.CharVector], *args: Any, wait: Optional[bool] = None, - sync: Optional[bool] = None + debug: bool = False, + **kwargs # since sync got removed this is added to ensure it doesn't break ) -> wrappers.K: """Run code in the q instance. @@ -212,7 +213,15 @@ def __call__(self, raise LicenseException("run q code via 'pykx.q'") if len(args) > 8: raise TypeError('Too many arguments - q queries cannot have more than 8 parameters') - result = _keval(bytes(wrappers.CharVector(query)), *[wrappers.K(x) for x in args]) + if debug: + result = _keval( + bytes(wrappers.CharVector( + '{[pykxquery] .Q.trp[value; pykxquery; {2@"backtrace:\n",.Q.sbt y;\'x}]}' + )), + wrappers.List([bytes(wrappers.CharVector(query)), *[wrappers.K(x) for x in args]]) + ) + else: + result = _keval(bytes(wrappers.CharVector(query)), *[wrappers.K(x) for x in args]) if wait is None or wait: return factory(result, False) return self('::', wait=True) diff --git a/src/pykx/ipc.py b/src/pykx/ipc.py index 1a69722..a9f9113 100644 --- a/src/pykx/ipc.py +++ b/src/pykx/ipc.py @@ -26,7 +26,7 @@ from abc import abstractmethod import asyncio from contextlib import nullcontext -from multiprocessing import Lock as multiprocessing_lock +from multiprocessing import Lock as multiprocessing_lock, RawValue import selectors import socket from threading import Lock as threading_lock @@ -92,12 +92,14 @@ class QFuture(asyncio.Future): _exception = None _callbacks = [] - def __init__(self, q_connection, timeout): + def __init__(self, q_connection, timeout, debug, poll_recv=None): self.q_connection = q_connection self._done = False self._cancelled = False self._cancelled_message = '' self._timeout = timeout + self.poll_recv = poll_recv + self._debug = debug super().__init__() def __await__(self) -> Any: @@ -113,12 +115,33 @@ def __await__(self) -> Any: if self.done(): return self.result() while not self.done(): - self.q_connection._recv(acceptAsync=True) + if self.poll_recv is not None: + try: + res = self.q_connection.poll_recv() + if res is not None: + self.set_result(res) + except BaseException as e: + self.set_exception(QError(str(e))) + else: + self.q_connection._recv(acceptAsync=True) yield from self super().__await__() return self.result() async def __async_await__(self) -> Any: + if self.done(): + return self.result() + while not self.done(): + await asyncio.sleep(0) + if self.poll_recv is not None: + try: + res = self.q_connection.poll_recv() + if res is not None: + self.set_result(res) + except BaseException as e: + self.set_exception(QError(str(e))) + else: + self.q_connection._recv(acceptAsync=True) return await self def _await(self) -> Any: @@ -167,6 +190,12 @@ def result(self) -> Any: if self._cancelled: raise FutureCancelled(self._cancelled_message) if self._result is not None: + if self._debug: + if self._result._unlicensed_getitem(0).py() == True: + print((self._result._unlicensed_getitem(1).py()).decode(), file=sys.stderr) + raise QError(self._result._unlicensed_getitem(2).py().decode()) + else: + return self._result._unlicensed_getitem(1) return self._result raise NoResults() @@ -434,6 +463,7 @@ def __call__(self, query: Union[str, bytes, CharVector], *args: Any, wait: Optional[bool] = None, + debug: bool = False, ) -> K: pass # nocov @@ -441,7 +471,8 @@ def _send(self, query, *params, wait: Optional[bool] = None, - error=False + error=False, + debug=False ): if self.closed: raise RuntimeError("Attempted to use a closed IPC connection") @@ -453,7 +484,20 @@ def _send(self, events = self._writer.select(timeout) for key, _mask in events: callback = key.data - return callback()(key.fileobj, query, *params, wait=wait, error=error) + if debug: + return callback()( + key.fileobj, + bytes(CharVector( + '{[pykxquery] .Q.trp[{[x] (0b; value x)}; pykxquery;' + '{(1b;"backtrace:\n",.Q.sbt y;x)}]}' + )), + CharVector(query) if len(params) == 0 else List((CharVector(query), *params)), + wait=wait, + error=error, + debug=debug + ) + else: + return callback()(key.fileobj, query, *params, wait=wait, error=error, debug=debug) def _ipc_query_builder(self, query, *params): data = bytes(query, 'utf-8') if isinstance(query, str) else query @@ -480,7 +524,8 @@ def _send_sock(self, query, *params, wait: Optional[bool] = None, - error=False + error=False, + debug=False ): if len(params) > 8: raise TypeError('Too many parameters - q queries cannot have more than 8 parameters') @@ -513,11 +558,11 @@ def _send_sock(self, if isinstance(self, SyncQConnection) or isinstance(self, RawQConnection): return if wait: - q_future = QFuture(self, self._connection_info['timeout']) + q_future = QFuture(self, self._connection_info['timeout'], debug) self._call_stack.append(q_future) return q_future else: - q_future = QFuture(self, self._connection_info['timeout']) + q_future = QFuture(self, self._connection_info['timeout'], debug) q_future.set_result(K(None)) return q_future @@ -779,6 +824,7 @@ def __call__(self, query: Union[str, bytes, CharVector], *args: Any, wait: Optional[bool] = None, + debug: bool = False, ) -> K: """Evaluate a query on the connected q process over IPC. @@ -840,17 +886,25 @@ def __call__(self, if wait is None: wait = self._connection_info['wait'] with self._lock if self._lock is not None else nullcontext(): - return self._call(query, *args, wait=wait) + return self._call(query, *args, wait=wait, debug=debug) def _call(self, query: Union[str, bytes], *args: Any, wait: Optional[bool] = None, + debug: bool = False, ) -> K: - self._send(query, *args, wait=wait) + self._send(query, *args, wait=wait, debug=debug) if not wait: return K(None) - return self._recv(locked=True) + res = self._recv(locked=True) + if not debug: + return res + if res._unlicensed_getitem(0).py() == True: + print((res._unlicensed_getitem(1).py()).decode(), file=sys.stderr) + raise QError(res._unlicensed_getitem(2).py().decode()) + else: + return res._unlicensed_getitem(1) def __enter__(self): return self @@ -1059,6 +1113,7 @@ def __call__(self, *args: Any, wait: bool = True, reuse: bool = True, + debug: bool = False, ) -> QFuture: """Evaluate a query on the connected q process over IPC. @@ -1134,7 +1189,7 @@ def __call__(self, unix=self._stored_args['unix'], wait=self._stored_args['wait'], no_ctx=self._stored_args['no_ctx']) - q_future = conn(query, *args, wait=wait) + q_future = conn(query, *args, wait=wait, debug=debug) q_future.add_done_callback(lambda x: conn.close()) if self._loop is None: return q_future @@ -1143,7 +1198,7 @@ def __call__(self, if not self._initialized: raise UninitializedConnection() with self._lock if self._lock is not None else nullcontext(): - q_future = self._send(query, *args, wait=wait) + q_future = self._send(query, *args, wait=wait, debug=debug) if self._loop is None: return q_future return self._loop.create_task(q_future.__async_await__()) @@ -1152,9 +1207,10 @@ def _call(self, query: Union[str, bytes], *args: Any, wait: Optional[bool] = None, + debug: bool = False, ): with self._lock if self._lock is not None else nullcontext(): - return self._send(query, *args, wait=wait)._await() + return self._send(query, *args, wait=wait, debug=debug)._await() async def __aenter__(self): return await self @@ -1243,15 +1299,17 @@ def __call__(self, query: Union[str, bytes, CharVector], *args: Any, wait: Optional[bool] = None, + debug: bool = False, ) -> K: - return self._send(query, *args, wait=wait) + return self._send(query, *args, wait=wait, debug=debug) def _call(self, query: Union[str, bytes], *args: Any, wait: Optional[bool] = None, + debug: bool = False, ): - return self._send(query, *args, wait=wait)._await() + return self._send(query, *args, wait=wait, debug=debug)._await() def close(self) -> None: if not self.closed: # nocov @@ -1488,6 +1546,7 @@ def __call__(self, query: Union[str, bytes, CharVector], *args: Any, wait: bool = True, + debug: bool = False, ) -> QFuture: """Evaluate a query on the connected q process over IPC. @@ -1557,8 +1616,8 @@ def __call__(self, """ if not self._initialized: raise UninitializedConnection() - res = QFuture(self, self._connection_info['timeout']) - self._send_stack.append({'query': query, 'args': args, 'wait': wait}) + res = QFuture(self, self._connection_info['timeout'], debug) + self._send_stack.append({'query': query, 'args': args, 'wait': wait, 'debug': debug}) self._call_stack.append(res) return res @@ -1566,6 +1625,7 @@ def _call(self, query: Union[str, bytes], *args: Any, wait: Optional[bool] = None, + debug: bool = False, ): conn = _DeferredQConnection(self._stored_args['host'], self._stored_args['port'], @@ -1578,7 +1638,7 @@ def _call(self, unix=self._stored_args['unix'], wait=self._stored_args['wait'], no_ctx=self._stored_args['no_ctx']) - q_future = conn(query, *args, wait=wait) + q_future = conn(query, *args, wait=wait, debug=debug) q_future.add_done_callback(lambda x: conn.close()) return q_future._await() @@ -1631,7 +1691,7 @@ def poll_send(self, amount: int = 1): if len(self._send_stack) == 0: return to_send = self._send_stack.pop(0) - self._send(to_send['query'], *to_send['args'], wait=to_send['wait']) + self._send(to_send['query'], *to_send['args'], wait=to_send['wait'], debug=to_send['debug']) count -= 1 def _serialize_response(self, response, level): @@ -1793,6 +1853,29 @@ def clean_open_connections(self): self.open_cons[i][2].close() self.open_cons.pop(i) + def poll_recv_async(self): + """Asynchronously recieve a query from the process connected to over IPC. + + Raises: + QError: Query timed out, may be raised if the time taken to make or receive a query goes + over the timeout limit. + + Examples: + + Receive a single queued message. + + ```python + q = await pykx.RawQConnection(host='localhost', port=5002) + q_fut = q('til 10') # not sent yet + q.poll_send() # message is sent + await q.poll_recv_async() # message response is received + ``` + """ + q_future = QFuture(self, self._connection_info['timeout'], False, poll_recv=True) + if self._loop is not None: + return self._loop.create_task(q_future.__async_await__()) + return q_future + def poll_recv(self, amount: int = 1): """Recieve queries from the process connected to over IPC. @@ -2012,6 +2095,7 @@ def __call__(self, query: Union[str, bytes, CharVector], *args: Any, wait: Optional[bool] = None, + debug: bool = False, ) -> K: """Evaluate a query on the connected q process over IPC. @@ -2070,12 +2154,13 @@ def __call__(self, q('{x set y+til z}', 'async_query', 10, 5, wait=True) ``` """ - return self._call(query, *args, wait=wait) + return self._call(query, *args, wait=wait, debug=debug) def _call(self, query: Union[str, bytes], *args: Any, wait: Optional[bool] = None, + debug: bool = False, ) -> K: if wait is None: wait = self._connection_info['wait'] @@ -2090,7 +2175,24 @@ def _call(self, if issubclass(b, Function) and not issubclass(a, Function): raise ValueError('Cannot send Python function over IPC') handler = self._licensed_call if licensed else self._unlicensed_call + with self._lock if self._lock is not None else nullcontext(): + if debug: + res = handler( + handle, + normalize_to_bytes( + '{[pykxquery] .Q.trp[{[x] (0b; value x)}; pykxquery;' + '{(1b; "backtrace:\n",.Q.sbt y; x)}]}', + 'Query' + ), + [K(normalize_to_bytes(query, 'Query'))] if len(args) == 0 else [List([K(normalize_to_bytes(query, 'Query')), *args])], + wait, + ) + if res._unlicensed_getitem(0).py() == True: + print((res._unlicensed_getitem(1).py()).decode(), file=sys.stderr) + raise QError(res._unlicensed_getitem(2).py().decode()) + else: + return res._unlicensed_getitem(1) return handler(handle, normalize_to_bytes(query, 'Query'), args, wait) def __enter__(self): diff --git a/src/pykx/lib/kxic.k b/src/pykx/lib/kxic.k index 7e8eda1..b7239f3 100644 --- a/src/pykx/lib/kxic.k +++ b/src/pykx/lib/kxic.k @@ -2,10 +2,16 @@ .comkxic.Kf:{[v]r:,/'$2\:'ri:"I"$raw:"."\:v:v@&v in .Q.n,".";if[max "b"$ri>1023;'version];vi:sum/ 2/:'.:'"0"^(r[0],20#"0";-20$r[1],10#"0";-10$r 2),\:"b";"***III"$`K`k`Kl`major`minor`patch!enlist[v],enlist["D"$"2022.03.10"],vi,raw}; .[`.comkxic;();,;.comkxic.Kf "3.0.0"]; .[`.comkxic.libs;();,;`nelf`kurl`objstor`qlog`restserver`bq`sql`l64!("6a93b2b";"7fa6ae8";"32363d0";"a49dc65";"00a5b89";"14d547d";"99f3988";"3fa87f1")]; -if[(.z.o~`l64)&$[0=#.z.x;1b;~"-nokurl"in .z.x]; +if[(.z.o~`l64)&$[0=#.z.x;1b;~"--no-qce"in .z.x]; {[] lf:{@[{."\\l ",$[""~x:getenv`QHOME;getenv[`HOME],"/q";x],"/",($x),"_"};x;{.pykx.i.kxic.loadfailed,:enlist[x]!enlist y}[x]]}; d:`kurl.q`objstor.q`qlog.q`s.k!$`insights.lib.kurl`insights.lib.objstore`insights.lib.qlog`insights.lib.sql; - lf' (!d) @ & (.:d) in " "\:.z.l 4; / only load enabled features + if[`s in !`;d:d _ `s.k]; + if[~0=#.z.x; + if["--no-kurl"in .z.x;d:d _ `kurl.q]; + if["--no-objstor"in .z.x;d:d _ `objstor.q]; + if["--no-qlog"in .z.x;d:d _ `qlog.q]; + if["--no-sql"in .z.x;d:d _ `s.k]]; + if[~0=#d;lf' (!d) @ & (.:d) in " "\:.z.l 4;]; / only load enabled features }[] ]; diff --git a/src/pykx/nbextension.py b/src/pykx/nbextension.py index e0531d4..2ae9fc1 100644 --- a/src/pykx/nbextension.py +++ b/src/pykx/nbextension.py @@ -1,18 +1,19 @@ import pykx as kx - +from IPython.display import display def q(instructions, code): # noqa ld = kx.SymbolAtom('.Q.pykxld') + host = 'localhost' + port = None + username = '' + password = '' + timeout = 0.0 + large_messages = True + tls = False + unix = False + no_ctx = False + displayRet = False if len(instructions)>0: - host = 'localhost' - port = None - username = '' - password = '' - timeout = 0.0 - large_messages = True - tls = False - unix = False - no_ctx = False instructions = instructions.split(' ') while True: @@ -59,15 +60,16 @@ def q(instructions, code): # noqa no_ctx = True instructions.pop(0) continue + elif instructions[0] == '--display': + displayRet = True + instructions.pop(0) + continue else: raise kx.QError( - f'Received unknown argument {instructions[0]} for construction of ' - 'IPC connection in magic command.' + f'Received unknown argument "{instructions[0]}" in %%q magic command' ) - if port is None: - raise kx.QError('--port must be set to create IPC connection in magic command.') - + if port is not None: _q = kx.SyncQConnection( host, port, @@ -96,15 +98,18 @@ def q(instructions, code): # noqa _q = kx.q code = [kx.CharVector(x) for x in code.split('\n')][:-1] ret = _q( - "{[ld;code;file] value (@';\"q\";enlist[file],/:value(ld;code))}", + "{[ld;code;file] {x where not (::)~/:x} value (@';\"q\";enlist[file],/:value(ld;code))}", ld, code, b'jupyter_cell.q' ) if not kx.licensed: ret = ret.py() - for i in range(len(ret)): - print(_q('{x y}', ret, i)) + for r in ret: + display(r) if displayRet else print(r) + else: + for i in range(len(ret)): + display(_q('{x y}', ret, i)) if displayRet else print(_q('{x y}', ret, i)) if issubclass(type(_q), kx.QConnection): _q.close() diff --git a/src/pykx/pandas_api/pandas_meta.py b/src/pykx/pandas_api/pandas_meta.py index d0e44ec..39668d5 100644 --- a/src/pykx/pandas_api/pandas_meta.py +++ b/src/pykx/pandas_api/pandas_meta.py @@ -1,4 +1,5 @@ -from pykx.pandas_api import api_return +from . import api_return +from ..exceptions import QError def _init(_q): @@ -60,7 +61,7 @@ def preparse_computations(tab, axis=0, skipna=True, numeric_only=False, bool_onl '{[tab;skipna;axis]' 'r:value flip tab;' 'if[not axis~0;r:flip r];' - 'if[skipna;r:{x where not null x} peach r];' + 'if[skipna;r:{x where not null x} each r];' 'r}', tab, skipna, @@ -255,3 +256,58 @@ def sum(self, axis=0, skipna=True, numeric_only=False, min_count=0): res, min_count ), cols) + + def agg(self, func, axis=0, *args, **kwargs): # noqa: C901 + if 'KeyedTable' in str(type(self)): + raise NotImplementedError("'agg' method not presently supported for KeyedTable") + if 'GroupbyTable' not in str(type(self)): + if 0 == len(self): + raise QError("Application of 'agg' method not supported for on tabular data with 0 rows") # noqa: E501 + keyname = q('()') + data = q('()') + if axis != 0: + raise NotImplementedError('axis parameter only presently supported for axis=0') + if isinstance(func, str): + return getattr(self, func)() + elif callable(func): + return self.apply(func, *args, **kwargs) + elif isinstance(func, list): + for i in func: + if isinstance(i, str): + keyname = q('{x,y}', keyname, i) + data = q('{x, enlist y}', data, getattr(self, i)()) + elif callable(i): + keyname = q('{x,y}', keyname, i.__name__) + data = q('{x, enlist y}', data, self.apply(i, *args, **kwargs)) + if 'GroupbyTable' in str(type(self)): + return q('{x!y}', keyname, data) + elif isinstance(func, dict): + if 'GroupbyTable' in str(type(self)): + raise NotImplementedError('Dictionary input func not presently supported for GroupbyTable') # noqa: E501 + data = q('{(flip enlist[`function]!enlist ())!' + 'flip ($[1~count x;enlist;]x)!' + '$[1~count x;enlist;]count[x]#()}', self.keys()) + for key, value in func.items(): + data_name = [key] + if isinstance(value, str): + valname = value + keyname = q('{x, y}', keyname, value) + exec_data = getattr(self[data_name], value)() + elif callable(value): + valname = value.__name__ + keyname = q('{x, y}', keyname, valname) + exec_data = self[data_name].apply(value, *args, **kwargs) + else: + raise NotImplementedError(f"Unsupported type '{type(value)}' supplied as dictionary value") # noqa: E501 + data = q('{[x;y;z;k]x upsert(enlist enlist[`function]!enlist[k])!enlist z}', + data, + self.keys(), + exec_data, + valname) + return data + else: + raise NotImplementedError(f"func type: {type(func)} unsupported") + if 'GroupbyTable' in str(type(self)): + return data + else: + return (q('{(flip enlist[`function]!enlist x)!y}', keyname, data)) diff --git a/src/pykx/pykx.q b/src/pykx/pykx.q index 4acdee5..4aae73e 100644 --- a/src/pykx/pykx.q +++ b/src/pykx/pykx.q @@ -29,19 +29,17 @@ util.os:first string .z.o; // @type {dict} util.startup:.Q.opt .z.x -util.whichPython:() - // @private // @desc Retrieval of PyKX initialization directory on first initialization if[not "true"~lower getenv`PYKX_LOADED_UNDER_Q; - util.redirect:"\" 2>",$[util.os="w";"nul >> import pykx as kx + >>> kx.TimeAtom.null + pykx.TimeAtom(pykx.q('0Nt')) + >>> kx.GUIDAtom.null + pykx.GUIDAtom(pykx.q('00000000-0000-0000-0000-000000000000')) + ``` + """ + if licensed and x is not None: + return q(f'{x}') + elif not licensed: + raise QError('Generation of null data not supported in unlicensed mode') + else: + raise NotImplementedError('Retrieval of null values not supported for this type') + return null + + +def _inf_gen(x): + def inf(neg=False): + """Generate the pykx infinite value associated with an atom type + + Parameters: + neg: Should the return value produce the negative infinity value + + Examples: + + ```python + >>> import pykx as kx + >>> kx.TimeAtom.inf + pykx.TimeAtom(pykx.q('0Wt')) + >>> kx.TimeAtom.inf(neg=True) + pykx.TimeAtom(pykx.q('-0Wt')) + ``` + """ + if licensed and x is not None: + return q('{[p]$[p;neg;]'+f'{x}'+'}', neg) + elif not licensed: + raise QError('Generation of infinite data not supported in unlicensed mode') + else: + raise NotImplementedError('Retrieval of infinite values not supported for this type') + return inf + + # HACK: This gets overwritten by the toq module to avoid a circular import error. def toq(*args, **kwargs): # nocov raise NotImplementedError @@ -616,12 +666,22 @@ def pd(self, *, raw: bool = False, has_nulls: Optional[bool] = None): class TimeAtom(TemporalSpanAtom): """Wrapper for q time atoms.""" t = -19 + _null = '0Nt' + _inf = '0Wt' _np_type = 'ms' _np_dtype = 'timedelta64[ms]' def _prototype(self=None): return TimeAtom(np.timedelta64(59789214, 'ms')) + @classproperty + def null(cls): # noqa: B902 + return _null_gen(cls._null)() + + @classproperty + def inf(cls): # noqa: B902 + return _inf_gen(cls._inf)() + @property def is_null(self) -> bool: return _wrappers.k_i(self) == NULL_INT32 @@ -634,12 +694,22 @@ def is_inf(self) -> bool: class SecondAtom(TemporalSpanAtom): """Wrapper for q second atoms.""" t = -18 + _null = '0Nv' + _inf = '0Wv' _np_type = 's' _np_dtype = 'timedelta64[s]' def _prototype(self=None): return SecondAtom(np.timedelta64(13019, 's')) + @classproperty + def null(cls): # noqa: B902 + return _null_gen(cls._null)() + + @classproperty + def inf(cls): # noqa: B902 + return _inf_gen(cls._inf)() + @property def is_null(self) -> bool: return _wrappers.k_i(self) == NULL_INT32 @@ -652,12 +722,22 @@ def is_inf(self) -> bool: class MinuteAtom(TemporalSpanAtom): """Wrapper for q minute atoms.""" t = -17 + _null = '0Nu' + _inf ='0Wu' _np_type = 'm' _np_dtype = 'timedelta64[m]' def _prototype(self=None): return MinuteAtom(np.timedelta64(216, 'm')) + @classproperty + def null(cls): # noqa: B902 + return _null_gen(cls._null)() + + @classproperty + def inf(cls): # noqa: B902 + return _inf_gen(cls._inf)() + @property def is_null(self) -> bool: return _wrappers.k_i(self) == NULL_INT32 @@ -670,12 +750,22 @@ def is_inf(self) -> bool: class TimespanAtom(TemporalSpanAtom): """Wrapper for q timespan atoms.""" t = -16 + _null = '0Nn' + _inf = '0Wn' _np_type = 'ns' _np_dtype = 'timedelta64[ns]' def _prototype(self=None): return TimespanAtom(np.timedelta64(3796312051664551936, 'ns')) + @classproperty + def null(cls): # noqa: B902 + return _null_gen(cls._null)() + + @classproperty + def inf(cls): # noqa: B902 + return _inf_gen(cls._inf)() + @property def is_null(self) -> bool: return _wrappers.k_j(self) == NULL_INT64 @@ -693,8 +783,18 @@ class DatetimeAtom(TemporalFixedAtom): using it whenever possible. """ t = -15 + _null = '0Nz' + _inf = '0Wz' _np_dtype = None + @classproperty + def null(cls): # noqa: B902 + return _null_gen(cls._null)() + + @classproperty + def inf(cls): # noqa: B902 + return _inf_gen(cls._inf)() + def __init__(self, *args, **kwargs): warnings.warn('The q datetime type is deprecated', DeprecationWarning) super().__init__(*args, **kwargs) @@ -716,12 +816,22 @@ class DateAtom(TemporalFixedAtom): """Wrapper for q date atoms.""" t = -14 _np_type = 'D' + _null = '0Nd' + _inf = '0Wd' _epoch_offset = DATE_OFFSET _np_dtype = 'datetime64[D]' def _prototype(self=None): return DateAtom(np.datetime64('1972-05-31', 'D')) + @classproperty + def null(cls): # noqa: B902 + return _null_gen(cls._null)() + + @classproperty + def inf(cls): # noqa: B902 + return _inf_gen(cls._inf)() + @property def is_null(self) -> bool: return _wrappers.k_i(self) == NULL_INT32 @@ -734,6 +844,8 @@ def is_inf(self) -> bool: class MonthAtom(TemporalFixedAtom): """Wrapper for q month atoms.""" t = -13 + _null = '0Nm' + _inf = '0Wm' _np_type = 'M' _epoch_offset = MONTH_OFFSET _np_dtype = 'datetime64[M]' @@ -741,6 +853,14 @@ class MonthAtom(TemporalFixedAtom): def _prototype(self=None): return MonthAtom(np.datetime64('1972-05', 'M')) + @classproperty + def null(cls): # noqa: B902 + return _null_gen(cls._null)() + + @classproperty + def inf(cls): # noqa: B902 + return _inf_gen(cls._inf)() + @property def is_null(self) -> bool: return _wrappers.k_i(self) == NULL_INT32 @@ -753,6 +873,8 @@ def is_inf(self) -> bool: class TimestampAtom(TemporalFixedAtom): """Wrapper for q timestamp atoms.""" t = -12 + _null = '0Np' + _inf = '0Wp' _np_type = 'ns' _epoch_offset = TIMESTAMP_OFFSET _np_dtype = 'datetime64[ns]' @@ -760,6 +882,14 @@ class TimestampAtom(TemporalFixedAtom): def _prototype(self=None): return TimestampAtom(datetime(2150, 10, 22, 20, 31, 15, 70713)) + @classproperty + def null(cls): # noqa: B902 + return _null_gen(cls._null)() + + @classproperty + def inf(cls): # noqa: B902 + return _inf_gen(cls._inf)() + @property def is_null(self) -> bool: return _wrappers.k_j(self) == NULL_INT64 @@ -768,6 +898,38 @@ def is_null(self) -> bool: def is_inf(self) -> bool: return abs(_wrappers.k_j(self)) == INF_INT64 + @property + def date(self): + return q('{`date$x}', self) + + @property + def time(self): + return q('{`time$x}', self) + + @property + def year(self): + return IntAtom(q('{`year$x}', self)) + + @property + def month(self): + return IntAtom(q('{`mm$x}', self)) + + @property + def day(self): + return IntAtom(q('{`dd$x}', self)) + + @property + def hour(self): + return IntAtom(q('{`hh$x}', self)) + + @property + def minute(self): + return IntAtom(q('{`uu$x}', self)) + + @property + def second(self): + return IntAtom(q('{`ss$x}', self)) + def py(self, *, raw: bool = False, @@ -809,11 +971,21 @@ class SymbolAtom(Atom): [`pykx.CharVector`][] """ t = -11 + _null = '`' + _inf = None _np_dtype = None def _prototype(self=None):# noqa return SymbolAtom('') + @classproperty + def null(cls): # noqa: B902 + return _null_gen(cls._null)() + + @classproperty + def inf(cls): # noqa: B902 + return _inf_gen(cls._inf)() + @property def is_null(self) -> bool: return str(self) == '' @@ -852,11 +1024,21 @@ def py(self, *, raw: bool = False, has_nulls: Optional[bool] = None, stdlib: boo class CharAtom(Atom): """Wrapper for q char (i.e. 8 bit ASCII value) atoms.""" t = -10 + _null = '" "' + _inf = None _np_dtype = None def _prototype(self=None): return CharAtom(b' ') + @classproperty + def null(cls): # noqa: B902 + return _null_gen(cls._null)() + + @classproperty + def inf(cls): # noqa: B902 + return _inf_gen(cls._inf)() + @property def is_null(self) -> bool: return 32 == _wrappers.k_g(self) @@ -868,6 +1050,16 @@ def is_inf(self) -> bool: def __bytes__(self): return self.py() + def __len__(self): + return 1 + + def __getitem__(self, key): + if not licensed: + raise LicenseException('index into K object') + if key != 0: + raise IndexError('index out of range') + return q('first', self) + def py(self, *, raw: bool = False, has_nulls: Optional[bool] = None, stdlib: bool = True): if raw: return _wrappers.k_g(self) @@ -926,11 +1118,21 @@ def __rpow__(self, other, mod=None): class FloatAtom(NonIntegralNumericAtom): """Wrapper for q float (i.e. 64 bit float) atoms.""" t = -9 + _null = '0n' + _inf = '0w' _np_dtype = np.float64 def _prototype(self=None): return FloatAtom(0.0) + @classproperty + def null(cls): # noqa: B902 + return _null_gen(cls._null)() + + @classproperty + def inf(cls): # noqa: B902 + return _inf_gen(cls._inf)() + def py(self, *, raw: bool = False, has_nulls: Optional[bool] = None, stdlib: bool = True): return _wrappers.k_f(self) @@ -941,11 +1143,21 @@ def np(self, *, raw: bool = False, has_nulls: Optional[bool] = None): class RealAtom(NonIntegralNumericAtom): """Wrapper for q real (i.e. 32 bit float) atoms.""" t = -8 + _null = '0Ne' + _inf = '0We' _np_dtype = np.float32 def _prototype(self=None): return RealAtom(0.0) + @classproperty + def null(cls): # noqa: B902 + return _null_gen(cls._null)() + + @classproperty + def inf(cls): # noqa: B902 + return _inf_gen(cls._inf)() + def py(self, *, raw: bool = False, has_nulls: Optional[bool] = None, stdlib: bool = True): return _wrappers.k_e(self) @@ -997,11 +1209,21 @@ def _np_null_or_inf(self, default, raw: bool): class LongAtom(IntegralNumericAtom): """Wrapper for q long (i.e. 64 bit signed integer) atoms.""" t = -7 + _null = '0Nj' + _inf = '0Wj' _np_dtype = np.int64 def _prototype(self=None): return LongAtom(0) + @classproperty + def null(cls): # noqa: B902 + return _null_gen(cls._null)() + + @classproperty + def inf(cls): # noqa: B902 + return _inf_gen(cls._inf)() + @property def is_null(self) -> bool: return _wrappers.k_j(self) == NULL_INT64 @@ -1020,11 +1242,21 @@ def np(self, *, raw: bool = False, has_nulls: Optional[bool] = None): class IntAtom(IntegralNumericAtom): """Wrapper for q int (i.e. 32 bit signed integer) atoms.""" t = -6 + _null = '0Ni' + _inf = '0Wi' _np_dtype = np.int32 def _prototype(self=None): return IntAtom(0) + @classproperty + def null(cls): # noqa: B902 + return _null_gen(cls._null)() + + @classproperty + def inf(cls): # noqa: B902 + return _inf_gen(cls._inf)() + @property def is_null(self) -> bool: return _wrappers.k_i(self) == NULL_INT32 @@ -1043,11 +1275,21 @@ def np(self, *, raw: bool = False, has_nulls: Optional[bool] = None): class ShortAtom(IntegralNumericAtom): """Wrapper for q short (i.e. 16 bit signed integer) atoms.""" t = -5 + _null = '0Nh' + _inf = '0Wh' _np_dtype = np.int16 def _prototype(self=None): return ShortAtom(0) + @classproperty + def null(cls): # noqa: B902 + return _null_gen(cls._null)() + + @classproperty + def inf(cls): # noqa: B902 + return _inf_gen(cls._inf)() + @property def is_null(self) -> bool: return _wrappers.k_h(self) == NULL_INT16 @@ -1066,11 +1308,21 @@ def np(self, *, raw: bool = False, has_nulls: Optional[bool] = None): class ByteAtom(IntegralNumericAtom): """Wrapper for q byte (i.e. 8 bit unsigned integer) atoms.""" t = -4 + _null = None + _inf = None _np_dtype = np.uint8 def _prototype(self=None): return ByteAtom(0) + @classproperty + def null(cls): # noqa: B902 + return _null_gen(cls._null)() + + @classproperty + def inf(cls): # noqa: B902 + return _inf_gen(cls._inf)() + @property def is_null(self) -> bool: return False @@ -1089,11 +1341,21 @@ def np(self, *, raw: bool = False, has_nulls: Optional[bool] = None): class GUIDAtom(Atom): """Wrapper for q GUID atoms.""" t = -2 + _null = '0Ng' + _inf = None _np_dtype = None def _prototype(self=None): return GUIDAtom(UUID(int=0)) + @classproperty + def null(cls): # noqa: B902 + return _null_gen(cls._null)() + + @classproperty + def inf(cls): # noqa: B902 + return _inf_gen(cls._inf)() + @property def is_null(self) -> bool: return self.py(raw=True) == 0j @@ -1109,11 +1371,21 @@ def py(self, *, raw: bool = False, has_nulls: Optional[bool] = None, stdlib: boo class BooleanAtom(IntegralNumericAtom): """Wrapper for q boolean atoms.""" t = -1 + _null = None + _inf = None _np_dtype = None def _prototype(self=None): return BooleanAtom(True) + @classproperty + def null(cls): # noqa: B902 + return _null_gen(cls._null)() + + @classproperty + def inf(cls): # noqa: B902 + return _inf_gen(cls._inf)() + @property def is_null(self) -> bool: return False @@ -1194,6 +1466,17 @@ def __reversed__(self): for i in reversed(range(_wrappers.k_n(self))): yield self._unlicensed_getitem(i) + def __setitem__(self, key, val): + try: + update_assign = q('{@[x;y;:;z]}', self, _idx_to_k(key, _wrappers.k_n(self)), val) + except QError as err: + if 'type' == str(err): + raise QError(f'Failed to assign value of type: {type(val)} ' + f'to list of type: {type(self)}') + else: + raise QError(str(err)) + self.__dict__.update(update_assign.__dict__) + def __array__(self): # The `__array__` method must return a `np.ndarray`, not a `np.ma.masked_array`. As a # result, the null check we currently perform (by default) is a waste of time and memory, @@ -1244,6 +1527,185 @@ def apply(self, func, *args, **kwargs): kwargs ) + def sum(self): + return q.sum(self) + + def prod(self): + return q.prod(self) + + def min(self): + return q.min(self) + + def max(self): + return q.max(self) + + def mean(self): + return q.avg(self) + + def median(self): + return q.med(self) + + def mode(self): + return q('{where max[c]=c:count each d:group x}', self) + + def append(self, data): + """Append object to the end of a vector. + + Parameters: + self: PyKX Vector/List object + data: Data to be used when appending to a list/vector, when + appending to a typed list this must be an object with a + type which converts to an equivalent vector type. + When appending to a List any type object can be appended. + + Raises: + PyKXException: When dealing with typed vectors appending to this vector + with data of a different type is unsupported and will raise an + error + + Examples: + + Append to a vector object with an atom + + ```python + >>> import pykx as kx + >>> qvec = kx.q.til(3) + >>> qvec + pykx.LongVector(pykx.q('0 1 2')) + >>> qvec.append(3) + >>> qvec + pykx.LongVector(pykx.q('0 1 2 3')) + ``` + + Attempt to append a vector object with an incorrect type: + + ```python + >>> import pykx as kx + >>> qvec = kx.q.til(3) + >>> qvec.append([1, 2, 3]) + Traceback (most recent call last): + File "", line 1, in + File "/usr/local/anaconda3/lib/python3.8/site-packages/pykx/wrappers.py", line 1262, ... + raise QError(f'Appending data of type: {type(data)} ' + pykx.exceptions.QError: Appending data of type: ... + ``` + + Append to a list object with an atom and list: + + ```python + >>> import pykx as kx + >>> qlist = kx.toq([1, 2.0]) + >>> qlist + pykx.List(pykx.q(' + 1 + 2f + ')) + >>> qlist.append('a') + >>> qlist + pykx.List(pykx.q(' + 1 + 2f + `a + ')) + >>> qlist.append([1, 2]) + >>> qlist + pykx.List(pykx.q(' + 1 + 2f + `a + 1 2 + ')) + ``` + """ + if not isinstance(self, List): + if not q('{(0>type[y])& type[x]=abs type y}', self, data): + raise QError(f'Appending data of type: {type(K(data))} ' + f'to vector of type: {type(self)} not supported') + append_vec = q('{[orig;app]orig,$[0>> import pykx as kx + >>> qvec = kx.q.til(3) + >>> qvec + pykx.LongVector(pykx.q('0 1 2')) + >>> qvec.extend(3) + >>> qvec + pykx.LongVector(pykx.q('0 1 2 3')) + >>> qvec.extend([4, 5, 6]) + >>> qvec + pykx.LongVector(pykx.q('0 1 2 3 4 5 6')) + ``` + + Attempt to extend a vector object with an incorrect type: + + ```python + >>> import pykx as kx + >>> qvec = kx.q.til(3) + >>> qvec.extend('a') + Traceback (most recent call last): + File "", line 1, in + File "/usr/local/anaconda3/lib/python3.8/site-packages/pykx/wrappers.py", line 1271, ... + raise QError(f'Extending data of type: {type(data)} ' + pykx.exceptions.QError: Extending data of type: ... + ``` + + Extend a list object with an atom and list: + + ```python + >>> import pykx as kx + >>> qlist = kx.toq([1, 2.0]) + >>> qlist + pykx.List(pykx.q(' + 1 + 2f + ')) + >>> qlist.extend('a') + >>> qlist + pykx.List(pykx.q(' + 1 + 2f + `a + ')) + >>> qlist.extend([1, 2]) + >>> qlist + pykx.List(pykx.q(' + 1 + 2f + `a + 1 + 2 + ')) + ``` + """ + if not isinstance(self, List): + if q('{type[x]<>abs type y}', self, data): + raise QError(f'Extending data of type: {type(K(data))} ' + f'to vector of type: {type(self)} not supported') + extend_vec = q(''' + {[orig;ext] + ret:t!t:orig,$[type[ext]in 99 98h;enlist;]ext; + value $[9h~type first t;count[t]#0x0;]ret} + ''', self, data) + self.__dict__.update(extend_vec.__dict__) + def index(self, x, start=None, end=None): for i in slice_to_range(slice(start, end), _wrappers.k_n(self)): if self[i] == x: @@ -1895,6 +2357,38 @@ def _prototype(self=None): datetime(2150, 10, 22, 20, 31, 15, 70713)] ) + @property + def date(self): + return q('{`date$x}', self) + + @property + def time(self): + return q('{`time$x}', self) + + @property + def year(self): + return IntVector(q('{`year$x}', self)) + + @property + def month(self): + return IntVector(q('{`mm$x}', self)) + + @property + def day(self): + return IntVector(q('{`dd$x}', self)) + + @property + def hour(self): + return IntVector(q('{`hh$x}', self)) + + @property + def minute(self): + return IntVector(q('{`uu$x}', self)) + + @property + def second(self): + return IntVector(q('{`ss$x}', self)) + def py(self, *, raw: bool = False, @@ -2424,7 +2918,7 @@ def _repr_html_(self): $[n>c[0];?[t;enlist(=;`i;(last;`i));0b;{x!x}cls];()]; h }''', console, self) - df = pd.read_json(qtab.py().decode("utf-8"), orient='records', + df = pd.read_json(StringIO(qtab.py().decode("utf-8")), orient='records', convert_dates=False, dtype=False) if len(df) == 0: columns = self.columns.py() @@ -2502,7 +2996,7 @@ def _repr_html_(self): $[n>c[0];?[t;enlist(=;`i;(last;`i));0b;{x!x}cls];()]; h }''', console, self) - df = pd.read_json(qtab.py().decode("utf-8"), orient='records', + df = pd.read_json(StringIO(qtab.py().decode("utf-8")), orient='records', convert_dates=False, dtype=False) if len(df) == 0: columns = self.columns.py() @@ -2594,7 +3088,7 @@ def _repr_html_(self): each flip x} r; h }''', console, self) - df = pd.read_json(qtab.py().decode("utf-8"), orient='records', + df = pd.read_json(StringIO(qtab.py().decode("utf-8")), orient='records', convert_dates=False, dtype=False) if len(df) == 0: columns = self.columns.py() @@ -2640,6 +3134,11 @@ def __getitem__(self, key): _check_k_mapping_key(original_key, key, self._keys) return super().__getitem__(key) + def __setitem__(self, key, val): + if isinstance(key, tuple): + raise NotImplementedError("pykx.Dictionary objects do not support tuple key assignment") + self.__dict__.update(q('{x,((),y)!((),z)}', self, key, val).__dict__) + def _repr_html_(self): if not licensed: return self.__repr__() @@ -2667,7 +3166,7 @@ def _repr_html_(self): each flip x} ?[t;enlist(<;`i;c[0]);0b;{x!x}cls]; h }''', console, self) - df = pd.read_json(qtab.py().decode("utf-8"), orient='records', + df = pd.read_json(StringIO(qtab.py().decode("utf-8")), orient='records', convert_dates=False, dtype=False) df.set_index('pykxDictionaryKeys', inplace=True) if df.columns.values[0] == 'pykxDictionaryValues': @@ -2698,7 +3197,6 @@ class KeyedTable(Dictionary, PandasAPI): Mortals](https://code.kx.com/q4m3/8_Tables/#84-primary-keys-and-keyed-tables) for more information about q keyed tables. """ - # TODO: `cast` should be set to False at the next major release (KXI-12945) def __new__(cls, *args, cast: bool = None, **kwargs): tab = None @@ -3045,7 +3543,7 @@ def _repr_html_(self): $[n>c[0];?[t;enlist(=;`i;(last;`i));0b;{x!x}cls];()]; h }''', console, self) - df = pd.read_json(qtab.py().decode("utf-8"), orient='records', + df = pd.read_json(StringIO(qtab.py().decode("utf-8")), orient='records', convert_dates=False, dtype=False) columns = q('cols', self).py() if len(keys)>=console[1]: @@ -3065,7 +3563,6 @@ def _repr_html_(self): class GroupbyTable(PandasAPI): - def __init__(self, tab, as_index, was_keyed, as_vector=None): self.tab = tab self.as_index = as_index @@ -3078,6 +3575,20 @@ def q(self): def ungroup(self): return q.ungroup(self.tab) + def apply(self, func, *args, **kwargs): + tab = self.q() + key = q.key(tab) + data = q.value(tab) + return q('{[k; t; f]' + ' ff:flip t;' + ' d:value ff;' + ' agg:{[row; f] f each row}[;f] each d;' + 'k!((key ff)!/:(flip agg))}', + key, + data, + func + ) + def __getitem__(self, item): keys = q.keys(self.tab).py() if isinstance(item, list): @@ -3193,6 +3704,10 @@ class Lambda(Function): """ t = 100 + @property + def __name__(self): + return 'pykx.Lambda' + @cached_property def params(self): # Strip "PyKXParam" from all param names if it is a prefix for all @@ -3212,6 +3727,10 @@ class UnaryPrimitive(Function): """ t = 101 + @property + def __name__(self): + return 'pykx.UnaryPrimitive' + def __call__(self, *args, **kwargs): if kwargs: raise TypeError('Cannot use kwargs on a unary primitive') @@ -3396,6 +3915,10 @@ class Composition(Function): """ t = 105 + @property + def __name__(self): + return 'pykx.Composition' + @property def params(self): if q('{.pykx.util.isw x}', self).py(): @@ -3475,6 +3998,10 @@ class AppliedIterator(Function): themselves are of the type [`pykx.Iterator`][], but when applied to a function a new type (which is a subclass of `AppliedIterator`) is created depending on what iterator was used. """ + @property + def __name__(self): + return 'pykx.AppliedIterator' + @cached_property def params(self): return self.func.params @@ -3562,6 +4089,10 @@ def params(self): class SymbolicFunction(Function, SymbolAtom): """Special wrapper type representing a symbol atom that can be used as a function.""" + @property + def __name__(self): + return 'pykx.SymbolicFunction' + def __init__(self, *args, **kwargs): self.execution_ctx = q # Default to executing in the embedded q instance. super().__init__(*args, **kwargs) diff --git a/tests/conftest.py b/tests/conftest.py index d983345..a8b8738 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -54,10 +54,26 @@ def kx(request): @pytest.fixture def q(request, kx, q_init): if request.param == 'embedded': + kx.q._inter_test_q = kx.q.__call__ + kx.q.__call__ = ( + lambda *args, **kwargs: kx.q._inter_test_q( + *args, + **kwargs, + debug=True if randint(0, 1) == 0 else False + ) + ) yield kx.q elif request.param == 'ipc': with q_proc(q_init) as port: with kx.QConnection(port=port) as conn: + _inter_test_q = conn.__call__ + conn.__call__ = ( + lambda *args, **kwargs: _inter_test_q( + *args, + **kwargs, + debug=True if randint(0, 1) == 0 else False + ) + ) yield conn @@ -142,7 +158,7 @@ def isolate_test(item): out_pipe, in_pipe = mp_ctx.Pipe(duplex=False) proc = mp_ctx.Process(target=isolated_test_runner, args=(item, in_pipe)) proc.start() - timeout = 20.0 + timeout = 50.0 for x in item.own_markers: if x.name == 'timeout': timeout = x.args[0] diff --git a/tests/qcumber_tests/conversions.quke b/tests/qcumber_tests/conversions.quke index 2bb3f5a..2666690 100644 --- a/tests/qcumber_tests/conversions.quke +++ b/tests/qcumber_tests/conversions.quke @@ -41,6 +41,15 @@ feature default conversions t::.pykx.eval["lambda x: bytes(str(type(x)), 'utf-8')"][<]; after each .pykx.setdefault["default"]; + should support default default + expect default for LongVector + .qu.compare[""; t til 10]; + expect default for dict + .qu.compare[""; t `a`b!1 2]; + expect default for table + .qu.compare[""; t ([]a:2 3; b:4 5)]; + expect default for keyed table + .qu.compare[""; t ([a:2 3] b:4 5)]; should support python default expect python default .pykx.setdefault["python"]; diff --git a/tests/test_ctx.py b/tests/test_ctx.py index e79907f..a5a0908 100644 --- a/tests/test_ctx.py +++ b/tests/test_ctx.py @@ -85,6 +85,15 @@ def test_find_namespace_in_q_file(q, tmp_path): assert q.testnamespace.true +def test_reserved_in_ctx(q, tmp_path): + with pytest.raises(AttributeError): + q.testnamespace + with cd(tmp_path): + with open('name.q', 'w') as f: + f.write('.name.test.update:{x+1}\n') + assert q.name.test.update(1) == 2 + + def test_python_keyword_as_q_fn(q): assert q.except_ == q('except') assert q._except == q('except') diff --git a/tests/test_ipc.py b/tests/test_ipc.py index c318cfe..28451fa 100644 --- a/tests/test_ipc.py +++ b/tests/test_ipc.py @@ -655,3 +655,71 @@ def test_large_IPC(kx, q_port): size = 4294967296 # Exceed 32 bit unsigned res = q('{x#0x0}', size) assert size == len(res) + + +@pytest.mark.unlicensed +def test_debug_kwarg(kx, q_port): + with kx.SyncQConnection(port=q_port) as q: + assert q('til 10', debug=True).py() == list(range(10)) + with pytest.raises(kx.QError) as e: + q('til "asd"') + assert '[1]' in str(e) + assert q('{[x] til x}', 10, debug=True).py() == list(range(10)) + with pytest.raises(kx.QError) as e: + q('{[x] til x}', b'asd') + assert '[1]' in str(e) + assert q('{[x; y] .[mavg; (x; til y)]}', 3, 10, debug=True).py() ==\ + [0.0, 0.5, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0] + with pytest.raises(kx.QError) as e: + q('{[x; y] .[mavg; (x; til y)]}', 3, b'asd') + assert '[1]' in str(e) + with kx.SecureQConnection(port=q_port) as q: + assert q('til 10', debug=True).py() == list(range(10)) + with pytest.raises(kx.QError) as e: + q('til "asd"') + assert '[1]' in str(e) + assert q('{[x] til x}', 10, debug=True).py() == list(range(10)) + with pytest.raises(kx.QError) as e: + q('{[x] til x}', b'asd') + assert '[1]' in str(e) + assert q('{[x; y] .[mavg; (x; til y)]}', 3, 10, debug=True).py() ==\ + [0.0, 0.5, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0] + with pytest.raises(kx.QError) as e: + q('{[x; y] .[mavg; (x; til y)]}', 3, b'asd') + assert '[1]' in str(e) + + +@pytest.mark.asyncio +@pytest.mark.unlicensed +async def test_debug_kwarg_async(kx, q_port): + async with kx.AsyncQConnection(port=q_port) as q: + assert (await q('til 10', debug=True)).py() == list(range(10)) + with pytest.raises(kx.QError) as e: + await q('til "asd"') + assert '[1]' in str(e) + assert (await q('{[x] til x}', 10, debug=True)).py() == list(range(10)) + with pytest.raises(kx.QError) as e: + await q('{[x] til x}', b'asd') + assert '[1]' in str(e) + assert (await q('{[x; y] .[mavg; (x; til y)]}', 3, 10, debug=True)).py()\ + == [0.0, 0.5, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0] + with pytest.raises(kx.QError) as e: + await q('{[x; y] .[mavg; (x; til y)]}', 3, b'asd') + assert '[1]' in str(e) + + +@pytest.mark.embedded +async def test_debug_kwarg_embedded(kx, q): + assert q('til 10', debug=True).py() == list(range(10)) + with pytest.raises(kx.QError) as e: + q('til "asd"') + assert '[1]' in str(e) + assert q('{[x] til x}', 10, debug=True).py() == list(range(10)) + with pytest.raises(kx.QError) as e: + q('{[x] til x}', b'asd') + assert '[1]' in str(e) + assert q('{[x; y] .[mavg; (x; til y)]}', 3, 10, debug=True).py() ==\ + [0.0, 0.5, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0] + with pytest.raises(kx.QError) as e: + q('{[x; y] .[mavg; (x; til y)]}', 3, b'asd') + assert '[1]' in str(e) diff --git a/tests/test_license.py b/tests/test_license.py index 0a5e526..0f8df89 100644 --- a/tests/test_license.py +++ b/tests/test_license.py @@ -1,6 +1,7 @@ import base64 from io import StringIO import os +import shutil import re # Do not import pykx here - use the `kx` fixture instead! @@ -107,6 +108,13 @@ def test_use_both_licensed_and_unlicensed_flags(QARGS): import pykx # noqa: F401 +def test_env_combination(): + os.environ['QARGS'] = '--licensed' + os.environ['PYKX_UNLICENSED'] = 'true' + with pytest.raises(Exception, match="(?i)'licensed' and 'unlicensed' behaviour"): + import pykx # noqa: F401 + + def test_check_license_invalid_file(kx): with patch('sys.stdout', new=StringIO()) as test_out: kx.license.check('/test/test.blah') @@ -138,6 +146,28 @@ def test_check_license_success_b64(kx): assert kx.license.check(license, format='STRING') +@pytest.mark.xfail(reason="Manual testing works correctly, seems to be a persistance issue") +@pytest.mark.skipif('KDB_LICENSE_EXPIRED' not in os.environ, + reason='Test required KDB_LICENSE_EXPIRED environment variable to be set') +def test_exp_license(kx): + exp_lic = os.environ['KDB_LICENSE_EXPIRED'] + lic_folder = '/tmp/license' + os.makedirs(lic_folder, exist_ok=True) + with open(lic_folder + '/k4.lic', 'wb') as binary_file: + binary_file.write(base64.b64decode(exp_lic)) + qhome_loc = os.environ['QHOME'] + os.environ['QLIC'] = os.environ['QHOME'] = lic_folder + pattern = re.compile('Your PyKX license has now.*') + with patch('sys.stdout', new=StringIO()) as test_out: + try: + import pykx # noqa: F401 + except Exception as e: + assert str(e) == "EOF when reading a line" + shutil.rmtree(lic_folder) + os.environ['QLIC'] = os.environ['QHOME'] = qhome_loc + assert pattern.match(test_out.getvalue()) + + def test_check_license_invalid(kx): pattern = re.compile("Supplied license information does not match.*") with patch('sys.stdout', new=StringIO()) as test_out: diff --git a/tests/test_pandas_agg.py b/tests/test_pandas_agg.py new file mode 100644 index 0000000..d82c7e4 --- /dev/null +++ b/tests/test_pandas_agg.py @@ -0,0 +1,98 @@ +"""Tests for the Pandas API agg functionality""" + +import pytest + +import statistics + + +def test_single_func(q, kx): + tab = q('([] til 10; 1)') + gtab = q('([]sym:`a`a`a`b`b;x:1 2 2 3 3)') + df = tab.pd() + gdf = gtab.pd() + + def mode(x): + return statistics.mode(x) + + min_tab = tab.agg('min') + assert all(min_tab == tab.min().values()) + mode_tab = tab.agg(mode) + mode_df = df.agg(mode) + assert all(kx.toq(mode_df) == mode_tab.values()) + + min_gtab = gtab.groupby('sym').agg('min') + min_df = gdf.groupby('sym').agg('min') + assert q('{x~y}', min_gtab, min_df) + mode_gtab = gtab.groupby('sym').agg(mode) + mode_gdf = gdf.groupby('sym').agg(mode) + assert q('{x~y}', mode_gtab, mode_gdf) + + +def test_list_funcs(q, kx): + tab = q('([] til 10; 1)') + gtab = q('([]sym:`a`a`a`b`b;x:1 2 2 3 3)') + df = tab.pd() + gdf = gtab.pd() + + def mode(x): + return statistics.mode(x) + + lst_tab = tab.agg(['min', mode]) + lst_df = df.agg(['min', mode]) + assert q('{key[x][`function]~key[y][`0]}', lst_tab, lst_df) + assert q('{value[x]~value[y]}', lst_tab, lst_df) + + lst_gtab = gtab.groupby('sym').agg(['min', mode]) + lst_gdf = gdf.groupby('sym').agg(['min', mode]) + assert q('{x~y}', lst_gdf['x']['min'], lst_gtab['min'].values()['x']) + assert q('{x~y}', lst_gdf['x']['mode'], lst_gtab['mode'].values()['x']) + + +def test_dict_funcs(q, kx): + tab = q('([] til 10; 1)') + dict_str = tab.agg({'x': 'min', 'x1': 'max'}) + max_ret = q('(enlist enlist[`function]!enlist[`max])!enlist enlist[`x1]!enlist 1') + min_ret = q('(enlist enlist[`function]!enlist[`min])!enlist enlist[`x]!enlist 0') + dict_str_max = kx.q.qsql.select(dict_str['x1'], where=['function=`max']) + dict_str_min = kx.q.qsql.select(dict_str['x'], where=['function=`min']) + assert isinstance(dict_str, kx.KeyedTable) + assert q('{x~y}', dict_str_max, max_ret) + assert q('{x~y}', dict_str_min, min_ret) + + def mode(x): + return statistics.mode(x) + + dict_mode = tab.agg({'x': 'min', 'x1': mode}) + mode_ret = q('(enlist enlist[`function]!enlist[`mode])!enlist enlist[`x1]!enlist 1') + min_ret = q('(enlist enlist[`function]!enlist[`min])!enlist enlist[`x]!enlist 0') + dict_func_mode = kx.q.qsql.select(dict_mode['x1'], where=['function=`mode']) + dict_func_min = kx.q.qsql.select(dict_mode['x'], where=['function=`min']) + assert isinstance(dict_mode, kx.KeyedTable) + assert q('{x~y}', dict_func_mode, mode_ret) + assert q('{x~y}', dict_func_min, min_ret) + + +def test_errors(q, kx): + tab = q('([]til 10;1)') + ktab = q('([til 10] til 10; 1)') + gtab = q('([]sym:`a`a`a`b`b;x:1 2 2 3 3)').groupby('sym') + + with pytest.raises(NotImplementedError) as err: + ktab.agg('min') + assert "KeyedTable" in str(err.value) + + with pytest.raises(NotImplementedError) as err: + tab.agg('min', axis=1) + assert 'axis parameter only presently supported' in str(err.value) + + with pytest.raises(NotImplementedError) as err: + gtab.agg({'x': 'min'}) + assert 'Dictionary input func not presently supported for GroupbyTable' in str(err.value) + + with pytest.raises(NotImplementedError) as err: + tab.agg({'x': ['min', 'max']}) + assert "Unsupported type '' supplied as dictionary value" in str(err.value) + + with pytest.raises(kx.QError) as err: + q('0#([]10?1f;10?1f)').agg('mean') + assert "Application of 'agg' method not supported" in str(err.value) diff --git a/tests/test_pykx.py b/tests/test_pykx.py index 13f18dd..553ab55 100644 --- a/tests/test_pykx.py +++ b/tests/test_pykx.py @@ -39,6 +39,25 @@ def test_qargs_q_flag(): assert kx.q('2 + 2') == 4 +@pytest.mark.isolate +def test_no_sql(): + os.environ['QARGS'] = '--no-sql' + import pykx as kx + with pytest.raises(kx.QError) as err: + kx.q.sql('select 42') + assert '.s.sp' in str(err) + + +@pytest.mark.isolate +@pytest.mark.skipif(not sys.platform.startswith('linux'), reason="qlog only supported on Linux") +def test_no_qlog(): + os.environ['QARGS'] = '--no-qlog' + import pykx as kx + with pytest.raises(kx.QError) as err: + kx.q('.qlog') + assert '.qlog' in str(err) + + @pytest.mark.isolate def test_qinit_startup(): # PyKX would not initialise appropriately if QINIT was set to a file containing show statement @@ -47,6 +66,14 @@ def test_qinit_startup(): assert kx.q('2 + 2') == 4 +@pytest.mark.isolate +def test_unlicensed_env(): + os.environ['PYKX_UNLICENSED'] = 'true' + import pykx as kx + assert not kx.licensed + assert 1 == kx.toq(1).py() + + @pytest.mark.isolate def test_qinit_qq_startup(): # PyKX would not initialise appropriately if q.q exists in QHOME containing a show statement @@ -193,3 +220,14 @@ def test_pykx_safe_reimport(): text=True, ).stdout.strip() assert output.split('\n')[-1] == "0 1 2 3 4 5 6 7 8 9" + + +@pytest.mark.isolate +def test_pykx_star(): + output = subprocess.run( + (str(Path(sys.executable).as_posix()), '-c', 'from pykx import *; print(q("til 10"))'), + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + ).stdout.strip() + assert output.split('\n')[-1] == "0 1 2 3 4 5 6 7 8 9" diff --git a/tests/test_system.py b/tests/test_system.py index d2d482e..6f2da19 100644 --- a/tests/test_system.py +++ b/tests/test_system.py @@ -106,8 +106,8 @@ def test_system_variables(): kx.q('a: 5') assert all(kx.q.system.variables() == kx.q('enlist `a')) print(kx.q.system.variables('.pykx')) - assert all(kx.q.system.variables('.pykx') == kx.q('`debug`i`pykxDir`util')) - assert all(kx.q.system.variables('pykx') == kx.q('`debug`i`pykxDir`util')) + assert all(kx.q.system.variables('.pykx') == kx.q('`debug`i`pykxDir`pykxExecutable`util')) + assert all(kx.q.system.variables('pykx') == kx.q('`debug`i`pykxDir`pykxExecutable`util')) @pytest.mark.isolate diff --git a/tests/test_wrappers.py b/tests/test_wrappers.py index 67ef283..5e4e4ac 100644 --- a/tests/test_wrappers.py +++ b/tests/test_wrappers.py @@ -235,6 +235,15 @@ def test_equality(self, q): class Test_Atom: + def test_char_atom(self, kx, q): + atom = kx.CharAtom('a') + assert 1 == len(atom) + assert atom[0].py() == b'a' + with pytest.raises(IndexError): + atom[1] + with pytest.raises(IndexError): + atom[-1] + def test_boolean_atom(self, q): t, f = q('1b'), q('0b') assert t == True # noqa @@ -248,6 +257,71 @@ def test_boolean_atom(self, q): assert bool(t) is True assert bool(f) is False + def test_null_gen_lic(self, kx): + qtypes = [kx.GUIDAtom, kx.ShortAtom, kx.IntAtom, + kx.LongAtom, kx.RealAtom, kx.FloatAtom, + kx.CharAtom, kx.SymbolAtom, kx.TimestampAtom, + kx.MonthAtom, kx.DateAtom, kx.DatetimeAtom, + kx.TimespanAtom, kx.MinuteAtom, kx.SecondAtom, + kx.TimeAtom] + for i in qtypes: + null_val = getattr(i, 'null') # noqa: B009 + assert type(null_val) == i + assert null_val.is_null + + def test_inf_pos_lic(self, kx): + qtypes = [kx.ShortAtom, kx.IntAtom, + kx.LongAtom, kx.RealAtom, kx.FloatAtom, + kx.TimestampAtom, kx.MonthAtom, kx.DateAtom, + kx.DatetimeAtom, kx.TimespanAtom, kx.MinuteAtom, + kx.SecondAtom, kx.TimeAtom] + for i in qtypes: + inf_val = getattr(i, 'inf') # noqa: B009 + assert type(inf_val) == i + assert inf_val>0 + assert inf_val.is_inf + + def test_inf_neg_lic(self, kx): + qtypes = [kx.ShortAtom, kx.IntAtom, + kx.LongAtom, kx.RealAtom, kx.FloatAtom, + kx.TimestampAtom, kx.MonthAtom, kx.DateAtom, + kx.DatetimeAtom, kx.TimespanAtom, kx.MinuteAtom, + kx.SecondAtom, kx.TimeAtom] + for i in qtypes: + inf_val = -getattr(i, 'inf') # noqa: B009 + assert type(inf_val) == i + assert inf_val<0 + assert inf_val.is_inf + + def test_null_fail(self, kx): + qtypes = [kx.BooleanAtom, kx.ByteAtom] + for i in qtypes: + with pytest.raises(NotImplementedError) as err: + getattr(i, 'null') # noqa: B009 + assert 'Retrieval of null values' in str(err) + + def test_inf_fail(self, kx): + qtypes = [kx.BooleanAtom, kx.ByteAtom, kx.GUIDAtom, + kx.CharAtom, kx.SymbolAtom] + for i in qtypes: + with pytest.raises(NotImplementedError) as err: + getattr(i, 'inf') # noqa: B009 + assert 'Retrieval of infinite values' in str(err) + + @pytest.mark.unlicensed(unlicensed_only=True) + def test_null_inf_unlic(self, kx): + qtypes = [kx.ByteAtom, kx.GUIDAtom, kx.ShortAtom, + kx.IntAtom, kx.LongAtom, kx.RealAtom, + kx.FloatAtom, kx.CharAtom, kx.SymbolAtom, + kx.TimestampAtom, kx.MonthAtom, kx.DateAtom, + kx.DatetimeAtom, kx.TimespanAtom, kx.MinuteAtom, + kx.SecondAtom, kx.TimeAtom] + for i in qtypes: + for j in ['null', 'inf']: + with pytest.raises(kx.QError) as err: + getattr(i, j)() + assert 'not supported in unlicensed mode' in str(err) + def test_is_null_and_is_inf(self, q): assert q('0Ng').is_null assert not q('first 1?0Ng').is_null @@ -874,6 +948,67 @@ def test_getting(self, q): assert f[q('1 2')].py() == [2.0, 3.0] assert f._unlicensed_getitem(-1) == 3.0 + def test_setting(self, q, kx): + v = q.til(10) + with pytest.raises(IndexError): + v[10] = 10 + with pytest.raises(IndexError): + v[-11] = 10 + with pytest.raises(kx.QError) as err: + v[2] = 'a' + assert "Failed to assign value of type: " in str(err) + for i in range(3): + v[0]+=i + assert v[0]>0 + v[1] = 2 + assert v[1] == 2 + v[-1] = 20 + assert v[9] == 20 + vlist = kx.List([1, 0.1, 3]) + vlist[2] = 'a' + assert vlist[2] == 'a' + vlist[:2] = 0.1 + assert vlist[0] == 0.1 + assert vlist[1] == 0.1 + + def test_append(self, q, kx): + p0 = [1, 2, 3] + q0 = kx.toq(p0) + p0.append(1) + q0.append(1) + assert all(p0 == q0) + with pytest.raises(kx.QError) as err: + q0.append(2.0) + assert "Appending data of type: