From 6fa9920094ccc7338d31e6261d257b20bfbaa49a Mon Sep 17 00:00:00 2001 From: EtWn <34377743+EtWnn@users.noreply.github.com> Date: Mon, 8 Mar 2021 11:46:41 +0100 Subject: [PATCH 01/17] Base setup (#1) * add env file * add tests logic * add README * add credentials example --- README.md | 4 ++ data/credentials.json.example | 9 +++ environment.yml | 13 ++++ tests/__init__.py | 4 ++ tests/__main__.py | 114 ++++++++++++++++++++++++++++++++++ tests/print_utils.py | 80 ++++++++++++++++++++++++ 6 files changed, 224 insertions(+) create mode 100644 README.md create mode 100644 data/credentials.json.example create mode 100644 environment.yml create mode 100644 tests/__init__.py create mode 100644 tests/__main__.py create mode 100644 tests/print_utils.py diff --git a/README.md b/README.md new file mode 100644 index 0000000..74f1470 --- /dev/null +++ b/README.md @@ -0,0 +1,4 @@ +# BinanceWatch + +This repo is made to keep track locally of the transactions made by a +user on his binance account. \ No newline at end of file diff --git a/data/credentials.json.example b/data/credentials.json.example new file mode 100644 index 0000000..0c79217 --- /dev/null +++ b/data/credentials.json.example @@ -0,0 +1,9 @@ +{ + "CoinAPI": { + "api_key": "MY_API_KEY" + }, + "Binance": { + "api_key": "MY_API_KEY", + "api_secret":"MY_API_SECRET" + } +} \ No newline at end of file diff --git a/environment.yml b/environment.yml new file mode 100644 index 0000000..7eb9581 --- /dev/null +++ b/environment.yml @@ -0,0 +1,13 @@ +name: binancewatch +channels: + - defaults +dependencies: + - jupyter + - numpy + - python=3.7.9 + - tqdm + - pip=20.3.3 + - pip: + - dateparser + - python-binance==0.7.9 + - requests diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..dab199c --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1,4 @@ +""" +modules needs to be imported here to be detected in the __main__ file for auto import. +If a module is not imported here, it won't be tested by the tests package +""" diff --git a/tests/__main__.py b/tests/__main__.py new file mode 100644 index 0000000..fa6df61 --- /dev/null +++ b/tests/__main__.py @@ -0,0 +1,114 @@ +import argparse +import traceback +from types import ModuleType +from typing import List, Optional, Dict + +import tests.print_utils as pu +import tests + + +def get_test_modules(module_names: Optional[List[str]] = None) -> List[ModuleType]: + """ + from a list of module names, will return the list of module objects belonging to the tests package + if the list is empty, will return all the modules with 'test_' in the name + :param module_names: list of modules from the tests package to fetch + :return: + """ + if module_names is None: + module_names = [m for m in dir(tests) if 'test_' in m] + test_modules = [] + for module_name in module_names: + try: + test_modules.append(getattr(tests, module_name)) + except AttributeError: + message = f"The test module {module_name} was skipped because the file was not found in the tests package" + print(pu.s_to_warning(message)) + return test_modules + + +def run_test_module(module: ModuleType, common_kwargs=None, verbose=1) -> Dict: + """ + will run all the functions from a module whose name has 'test_' in it and return an execution report + :param module: module that host the functions + :param common_kwargs: kwargs to provide to all functions + :param verbose: the higher the more information are printed + :return: dictionary with execution report ex: {'success': 10, 'failure': 1, 'total': 11} + """ + if common_kwargs is None: + common_kwargs = {} + n_success = 0 + n_failure = 0 + tests_names = [f_name for f_name in dir(module) if 'test_' in f_name and callable(getattr(module, f_name))] + for test_name in tests_names: + if verbose: + print(pu.get_blue_title(module.__name__, test_name)) + try: + getattr(module, test_name)(**common_kwargs) + n_success += 1 + except Exception: # general catch to display the error reports without stopping the tests + n_failure += 1 + if verbose: + print(pu.s_to_fail(traceback.format_exc())) + pass + if verbose: + print(pu.get_blue_sep()) + return {'success': n_success, 'failure': n_failure, 'total': len(tests_names)} + + +def print_report(report: Dict): + """ + print nicely a report of test execution + :param report: + """ + print("\n\nSummary:\n") + total_success = 0 + total_tests = 0 + for module_name, dico in report.items(): + if dico['total']: + total_success += dico['success'] + total_tests += dico['total'] + rate = dico['success'] / dico['total'] + if rate == 1: + s_result = pu.s_to_pass(f"{dico['success'] / dico['total']:.0%}") + else: + s_result = pu.s_to_fail(f"{dico['success'] / dico['total']:.0%}") + print(f"\t- {module_name}: {dico['success']}/{dico['total']} ({s_result}) tests were passed") + else: + print(f"\t- {module_name}: no tests were executed") + + +def run(tests_names=None, common_kwargs=None): + """ + main function, launch the tests function from a list of test modules names + :param tests_names: testers to execute + :param common_kwargs: kwargs for all modules' functions + :return: + """ + if common_kwargs is None: + common_kwargs = {} + test_modules = get_test_modules(tests_names) + + report = {} + for test_module in test_modules: + report[test_module.__name__] = run_test_module(test_module, common_kwargs) + + if len(report): + print_report(report) + else: + print("No report to print") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument('-t', '--test_modules', metavar="", required=False, + help='names of the test modules to be executed', type=str, nargs='+') + parser.add_argument('-v', '--verbose', metavar="", required=False, + help='verbose: the higher the more information are printed', type=int) + + args = parser.parse_args() + + testers_args = {'verbose': 0} + if args.verbose: + testers_args['verbose'] = args.verbose + + run(args.test_modules, testers_args) diff --git a/tests/print_utils.py b/tests/print_utils.py new file mode 100644 index 0000000..7f4115f --- /dev/null +++ b/tests/print_utils.py @@ -0,0 +1,80 @@ +import os +import platform +from enum import Enum +from collections.abc import Iterable + +if platform.system() == "Windows": + os.system('color') # enable colors in consoles + + +class ColorEnum(Enum): + """ + Class to store different values to get colors in a console print + """ + HEADER = '\033[95m' + OKBLUE = '\033[94m' + OKGREEN = '\033[92m' + WARNING = '\033[93m' + FAIL = '\033[91m' + ENDC = '\033[0m' + BOLD = '\033[1m' + UNDERLINE = '\033[4m' + + +def color_string(s: str, color: ColorEnum): + return color.value + s + ColorEnum.ENDC.value + + +def s_to_warning(message: str): + return color_string(f"WARNING: {message}", ColorEnum.WARNING) + + +def s_to_fail(message: str): + return color_string(message, ColorEnum.FAIL) + + +def s_to_pass(message: str): + return color_string(message, ColorEnum.OKGREEN) + + +def get_blue_sep(n: int = 70): + return color_string(n * '-', ColorEnum.OKBLUE) + + +def get_blue_title(tester_name: str, method_name: str, n: int = 10): + message = f"{n * '#'} {tester_name}.{method_name} {n * '#'}" + return color_string(message, ColorEnum.OKBLUE) + + +def string_status(instance): + """ + transform all the attributes and the values of the instance of a class in a string + :param instance: + :return: None + """ + if isinstance(instance, Iterable) and not isinstance(instance, str): + s = "[\n" + "\n".join([string_status(e) for e in instance]) + "\n]" + else: + if '__class__' not in dir(instance): + s = str(instance) + else: + attribute_names = [m for m in dir(instance) if not m.startswith('__') and not callable(getattr(instance, m))] + if len(attribute_names): + s = "" + for attribute_name in attribute_names: + attribute = getattr(instance, attribute_name) + if isinstance(attribute, Enum): + attribute = attribute.name + s = s + f"{attribute_name} -> {attribute}\n" + else: + s = str(instance) + return s + + +def print_status(instance): + """ + print all the attributes and the values of the instance of a class + :param instance: + :return: None + """ + print(string_status(instance)) From 01262d55d8ae4e1d15bbf5f4bbceda1e0045dc6e Mon Sep 17 00:00:00 2001 From: EtWn <34377743+EtWnn@users.noreply.github.com> Date: Mon, 8 Mar 2021 17:02:50 +0100 Subject: [PATCH 02/17] Database (#2) * add LoggerGenerator and credentials manager * add Tables and DataBase class * generalize fetching in database * add test_DataBase --- src/storage/DataBase.py | 157 +++++++++++++++++++++++++++++++++++ src/storage/tables.py | 20 +++++ src/utils/LoggerGenerator.py | 36 ++++++++ src/utils/credentials.py | 23 +++++ src/utils/time_utils.py | 24 ++++++ tests/__init__.py | 1 + tests/test_DataBase.py | 42 ++++++++++ 7 files changed, 303 insertions(+) create mode 100644 src/storage/DataBase.py create mode 100644 src/storage/tables.py create mode 100644 src/utils/LoggerGenerator.py create mode 100644 src/utils/credentials.py create mode 100644 src/utils/time_utils.py create mode 100644 tests/test_DataBase.py diff --git a/src/storage/DataBase.py b/src/storage/DataBase.py new file mode 100644 index 0000000..c704f74 --- /dev/null +++ b/src/storage/DataBase.py @@ -0,0 +1,157 @@ +import sys +import os +from enum import Enum +from typing import List, Tuple, Optional, Any +import sqlite3 + +from src.storage.tables import Table +from src.utils.LoggerGenerator import LoggerGenerator + + +class SQLConditionEnum(Enum): + equal = '=' + greater_equal = '>=' + greater = '>' + lower = '<' + lower_equal = '<=' + + +class DataBase: + """ + This class will be used to interact with sqlite3 databases without having to generates sqlite commands + """ + + def __init__(self, name: str): + self.name = name + self.logger = LoggerGenerator.get_logger(self.name) + self.save_path = f"data/{name}.db" + self.db_conn = sqlite3.connect(self.save_path) + self.db_cursor = self.db_conn.cursor() + + def _fetch_rows(self, execution_cmd: str): + """ + execute a command to fetch some rows and return them + :param execution_cmd: the command to execute + :return: + """ + rows = [] + try: + self.db_cursor.execute(execution_cmd) + except sqlite3.OperationalError: + return rows + while True: + row = self.db_cursor.fetchone() + if row is None: + break + rows.append(row) + return rows + + def get_row_by_key(self, table: Table, key_value) -> Optional[Tuple]: + """ + get the row identified by a primary key value from a table + :param table: table to fetch the row from + :param key_value: key value of the row + :return: None or the row of value + """ + conditions_list = [(table.columns_names[0], SQLConditionEnum.equal, key_value)] + rows = self.get_conditions_rows(table, conditions_list) + if len(rows): + return rows[0] + + def get_conditions_rows(self, table: Table, + conditions_list: Optional[List[Tuple[str, SQLConditionEnum, Any]]] = None) -> List: + if conditions_list is None: + conditions_list = [] + execution_cmd = f"SELECT * from {table.name}" + execution_cmd = self._add_conditions(execution_cmd, conditions_list) + return self._fetch_rows(execution_cmd) + + def get_all_rows(self, table: Table) -> List: + return self.get_conditions_rows(table) + + def add_row(self, table: Table, row: Tuple, auto_commit: bool = True, update_if_exists: bool = False): + row_s = ", ".join(f"'{v}'" for v in row) + row_s = f'({row_s})' + execution_order = f"INSERT INTO {table.name} VALUES {row_s}" + try: + self.db_cursor.execute(execution_order) + if auto_commit: + self.commit() + except sqlite3.OperationalError: + self.create_table(table) + self.db_cursor.execute(execution_order) + if auto_commit: + self.commit() + except sqlite3.IntegrityError as err: + if update_if_exists: + self.update_row(table, row, auto_commit) + else: + raise err + + def add_rows(self, table: Table, rows: List[Tuple], auto_commit: bool = True, update_if_exists: bool = False): + for row in rows: + self.add_row(table, row, auto_commit=False, update_if_exists=update_if_exists) + if auto_commit: + self.commit() + + def update_row(self, table: Table, row: Tuple, auto_commit=True): + row_s = ", ".join(f"{n} = {v}" for n, v in zip(table.columns_names[1:], row[1:])) + execution_order = f"UPDATE {table.name} SET {row_s} WHERE {table.columns_names[0]} = {row[0]}" + self.db_cursor.execute(execution_order) + if auto_commit: + self.commit() + + def create_table(self, table: Table): + """ + create a table in the database + :param table: Table instance with the config of the table to create + :return: + """ + create_cmd = self.get_create_cmd(table) + self.db_cursor.execute(create_cmd) + self.db_conn.commit() + + def drop_table(self, table: Table): + """ + delete a table from the database + :param table: Table instance with the config of the table to drop + :return: + """ + execution_order = f"DROP TABLE IF EXISTS {table.name}" + self.db_cursor.execute(execution_order) + self.db_conn.commit() + + def commit(self): + """ + submit and save the database state + :return: + """ + self.db_conn.commit() + + @staticmethod + def _add_conditions(execution_cmd: str, conditions_list: List[Tuple[str, SQLConditionEnum, Any]]): + """ + add a list of condition to an SQL command + :param execution_cmd: string with 'WHERE' statement + :param conditions_list: + :return: + """ + if len(conditions_list): + add_cmd = ' WHERE' + for column_name, condition, value in conditions_list: + add_cmd = add_cmd + f" {column_name} {condition.value} '{value}' AND" + return execution_cmd + add_cmd[:-4] + else: + return execution_cmd + + @staticmethod + def get_create_cmd(table: Table): + """ + return the command in string format to create a table in the database + :param table: Table instance with the config if the table to create + :return: execution command for the table creation + """ + cmd = f"[{table.columns_names[0]}] {table.columns_sql_types[0]} PRIMARY KEY, " + for arg_name, arg_type in zip(table.columns_names[1:], table.columns_sql_types[1:]): + cmd = cmd + f"[{arg_name}] {arg_type}, " + return f"CREATE TABLE {table.name}\n({cmd[:-2]})" diff --git a/src/storage/tables.py b/src/storage/tables.py new file mode 100644 index 0000000..991a1d1 --- /dev/null +++ b/src/storage/tables.py @@ -0,0 +1,20 @@ +from dataclasses import dataclass +from typing import List + + +@dataclass +class Table: + name: str + columns_names: List[str] + columns_sql_types: List[str] + + +SpotTradeTable = Table( + 'spot_trade', + [ + + ], + [ + + ] +) diff --git a/src/utils/LoggerGenerator.py b/src/utils/LoggerGenerator.py new file mode 100644 index 0000000..5d91774 --- /dev/null +++ b/src/utils/LoggerGenerator.py @@ -0,0 +1,36 @@ +import logging + + +class LoggerGenerator: + logger_count = 0 + global_log_level = logging.WARNING + + @staticmethod + def get_logger(logger_name, create_file=False, log_level=None): + if log_level is None: + log_level = LoggerGenerator.global_log_level + + # create logger for prd_ci + log = logging.getLogger(f"lg_{LoggerGenerator.logger_count}_{logger_name}") + log.setLevel(level=log_level) + LoggerGenerator.logger_count += 1 + + # create formatter and add it to the handlers + log_format = '[%(asctime)s %(name)s %(levelname)s] %(message)s [%(pathname)s:%(lineno)d in %(funcName)s]' + formatter = logging.Formatter(log_format) + + if create_file: + # create file handler for logger. + fh = logging.FileHandler('SPOT.log') + fh.setLevel(level=log_level) + fh.setFormatter(formatter) + log.addHandler(fh) + + # create console handler for logger. + ch = logging.StreamHandler() + ch.setLevel(level=log_level) + ch.setFormatter(formatter) + log.addHandler(ch) + + return log + diff --git a/src/utils/credentials.py b/src/utils/credentials.py new file mode 100644 index 0000000..cc980e8 --- /dev/null +++ b/src/utils/credentials.py @@ -0,0 +1,23 @@ +import json +from typing import Optional, Dict + +from src.utils.LoggerGenerator import LoggerGenerator + + +class CredentialManager: + + logger = LoggerGenerator.get_logger("crendentials_manager") + + @staticmethod + def get_api_credentials(api_name) -> Optional[Dict]: + try: + with open("data/credentials.json") as file: + credentials = json.load(file) + return credentials[api_name] + except FileNotFoundError as ex: + CredentialManager.logger.error("Could not find the credentials file") + raise ex + except KeyError as ex: + CredentialManager.logger.error(f"there is no key registered in the credentials file " + f"the name {api_name}") + raise ex diff --git a/src/utils/time_utils.py b/src/utils/time_utils.py new file mode 100644 index 0000000..c91512e --- /dev/null +++ b/src/utils/time_utils.py @@ -0,0 +1,24 @@ +import datetime + + +def millistamp_to_round_min(millitstamp: int): + return millitstamp - millitstamp % 60000 + + +def millistamp_to_upper_min(millitstamp: int): + over_min = millitstamp % 60000 + if over_min: + return millitstamp - over_min + 60000 + return millitstamp + + +def datetime_to_round_min(date_time: datetime.datetime): + return date_time - datetime.timedelta(seconds=date_time.second, microseconds=date_time.microsecond) + + +def millistamp_to_datetime(millitstamp: int) -> datetime.datetime: + return datetime.datetime.fromtimestamp(millitstamp / 1000, tz=datetime.timezone.utc) + + +def datetime_to_millistamp(date_time: datetime.datetime) -> int: + return int(1000 * date_time.timestamp()) diff --git a/tests/__init__.py b/tests/__init__.py index dab199c..c638c87 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -2,3 +2,4 @@ modules needs to be imported here to be detected in the __main__ file for auto import. If a module is not imported here, it won't be tested by the tests package """ +import tests.test_DataBase diff --git a/tests/test_DataBase.py b/tests/test_DataBase.py new file mode 100644 index 0000000..b1e57b9 --- /dev/null +++ b/tests/test_DataBase.py @@ -0,0 +1,42 @@ +from src.storage.DataBase import DataBase, SQLConditionEnum +from src.storage.tables import Table + +table = Table( + 'test_table', + [ + 'Key', + 'age', + 'name', + 'weight' + ], + [ + 'INTEGER', + 'INTEGER', + 'TEXT', + 'REAL' + ] +) + + +def test_inserts_search(verbose=0, **kwargs): + db = DataBase("test_table") + db.drop_table(table) + rows = [ + (1, 15, 'Karl', 55.5), + (2, 18, 'Kitty', 61.1), + (3, 18, 'Marc', 48.1), + (8, 55, 'Jean', 78.1) + ] + db.add_rows(table, rows) + + retrieved_row = db.get_row_by_key(table, 1) + if verbose: + print(f"retrieved the row: {retrieved_row} when looking for the row: {rows[0]}") + assert rows[0] == retrieved_row + + conditions = [ + (table.columns_names[1], SQLConditionEnum.equal, 18), + (table.columns_names[3], SQLConditionEnum.greater_equal, 55), + ] + retrieved_rows = db.get_conditions_rows(table, conditions) + assert rows[1:2] == retrieved_rows From d2d9b16c41ceb8fb820914bc2b2c63e9068ad7c5 Mon Sep 17 00:00:00 2001 From: EtWn <34377743+EtWnn@users.noreply.github.com> Date: Wed, 10 Mar 2021 11:42:31 +0100 Subject: [PATCH 03/17] Binance spot (#3) * add LoggerGenerator and credentials manager * add Tables and DataBase class * generalize fetching in database * add test_DataBase * add binance spot table and Binance database * add selection for SQL request * add unique key for trade * remove Binance from tables name * add deposits in the database, elaborate comments * add BinanceManager * specify latest python-binance version * add dividend to the database * add get_last_spot_dividend_time * add update_spot_dividends * add spot dusts * add lending interests * add drop_lending_interest_table and generalize drop_all_tables --- environment.yml | 2 +- src/storage/BinanceDataBase.py | 463 +++++++++++++++++++++++++++++++++ src/storage/BinanceManager.py | 313 ++++++++++++++++++++++ src/storage/DataBase.py | 17 +- src/storage/tables.py | 112 +++++++- tests/test_DataBase.py | 5 +- 6 files changed, 905 insertions(+), 7 deletions(-) create mode 100644 src/storage/BinanceDataBase.py create mode 100644 src/storage/BinanceManager.py diff --git a/environment.yml b/environment.yml index 7eb9581..5b2d958 100644 --- a/environment.yml +++ b/environment.yml @@ -9,5 +9,5 @@ dependencies: - pip=20.3.3 - pip: - dateparser - - python-binance==0.7.9 + - python-binance - requests diff --git a/src/storage/BinanceDataBase.py b/src/storage/BinanceDataBase.py new file mode 100644 index 0000000..0616ed8 --- /dev/null +++ b/src/storage/BinanceDataBase.py @@ -0,0 +1,463 @@ +import datetime +from typing import Optional + +from src.storage.DataBase import DataBase, SQLConditionEnum +from src.storage import tables +from src.utils.time_utils import datetime_to_millistamp + + +class BinanceDataBase(DataBase): + """ + Handles the recording of the binance account in a local database + """ + + def __init__(self, name: str = 'binance_db'): + super().__init__(name) + + def add_lending_interest(self, int_id: str, time: int, lending_type: str, asset: str, amount: float, + auto_commit: bool = True): + """ + add an lending interest to the database + + :param int_id: if for the interest + :type int_id: str + :param time: millitstamp of the operation + :type time: int + :param lending_type: either 'DAILY', 'ACTIVITY' or 'CUSTOMIZED_FIXED' + :type lending_type: str + :param asset: asset that got converted to BNB + :type asset: str + :param amount: amount of asset received + :type amount: float + :param auto_commit: if the database should commit the change made, default True + :type auto_commit: bool + :return: None + :rtype: None + """ + row = (int_id, time, lending_type, asset, amount) + self.add_row(tables.LENDING_INTEREST_TABLE, row, auto_commit=auto_commit) + + def get_lending_interests(self, lending_type: Optional[str] = None, asset: Optional[str] = None, + start_time: Optional[int] = None, end_time: Optional[int] = None): + """ + return lending interests stored in the database. Asset type and time filters can be used + + :param lending_type:fetch only interests from this lending type + :type lending_type: Optional[str] + :param asset: fetch only interests from this asset + :type asset: Optional[str] + :param start_time: fetch only interests after this millistamp + :type start_time: Optional[int] + :param end_time: fetch only interests before this millistamp + :type end_time: Optional[int] + :return: The raw rows selected as saved in the database + :rtype: List[Tuple] + """ + conditions_list = [] + if lending_type is not None: + conditions_list.append((tables.LENDING_INTEREST_TABLE.columns_names[2], + SQLConditionEnum.equal, + lending_type)) + if asset is not None: + conditions_list.append((tables.LENDING_INTEREST_TABLE.columns_names[3], + SQLConditionEnum.equal, + asset)) + if start_time is not None: + conditions_list.append((tables.LENDING_INTEREST_TABLE.columns_names[1], + SQLConditionEnum.greater_equal, + start_time)) + if end_time is not None: + conditions_list.append((tables.LENDING_INTEREST_TABLE.columns_names[1], + SQLConditionEnum.lower, + end_time)) + return self.get_conditions_rows(tables.LENDING_INTEREST_TABLE, conditions_list=conditions_list) + + def get_last_lending_interest_time(self, lending_type: Optional[str] = None): + """ + return the latest time when an interest was recieved. + If None, return the millistamp corresponding to 2017/01/01 + + :param lending_type: type of lending + :type lending_type: str + :return: millistamp + :rtype: int + """ + conditions_list = [] + if lending_type is not None: + conditions_list.append((tables.LENDING_INTEREST_TABLE.columns_names[2], + SQLConditionEnum.equal, + lending_type)) + selection = f"MAX({tables.LENDING_INTEREST_TABLE.columns_names[1]})" + result = self.get_conditions_rows(tables.LENDING_INTEREST_TABLE, + selection=selection, + conditions_list=conditions_list) + default = datetime_to_millistamp(datetime.datetime(2017, 1, 1, tzinfo=datetime.timezone.utc)) + try: + result = result[0][0] + except IndexError: + return default + if result is None: + return default + return result + + def add_dust(self, dust_id: str, time: int, asset: str, asset_amount: float, bnb_amount: float, bnb_fee: float, + auto_commit: bool = True): + """ + add dust operation to the database + + :param dust_id: id of the operation + :type dust_id: str + :param time: millitstamp of the operation + :type time: int + :param asset: asset that got converted to BNB + :type asset: str + :param asset_amount: amount of asset that got converted + :type asset_amount: float + :param bnb_amount: amount received from the conversion + :type bnb_amount: float + :param bnb_fee: fee amount in BNB + :type bnb_fee: float + :param auto_commit: if the database should commit the change made, default True + :type auto_commit: bool + :return: None + :rtype: None + """ + + row = (dust_id, time, asset, asset_amount, bnb_amount, bnb_fee) + self.add_row(tables.SPOT_DUST_TABLE, row, auto_commit=auto_commit) + + def get_spot_dusts(self, asset: Optional[str] = None, start_time: Optional[int] = None, + end_time: Optional[int] = None): + """ + return dusts stored in the database. Asset type and time filters can be used + + :param asset: fetch only dusts from this asset + :type asset: Optional[str] + :param start_time: fetch only dusts after this millistamp + :type start_time: Optional[int] + :param end_time: fetch only dusts before this millistamp + :type end_time: Optional[int] + :return: The raw rows selected as saved in the database + :rtype: List[Tuple] + """ + conditions_list = [] + if asset is not None: + conditions_list.append((tables.SPOT_DIVIDEND_TABLE.columns_names[2], + SQLConditionEnum.equal, + asset)) + if start_time is not None: + conditions_list.append((tables.SPOT_DIVIDEND_TABLE.columns_names[1], + SQLConditionEnum.greater_equal, + start_time)) + if end_time is not None: + conditions_list.append((tables.SPOT_DIVIDEND_TABLE.columns_names[1], + SQLConditionEnum.lower, + end_time)) + return self.get_conditions_rows(tables.SPOT_DUST_TABLE, conditions_list=conditions_list) + + def add_dividend(self, div_id: int, div_time: int, asset: str, amount: float, auto_commit: bool = True): + """ + add a dividend to the database + + :param div_id: dividend id + :type div_id: int + :param div_time: millistamp of dividend reception + :type div_time: int + :param asset: name of the dividend unit + :type asset: str + :param amount: amount of asset distributed + :type amount: float + :param auto_commit: if the database should commit the change made, default True + :type auto_commit: bool + :return: None + :rtype: None + """ + row = (div_id, div_time, asset, amount) + self.add_row(tables.SPOT_DIVIDEND_TABLE, row, auto_commit=auto_commit) + + def get_spot_dividends(self, asset: Optional[str] = None, start_time: Optional[int] = None, + end_time: Optional[int] = None): + """ + return dividends stored in the database. Asset type and time filters can be used + + :param asset: fetch only dividends of this asset + :type asset: Optional[str] + :param start_time: fetch only dividends after this millistamp + :type start_time: Optional[int] + :param end_time: fetch only dividends before this millistamp + :type end_time: Optional[int] + :return: The raw rows selected as saved in the database + :rtype: List[Tuple] + """ + conditions_list = [] + if asset is not None: + conditions_list.append((tables.SPOT_DIVIDEND_TABLE.columns_names[2], + SQLConditionEnum.equal, + asset)) + if start_time is not None: + conditions_list.append((tables.SPOT_DIVIDEND_TABLE.columns_names[1], + SQLConditionEnum.greater_equal, + start_time)) + if end_time is not None: + conditions_list.append((tables.SPOT_DIVIDEND_TABLE.columns_names[1], + SQLConditionEnum.lower, + end_time)) + return self.get_conditions_rows(tables.SPOT_DIVIDEND_TABLE, conditions_list=conditions_list) + + def get_last_spot_dividend_time(self) -> int: + """ + fetch the latest time a dividend has been distributed on the spot account. If None is found, + return the millistamp corresponding to 2017/1/1 + + :return: + """ + selection = f"MAX({tables.SPOT_DIVIDEND_TABLE.columns_names[1]})" + result = self.get_conditions_rows(tables.SPOT_WITHDRAW_TABLE, + selection=selection) + default = datetime_to_millistamp(datetime.datetime(2017, 1, 1, tzinfo=datetime.timezone.utc)) + try: + result = result[0][0] + except IndexError: + return default + if result is None: + return default + return result + + def add_withdraw(self, withdraw_id: str, tx_id: str, apply_time: int, asset: str, amount: float, fee: float, + auto_commit: bool = True): + """ + add a withdraw to the database + + :param withdraw_id: binance if of the withdraw + :type withdraw_id: str + :param tx_id: transaction id + :type tx_id: str + :param apply_time: millistamp when the withdraw was requested + :type apply_time: int + :param asset: name of the token + :type asset: str + :param amount: amount of token withdrawn + :type amount: float + :param fee: amount of the asset paid for the withdraw + :type fee: float + :param auto_commit: if the database should commit the change made, default True + :type auto_commit: bool + :return: None + :rtype: None + """ + row = (withdraw_id, tx_id, apply_time, asset, amount, fee) + self.add_row(tables.SPOT_WITHDRAW_TABLE, row, auto_commit=auto_commit) + + def get_spot_withdraws(self, asset: Optional[str] = None, start_time: Optional[int] = None, + end_time: Optional[int] = None): + """ + return withdraws stored in the database. Asset type and time filters can be used + + :param asset: fetch only withdraws of this asset + :type asset: Optional[str] + :param start_time: fetch only withdraws after this millistamp + :type start_time: Optional[int] + :param end_time: fetch only withdraws before this millistamp + :type end_time: Optional[int] + :return: The raw rows selected as saved in the database + :rtype: List[Tuple] + """ + conditions_list = [] + if asset is not None: + conditions_list.append((tables.SPOT_WITHDRAW_TABLE.columns_names[3], + SQLConditionEnum.equal, + asset)) + if start_time is not None: + conditions_list.append((tables.SPOT_WITHDRAW_TABLE.columns_names[2], + SQLConditionEnum.greater_equal, + start_time)) + if end_time is not None: + conditions_list.append((tables.SPOT_WITHDRAW_TABLE.columns_names[2], + SQLConditionEnum.lower, + end_time)) + return self.get_conditions_rows(tables.SPOT_WITHDRAW_TABLE, conditions_list=conditions_list) + + def get_last_spot_withdraw_time(self) -> int: + """ + fetch the latest time a withdraw has been made on the spot account. If None is found, return the millistamp + corresponding to 2017/1/1 + + :return: + """ + selection = f"MAX({tables.SPOT_WITHDRAW_TABLE.columns_names[2]})" + result = self.get_conditions_rows(tables.SPOT_WITHDRAW_TABLE, + selection=selection) + default = datetime_to_millistamp(datetime.datetime(2017, 1, 1, tzinfo=datetime.timezone.utc)) + try: + result = result[0][0] + except IndexError: + return default + if result is None: + return default + return result + + def add_deposit(self, tx_id: str, insert_time: int, amount: float, asset: str, auto_commit=True): + """ + add a deposit to the database + + :param tx_id: transaction id + :type tx_id: str + :param insert_time: millistamp when the deposit arrived on binance + :type insert_time: int + :param amount: amount of token deposited + :type amount: float + :param asset: name of the token + :type asset: str + :param auto_commit: if the database should commit the change made, default True + :type auto_commit: bool + :return: None + :rtype: None + """ + row = (tx_id, insert_time, asset, amount) + self.add_row(tables.SPOT_DEPOSIT_TABLE, row, auto_commit) + + def get_spot_deposits(self, asset: Optional[str] = None, start_time: Optional[int] = None, + end_time: Optional[int] = None): + """ + return deposits stored in the database. Asset type and time filters can be used + + :param asset: fetch only deposits of this asset + :type asset: Optional[str] + :param start_time: fetch only deposits after this millistamp + :type start_time: Optional[int] + :param end_time: fetch only deposits before this millistamp + :type end_time: Optional[int] + :return: The raw rows selected as saved in the database + :rtype: List[Tuple] + """ + conditions_list = [] + if asset is not None: + conditions_list.append((tables.SPOT_DEPOSIT_TABLE.columns_names[2], + SQLConditionEnum.equal, + asset)) + if start_time is not None: + conditions_list.append((tables.SPOT_DEPOSIT_TABLE.columns_names[1], + SQLConditionEnum.greater_equal, + start_time)) + if end_time is not None: + conditions_list.append((tables.SPOT_DEPOSIT_TABLE.columns_names[1], + SQLConditionEnum.lower, + end_time)) + return self.get_conditions_rows(tables.SPOT_DEPOSIT_TABLE, conditions_list=conditions_list) + + def get_last_spot_deposit_time(self) -> int: + """ + fetch the latest time a deposit has been made on the spot account. If None is found, return the millistamp + corresponding to 2017/1/1 + + :return: + """ + selection = f"MAX({tables.SPOT_DEPOSIT_TABLE.columns_names[1]})" + result = self.get_conditions_rows(tables.SPOT_DEPOSIT_TABLE, + selection=selection) + default = datetime_to_millistamp(datetime.datetime(2017, 1, 1, tzinfo=datetime.timezone.utc)) + try: + result = result[0][0] + except IndexError: + return default + if result is None: + return default + return result + + def add_spot_trade(self, trade_id: int, millistamp: int, asset: str, ref_asset: str, qty: float, price: float, + fee: float, fee_asset: str, is_buyer: bool, auto_commit=True): + """ + add a trade to the database + + :param trade_id: id of the trade (binance id, unique per trading pair) + :type trade_id: int + :param millistamp: millistamp of the trade + :type millistamp: int + :param asset: name of the asset in the trading pair (ex 'BTC' for 'BTCUSDT') + :type asset: string + :param ref_asset: name of the reference asset in the trading pair (ex 'USDT' for 'BTCUSDT') + :type ref_asset: string + :param qty: quantity of asset exchanged + :type qty: float + :param price: price of the asset regarding the ref_asset + :type price: float + :param fee: amount kept by the exchange + :type fee: float + :param fee_asset:token unit for the fee + :type fee_asset: str + :param is_buyer: if the trade is a buy or a sell + :type is_buyer: bool + :param auto_commit: if the database should commit the change made, default True + :type auto_commit: bool + :return: None + :rtype: None + """ + key = f'{asset}{ref_asset}{trade_id}' + row = (key, trade_id, millistamp, asset, ref_asset, qty, price, fee, fee_asset, int(is_buyer)) + self.add_row(tables.SPOT_TRADE_TABLE, row, auto_commit) + + def get_spot_trades(self, start_time: Optional[int] = None, end_time: Optional[int] = None, + asset: Optional[str] = None, ref_asset: Optional[str] = None): + """ + return trades stored in the database. asset type, ref_asset type and time filters can be used + + :param start_time: fetch only trades after this millistamp + :type start_time: Optional[int] + :param end_time: fetch only trades before this millistamp + :type end_time: Optional[int] + :param asset: fetch only trades with this asset + :type asset: Optional[str] + :param ref_asset: fetch only trades with this ref_asset + :type ref_asset: Optional[str] + :return: The raw rows selected as saved in the database + :rtype: List[Tuple] + """ + conditions_list = [] + if start_time is not None: + conditions_list.append((tables.SPOT_TRADE_TABLE.columns_names[2], + SQLConditionEnum.greater_equal, + start_time)) + if end_time is not None: + conditions_list.append((tables.SPOT_TRADE_TABLE.columns_names[2], + SQLConditionEnum.lower, + end_time)) + if asset is not None: + conditions_list.append((tables.SPOT_TRADE_TABLE.columns_names[3], + SQLConditionEnum.equal, + asset)) + if ref_asset is not None: + conditions_list.append((tables.SPOT_TRADE_TABLE.columns_names[4], + SQLConditionEnum.equal, + ref_asset)) + return self.get_conditions_rows(tables.SPOT_TRADE_TABLE, conditions_list=conditions_list) + + def get_max_trade_id(self, asset: str, ref_asset: str) -> int: + """ + return the latest trade id for a trading pair. If none is found, return -1 + + :param asset: name of the asset in the trading pair (ex 'BTC' for 'BTCUSDT') + :type asset: string + :param ref_asset: name of the reference asset in the trading pair (ex 'USDT' for 'BTCUSDT') + :type ref_asset: string + :return: latest trade id + :rtype: int + """ + selection = f"MAX({tables.SPOT_TRADE_TABLE.columns_names[1]})" + conditions_list = [ + (tables.SPOT_TRADE_TABLE.columns_names[3], + SQLConditionEnum.equal, + asset), + (tables.SPOT_TRADE_TABLE.columns_names[4], + SQLConditionEnum.equal, + ref_asset) + ] + result = self.get_conditions_rows(tables.SPOT_TRADE_TABLE, + selection=selection, + conditions_list=conditions_list) + try: + result = result[0][0] + except IndexError: + return -1 + if result is None: + return -1 + return result diff --git a/src/storage/BinanceManager.py b/src/storage/BinanceManager.py new file mode 100644 index 0000000..69097aa --- /dev/null +++ b/src/storage/BinanceManager.py @@ -0,0 +1,313 @@ +import datetime +import math + +import dateparser +from binance.client import Client +from tqdm import tqdm + +from src.utils.time_utils import datetime_to_millistamp +from src.storage.BinanceDataBase import BinanceDataBase +from src.utils.credentials import CredentialManager +from src.storage import tables + + +class BinanceManager: + """ + This class is in charge of filling the database by calling the binance API + """ + + def __init__(self): + self.db = BinanceDataBase() + credentials = CredentialManager.get_api_credentials("Binance") + self.client = Client(**credentials) + + def update_lending_interests(self): + """ + update the lending interests database. + for each update + + :return: None + :rtype: None + """ + lending_types = ['DAILY', 'ACTIVITY', 'CUSTOMIZED_FIXED'] + pbar = tqdm(total=3) + for lending_type in lending_types: + pbar.set_description(f"fetching lending type {lending_type}") + latest_time = self.db.get_last_lending_interest_time(lending_type=lending_type) + 3600 * 1000 # add 1 hour + current = 1 + while True: + lending_interests = self.client.get_lending_interest_history(lendingType=lending_type, + startTime=latest_time, + current=current, + limit=100) + for li in lending_interests: + print(li) + self.db.add_lending_interest(int_id=str(li['time']) + li['asset'] + li['lendingType'], + time=li['time'], + lending_type=li['lendingType'], + asset=li['asset'], + amount=li['interest'] + ) + + if lending_interests: + current += 1 # next page + self.db.commit() + else: + break + pbar.update() + pbar.close() + + def update_spot_dusts(self): + """ + update the dust database. As there is no way to get the dust by id or timeframe, the table is cleared + for each update + + :return: None + :rtype: None + """ + self.drop_dust_table() + + result = self.client.get_dust_log() + dusts = result['results'] + pbar = tqdm(total=dusts['total']) + pbar.set_description("fetching dusts") + for d in dusts['rows']: + for sub_dust in d['logs']: + date_time = dateparser.parse(sub_dust['operateTime'] + 'Z') + self.db.add_dust(dust_id=str(sub_dust['tranId']) + sub_dust['fromAsset'], + time=datetime_to_millistamp(date_time), + asset=sub_dust['fromAsset'], + asset_amount=sub_dust['amount'], + bnb_amount=sub_dust['transferedAmount'], + bnb_fee=sub_dust['serviceChargeAmount'], + auto_commit=False + ) + pbar.update() + self.db.commit() + pbar.close() + + def update_spot_dividends(self, day_jump: float = 90, limit: int = 500): + limit = min(500, limit) + delta_jump = min(day_jump, 90) * 24 * 3600 * 1000 + start_time = self.db.get_last_spot_dividend_time() + 1 + now_millistamp = datetime_to_millistamp(datetime.datetime.now(tz=datetime.timezone.utc)) + pbar = tqdm(total=math.ceil((now_millistamp - start_time) / delta_jump)) + pbar.set_description("fetching spot dividends") + while start_time < now_millistamp: + params = { + 'startTime': start_time, + 'endTime': start_time + delta_jump, + 'limit': limit + } + # the stable working version of client.get_asset_dividend_history is not released yet, + # for now it has a post error, so this protected member is used in the meantime + result = self.client._request_margin_api('get', + 'asset/assetDividend', + True, + data=params + ) + dividends = result['rows'] + for div in dividends: + self.db.add_dividend(div_id=int(div['tranId']), + div_time=int(div['divTime']), + asset=div['asset'], + amount=float(div['amount']), + auto_commit=False + ) + pbar.update() + if len(dividends) < limit: + start_time += delta_jump + else: # limit was reached before the end of the time windows + start_time = int(dividends[0]['divTime']) + 1 + if len(dividends): + self.db.commit() + pbar.close() + + def update_spot_withdraws(self, day_jump: float = 90): + """ + This fetch the crypto withdraws made on the spot account from the last withdraw time in the database to now. + It is done with multiple call, each having a time window of day_jump days. + The withdraws are then saved in the database. + Only successful withdraws are fetched. + + :param day_jump: length of the time window for each call (max 90) + :type day_jump: float + :return: None + :rtype: None + """ + delta_jump = min(day_jump, 90) * 24 * 3600 * 1000 + start_time = self.db.get_last_spot_withdraw_time() + 1 + now_millistamp = datetime_to_millistamp(datetime.datetime.now(tz=datetime.timezone.utc)) + pbar = tqdm(total=math.ceil((now_millistamp - start_time) / delta_jump)) + pbar.set_description("fetching spot withdraws") + while start_time < now_millistamp: + result = self.client.get_withdraw_history(startTime=start_time, endTime=start_time + delta_jump, status=6) + withdraws = result['withdrawList'] + for withdraw in withdraws: + self.db.add_withdraw(withdraw_id=withdraw['id'], + tx_id=withdraw['txId'], + apply_time=int(withdraw['applyTime']), + asset=withdraw['asset'], + amount=float(withdraw['amount']), + fee=float(withdraw['transactionFee']), + auto_commit=False + ) + pbar.update() + start_time += delta_jump + if len(withdraws): + self.db.commit() + pbar.close() + + def update_spot_deposits(self, day_jump: float = 90): + """ + This fetch the crypto deposit made on the spot account from the last deposit time in the database to now. + It is done with multiple call, each having a time window of day_jump days. + The deposits are then saved in the database. + Only successful deposits are fetched. + + :param day_jump: length of the time window for each call (max 90) + :type day_jump: float + :return: None + :rtype: None + """ + delta_jump = min(day_jump, 90) * 24 * 3600 * 1000 + start_time = self.db.get_last_spot_deposit_time() + 1 + now_millistamp = datetime_to_millistamp(datetime.datetime.now(tz=datetime.timezone.utc)) + pbar = tqdm(total=math.ceil((now_millistamp - start_time) / delta_jump)) + pbar.set_description("fetching spot deposits") + while start_time < now_millistamp: + result = self.client.get_deposit_history(startTime=start_time, endTime=start_time + delta_jump, status=1) + deposits = result['depositList'] + for deposit in deposits: + self.db.add_deposit(tx_id=deposit['txId'], + asset=deposit['asset'], + insert_time=int(deposit['insertTime']), + amount=float(deposit['amount']), + auto_commit=False) + pbar.update() + start_time += delta_jump + if len(deposits): + self.db.commit() + pbar.close() + + def update_spot_symbol_trades(self, asset: str, ref_asset: str, limit: int = 1000): + """ + This update the spot trades in the database for a single trading pair. It will check the last trade id and will + requests the all trades after this trade_id. + + :param asset: name of the asset in the trading pair (ex 'BTC' for 'BTCUSDT') + :type asset: string + :param ref_asset: name of the reference asset in the trading pair (ex 'USDT' for 'BTCUSDT') + :type ref_asset: string + :param limit: max size of each trade requests + :type limit: int + :return: None + :rtype: None + """ + limit = min(1000, limit) + symbol = asset + ref_asset + last_trade_id = self.db.get_max_trade_id(asset, ref_asset) + while True: + new_trades = self.client.get_my_trades(symbol=symbol, fromId=last_trade_id + 1, limit=limit) + for trade in new_trades: + self.db.add_spot_trade(trade_id=int(trade['id']), + millistamp=int(trade['time']), + asset=asset, + ref_asset=ref_asset, + qty=float(trade['qty']), + price=float(trade['price']), + fee=float(trade['commission']), + fee_asset=trade['commissionAsset'], + is_buyer=trade['isBuyer'], + auto_commit=False + ) + last_trade_id = max(last_trade_id, int(trade['id'])) + if len(new_trades): + self.db.commit() + if len(new_trades) < limit: + break + + def update_all_spot_trades(self, limit: int = 1000): + """ + This update the spot trades in the database for every trading pairs + + :param limit: max size of each trade requests + :type limit: int + :return: None + :rtype: None + """ + symbols_info = self.client.get_exchange_info()['symbols'] + pbar = tqdm(total=len(symbols_info)) + for symbol_info in symbols_info: + pbar.set_description(f"fetching {symbol_info['symbol']}") + self.update_spot_symbol_trades(asset=symbol_info['baseAsset'], + ref_asset=symbol_info['quoteAsset'], + limit=limit) + pbar.update() + pbar.close() + + def drop_spot_trade_table(self): + """ + erase the spot trades table + + :return: None + :rtype: None + """ + self.db.drop_table(tables.SPOT_TRADE_TABLE) + + def drop_spot_deposit_table(self): + """ + erase the spot deposits table + + :return: None + :rtype: None + """ + self.db.drop_table(tables.SPOT_DEPOSIT_TABLE) + + def drop_spot_withdraw_table(self): + """ + erase the spot withdraws table + + :return: None + :rtype: None + """ + self.db.drop_table(tables.SPOT_WITHDRAW_TABLE) + + def drop_spot_dividends_table(self): + """ + erase the spot dividends table + + :return: None + :rtype: None + """ + self.db.drop_table(tables.SPOT_DIVIDEND_TABLE) + + def drop_dust_table(self): + """ + erase the spot dust table + + :return: None + :rtype: None + """ + self.db.drop_table(tables.SPOT_DUST_TABLE) + + def drop_lending_interest_table(self): + """ + erase the lending interests + + :return: None + :rtype: None + """ + self.db.drop_table(tables.LENDING_INTEREST_TABLE) + + def drop_all_tables(self): + """ + erase all the tables of the database by calling all the methods having 'drop' and 'table' in their names + + :return: None + :rtype: None + """ + methods = [m for m in dir(self) if 'drop' in m and 'table' in m and callable(getattr(self, m))] + for m in methods: + if m != "drop_all_tables": + getattr(self, m)() diff --git a/src/storage/DataBase.py b/src/storage/DataBase.py index c704f74..16a69b9 100644 --- a/src/storage/DataBase.py +++ b/src/storage/DataBase.py @@ -1,7 +1,7 @@ import sys import os from enum import Enum -from typing import List, Tuple, Optional, Any +from typing import List, Tuple, Optional, Any, Union import sqlite3 from src.storage.tables import Table @@ -9,11 +9,15 @@ class SQLConditionEnum(Enum): + """ + https://www.techonthenet.com/sqlite/comparison_operators.php + """ equal = '=' greater_equal = '>=' greater = '>' lower = '<' lower_equal = '<=' + diff = '!=' class DataBase: @@ -54,16 +58,21 @@ def get_row_by_key(self, table: Table, key_value) -> Optional[Tuple]: :return: None or the row of value """ conditions_list = [(table.columns_names[0], SQLConditionEnum.equal, key_value)] - rows = self.get_conditions_rows(table, conditions_list) + rows = self.get_conditions_rows(table, conditions_list=conditions_list) if len(rows): return rows[0] def get_conditions_rows(self, table: Table, + selection: Optional[Union[str, List[str]]] = None, conditions_list: Optional[List[Tuple[str, SQLConditionEnum, Any]]] = None) -> List: + if selection is None: + selection = '*' + elif isinstance(selection, List): + selection = ','.join(selection) if conditions_list is None: conditions_list = [] - execution_cmd = f"SELECT * from {table.name}" - execution_cmd = self._add_conditions(execution_cmd, conditions_list) + execution_cmd = f"SELECT {selection} from {table.name}" + execution_cmd = self._add_conditions(execution_cmd, conditions_list=conditions_list) return self._fetch_rows(execution_cmd) def get_all_rows(self, table: Table) -> List: diff --git a/src/storage/tables.py b/src/storage/tables.py index 991a1d1..5e5dd55 100644 --- a/src/storage/tables.py +++ b/src/storage/tables.py @@ -9,12 +9,122 @@ class Table: columns_sql_types: List[str] -SpotTradeTable = Table( +SPOT_TRADE_TABLE = Table( 'spot_trade', [ + 'key', + 'id', + 'millistamp', + 'asset', + 'ref_asset', + 'qty', + 'price', + 'fee', + 'fee_asset', + 'isBuyer' ], [ + 'TEXT', + 'INTEGER', + 'INTEGER', + 'TEXT', + 'TEXT', + 'REAL', + 'REAL', + 'REAL', + 'TEXT', + 'INTEGER' + ] +) + +SPOT_DEPOSIT_TABLE = Table( + 'spot_deposit', + [ + 'txId', + 'insertTime', + 'asset', + 'amount', + ], + [ + 'TEXT', + 'INTEGER', + 'TEXT', + 'REAL' + ] +) + + +SPOT_WITHDRAW_TABLE = Table( + 'spot_withdraw', + [ + 'id', + 'txId', + 'applyTime', + 'asset', + 'amount', + 'fee' + ], + [ + 'TEXT', + 'TEXT', + 'INTEGER', + 'TEXT', + 'REAL', + 'REAL' + ] +) +SPOT_DIVIDEND_TABLE = Table( + 'spot_dividend_table', + [ + 'id', + 'divTime', + 'asset', + 'amount' + ], + [ + 'INTEGER', + 'INTEGER', + 'TEXT', + 'REAL' + ] +) + +SPOT_DUST_TABLE = Table( + 'spot_dust_table', + [ + 'id', + 'time', + 'asset', + 'asset_amount', + 'bnb_amount', + 'bnb_fee', + ], + [ + 'TEXT', + 'INTEGER', + 'TEXT', + 'REAL', + 'REAL', + 'REAL' + ] +) + +LENDING_INTEREST_TABLE = Table( + 'lending_interest_table', + [ + 'id', + 'time', + 'lending_type', + 'asset', + 'amount', + ], + [ + 'TEXT', + 'INTEGER', + 'TEXT', + 'TEXT', + 'REAL', ] ) diff --git a/tests/test_DataBase.py b/tests/test_DataBase.py index b1e57b9..9de37c7 100644 --- a/tests/test_DataBase.py +++ b/tests/test_DataBase.py @@ -38,5 +38,8 @@ def test_inserts_search(verbose=0, **kwargs): (table.columns_names[1], SQLConditionEnum.equal, 18), (table.columns_names[3], SQLConditionEnum.greater_equal, 55), ] - retrieved_rows = db.get_conditions_rows(table, conditions) + retrieved_rows = db.get_conditions_rows(table, conditions_list=conditions) assert rows[1:2] == retrieved_rows + + # check max weight is right + assert max([r[-1] for r in rows]) == db.get_conditions_rows(table, selection="MAX(weight)")[0][0] From 1644772666b93730ac88dcbd132b9f8db248823f Mon Sep 17 00:00:00 2001 From: EtWn <34377743+EtWnn@users.noreply.github.com> Date: Mon, 15 Mar 2021 14:56:35 +0100 Subject: [PATCH 04/17] Re table (#4) * upgrade Table class with setaatr for cols and key attributes * apdate database to the new Table class and update the tests * reformat columns name for SQL convs * reformat BinanceDataBase and BinanceManager for the new table format --- src/storage/BinanceDataBase.py | 111 ++++++++++++++++++--------------- src/storage/BinanceManager.py | 6 +- src/storage/DataBase.py | 14 +++-- src/storage/tables.py | 73 +++++++++++++--------- tests/test_DataBase.py | 86 ++++++++++++++++++++----- 5 files changed, 186 insertions(+), 104 deletions(-) diff --git a/src/storage/BinanceDataBase.py b/src/storage/BinanceDataBase.py index 0616ed8..eaeef46 100644 --- a/src/storage/BinanceDataBase.py +++ b/src/storage/BinanceDataBase.py @@ -14,13 +14,11 @@ class BinanceDataBase(DataBase): def __init__(self, name: str = 'binance_db'): super().__init__(name) - def add_lending_interest(self, int_id: str, time: int, lending_type: str, asset: str, amount: float, + def add_lending_interest(self, time: int, lending_type: str, asset: str, amount: float, auto_commit: bool = True): """ add an lending interest to the database - :param int_id: if for the interest - :type int_id: str :param time: millitstamp of the operation :type time: int :param lending_type: either 'DAILY', 'ACTIVITY' or 'CUSTOMIZED_FIXED' @@ -34,7 +32,7 @@ def add_lending_interest(self, int_id: str, time: int, lending_type: str, asset: :return: None :rtype: None """ - row = (int_id, time, lending_type, asset, amount) + row = (time, lending_type, asset, amount) self.add_row(tables.LENDING_INTEREST_TABLE, row, auto_commit=auto_commit) def get_lending_interests(self, lending_type: Optional[str] = None, asset: Optional[str] = None, @@ -54,27 +52,28 @@ def get_lending_interests(self, lending_type: Optional[str] = None, asset: Optio :rtype: List[Tuple] """ conditions_list = [] + table = tables.LENDING_INTEREST_TABLE if lending_type is not None: - conditions_list.append((tables.LENDING_INTEREST_TABLE.columns_names[2], + conditions_list.append((table.lendingType, SQLConditionEnum.equal, lending_type)) if asset is not None: - conditions_list.append((tables.LENDING_INTEREST_TABLE.columns_names[3], + conditions_list.append((table.asset, SQLConditionEnum.equal, asset)) if start_time is not None: - conditions_list.append((tables.LENDING_INTEREST_TABLE.columns_names[1], + conditions_list.append((table.interestTime, SQLConditionEnum.greater_equal, start_time)) if end_time is not None: - conditions_list.append((tables.LENDING_INTEREST_TABLE.columns_names[1], + conditions_list.append((table.interestTime, SQLConditionEnum.lower, end_time)) - return self.get_conditions_rows(tables.LENDING_INTEREST_TABLE, conditions_list=conditions_list) + return self.get_conditions_rows(table, conditions_list=conditions_list) def get_last_lending_interest_time(self, lending_type: Optional[str] = None): """ - return the latest time when an interest was recieved. + return the latest time when an interest was received. If None, return the millistamp corresponding to 2017/01/01 :param lending_type: type of lending @@ -83,12 +82,13 @@ def get_last_lending_interest_time(self, lending_type: Optional[str] = None): :rtype: int """ conditions_list = [] + table = tables.LENDING_INTEREST_TABLE if lending_type is not None: - conditions_list.append((tables.LENDING_INTEREST_TABLE.columns_names[2], + conditions_list.append((table.lendingType, SQLConditionEnum.equal, lending_type)) - selection = f"MAX({tables.LENDING_INTEREST_TABLE.columns_names[1]})" - result = self.get_conditions_rows(tables.LENDING_INTEREST_TABLE, + selection = f"MAX({table.interestTime})" + result = self.get_conditions_rows(table, selection=selection, conditions_list=conditions_list) default = datetime_to_millistamp(datetime.datetime(2017, 1, 1, tzinfo=datetime.timezone.utc)) @@ -100,13 +100,14 @@ def get_last_lending_interest_time(self, lending_type: Optional[str] = None): return default return result - def add_dust(self, dust_id: str, time: int, asset: str, asset_amount: float, bnb_amount: float, bnb_fee: float, + def add_dust(self, tran_id: str, time: int, asset: str, asset_amount: float, bnb_amount: float, bnb_fee: float, auto_commit: bool = True): """ add dust operation to the database + https://binance-docs.github.io/apidocs/spot/en/#dustlog-user_data - :param dust_id: id of the operation - :type dust_id: str + :param tran_id: id of the transaction (non unique) + :type tran_id: str :param time: millitstamp of the operation :type time: int :param asset: asset that got converted to BNB @@ -123,7 +124,7 @@ def add_dust(self, dust_id: str, time: int, asset: str, asset_amount: float, bnb :rtype: None """ - row = (dust_id, time, asset, asset_amount, bnb_amount, bnb_fee) + row = (tran_id, time, asset, asset_amount, bnb_amount, bnb_fee) self.add_row(tables.SPOT_DUST_TABLE, row, auto_commit=auto_commit) def get_spot_dusts(self, asset: Optional[str] = None, start_time: Optional[int] = None, @@ -141,19 +142,20 @@ def get_spot_dusts(self, asset: Optional[str] = None, start_time: Optional[int] :rtype: List[Tuple] """ conditions_list = [] + table = tables.SPOT_DUST_TABLE if asset is not None: - conditions_list.append((tables.SPOT_DIVIDEND_TABLE.columns_names[2], + conditions_list.append((table.asset, SQLConditionEnum.equal, asset)) if start_time is not None: - conditions_list.append((tables.SPOT_DIVIDEND_TABLE.columns_names[1], + conditions_list.append((table.dustTime, SQLConditionEnum.greater_equal, start_time)) if end_time is not None: - conditions_list.append((tables.SPOT_DIVIDEND_TABLE.columns_names[1], + conditions_list.append((table.dustTime, SQLConditionEnum.lower, end_time)) - return self.get_conditions_rows(tables.SPOT_DUST_TABLE, conditions_list=conditions_list) + return self.get_conditions_rows(table, conditions_list=conditions_list) def add_dividend(self, div_id: int, div_time: int, asset: str, amount: float, auto_commit: bool = True): """ @@ -190,19 +192,20 @@ def get_spot_dividends(self, asset: Optional[str] = None, start_time: Optional[i :rtype: List[Tuple] """ conditions_list = [] + table = tables.SPOT_DIVIDEND_TABLE if asset is not None: - conditions_list.append((tables.SPOT_DIVIDEND_TABLE.columns_names[2], + conditions_list.append((table.asset, SQLConditionEnum.equal, asset)) if start_time is not None: - conditions_list.append((tables.SPOT_DIVIDEND_TABLE.columns_names[1], + conditions_list.append((table.divTime, SQLConditionEnum.greater_equal, start_time)) if end_time is not None: - conditions_list.append((tables.SPOT_DIVIDEND_TABLE.columns_names[1], + conditions_list.append((table.divTime, SQLConditionEnum.lower, end_time)) - return self.get_conditions_rows(tables.SPOT_DIVIDEND_TABLE, conditions_list=conditions_list) + return self.get_conditions_rows(table, conditions_list=conditions_list) def get_last_spot_dividend_time(self) -> int: """ @@ -211,9 +214,11 @@ def get_last_spot_dividend_time(self) -> int: :return: """ - selection = f"MAX({tables.SPOT_DIVIDEND_TABLE.columns_names[1]})" - result = self.get_conditions_rows(tables.SPOT_WITHDRAW_TABLE, + table = tables.SPOT_DIVIDEND_TABLE + selection = f"MAX({table.divTime})" + result = self.get_conditions_rows(table, selection=selection) + default = datetime_to_millistamp(datetime.datetime(2017, 1, 1, tzinfo=datetime.timezone.utc)) try: result = result[0][0] @@ -263,19 +268,20 @@ def get_spot_withdraws(self, asset: Optional[str] = None, start_time: Optional[i :rtype: List[Tuple] """ conditions_list = [] + table = tables.SPOT_WITHDRAW_TABLE if asset is not None: - conditions_list.append((tables.SPOT_WITHDRAW_TABLE.columns_names[3], + conditions_list.append((table.asset, SQLConditionEnum.equal, asset)) if start_time is not None: - conditions_list.append((tables.SPOT_WITHDRAW_TABLE.columns_names[2], + conditions_list.append((table.applyTime, SQLConditionEnum.greater_equal, start_time)) if end_time is not None: - conditions_list.append((tables.SPOT_WITHDRAW_TABLE.columns_names[2], + conditions_list.append((table.applyTime, SQLConditionEnum.lower, end_time)) - return self.get_conditions_rows(tables.SPOT_WITHDRAW_TABLE, conditions_list=conditions_list) + return self.get_conditions_rows(table, conditions_list=conditions_list) def get_last_spot_withdraw_time(self) -> int: """ @@ -284,8 +290,9 @@ def get_last_spot_withdraw_time(self) -> int: :return: """ - selection = f"MAX({tables.SPOT_WITHDRAW_TABLE.columns_names[2]})" - result = self.get_conditions_rows(tables.SPOT_WITHDRAW_TABLE, + table = tables.SPOT_WITHDRAW_TABLE + selection = f"MAX({table.applyTime})" + result = self.get_conditions_rows(table, selection=selection) default = datetime_to_millistamp(datetime.datetime(2017, 1, 1, tzinfo=datetime.timezone.utc)) try: @@ -331,19 +338,20 @@ def get_spot_deposits(self, asset: Optional[str] = None, start_time: Optional[in :rtype: List[Tuple] """ conditions_list = [] + table = tables.SPOT_DEPOSIT_TABLE if asset is not None: - conditions_list.append((tables.SPOT_DEPOSIT_TABLE.columns_names[2], + conditions_list.append((table.asset, SQLConditionEnum.equal, asset)) if start_time is not None: - conditions_list.append((tables.SPOT_DEPOSIT_TABLE.columns_names[1], + conditions_list.append((table.insertTime, SQLConditionEnum.greater_equal, start_time)) if end_time is not None: - conditions_list.append((tables.SPOT_DEPOSIT_TABLE.columns_names[1], + conditions_list.append((table.insertTime, SQLConditionEnum.lower, end_time)) - return self.get_conditions_rows(tables.SPOT_DEPOSIT_TABLE, conditions_list=conditions_list) + return self.get_conditions_rows(table, conditions_list=conditions_list) def get_last_spot_deposit_time(self) -> int: """ @@ -352,9 +360,11 @@ def get_last_spot_deposit_time(self) -> int: :return: """ - selection = f"MAX({tables.SPOT_DEPOSIT_TABLE.columns_names[1]})" - result = self.get_conditions_rows(tables.SPOT_DEPOSIT_TABLE, + table = tables.SPOT_DEPOSIT_TABLE + selection = f"MAX({table.insertTime})" + result = self.get_conditions_rows(table, selection=selection) + default = datetime_to_millistamp(datetime.datetime(2017, 1, 1, tzinfo=datetime.timezone.utc)) try: result = result[0][0] @@ -392,8 +402,7 @@ def add_spot_trade(self, trade_id: int, millistamp: int, asset: str, ref_asset: :return: None :rtype: None """ - key = f'{asset}{ref_asset}{trade_id}' - row = (key, trade_id, millistamp, asset, ref_asset, qty, price, fee, fee_asset, int(is_buyer)) + row = (trade_id, millistamp, asset, ref_asset, qty, price, fee, fee_asset, int(is_buyer)) self.add_row(tables.SPOT_TRADE_TABLE, row, auto_commit) def get_spot_trades(self, start_time: Optional[int] = None, end_time: Optional[int] = None, @@ -413,20 +422,21 @@ def get_spot_trades(self, start_time: Optional[int] = None, end_time: Optional[i :rtype: List[Tuple] """ conditions_list = [] + table = tables.SPOT_TRADE_TABLE if start_time is not None: - conditions_list.append((tables.SPOT_TRADE_TABLE.columns_names[2], + conditions_list.append((table.tdTime, SQLConditionEnum.greater_equal, start_time)) if end_time is not None: - conditions_list.append((tables.SPOT_TRADE_TABLE.columns_names[2], + conditions_list.append((table.tdTime, SQLConditionEnum.lower, end_time)) if asset is not None: - conditions_list.append((tables.SPOT_TRADE_TABLE.columns_names[3], + conditions_list.append((table.asset, SQLConditionEnum.equal, asset)) if ref_asset is not None: - conditions_list.append((tables.SPOT_TRADE_TABLE.columns_names[4], + conditions_list.append((table.refAsset, SQLConditionEnum.equal, ref_asset)) return self.get_conditions_rows(tables.SPOT_TRADE_TABLE, conditions_list=conditions_list) @@ -442,18 +452,17 @@ def get_max_trade_id(self, asset: str, ref_asset: str) -> int: :return: latest trade id :rtype: int """ - selection = f"MAX({tables.SPOT_TRADE_TABLE.columns_names[1]})" + table = tables.SPOT_TRADE_TABLE + selection = f"MAX({table.tradeId})" conditions_list = [ - (tables.SPOT_TRADE_TABLE.columns_names[3], + (table.asset, SQLConditionEnum.equal, asset), - (tables.SPOT_TRADE_TABLE.columns_names[4], + (table.refAsset, SQLConditionEnum.equal, ref_asset) ] - result = self.get_conditions_rows(tables.SPOT_TRADE_TABLE, - selection=selection, - conditions_list=conditions_list) + result = self.get_conditions_rows(table, selection=selection, conditions_list=conditions_list) try: result = result[0][0] except IndexError: diff --git a/src/storage/BinanceManager.py b/src/storage/BinanceManager.py index 69097aa..6307596 100644 --- a/src/storage/BinanceManager.py +++ b/src/storage/BinanceManager.py @@ -41,9 +41,7 @@ def update_lending_interests(self): current=current, limit=100) for li in lending_interests: - print(li) - self.db.add_lending_interest(int_id=str(li['time']) + li['asset'] + li['lendingType'], - time=li['time'], + self.db.add_lending_interest(time=li['time'], lending_type=li['lendingType'], asset=li['asset'], amount=li['interest'] @@ -74,7 +72,7 @@ def update_spot_dusts(self): for d in dusts['rows']: for sub_dust in d['logs']: date_time = dateparser.parse(sub_dust['operateTime'] + 'Z') - self.db.add_dust(dust_id=str(sub_dust['tranId']) + sub_dust['fromAsset'], + self.db.add_dust(tran_id=sub_dust['tranId'], time=datetime_to_millistamp(date_time), asset=sub_dust['fromAsset'], asset_amount=sub_dust['amount'], diff --git a/src/storage/DataBase.py b/src/storage/DataBase.py index 16a69b9..54b397d 100644 --- a/src/storage/DataBase.py +++ b/src/storage/DataBase.py @@ -57,7 +57,9 @@ def get_row_by_key(self, table: Table, key_value) -> Optional[Tuple]: :param key_value: key value of the row :return: None or the row of value """ - conditions_list = [(table.columns_names[0], SQLConditionEnum.equal, key_value)] + if table.primary_key is None: + raise ValueError(f"table {table.name} has no explicit primary key") + conditions_list = [(table.primary_key, SQLConditionEnum.equal, key_value)] rows = self.get_conditions_rows(table, conditions_list=conditions_list) if len(rows): return rows[0] @@ -104,8 +106,8 @@ def add_rows(self, table: Table, rows: List[Tuple], auto_commit: bool = True, up self.commit() def update_row(self, table: Table, row: Tuple, auto_commit=True): - row_s = ", ".join(f"{n} = {v}" for n, v in zip(table.columns_names[1:], row[1:])) - execution_order = f"UPDATE {table.name} SET {row_s} WHERE {table.columns_names[0]} = {row[0]}" + row_s = ", ".join(f"{n} = {v}" for n, v in zip(table.columns_names, row)) + execution_order = f"UPDATE {table.name} SET {row_s} WHERE {table.primary_key} = {row[0]}" self.db_cursor.execute(execution_order) if auto_commit: self.commit() @@ -160,7 +162,9 @@ def get_create_cmd(table: Table): :param table: Table instance with the config if the table to create :return: execution command for the table creation """ - cmd = f"[{table.columns_names[0]}] {table.columns_sql_types[0]} PRIMARY KEY, " - for arg_name, arg_type in zip(table.columns_names[1:], table.columns_sql_types[1:]): + cmd = "" + if table.primary_key is not None: + cmd = f"[{table.primary_key}] {table.primary_key_sql_type} PRIMARY KEY, " + for arg_name, arg_type in zip(table.columns_names, table.columns_sql_types): cmd = cmd + f"[{arg_name}] {arg_type}, " return f"CREATE TABLE {table.name}\n({cmd[:-2]})" diff --git a/src/storage/tables.py b/src/storage/tables.py index 5e5dd55..7304d54 100644 --- a/src/storage/tables.py +++ b/src/storage/tables.py @@ -1,31 +1,46 @@ from dataclasses import dataclass -from typing import List +from typing import List, Optional -@dataclass class Table: - name: str - columns_names: List[str] - columns_sql_types: List[str] + """ + @DynamicAttrs + """ + + def __init__(self, name: str, columns_names: List[str], columns_sql_types: List[str], + primary_key: Optional[str] = None, primary_key_sql_type: Optional[str] = None): + self.name = name + self.columns_names = columns_names + self.columns_sql_types = columns_sql_types + self.primary_key = primary_key + self.primary_key_sql_type = primary_key_sql_type + + for column_name in self.columns_names: + try: + value = getattr(self, column_name) + raise ValueError(f"the name {column_name} conflicts with an existing attribute of value {value}") + except AttributeError: + setattr(self, column_name, column_name) + + if self.primary_key is not None: + setattr(self, self.primary_key, self.primary_key) SPOT_TRADE_TABLE = Table( 'spot_trade', [ - 'key', - 'id', - 'millistamp', + 'tradeId', + 'tdTime', 'asset', - 'ref_asset', + 'refAsset', 'qty', 'price', 'fee', - 'fee_asset', + 'feeAsset', 'isBuyer' ], [ - 'TEXT', 'INTEGER', 'INTEGER', 'TEXT', @@ -41,24 +56,23 @@ class Table: SPOT_DEPOSIT_TABLE = Table( 'spot_deposit', [ - 'txId', 'insertTime', 'asset', 'amount', ], [ - 'TEXT', 'INTEGER', 'TEXT', 'REAL' - ] + ], + primary_key='txId', + primary_key_sql_type='TEXT' ) SPOT_WITHDRAW_TABLE = Table( 'spot_withdraw', [ - 'id', 'txId', 'applyTime', 'asset', @@ -66,43 +80,44 @@ class Table: 'fee' ], [ - 'TEXT', 'TEXT', 'INTEGER', 'TEXT', 'REAL', 'REAL' - ] + ], + primary_key='withdrawId', + primary_key_sql_type='TEXT' ) SPOT_DIVIDEND_TABLE = Table( 'spot_dividend_table', [ - 'id', 'divTime', 'asset', 'amount' ], [ - 'INTEGER', 'INTEGER', 'TEXT', 'REAL' - ] + ], + primary_key='divId', + primary_key_sql_type='INTEGER' ) SPOT_DUST_TABLE = Table( 'spot_dust_table', [ - 'id', - 'time', + 'tranId', + 'dustTime', 'asset', - 'asset_amount', - 'bnb_amount', - 'bnb_fee', + 'assetAmount', + 'bnbAmount', + 'bnbFee', ], [ - 'TEXT', + 'INTEGER', 'INTEGER', 'TEXT', 'REAL', @@ -114,14 +129,12 @@ class Table: LENDING_INTEREST_TABLE = Table( 'lending_interest_table', [ - 'id', - 'time', - 'lending_type', + 'interestTime', + 'lendingType', 'asset', 'amount', ], [ - 'TEXT', 'INTEGER', 'TEXT', 'TEXT', diff --git a/tests/test_DataBase.py b/tests/test_DataBase.py index 9de37c7..84a251a 100644 --- a/tests/test_DataBase.py +++ b/tests/test_DataBase.py @@ -1,45 +1,103 @@ +from pandas import DataFrame + from src.storage.DataBase import DataBase, SQLConditionEnum from src.storage.tables import Table -table = Table( - 'test_table', +db = DataBase("test_table") + +table1 = Table( + 'test_table1', [ - 'Key', 'age', - 'name', + 'surname', 'weight' ], [ 'INTEGER', + 'TEXT', + 'REAL' + ], + primary_key='key', + primary_key_sql_type='INTEGER' +) + +table2 = Table( + 'test_table2', + [ + 'age', + 'surname', + 'weight' + ], + [ 'INTEGER', 'TEXT', 'REAL' - ] + ], ) -def test_inserts_search(verbose=0, **kwargs): - db = DataBase("test_table") - db.drop_table(table) +def test_create_cmd(verbose=0, **kwargs): + create_cmd = db.get_create_cmd(table1) + if verbose: + print(create_cmd) + assert create_cmd == "CREATE TABLE test_table1\n([key] INTEGER PRIMARY KEY, [age] INTEGER, [surname] TEXT," \ + " [weight] REAL)" + + create_cmd = db.get_create_cmd(table2) + if verbose: + print(create_cmd) + assert create_cmd == "CREATE TABLE test_table2\n([age] INTEGER, [surname] TEXT, [weight] REAL)" + + +def test_inserts_search_1(verbose=0, **kwargs): + db.drop_table(table1) rows = [ (1, 15, 'Karl', 55.5), (2, 18, 'Kitty', 61.1), (3, 18, 'Marc', 48.1), (8, 55, 'Jean', 78.1) ] - db.add_rows(table, rows) + db.add_rows(table1, rows) - retrieved_row = db.get_row_by_key(table, 1) + retrieved_row = db.get_row_by_key(table1, 1) if verbose: print(f"retrieved the row: {retrieved_row} when looking for the row: {rows[0]}") assert rows[0] == retrieved_row conditions = [ - (table.columns_names[1], SQLConditionEnum.equal, 18), - (table.columns_names[3], SQLConditionEnum.greater_equal, 55), + (table1.age, SQLConditionEnum.equal, 18), + (table1.weight, SQLConditionEnum.greater_equal, 55), ] - retrieved_rows = db.get_conditions_rows(table, conditions_list=conditions) + retrieved_rows = db.get_conditions_rows(table1, conditions_list=conditions) assert rows[1:2] == retrieved_rows # check max weight is right - assert max([r[-1] for r in rows]) == db.get_conditions_rows(table, selection="MAX(weight)")[0][0] + assert max([r[-1] for r in rows]) == db.get_conditions_rows(table1, selection=f"MAX({table1.weight})")[0][0] + + +def test_inserts_search_2(verbose=0, **kwargs): + db.drop_table(table2) + rows = [ + (15, 'Karl', 55.5), + (18, 'Kitty', 61.1), + (18, 'Kitty', 61.1), # no set primary key -> should allow identical rows + (18, 'Marc', 48.1), + (55, 'Jean', 78.1) + ] + db.add_rows(table2, rows) + + try: + db.get_row_by_key(table2, 1) + raise RuntimeError("the above line should throw an error as no primary key is defined") + except ValueError: + pass + + conditions = [ + (table2.age, SQLConditionEnum.equal, 18), + (table2.weight, SQLConditionEnum.greater_equal, 55), + ] + retrieved_rows = db.get_conditions_rows(table2, conditions_list=conditions) + assert rows[1:3] == retrieved_rows + + # check min ge is right + assert max([r[0] for r in rows]) == db.get_conditions_rows(table2, selection=f"MAX({table2.age})")[0][0] From 2fbb16959637d3b932484389a76dac22cd99c7e9 Mon Sep 17 00:00:00 2001 From: EtWn <34377743+EtWnn@users.noreply.github.com> Date: Tue, 16 Mar 2021 11:34:23 +0100 Subject: [PATCH 05/17] Cross margin (#5) * add cross_margin trades * add order by possibility for database * add loans to database and update_asset_loans to BinanceManager * set txId for loans to primary key * add update_cross_margin_loans to BinanceManager * add sources to the documentation * add cross margin repay * add margin interest --- src/storage/BinanceDataBase.py | 360 +++++++++++++++++++++++++++++++-- src/storage/BinanceManager.py | 323 +++++++++++++++++++++++++++-- src/storage/DataBase.py | 29 ++- src/storage/tables.py | 79 ++++++++ tests/test_DataBase.py | 12 +- 5 files changed, 773 insertions(+), 30 deletions(-) diff --git a/src/storage/BinanceDataBase.py b/src/storage/BinanceDataBase.py index eaeef46..3e9c7bf 100644 --- a/src/storage/BinanceDataBase.py +++ b/src/storage/BinanceDataBase.py @@ -14,6 +14,319 @@ class BinanceDataBase(DataBase): def __init__(self, name: str = 'binance_db'): super().__init__(name) + def add_margin_interest(self, margin_type: str, interest_time: int, asset: str, interest: float, + interest_type: str, auto_commit: bool = True): + """ + add a repay to the database + + :param margin_type: either 'cross' or 'isolated' + :type margin_type: str + :param interest_time: millistamp of the operation + :type interest_time: int + :param asset: asset that got repaid + :type asset: str + :param interest: amount of interest accrued + :type interest: float + :param interest_type: one of (PERIODIC, ON_BORROW, PERIODIC_CONVERTED, ON_BORROW_CONVERTED) + :type interest_type: str + :param auto_commit: if the database should commit the change made, default True + :type auto_commit: bool + :return: None + :rtype: None + """ + if margin_type == 'cross': + table = tables.CROSS_MARGIN_INTEREST_TABLE + elif margin_type == 'isolated': + raise NotImplementedError + else: + raise ValueError(f"margin type should be 'cross' or 'isolated' but {margin_type} was received") + + row = (interest_time, asset, interest, interest_type) + self.add_row(table, row, auto_commit=auto_commit) + + def get_margin_interests(self, margin_type: str, asset: Optional[str] = None, start_time: Optional[int] = None, + end_time: Optional[int] = None): + """ + return margin interests stored in the database. Asset type and time filters can be used + + :param margin_type: either 'cross' or 'isolated' + :type margin_type: + :param asset: fetch only interests in this asset + :type asset: Optional[str] + :param start_time: fetch only interests after this millistamp + :type start_time: Optional[int] + :param end_time: fetch only interests before this millistamp + :type end_time: Optional[int] + :return: The raw rows selected as saved in the database + :rtype: List[Tuple] + """ + if margin_type == 'cross': + table = tables.CROSS_MARGIN_INTEREST_TABLE + elif margin_type == 'isolated': + raise NotImplementedError + else: + raise ValueError(f"margin type should be 'cross' or 'isolated' but {margin_type} was received") + + conditions_list = [] + if asset is not None: + conditions_list.append((table.asset, + SQLConditionEnum.equal, + asset)) + if start_time is not None: + conditions_list.append((table.repayTime, + SQLConditionEnum.greater_equal, + start_time)) + if end_time is not None: + conditions_list.append((table.interestTime, + SQLConditionEnum.lower, + end_time)) + return self.get_conditions_rows(table, conditions_list=conditions_list) + + def get_last_margin_interest_time(self, margin_type: str, asset: Optional[str] = None): + """ + return the latest time when a margin interest was accured on a defined asset or on all assets + If None, return the millistamp corresponding to 2017/01/01 + + :param asset: name of the asset charged as interest + :type asset: Optional[str] + :param margin_type: either 'cross' or 'isolated' + :type margin_type: + :return: millistamp + :rtype: int + """ + if margin_type == 'cross': + table = tables.CROSS_MARGIN_INTEREST_TABLE + elif margin_type == 'isolated': + raise NotImplementedError + else: + raise ValueError(f"margin type should be 'cross' or 'isolated' but {margin_type} was received") + + conditions_list = [] + if asset is not None: + conditions_list = [(table.asset, + SQLConditionEnum.equal, + asset)] + selection = f"MAX({table.interestTime})" + result = self.get_conditions_rows(table, + selection=selection, + conditions_list=conditions_list) + default = datetime_to_millistamp(datetime.datetime(2017, 1, 1, tzinfo=datetime.timezone.utc)) + try: + result = result[0][0] + except IndexError: + return default + if result is None: + return default + return result + + def add_repay(self, margin_type: str, tx_id: int, repay_time: int, asset: str, principal: float, + interest: float, auto_commit: bool = True): + """ + add a repay to the database + + :param margin_type: either 'cross' or 'isolated' + :type margin_type: + :param tx_id: binance id for the transaction (uniqueness?) + :type tx_id: int + :param repay_time: millitstamp of the operation + :type repay_time: int + :param asset: asset that got repaid + :type asset: str + :param principal: principal amount repaid for the loan + :type principal: float + :param interest: amount of interest repaid for the loan + :type interest: + :param auto_commit: if the database should commit the change made, default True + :type auto_commit: bool + :return: None + :rtype: None + """ + if margin_type == 'cross': + table = tables.CROSS_MARGIN_REPAY_TABLE + elif margin_type == 'isolated': + raise NotImplementedError + else: + raise ValueError(f"margin type should be 'cross' or 'isolated' but {margin_type} was received") + + row = (tx_id, repay_time, asset, principal, interest) + self.add_row(table, row, auto_commit=auto_commit) + + def get_repays(self, margin_type: str, asset: Optional[str] = None, start_time: Optional[int] = None, + end_time: Optional[int] = None): + """ + return repays stored in the database. Asset type and time filters can be used + + :param margin_type: either 'cross' or 'isolated' + :type margin_type: + :param asset: fetch only repays of this asset + :type asset: Optional[str] + :param start_time: fetch only repays after this millistamp + :type start_time: Optional[int] + :param end_time: fetch only repays before this millistamp + :type end_time: Optional[int] + :return: The raw rows selected as saved in the database + :rtype: List[Tuple] + """ + if margin_type == 'cross': + table = tables.CROSS_MARGIN_REPAY_TABLE + elif margin_type == 'isolated': + raise NotImplementedError + else: + raise ValueError(f"margin type should be 'cross' or 'isolated' but {margin_type} was received") + + conditions_list = [] + if asset is not None: + conditions_list.append((table.asset, + SQLConditionEnum.equal, + asset)) + if start_time is not None: + conditions_list.append((table.repayTime, + SQLConditionEnum.greater_equal, + start_time)) + if end_time is not None: + conditions_list.append((table.repayTime, + SQLConditionEnum.lower, + end_time)) + return self.get_conditions_rows(table, conditions_list=conditions_list) + + def get_last_repay_time(self, asset: str, margin_type: str): + """ + return the latest time when a repay was made on a defined asset + If None, return the millistamp corresponding to 2017/01/01 + + :param asset: name of the asset repaid + :type asset: str + :param margin_type: either 'cross' or 'isolated' + :type margin_type: + :return: millistamp + :rtype: int + """ + if margin_type == 'cross': + table = tables.CROSS_MARGIN_REPAY_TABLE + elif margin_type == 'isolated': + raise NotImplementedError + else: + raise ValueError(f"margin type should be 'cross' or 'isolated' but {margin_type} was received") + + conditions_list = [(table.asset, + SQLConditionEnum.equal, + asset)] + selection = f"MAX({table.repayTime})" + result = self.get_conditions_rows(table, + selection=selection, + conditions_list=conditions_list) + default = datetime_to_millistamp(datetime.datetime(2017, 1, 1, tzinfo=datetime.timezone.utc)) + try: + result = result[0][0] + except IndexError: + return default + if result is None: + return default + return result + + def add_loan(self, margin_type: str, tx_id: int, loan_time: int, asset: str, principal: float, + auto_commit: bool = True): + """ + add a loan to the database + + :param margin_type: either 'cross' or 'isolated' + :type margin_type: + :param tx_id: binance id for the transaction (uniqueness?) + :type tx_id: int + :param loan_time: millitstamp of the operation + :type loan_time: int + :param asset: asset that got loaned + :type asset: str + :param principal: amount of loaned asset + :type principal: float + :param auto_commit: if the database should commit the change made, default True + :type auto_commit: bool + :return: None + :rtype: None + """ + if margin_type == 'cross': + table = tables.CROSS_MARGIN_LOAN_TABLE + elif margin_type == 'isolated': + raise NotImplementedError + else: + raise ValueError(f"margin type should be 'cross' or 'isolated' but {margin_type} was received") + + row = (tx_id, loan_time, asset, principal) + self.add_row(table, row, auto_commit=auto_commit) + + def get_loans(self, margin_type: str, asset: Optional[str] = None, start_time: Optional[int] = None, + end_time: Optional[int] = None): + """ + return loans stored in the database. Asset type and time filters can be used + + :param margin_type: either 'cross' or 'isolated' + :type margin_type: + :param asset: fetch only loans of this asset + :type asset: Optional[str] + :param start_time: fetch only loans after this millistamp + :type start_time: Optional[int] + :param end_time: fetch only loans before this millistamp + :type end_time: Optional[int] + :return: The raw rows selected as saved in the database + :rtype: List[Tuple] + """ + if margin_type == 'cross': + table = tables.CROSS_MARGIN_LOAN_TABLE + elif margin_type == 'isolated': + raise NotImplementedError + else: + raise ValueError(f"margin type should be 'cross' or 'isolated' but {margin_type} was received") + + conditions_list = [] + if asset is not None: + conditions_list.append((table.asset, + SQLConditionEnum.equal, + asset)) + if start_time is not None: + conditions_list.append((table.loanTime, + SQLConditionEnum.greater_equal, + start_time)) + if end_time is not None: + conditions_list.append((table.loanTime, + SQLConditionEnum.lower, + end_time)) + return self.get_conditions_rows(table, conditions_list=conditions_list) + + def get_last_loan_time(self, asset: str, margin_type: str): + """ + return the latest time when an loan was made on a defined asset + If None, return the millistamp corresponding to 2017/01/01 + + :param asset: name of the asset loaned + :type asset: str + :param margin_type: either 'cross' or 'isolated' + :type margin_type: + :return: millistamp + :rtype: int + """ + if margin_type == 'cross': + table = tables.CROSS_MARGIN_LOAN_TABLE + elif margin_type == 'isolated': + raise NotImplementedError + else: + raise ValueError(f"margin type should be 'cross' or 'isolated' but {margin_type} was received") + + conditions_list = [(table.asset, + SQLConditionEnum.equal, + asset)] + selection = f"MAX({table.loanTime})" + result = self.get_conditions_rows(table, + selection=selection, + conditions_list=conditions_list) + default = datetime_to_millistamp(datetime.datetime(2017, 1, 1, tzinfo=datetime.timezone.utc)) + try: + result = result[0][0] + except IndexError: + return default + if result is None: + return default + return result + def add_lending_interest(self, time: int, lending_type: str, asset: str, amount: float, auto_commit: bool = True): """ @@ -374,15 +687,17 @@ def get_last_spot_deposit_time(self) -> int: return default return result - def add_spot_trade(self, trade_id: int, millistamp: int, asset: str, ref_asset: str, qty: float, price: float, - fee: float, fee_asset: str, is_buyer: bool, auto_commit=True): + def add_trade(self, trade_type: str, trade_id: int, trade_time: int, asset: str, ref_asset: str, qty: float, + price: float, fee: float, fee_asset: str, is_buyer: bool, auto_commit=True): """ add a trade to the database + :param trade_type: type trade executed + :type trade_type: string, must be one of {'spot', 'cross_margin'} :param trade_id: id of the trade (binance id, unique per trading pair) :type trade_id: int - :param millistamp: millistamp of the trade - :type millistamp: int + :param trade_time: millistamp of the trade + :type trade_time: int :param asset: name of the asset in the trading pair (ex 'BTC' for 'BTCUSDT') :type asset: string :param ref_asset: name of the reference asset in the trading pair (ex 'USDT' for 'BTCUSDT') @@ -402,14 +717,22 @@ def add_spot_trade(self, trade_id: int, millistamp: int, asset: str, ref_asset: :return: None :rtype: None """ - row = (trade_id, millistamp, asset, ref_asset, qty, price, fee, fee_asset, int(is_buyer)) - self.add_row(tables.SPOT_TRADE_TABLE, row, auto_commit) + row = (trade_id, trade_time, asset, ref_asset, qty, price, fee, fee_asset, int(is_buyer)) + if trade_type == 'spot': + table = tables.SPOT_TRADE_TABLE + elif trade_type == 'cross_margin': + table = tables.CROSS_MARGIN_TRADE_TABLE + else: + raise ValueError(f"trade type should be one of ('spot', 'cross_margin') but {trade_type} was received") + self.add_row(table, row, auto_commit) - def get_spot_trades(self, start_time: Optional[int] = None, end_time: Optional[int] = None, - asset: Optional[str] = None, ref_asset: Optional[str] = None): + def get_trades(self, trade_type: str, start_time: Optional[int] = None, end_time: Optional[int] = None, + asset: Optional[str] = None, ref_asset: Optional[str] = None): """ return trades stored in the database. asset type, ref_asset type and time filters can be used + :param trade_type: type trade executed + :type trade_type: string, must be one of ('spot', 'cross_margin') :param start_time: fetch only trades after this millistamp :type start_time: Optional[int] :param end_time: fetch only trades before this millistamp @@ -421,8 +744,13 @@ def get_spot_trades(self, start_time: Optional[int] = None, end_time: Optional[i :return: The raw rows selected as saved in the database :rtype: List[Tuple] """ + if trade_type == 'spot': + table = tables.SPOT_TRADE_TABLE + elif trade_type == 'cross_margin': + table = tables.CROSS_MARGIN_TRADE_TABLE + else: + raise ValueError(f"trade type should be one of ('spot', 'cross_margin') but {trade_type} was received") conditions_list = [] - table = tables.SPOT_TRADE_TABLE if start_time is not None: conditions_list.append((table.tdTime, SQLConditionEnum.greater_equal, @@ -439,9 +767,9 @@ def get_spot_trades(self, start_time: Optional[int] = None, end_time: Optional[i conditions_list.append((table.refAsset, SQLConditionEnum.equal, ref_asset)) - return self.get_conditions_rows(tables.SPOT_TRADE_TABLE, conditions_list=conditions_list) + return self.get_conditions_rows(table, conditions_list=conditions_list, order_list=[table.tdTime]) - def get_max_trade_id(self, asset: str, ref_asset: str) -> int: + def get_max_trade_id(self, asset: str, ref_asset: str, trade_type: str) -> int: """ return the latest trade id for a trading pair. If none is found, return -1 @@ -449,10 +777,18 @@ def get_max_trade_id(self, asset: str, ref_asset: str) -> int: :type asset: string :param ref_asset: name of the reference asset in the trading pair (ex 'USDT' for 'BTCUSDT') :type ref_asset: string + :param trade_type: type trade executed + :type trade_type: string, must be one of {'spot', 'cross_margin'} :return: latest trade id :rtype: int """ - table = tables.SPOT_TRADE_TABLE + if trade_type == 'spot': + table = tables.SPOT_TRADE_TABLE + elif trade_type == 'cross_margin': + table = tables.CROSS_MARGIN_TRADE_TABLE + else: + raise ValueError(f"trade type should be one of {'spot', 'cross_margin'} but {trade_type} was received") + selection = f"MAX({table.tradeId})" conditions_list = [ (table.asset, diff --git a/src/storage/BinanceManager.py b/src/storage/BinanceManager.py index 6307596..c686585 100644 --- a/src/storage/BinanceManager.py +++ b/src/storage/BinanceManager.py @@ -1,5 +1,6 @@ import datetime import math +import time import dateparser from binance.client import Client @@ -21,10 +22,247 @@ def __init__(self): credentials = CredentialManager.get_api_credentials("Binance") self.client = Client(**credentials) + def update_cross_margin_interests(self): + """ + update the interests for all cross margin assets + + sources: + https://binance-docs.github.io/apidocs/spot/en/#query-repay-record-user_data + + :return: + :rtype: + """ + margin_type = 'cross' + latest_time = self.db.get_last_margin_interest_time(margin_type) + archived = 1000 * time.time() - latest_time > 1000 * 3600 * 24 * 30 * 3 + current = 1 + while True: + params = { + 'current': current, + 'startTime': latest_time + 1000, + 'size': 100, + 'archived': archived + } + # no built-in method yet in python-binance for margin/interestHistory + interests = self.client._request_margin_api('get', 'margin/interestHistory', signed=True, data=params) + + for interest in interests['rows']: + self.db.add_margin_interest(margin_type=margin_type, + interest_time=interest['interestAccuredTime'], + asset=interest['asset'], + interest=interest['interest'], + interest_type=interest['type'], + auto_commit=False) + + if len(interests['rows']): + current += 1 # next page + self.db.commit() + elif archived: # switching to non archived interests + current = 1 + archived = False + latest_time = self.db.get_last_margin_interest_time(margin_type) + else: + break + + def update_cross_margin_repays(self): + """ + update the repays for all cross margin assets + + :return: None + :rtype: None + """ + symbols_info = self.client._request_margin_api('get', 'margin/allPairs', data={}) # not built-in yet + assets = set() + for symbol_info in symbols_info: + assets.add(symbol_info['base']) + assets.add(symbol_info['quote']) + + pbar = tqdm(total=len(assets)) + for asset in assets: + pbar.set_description(f"fetching {asset} cross margin repays") + self.update_margin_asset_repay(asset=asset) + pbar.update() + pbar.close() + + def update_margin_asset_repay(self, asset: str, isolated_symbol=''): + """ + update the repays database for a specified asset. + + sources: + https://binance-docs.github.io/apidocs/spot/en/#query-repay-record-user_data + https://python-binance.readthedocs.io/en/latest/binance.html#binance.client.Client.get_margin_repay_details + + :param asset: asset for the repays + :type asset: str + :param isolated_symbol: the symbol must be specified of isolated margin, otherwise cross margin data is returned + :type isolated_symbol: str + :return: None + :rtype: None + """ + margin_type = 'cross' if isolated_symbol == '' else 'isolated' + latest_time = self.db.get_last_repay_time(asset=asset, margin_type=margin_type) + archived = 1000 * time.time() - latest_time > 1000 * 3600 * 24 * 30 * 3 + current = 1 + while True: + repays = self.client.get_margin_repay_details(asset=asset, + current=current, + startTime=latest_time + 1000, + archived=archived, + isolatedSymbol=isolated_symbol, + size=100) + for repay in repays['rows']: + if repay['status'] == 'CONFIRMED': + self.db.add_repay(margin_type=margin_type, + tx_id=repay['txId'], + repay_time=repay['timestamp'], + asset=repay['asset'], + principal=repay['principal'], + interest=repay['interest'], + auto_commit=False) + + if len(repays['rows']): + current += 1 # next page + self.db.commit() + elif archived: # switching to non archived repays + current = 1 + archived = False + latest_time = self.db.get_last_repay_time(asset=asset, margin_type=margin_type) + else: + break + + def update_cross_margin_loans(self): + """ + update the loans for all cross margin assets + + :return: None + :rtype: None + """ + symbols_info = self.client._request_margin_api('get', 'margin/allPairs', data={}) # not built-in yet + assets = set() + for symbol_info in symbols_info: + assets.add(symbol_info['base']) + assets.add(symbol_info['quote']) + + pbar = tqdm(total=len(assets)) + for asset in assets: + pbar.set_description(f"fetching {asset} cross margin loans") + self.update_margin_asset_loans(asset=asset) + pbar.update() + pbar.close() + + def update_margin_asset_loans(self, asset: str, isolated_symbol=''): + """ + update the loans database for a specified asset. + + sources: + https://binance-docs.github.io/apidocs/spot/en/#query-loan-record-user_data + https://python-binance.readthedocs.io/en/latest/binance.html#binance.client.Client.get_margin_loan_details + + :param asset: asset for the loans + :type asset: str + :param isolated_symbol: the symbol must be specified of isolated margin, otherwise cross margin data is returned + :type isolated_symbol: str + :return: None + :rtype: None + """ + margin_type = 'cross' if isolated_symbol == '' else 'isolated' + latest_time = self.db.get_last_loan_time(asset=asset, margin_type=margin_type) + archived = 1000 * time.time() - latest_time > 1000 * 3600 * 24 * 30 * 3 + current = 1 + while True: + loans = self.client.get_margin_loan_details(asset=asset, + current=current, + startTime=latest_time + 1000, + archived=archived, + isolatedSymbol=isolated_symbol, + size=100) + for loan in loans['rows']: + if loan['status'] == 'CONFIRMED': + self.db.add_loan(margin_type=margin_type, + tx_id=loan['txId'], + loan_time=loan['timestamp'], + asset=loan['asset'], + principal=loan['principal'], + auto_commit=False) + + if len(loans['rows']): + current += 1 # next page + self.db.commit() + elif archived: # switching to non archived loans + current = 1 + archived = False + latest_time = self.db.get_last_loan_time(asset=asset, margin_type=margin_type) + else: + break + + def update_cross_margin_symbol_trades(self, asset: str, ref_asset: str, limit: int = 1000): + """ + This update the cross_margin trades in the database for a single trading pair. + It will check the last trade id and will requests the all trades after this trade_id. + + sources: + https://binance-docs.github.io/apidocs/spot/en/#query-margin-account-39-s-trade-list-user_data + https://python-binance.readthedocs.io/en/latest/binance.html#binance.client.Client.get_margin_trades + + :param asset: name of the asset in the trading pair (ex 'BTC' for 'BTCUSDT') + :type asset: string + :param ref_asset: name of the reference asset in the trading pair (ex 'USDT' for 'BTCUSDT') + :type ref_asset: string + :param limit: max size of each trade requests + :type limit: int + :return: None + :rtype: None + """ + limit = min(1000, limit) + symbol = asset + ref_asset + last_trade_id = self.db.get_max_trade_id(asset, ref_asset, 'cross_margin') + while True: + new_trades = self.client.get_margin_trades(symbol=symbol, fromId=last_trade_id + 1, limit=limit) + for trade in new_trades: + self.db.add_trade(trade_type='cross_margin', + trade_id=int(trade['id']), + trade_time=int(trade['time']), + asset=asset, + ref_asset=ref_asset, + qty=float(trade['qty']), + price=float(trade['price']), + fee=float(trade['commission']), + fee_asset=trade['commissionAsset'], + is_buyer=trade['isBuyer'], + auto_commit=False + ) + last_trade_id = max(last_trade_id, int(trade['id'])) + if len(new_trades): + self.db.commit() + if len(new_trades) < limit: + break + + def update_all_cross_margin_trades(self, limit: int = 1000): + """ + This update the cross margin trades in the database for every trading pairs + + :param limit: max size of each trade requests + :type limit: int + :return: None + :rtype: None + """ + symbols_info = self.client._request_margin_api('get', 'margin/allPairs', data={}) # not built-in yet + pbar = tqdm(total=len(symbols_info)) + for symbol_info in symbols_info: + pbar.set_description(f"fetching {symbol_info['symbol']}") + self.update_cross_margin_symbol_trades(asset=symbol_info['base'], + ref_asset=symbol_info['quote'], + limit=limit) + pbar.update() + pbar.close() + def update_lending_interests(self): """ update the lending interests database. - for each update + + sources: + https://python-binance.readthedocs.io/en/latest/binance.html#binance.client.Client.get_lending_interest_history + https://binance-docs.github.io/apidocs/spot/en/#get-interest-history-user_data-2 :return: None :rtype: None @@ -47,7 +285,7 @@ def update_lending_interests(self): amount=li['interest'] ) - if lending_interests: + if len(lending_interests): current += 1 # next page self.db.commit() else: @@ -60,6 +298,10 @@ def update_spot_dusts(self): update the dust database. As there is no way to get the dust by id or timeframe, the table is cleared for each update + sources: + https://python-binance.readthedocs.io/en/latest/binance.html#binance.client.Client.get_dust_log + https://binance-docs.github.io/apidocs/spot/en/#dustlog-user_data + :return: None :rtype: None """ @@ -85,6 +327,19 @@ def update_spot_dusts(self): pbar.close() def update_spot_dividends(self, day_jump: float = 90, limit: int = 500): + """ + update the dividends database (earnings distributed by Binance) + sources: + https://python-binance.readthedocs.io/en/latest/binance.html#binance.client.Client.get_asset_dividend_history + https://binance-docs.github.io/apidocs/spot/en/#asset-dividend-record-user_data + + :param day_jump: length of the time window in days, max is 90 + :type day_jump: float + :param limit: max number of dividends to retrieve per call, max is 500 + :type limit: int + :return: None + :rtype: None + """ limit = min(500, limit) delta_jump = min(day_jump, 90) * 24 * 3600 * 1000 start_time = self.db.get_last_spot_dividend_time() + 1 @@ -128,6 +383,10 @@ def update_spot_withdraws(self, day_jump: float = 90): The withdraws are then saved in the database. Only successful withdraws are fetched. + sources: + https://python-binance.readthedocs.io/en/latest/binance.html#binance.client.Client.get_withdraw_history + https://binance-docs.github.io/apidocs/spot/en/#withdraw-history-user_data + :param day_jump: length of the time window for each call (max 90) :type day_jump: float :return: None @@ -163,6 +422,10 @@ def update_spot_deposits(self, day_jump: float = 90): The deposits are then saved in the database. Only successful deposits are fetched. + sources: + https://python-binance.readthedocs.io/en/latest/binance.html#binance.client.Client.get_deposit_history + https://binance-docs.github.io/apidocs/spot/en/#deposit-history-user_data + :param day_jump: length of the time window for each call (max 90) :type day_jump: float :return: None @@ -193,6 +456,10 @@ def update_spot_symbol_trades(self, asset: str, ref_asset: str, limit: int = 100 This update the spot trades in the database for a single trading pair. It will check the last trade id and will requests the all trades after this trade_id. + sources: + https://python-binance.readthedocs.io/en/latest/binance.html#binance.client.Client.get_my_trades + https://binance-docs.github.io/apidocs/spot/en/#account-trade-list-user_data + :param asset: name of the asset in the trading pair (ex 'BTC' for 'BTCUSDT') :type asset: string :param ref_asset: name of the reference asset in the trading pair (ex 'USDT' for 'BTCUSDT') @@ -204,21 +471,22 @@ def update_spot_symbol_trades(self, asset: str, ref_asset: str, limit: int = 100 """ limit = min(1000, limit) symbol = asset + ref_asset - last_trade_id = self.db.get_max_trade_id(asset, ref_asset) + last_trade_id = self.db.get_max_trade_id(asset, ref_asset, 'spot') while True: new_trades = self.client.get_my_trades(symbol=symbol, fromId=last_trade_id + 1, limit=limit) for trade in new_trades: - self.db.add_spot_trade(trade_id=int(trade['id']), - millistamp=int(trade['time']), - asset=asset, - ref_asset=ref_asset, - qty=float(trade['qty']), - price=float(trade['price']), - fee=float(trade['commission']), - fee_asset=trade['commissionAsset'], - is_buyer=trade['isBuyer'], - auto_commit=False - ) + self.db.add_trade(trade_type='spot', + trade_id=int(trade['id']), + trade_time=int(trade['time']), + asset=asset, + ref_asset=ref_asset, + qty=float(trade['qty']), + price=float(trade['price']), + fee=float(trade['commission']), + fee_asset=trade['commissionAsset'], + is_buyer=trade['isBuyer'], + auto_commit=False + ) last_trade_id = max(last_trade_id, int(trade['id'])) if len(new_trades): self.db.commit() @@ -298,6 +566,33 @@ def drop_lending_interest_table(self): """ self.db.drop_table(tables.LENDING_INTEREST_TABLE) + def drop_cross_margin_trade_table(self): + """ + erase the cross margin trades table + + :return: None + :rtype: None + """ + self.db.drop_table(tables.CROSS_MARGIN_TRADE_TABLE) + + def drop_cross_margin_loan_table(self): + """ + erase the cross margin loan table + + :return: None + :rtype: None + """ + self.db.drop_table(tables.CROSS_MARGIN_LOAN_TABLE) + + def drop_cross_margin_repay_table(self): + """ + erase the cross margin repay table + + :return: None + :rtype: None + """ + self.db.drop_table(tables.CROSS_MARGIN_REPAY_TABLE) + def drop_all_tables(self): """ erase all the tables of the database by calling all the methods having 'drop' and 'table' in their names diff --git a/src/storage/DataBase.py b/src/storage/DataBase.py index 54b397d..404d94c 100644 --- a/src/storage/DataBase.py +++ b/src/storage/DataBase.py @@ -66,15 +66,19 @@ def get_row_by_key(self, table: Table, key_value) -> Optional[Tuple]: def get_conditions_rows(self, table: Table, selection: Optional[Union[str, List[str]]] = None, - conditions_list: Optional[List[Tuple[str, SQLConditionEnum, Any]]] = None) -> List: + conditions_list: Optional[List[Tuple[str, SQLConditionEnum, Any]]] = None, + order_list: Optional[List[str]] = None) -> List: if selection is None: selection = '*' elif isinstance(selection, List): selection = ','.join(selection) if conditions_list is None: conditions_list = [] + if order_list is None: + order_list = [] execution_cmd = f"SELECT {selection} from {table.name}" execution_cmd = self._add_conditions(execution_cmd, conditions_list=conditions_list) + execution_cmd = self._add_order(execution_cmd, order_list=order_list) return self._fetch_rows(execution_cmd) def get_all_rows(self, table: Table) -> List: @@ -143,7 +147,8 @@ def commit(self): def _add_conditions(execution_cmd: str, conditions_list: List[Tuple[str, SQLConditionEnum, Any]]): """ add a list of condition to an SQL command - :param execution_cmd: string with 'WHERE' statement + :param execution_cmd: SQL command without 'WHERE' statement + :type execution_cmd: str :param conditions_list: :return: """ @@ -155,6 +160,26 @@ def _add_conditions(execution_cmd: str, conditions_list: List[Tuple[str, SQLCond else: return execution_cmd + @staticmethod + def _add_order(execution_cmd: str, order_list: List[str]): + """ + add an order specification to an SQL command + + :param execution_cmd: SQL command without 'ORDER BY' statement + :type execution_cmd: str + :param order_list: + :type order_list: + :return: + :rtype: + """ + if len(order_list): + add_cmd = ' ORDER BY' + for column_name in order_list: + add_cmd = add_cmd + f" {column_name}," + return execution_cmd + add_cmd[:-1] + ' ASC' + else: + return execution_cmd + @staticmethod def get_create_cmd(table: Table): """ diff --git a/src/storage/tables.py b/src/storage/tables.py index 7304d54..df0d2b1 100644 --- a/src/storage/tables.py +++ b/src/storage/tables.py @@ -141,3 +141,82 @@ def __init__(self, name: str, columns_names: List[str], columns_sql_types: List[ 'REAL', ] ) + +CROSS_MARGIN_TRADE_TABLE = Table( + 'cross_margin_trade', + [ + 'tradeId', + 'tdTime', + 'asset', + 'refAsset', + 'qty', + 'price', + 'fee', + 'feeAsset', + 'isBuyer' + + ], + [ + 'INTEGER', + 'INTEGER', + 'TEXT', + 'TEXT', + 'REAL', + 'REAL', + 'REAL', + 'TEXT', + 'INTEGER' + ] +) + +CROSS_MARGIN_LOAN_TABLE = Table( + "cross_margin_loan_table", + [ + 'loanTime', + 'asset', + 'principal', + ], + [ + 'INTEGER', + 'TEXT', + 'REAL' + ], + primary_key='txId', + primary_key_sql_type='INTEGER' + +) + +CROSS_MARGIN_REPAY_TABLE = Table( + "cross_margin_repay_table", + [ + 'repayTime', + 'asset', + 'principal', + 'interest', + ], + [ + 'INTEGER', + 'TEXT', + 'REAL', + 'REAL' + ], + primary_key='txId', + primary_key_sql_type='INTEGER' + +) + +CROSS_MARGIN_INTEREST_TABLE = Table( + "cross_margin_interest_table", + [ + 'interestTime', + 'asset', + 'interest', + 'interestType' + ], + [ + 'INTEGER', + 'TEXT', + 'REAL', + 'TEXT' + ] +) diff --git a/tests/test_DataBase.py b/tests/test_DataBase.py index 84a251a..2225479 100644 --- a/tests/test_DataBase.py +++ b/tests/test_DataBase.py @@ -81,6 +81,7 @@ def test_inserts_search_2(verbose=0, **kwargs): (15, 'Karl', 55.5), (18, 'Kitty', 61.1), (18, 'Kitty', 61.1), # no set primary key -> should allow identical rows + (18, 'Marcus', 58.2), (18, 'Marc', 48.1), (55, 'Jean', 78.1) ] @@ -96,8 +97,15 @@ def test_inserts_search_2(verbose=0, **kwargs): (table2.age, SQLConditionEnum.equal, 18), (table2.weight, SQLConditionEnum.greater_equal, 55), ] - retrieved_rows = db.get_conditions_rows(table2, conditions_list=conditions) - assert rows[1:3] == retrieved_rows + order_list = [table2.weight] + retrieved_rows = db.get_conditions_rows(table2, conditions_list=conditions, order_list=order_list) + sub_rows = list(rows[1:4]) + if verbose: + print("rows retrieved:") + for row in retrieved_rows: + print('\t', row) + sub_rows.sort(key=lambda x: x[-1]) + assert sub_rows == retrieved_rows # check min ge is right assert max([r[0] for r in rows]) == db.get_conditions_rows(table2, selection=f"MAX({table2.age})")[0][0] From 9dd0acace9f5a3f3147b9deeb69dcdd630b165b9 Mon Sep 17 00:00:00 2001 From: EtWn <34377743+EtWnn@users.noreply.github.com> Date: Tue, 16 Mar 2021 14:30:43 +0100 Subject: [PATCH 06/17] Universal transfert (#6) * add universal transfer table * add universal transfer to the binance database * add BinanceManager universal transfer handling --- src/storage/BinanceDataBase.py | 91 +++++++++++++++++++++++++++++++++- src/storage/BinanceManager.py | 46 +++++++++++++++++ src/storage/tables.py | 18 +++++++ 3 files changed, 154 insertions(+), 1 deletion(-) diff --git a/src/storage/BinanceDataBase.py b/src/storage/BinanceDataBase.py index 3e9c7bf..2fa9a59 100644 --- a/src/storage/BinanceDataBase.py +++ b/src/storage/BinanceDataBase.py @@ -14,6 +14,95 @@ class BinanceDataBase(DataBase): def __init__(self, name: str = 'binance_db'): super().__init__(name) + def add_universal_transfer(self, transfer_id: int, transfer_type: str, transfer_time: int, asset: str, + amount: float, auto_commit: bool = True): + """ + add a universal transfer to the database + + :param transfer_id: id of the transfer + :type transfer_id: int + :param transfer_type: enum of the transfer type (ex: 'MAIN_MARGIN') + :type transfer_type: str + :param transfer_time: millistamp of the operation + :type transfer_time: int + :param asset: asset that got transferred + :type asset: str + :param amount: amount transferred + :type amount: float + :param auto_commit: if the database should commit the change made, default True + :type auto_commit: bool + :return: None + :rtype: None + """ + table = tables.UNIVERSAL_TRANSFER_TABLE + + row = (transfer_id, transfer_type, transfer_time, asset, amount) + self.add_row(table, row, auto_commit=auto_commit) + + def get_universal_transfers(self, transfer_type: Optional[str] = None, asset: Optional[str] = None, + start_time: Optional[int] = None, end_time: Optional[int] = None): + """ + return universal transfers stored in the database. Transfer type, Asset type and time filters can be used + + :param transfer_type: enum of the transfer type (ex: 'MAIN_MARGIN') + :type transfer_type: Optional[str] + :param asset: fetch only interests in this asset + :type asset: Optional[str] + :param start_time: fetch only interests after this millistamp + :type start_time: Optional[int] + :param end_time: fetch only interests before this millistamp + :type end_time: Optional[int] + :return: The raw rows selected as saved in the database + :rtype: List[Tuple] + """ + table = tables.UNIVERSAL_TRANSFER_TABLE + + conditions_list = [] + if transfer_type is not None: + conditions_list.append((table.trfType, + SQLConditionEnum.equal, + transfer_type)) + if asset is not None: + conditions_list.append((table.asset, + SQLConditionEnum.equal, + asset)) + if start_time is not None: + conditions_list.append((table.trfTime, + SQLConditionEnum.greater_equal, + start_time)) + if end_time is not None: + conditions_list.append((table.trfTime, + SQLConditionEnum.lower, + end_time)) + return self.get_conditions_rows(table, conditions_list=conditions_list) + + def get_last_universal_transfer(self, transfer_type: str): + """ + return the latest time when a universal transfer was made + If None, return the millistamp corresponding to 2017/01/01 + + :param transfer_type: enum of the transfer type (ex: 'MAIN_MARGIN') + :type transfer_type: str + :return: millistamp + :rtype: int + """ + table = tables.UNIVERSAL_TRANSFER_TABLE + conditions_list = [(table.trfType, + SQLConditionEnum.equal, + transfer_type)] + selection = f"MAX({table.trfTime})" + result = self.get_conditions_rows(table, + selection=selection, + conditions_list=conditions_list) + default = datetime_to_millistamp(datetime.datetime(2017, 1, 1, tzinfo=datetime.timezone.utc)) + try: + result = result[0][0] + except IndexError: + return default + if result is None: + return default + return result + def add_margin_interest(self, margin_type: str, interest_time: int, asset: str, interest: float, interest_type: str, auto_commit: bool = True): """ @@ -73,7 +162,7 @@ def get_margin_interests(self, margin_type: str, asset: Optional[str] = None, st SQLConditionEnum.equal, asset)) if start_time is not None: - conditions_list.append((table.repayTime, + conditions_list.append((table.interestTime, SQLConditionEnum.greater_equal, start_time)) if end_time is not None: diff --git a/src/storage/BinanceManager.py b/src/storage/BinanceManager.py index c686585..609a7ff 100644 --- a/src/storage/BinanceManager.py +++ b/src/storage/BinanceManager.py @@ -22,6 +22,52 @@ def __init__(self): credentials = CredentialManager.get_api_credentials("Binance") self.client = Client(**credentials) + def update_universal_transfers(self): + """ + update the universal transfers database. + + sources: + https://python-binance.readthedocs.io/en/latest/binance.html#binance.client.Client.query_universal_transfer_history + https://binance-docs.github.io/apidocs/spot/en/#query-user-universal-transfer-history + + :return: None + :rtype: None + """ + transfers_types = ['MAIN_C2C', 'MAIN_UMFUTURE', 'MAIN_CMFUTURE', 'MAIN_MARGIN', 'MAIN_MINING', 'C2C_MAIN', + 'C2C_UMFUTURE', 'C2C_MINING', 'C2C_MARGIN', 'UMFUTURE_MAIN', 'UMFUTURE_C2C', + 'UMFUTURE_MARGIN', 'CMFUTURE_MAIN', 'CMFUTURE_MARGIN', 'MARGIN_MAIN', 'MARGIN_UMFUTURE', + 'MARGIN_CMFUTURE', 'MARGIN_MINING', 'MARGIN_C2C', 'MINING_MAIN', 'MINING_UMFUTURE', + 'MINING_C2C', 'MINING_MARGIN'] + pbar = tqdm(total=len(transfers_types)) + for transfer_type in transfers_types: + pbar.set_description(f"fetching transfer type {transfer_type}") + latest_time = self.db.get_last_universal_transfer(transfer_type=transfer_type) + 1 + current = 1 + while True: + universal_transfers = self.client.query_universal_transfer_history(type=transfer_type, + startTime=latest_time, + current=current, + size=100) + try: + universal_transfers = universal_transfers['rows'] + except KeyError: + break + for transfer in universal_transfers: + self.db.add_universal_transfer(transfer_id=transfer['tranId'], + transfer_type=transfer['type'], + transfer_time=transfer['timestamp'], + asset=transfer['asset'], + amount=float(transfer['amount']) + ) + + if len(universal_transfers): + current += 1 # next page + self.db.commit() + else: + break + pbar.update() + pbar.close() + def update_cross_margin_interests(self): """ update the interests for all cross margin assets diff --git a/src/storage/tables.py b/src/storage/tables.py index df0d2b1..da5541f 100644 --- a/src/storage/tables.py +++ b/src/storage/tables.py @@ -220,3 +220,21 @@ def __init__(self, name: str, columns_names: List[str], columns_sql_types: List[ 'TEXT' ] ) + +UNIVERSAL_TRANSFER_TABLE = Table( + "universal_transfer_table", + [ + 'trfType', + 'trfTime', + 'asset', + 'amount' + ], + [ + 'TEXT', + 'INTEGER', + 'TEXT', + 'REAL' + ], + primary_key='tranId', + primary_key_sql_type='INTEGER' +) From d30a7c51e8d111094ea9e98ab80085b304845a5c Mon Sep 17 00:00:00 2001 From: EtWn <34377743+EtWnn@users.noreply.github.com> Date: Mon, 22 Mar 2021 09:53:35 +0100 Subject: [PATCH 07/17] Data path (#7) * add utils.paths and the data path * add default logs folders in LoggerGenerator Improve LoggerGenerator documentation * update save path for DataBase remove unused imports --- src/storage/DataBase.py | 5 +-- src/utils/LoggerGenerator.py | 82 ++++++++++++++++++++++++++++++------ src/utils/paths.py | 26 ++++++++++++ tests/test_DataBase.py | 2 - 4 files changed, 97 insertions(+), 18 deletions(-) create mode 100644 src/utils/paths.py diff --git a/src/storage/DataBase.py b/src/storage/DataBase.py index 404d94c..1c389a2 100644 --- a/src/storage/DataBase.py +++ b/src/storage/DataBase.py @@ -1,11 +1,10 @@ -import sys -import os from enum import Enum from typing import List, Tuple, Optional, Any, Union import sqlite3 from src.storage.tables import Table from src.utils.LoggerGenerator import LoggerGenerator +from src.utils.paths import get_data_path class SQLConditionEnum(Enum): @@ -28,7 +27,7 @@ class DataBase: def __init__(self, name: str): self.name = name self.logger = LoggerGenerator.get_logger(self.name) - self.save_path = f"data/{name}.db" + self.save_path = get_data_path() / f"{name}.db" self.db_conn = sqlite3.connect(self.save_path) self.db_cursor = self.db_conn.cursor() diff --git a/src/utils/LoggerGenerator.py b/src/utils/LoggerGenerator.py index 5d91774..4b1e5fe 100644 --- a/src/utils/LoggerGenerator.py +++ b/src/utils/LoggerGenerator.py @@ -1,36 +1,92 @@ import logging +import os +from typing import Optional + +from src.utils.paths import get_data_path class LoggerGenerator: - logger_count = 0 - global_log_level = logging.WARNING + """ + This class is a utility to facilitate the creation of loggers for the different classes / files + """ + LOGS_FOLDER_PATH = get_data_path() / "logs" + + _default_log_level = logging.WARNING + _default_write_file = False + _logger_count = 0 + + @staticmethod + def set_global_log_level(log_level: int): + """ + set the default log level for loggers creation + + :param log_level: threshold to display the message + :type log_level: logging enum (ex: logging.WARNING) + :return: None + :rtype: None + """ + LoggerGenerator._default_log_level = log_level @staticmethod - def get_logger(logger_name, create_file=False, log_level=None): + def set_default_write_file(write_file: bool): + """ + set the default write level for loggers creation + + :param write_file: if the logger should save the message in a file + :type write_file: bool + :return: None + :rtype: None + """ + LoggerGenerator._default_write_file = write_file + + @staticmethod + def get_logger(logger_name: str, write_file: Optional[bool] = None, + log_level: Optional[int] = None) -> logging.Logger: + """ + create a logger that will display messages according to the log level threshold. If specified, it will + also save the messages in a file inside the logs folder + + :param logger_name: name of the logger (a unique logger id will be added after the name) + :type logger_name: str + :param write_file: if the logger should save the message in a file + :type write_file: bool + :param log_level: threshold to display the message + :type log_level: logging enum (ex: logging.WARNING) + :return: the logger object + :rtype: logging.Logger + """ if log_level is None: - log_level = LoggerGenerator.global_log_level + log_level = LoggerGenerator._default_log_level + if write_file is None: + write_file = LoggerGenerator._default_write_file - # create logger for prd_ci - log = logging.getLogger(f"lg_{LoggerGenerator.logger_count}_{logger_name}") - log.setLevel(level=log_level) - LoggerGenerator.logger_count += 1 + # create logger + logger = logging.getLogger(f"lg_{LoggerGenerator._logger_count}_{logger_name}") + logger.setLevel(level=log_level) + LoggerGenerator._logger_count += 1 # create formatter and add it to the handlers log_format = '[%(asctime)s %(name)s %(levelname)s] %(message)s [%(pathname)s:%(lineno)d in %(funcName)s]' formatter = logging.Formatter(log_format) - if create_file: + if write_file: # create file handler for logger. - fh = logging.FileHandler('SPOT.log') + log_file_path = LoggerGenerator.LOGS_FOLDER_PATH / f"{logger_name}.log" + fh = logging.FileHandler(log_file_path) fh.setLevel(level=log_level) fh.setFormatter(formatter) - log.addHandler(fh) + logger.addHandler(fh) # create console handler for logger. ch = logging.StreamHandler() ch.setLevel(level=log_level) ch.setFormatter(formatter) - log.addHandler(ch) + logger.addHandler(ch) + + return logger - return log +try: # create logs folder + os.makedirs(LoggerGenerator.LOGS_FOLDER_PATH) +except FileExistsError: + pass diff --git a/src/utils/paths.py b/src/utils/paths.py new file mode 100644 index 0000000..76ba2b1 --- /dev/null +++ b/src/utils/paths.py @@ -0,0 +1,26 @@ +import os +from pathlib import Path +from appdirs import AppDirs + +_app_dirs = AppDirs("BinanceWatch", "EtWnn") + + +def get_data_path(): + """ + Return the folder path where to store the data created by this project + It uses the library appdirs to follow the conventions across multi OS(MAc, Linux, Windows) + + https://pypi.org/project/appdirs/ + + :return: path of the folder to use for data saving + :rtype: pathlib.Path + """ + return Path(_app_dirs.user_data_dir) + + +try: # create the data folder path + os.makedirs(get_data_path()) +except FileExistsError: + pass + + diff --git a/tests/test_DataBase.py b/tests/test_DataBase.py index 2225479..925cceb 100644 --- a/tests/test_DataBase.py +++ b/tests/test_DataBase.py @@ -1,5 +1,3 @@ -from pandas import DataFrame - from src.storage.DataBase import DataBase, SQLConditionEnum from src.storage.tables import Table From 1f5825af5553853694a87e83630aa208a1102f91 Mon Sep 17 00:00:00 2001 From: EtWn <34377743+EtWnn@users.noreply.github.com> Date: Mon, 22 Mar 2021 12:34:42 +0100 Subject: [PATCH 08/17] externalize credentials (#8) * delete credentials.py and credentials example * update BinanceManager and .gitignore --- .gitignore | 2 -- data/credentials.json.example | 9 --------- src/storage/BinanceManager.py | 6 ++---- src/utils/credentials.py | 23 ----------------------- 4 files changed, 2 insertions(+), 38 deletions(-) delete mode 100644 data/credentials.json.example delete mode 100644 src/utils/credentials.py diff --git a/.gitignore b/.gitignore index bab857f..59921b0 100644 --- a/.gitignore +++ b/.gitignore @@ -141,8 +141,6 @@ dmypy.json # except README !**/README.md -# except credentials example -!data/credentials.json.example # except gitkeep !**/.gitkeep \ No newline at end of file diff --git a/data/credentials.json.example b/data/credentials.json.example deleted file mode 100644 index 0c79217..0000000 --- a/data/credentials.json.example +++ /dev/null @@ -1,9 +0,0 @@ -{ - "CoinAPI": { - "api_key": "MY_API_KEY" - }, - "Binance": { - "api_key": "MY_API_KEY", - "api_secret":"MY_API_SECRET" - } -} \ No newline at end of file diff --git a/src/storage/BinanceManager.py b/src/storage/BinanceManager.py index 609a7ff..64890fa 100644 --- a/src/storage/BinanceManager.py +++ b/src/storage/BinanceManager.py @@ -8,7 +8,6 @@ from src.utils.time_utils import datetime_to_millistamp from src.storage.BinanceDataBase import BinanceDataBase -from src.utils.credentials import CredentialManager from src.storage import tables @@ -17,10 +16,9 @@ class BinanceManager: This class is in charge of filling the database by calling the binance API """ - def __init__(self): + def __init__(self, api_key: str, api_secret: str): self.db = BinanceDataBase() - credentials = CredentialManager.get_api_credentials("Binance") - self.client = Client(**credentials) + self.client = Client(api_key=api_key, api_secret=api_secret) def update_universal_transfers(self): """ diff --git a/src/utils/credentials.py b/src/utils/credentials.py deleted file mode 100644 index cc980e8..0000000 --- a/src/utils/credentials.py +++ /dev/null @@ -1,23 +0,0 @@ -import json -from typing import Optional, Dict - -from src.utils.LoggerGenerator import LoggerGenerator - - -class CredentialManager: - - logger = LoggerGenerator.get_logger("crendentials_manager") - - @staticmethod - def get_api_credentials(api_name) -> Optional[Dict]: - try: - with open("data/credentials.json") as file: - credentials = json.load(file) - return credentials[api_name] - except FileNotFoundError as ex: - CredentialManager.logger.error("Could not find the credentials file") - raise ex - except KeyError as ex: - CredentialManager.logger.error(f"there is no key registered in the credentials file " - f"the name {api_name}") - raise ex From da9ec8d7d4151be28b59b29eb317d4c738f1ba4c Mon Sep 17 00:00:00 2001 From: EtWn <34377743+EtWnn@users.noreply.github.com> Date: Mon, 22 Mar 2021 14:11:36 +0100 Subject: [PATCH 09/17] Setup (#9) * rename src folder to BinanceWatch * add __init__ * add License * add setup.py * remove environment.yml --- BinanceWatch/__init__.py | 0 .../storage/BinanceDataBase.py | 6 ++--- .../storage/BinanceManager.py | 6 ++--- {src => BinanceWatch}/storage/DataBase.py | 6 ++--- {src => BinanceWatch}/storage/tables.py | 0 .../utils/LoggerGenerator.py | 2 +- {src => BinanceWatch}/utils/paths.py | 0 {src => BinanceWatch}/utils/time_utils.py | 0 LICENSE | 21 ++++++++++++++++++ environment.yml | 13 ----------- setup.py | 22 +++++++++++++++++++ tests/test_DataBase.py | 4 ++-- 12 files changed, 55 insertions(+), 25 deletions(-) create mode 100644 BinanceWatch/__init__.py rename {src => BinanceWatch}/storage/BinanceDataBase.py (99%) rename {src => BinanceWatch}/storage/BinanceManager.py (99%) rename {src => BinanceWatch}/storage/DataBase.py (97%) rename {src => BinanceWatch}/storage/tables.py (100%) rename {src => BinanceWatch}/utils/LoggerGenerator.py (98%) rename {src => BinanceWatch}/utils/paths.py (100%) rename {src => BinanceWatch}/utils/time_utils.py (100%) create mode 100644 LICENSE delete mode 100644 environment.yml create mode 100644 setup.py diff --git a/BinanceWatch/__init__.py b/BinanceWatch/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/storage/BinanceDataBase.py b/BinanceWatch/storage/BinanceDataBase.py similarity index 99% rename from src/storage/BinanceDataBase.py rename to BinanceWatch/storage/BinanceDataBase.py index 2fa9a59..80f4079 100644 --- a/src/storage/BinanceDataBase.py +++ b/BinanceWatch/storage/BinanceDataBase.py @@ -1,9 +1,9 @@ import datetime from typing import Optional -from src.storage.DataBase import DataBase, SQLConditionEnum -from src.storage import tables -from src.utils.time_utils import datetime_to_millistamp +from BinanceWatch.storage.DataBase import DataBase, SQLConditionEnum +from BinanceWatch.storage import tables +from BinanceWatch.utils.time_utils import datetime_to_millistamp class BinanceDataBase(DataBase): diff --git a/src/storage/BinanceManager.py b/BinanceWatch/storage/BinanceManager.py similarity index 99% rename from src/storage/BinanceManager.py rename to BinanceWatch/storage/BinanceManager.py index 64890fa..b4bbd92 100644 --- a/src/storage/BinanceManager.py +++ b/BinanceWatch/storage/BinanceManager.py @@ -6,9 +6,9 @@ from binance.client import Client from tqdm import tqdm -from src.utils.time_utils import datetime_to_millistamp -from src.storage.BinanceDataBase import BinanceDataBase -from src.storage import tables +from BinanceWatch.utils.time_utils import datetime_to_millistamp +from BinanceWatch.storage.BinanceDataBase import BinanceDataBase +from BinanceWatch.storage import tables class BinanceManager: diff --git a/src/storage/DataBase.py b/BinanceWatch/storage/DataBase.py similarity index 97% rename from src/storage/DataBase.py rename to BinanceWatch/storage/DataBase.py index 1c389a2..3c8b3ca 100644 --- a/src/storage/DataBase.py +++ b/BinanceWatch/storage/DataBase.py @@ -2,9 +2,9 @@ from typing import List, Tuple, Optional, Any, Union import sqlite3 -from src.storage.tables import Table -from src.utils.LoggerGenerator import LoggerGenerator -from src.utils.paths import get_data_path +from BinanceWatch.storage.tables import Table +from BinanceWatch.utils.LoggerGenerator import LoggerGenerator +from BinanceWatch.utils.paths import get_data_path class SQLConditionEnum(Enum): diff --git a/src/storage/tables.py b/BinanceWatch/storage/tables.py similarity index 100% rename from src/storage/tables.py rename to BinanceWatch/storage/tables.py diff --git a/src/utils/LoggerGenerator.py b/BinanceWatch/utils/LoggerGenerator.py similarity index 98% rename from src/utils/LoggerGenerator.py rename to BinanceWatch/utils/LoggerGenerator.py index 4b1e5fe..45ba69e 100644 --- a/src/utils/LoggerGenerator.py +++ b/BinanceWatch/utils/LoggerGenerator.py @@ -2,7 +2,7 @@ import os from typing import Optional -from src.utils.paths import get_data_path +from BinanceWatch.utils.paths import get_data_path class LoggerGenerator: diff --git a/src/utils/paths.py b/BinanceWatch/utils/paths.py similarity index 100% rename from src/utils/paths.py rename to BinanceWatch/utils/paths.py diff --git a/src/utils/time_utils.py b/BinanceWatch/utils/time_utils.py similarity index 100% rename from src/utils/time_utils.py rename to BinanceWatch/utils/time_utils.py diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..5f656ca --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2021 EtWnn + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/environment.yml b/environment.yml deleted file mode 100644 index 5b2d958..0000000 --- a/environment.yml +++ /dev/null @@ -1,13 +0,0 @@ -name: binancewatch -channels: - - defaults -dependencies: - - jupyter - - numpy - - python=3.7.9 - - tqdm - - pip=20.3.3 - - pip: - - dateparser - - python-binance - - requests diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..21fdc2b --- /dev/null +++ b/setup.py @@ -0,0 +1,22 @@ +from setuptools import setup + +setup( + name='BinanceWatch', + version='0.1', + packages=['BinanceWatch', 'tests'], + url='https://github.com/EtWnn/BinanceWatch', + author='EtWnn', + author_email='', + license='MIT', + description='Local tracker of a binance account', + install_requires=['numpy', 'tqdm', 'dateparser', 'requests', 'python-binance', 'appdirs'], + keywords='binance exchange wallet save tracking bitcoin ethereum btc eth neo', + classifiers=[ + 'Intended Audience :: Developers', + 'License :: OSI Approved :: MIT License', + 'Operating System :: OS Independent', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python', + 'Topic :: Software Development :: Libraries :: Python Modules', + ] +) diff --git a/tests/test_DataBase.py b/tests/test_DataBase.py index 925cceb..d713f86 100644 --- a/tests/test_DataBase.py +++ b/tests/test_DataBase.py @@ -1,5 +1,5 @@ -from src.storage.DataBase import DataBase, SQLConditionEnum -from src.storage.tables import Table +from BinanceWatch.storage.DataBase import DataBase, SQLConditionEnum +from BinanceWatch.storage.tables import Table db = DataBase("test_table") From 29ae2597edb2f450f1b98c180fa0429229775323 Mon Sep 17 00:00:00 2001 From: EtWn <34377743+EtWnn@users.noreply.github.com> Date: Mon, 22 Mar 2021 17:30:13 +0100 Subject: [PATCH 10/17] Readme (#10) * add the sections Note, Features, Quick Tour, Donation and Known Issues to the README * remove unnecessary new lines * remove unnecessary separations --- README.md | 91 +++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 88 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 74f1470..ff11abb 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,89 @@ -# BinanceWatch +# Welcome to BinanceWatch v0.1 -This repo is made to keep track locally of the transactions made by a -user on his binance account. \ No newline at end of file + +## Note + + +This library is under development by EtWnn, but feel free to drop your suggestions or remarks in +the discussion tab of this repo. You are also welcome to contribute by submitting PRs. + +This is an unofficial tracker for binance accounts. I am in no way affiliated with Binance, use at +your own risk. + +**Source Code:** https://github.com/EtWnn/BinanceWatch + + +## Features + + +If you used quite intensively Binance, it can take some time to retrieve everything that happened +on your account. This library is made to save locally the events of your account so that you don't +need to fetch your history from the beginning every time. + + +It currently supports: + +- Spot Trades +- Spot Crypto Deposits +- Spot Crypto Withdraws +- Spot Dividends +- Spot Interests +- Spot Dusts +- Universal Transfers + + +- Cross Margin Trades +- Cross Margin Repayment +- Cross Margin Loans +- Cross Margin Interests + +## Quick Tour + + +[Generate an API Key](https://www.binance.com/en/my/settings/api-management) in your binance account. Only read +permissions are needed. + + + +```python +from BinanceWatch.storage.BinanceManager import BinanceManager + +api_key = "" +api_secret = "" + +bm = BinanceManager(api_key, api_secret) + +# fetch the latest spot trades from Binance +bm.update_all_spot_trades() +``` +``` +Out -> fetching spot deposits: 100%|██████████████████████████████| 18/18 [00:08<00:00, 2.24it/s] +``` +```python +from datetime import datetime +from BinanceWatch.utils.time_utils import datetime_to_millistamp + + +start_time = datetime_to_millistamp(datetime(2018,1,1)) + +# get the locally saved spot trades made after 2018/01/01 +spot_trades = bm.db.get_trades('spot', start_time=start_time) +``` + +## Donation + + +If this library has helped you in any way, feel free to donate: +- **BTC**: 14ou4fMYoMVYbWEKnhADPJUNVytWQWx9HG +- **ETH**: 0xfb0ebcf8224ce561bfb06a56c3b9a43e1a4d1be2 +- **LTC**: LfHgc969RFUjnmyLn41SRDvmT146jUg9tE +- **EGLD**: erd1qk98xm2hgztvmq6s4jwtk06g6laattewp6vh20z393drzy5zzfrq0gaefh + + +## Known Issues: + + +Some endpoints are not yet provided by Binance, so they can't be implemented in this library: +- Fiat withdraws and deposits +- Locked stacking interests +- Direct purchases with debit card From 3393cb3130e91eae2d80cf881e8ade44dce1aeca Mon Sep 17 00:00:00 2001 From: EtWn <34377743+EtWnn@users.noreply.github.com> Date: Mon, 22 Mar 2021 18:26:04 +0100 Subject: [PATCH 11/17] correct output (#11) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ff11abb..983c7be 100644 --- a/README.md +++ b/README.md @@ -57,7 +57,7 @@ bm = BinanceManager(api_key, api_secret) bm.update_all_spot_trades() ``` ``` -Out -> fetching spot deposits: 100%|██████████████████████████████| 18/18 [00:08<00:00, 2.24it/s] +Out -> fetching BIFIBUSD: 100%|██████████████████████| 1349/1349 [06:24<00:00, 3.51it/s] ``` ```python from datetime import datetime From 59dc93dbd213bfa16885995d9f4461cc87469114 Mon Sep 17 00:00:00 2001 From: EtWn <34377743+EtWnn@users.noreply.github.com> Date: Mon, 22 Mar 2021 19:42:24 +0100 Subject: [PATCH 12/17] Update union (#12) * add update_spot, update_cross_margin and update_lending * add a progress bar for cross margin interest * update progress bar descriptions --- BinanceWatch/storage/BinanceManager.py | 48 ++++++++++++++++++++++++-- 1 file changed, 45 insertions(+), 3 deletions(-) diff --git a/BinanceWatch/storage/BinanceManager.py b/BinanceWatch/storage/BinanceManager.py index b4bbd92..cff0202 100644 --- a/BinanceWatch/storage/BinanceManager.py +++ b/BinanceWatch/storage/BinanceManager.py @@ -20,6 +20,44 @@ def __init__(self, api_key: str, api_secret: str): self.db = BinanceDataBase() self.client = Client(api_key=api_key, api_secret=api_secret) + def update_spot(self): + """ + call all update methods related to the spot account + + :return: None + :rtype: None + """ + self.update_all_spot_trades() + self.update_spot_deposits() + self.update_spot_withdraws() + self.update_spot_dusts() + self.update_spot_dividends() + self.update_universal_transfers() + + def update_cross_margin(self): + """ + call all update methods related to cross margin spot account + + :return: None + :rtype: None + """ + self.update_all_cross_margin_trades() + self.update_cross_margin_loans() + self.update_cross_margin_interests() + self.update_cross_margin_repays() + self.update_universal_transfers() + + def update_lending(self): + """ + call all update methods related to lending activities + + :return: None + :rtype: None + """ + self.update_lending_interests() + # TODO add update lending purchase + # TODO add update lending redemption + def update_universal_transfers(self): """ update the universal transfers database. @@ -80,6 +118,8 @@ def update_cross_margin_interests(self): latest_time = self.db.get_last_margin_interest_time(margin_type) archived = 1000 * time.time() - latest_time > 1000 * 3600 * 24 * 30 * 3 current = 1 + pbar = tqdm() + pbar.set_description("fetching cross margin interests") while True: params = { 'current': current, @@ -107,6 +147,8 @@ def update_cross_margin_interests(self): latest_time = self.db.get_last_margin_interest_time(margin_type) else: break + pbar.update() + pbar.close() def update_cross_margin_repays(self): """ @@ -293,7 +335,7 @@ def update_all_cross_margin_trades(self, limit: int = 1000): symbols_info = self.client._request_margin_api('get', 'margin/allPairs', data={}) # not built-in yet pbar = tqdm(total=len(symbols_info)) for symbol_info in symbols_info: - pbar.set_description(f"fetching {symbol_info['symbol']}") + pbar.set_description(f"fetching {symbol_info['symbol']} cross margin trades") self.update_cross_margin_symbol_trades(asset=symbol_info['base'], ref_asset=symbol_info['quote'], limit=limit) @@ -354,7 +396,7 @@ def update_spot_dusts(self): result = self.client.get_dust_log() dusts = result['results'] pbar = tqdm(total=dusts['total']) - pbar.set_description("fetching dusts") + pbar.set_description("fetching spot dusts") for d in dusts['rows']: for sub_dust in d['logs']: date_time = dateparser.parse(sub_dust['operateTime'] + 'Z') @@ -549,7 +591,7 @@ def update_all_spot_trades(self, limit: int = 1000): symbols_info = self.client.get_exchange_info()['symbols'] pbar = tqdm(total=len(symbols_info)) for symbol_info in symbols_info: - pbar.set_description(f"fetching {symbol_info['symbol']}") + pbar.set_description(f"fetching {symbol_info['symbol']} spot trades") self.update_spot_symbol_trades(asset=symbol_info['baseAsset'], ref_asset=symbol_info['quoteAsset'], limit=limit) From 712632dfb8d3a9fec013d3e111ea8983e12e2ee1 Mon Sep 17 00:00:00 2001 From: EtWn <34377743+EtWnn@users.noreply.github.com> Date: Tue, 23 Mar 2021 10:18:43 +0100 Subject: [PATCH 13/17] Lending (#13) * add lending purchase table * adapt keywords setup * add lending purchases to the database * BinanceManager update lending purchases * add redemption table * add lending redemptions to the database * Binance Manager update lending redemptions * update README Features with lending purchases and redemptions --- BinanceWatch/storage/BinanceDataBase.py | 176 +++++++++++++++++++++++- BinanceWatch/storage/BinanceManager.py | 85 +++++++++++- BinanceWatch/storage/tables.py | 34 +++++ README.md | 8 +- setup.py | 2 +- 5 files changed, 298 insertions(+), 7 deletions(-) diff --git a/BinanceWatch/storage/BinanceDataBase.py b/BinanceWatch/storage/BinanceDataBase.py index 80f4079..4833767 100644 --- a/BinanceWatch/storage/BinanceDataBase.py +++ b/BinanceWatch/storage/BinanceDataBase.py @@ -416,6 +416,180 @@ def get_last_loan_time(self, asset: str, margin_type: str): return default return result + def add_lending_redemption(self, redemption_time: int, lending_type: str, asset: str, amount: float, + auto_commit: bool = True): + """ + add a lending redemption to the database + + :param redemption_time: millitstamp of the operation + :type redemption_time: int + :param lending_type: either 'DAILY', 'ACTIVITY' or 'CUSTOMIZED_FIXED' + :type lending_type: str + :param asset: asset lent + :type asset: str + :param amount: amount of asset redeemed + :type amount: float + :param auto_commit: if the database should commit the change made, default True + :type auto_commit: bool + :return: None + :rtype: None + """ + row = (redemption_time, lending_type, asset, amount) + self.add_row(tables.LENDING_REDEMPTION_TABLE, row, auto_commit=auto_commit) + + def get_lending_redemptions(self, lending_type: Optional[str] = None, asset: Optional[str] = None, + start_time: Optional[int] = None, end_time: Optional[int] = None): + """ + return lending redemptions stored in the database. Asset type and time filters can be used + + :param lending_type:fetch only redemptions from this lending type + :type lending_type: Optional[str] + :param asset: fetch only redemptions from this asset + :type asset: Optional[str] + :param start_time: fetch only redemptions after this millistamp + :type start_time: Optional[int] + :param end_time: fetch only redemptions before this millistamp + :type end_time: Optional[int] + :return: The raw rows selected as saved in the database + :rtype: List[Tuple] + """ + conditions_list = [] + table = tables.LENDING_REDEMPTION_TABLE + if lending_type is not None: + conditions_list.append((table.lendingType, + SQLConditionEnum.equal, + lending_type)) + if asset is not None: + conditions_list.append((table.asset, + SQLConditionEnum.equal, + asset)) + if start_time is not None: + conditions_list.append((table.redemptionTime, + SQLConditionEnum.greater_equal, + start_time)) + if end_time is not None: + conditions_list.append((table.redemptionTime, + SQLConditionEnum.lower, + end_time)) + return self.get_conditions_rows(table, conditions_list=conditions_list) + + def get_last_lending_redemption_time(self, lending_type: Optional[str] = None): + """ + return the latest time when an lending redemption was made. + If None, return the millistamp corresponding to 2017/01/01 + + :param lending_type: type of lending + :type lending_type: str + :return: millistamp + :rtype: int + """ + conditions_list = [] + table = tables.LENDING_REDEMPTION_TABLE + if lending_type is not None: + conditions_list.append((table.lendingType, + SQLConditionEnum.equal, + lending_type)) + selection = f"MAX({table.redemptionTime})" + result = self.get_conditions_rows(table, + selection=selection, + conditions_list=conditions_list) + default = datetime_to_millistamp(datetime.datetime(2017, 1, 1, tzinfo=datetime.timezone.utc)) + try: + result = result[0][0] + except IndexError: + return default + if result is None: + return default + return result + + def add_lending_purchase(self, purchase_id: int, purchase_time: int, lending_type: str, asset: str, amount: float, + auto_commit: bool = True): + """ + add a lending purchase to the database + + :param purchase_id: id of the purchase + :type purchase_id: int + :param purchase_time: millitstamp of the operation + :type purchase_time: int + :param lending_type: either 'DAILY', 'ACTIVITY' or 'CUSTOMIZED_FIXED' + :type lending_type: str + :param asset: asset lent + :type asset: str + :param amount: amount of asset lent + :type amount: float + :param auto_commit: if the database should commit the change made, default True + :type auto_commit: bool + :return: None + :rtype: None + """ + row = (purchase_id, purchase_time, lending_type, asset, amount) + self.add_row(tables.LENDING_PURCHASE_TABLE, row, auto_commit=auto_commit) + + def get_lending_purchases(self, lending_type: Optional[str] = None, asset: Optional[str] = None, + start_time: Optional[int] = None, end_time: Optional[int] = None): + """ + return lending purchases stored in the database. Asset type and time filters can be used + + :param lending_type:fetch only purchases from this lending type + :type lending_type: Optional[str] + :param asset: fetch only purchases from this asset + :type asset: Optional[str] + :param start_time: fetch only purchases after this millistamp + :type start_time: Optional[int] + :param end_time: fetch only purchases before this millistamp + :type end_time: Optional[int] + :return: The raw rows selected as saved in the database + :rtype: List[Tuple] + """ + conditions_list = [] + table = tables.LENDING_PURCHASE_TABLE + if lending_type is not None: + conditions_list.append((table.lendingType, + SQLConditionEnum.equal, + lending_type)) + if asset is not None: + conditions_list.append((table.asset, + SQLConditionEnum.equal, + asset)) + if start_time is not None: + conditions_list.append((table.purchaseTime, + SQLConditionEnum.greater_equal, + start_time)) + if end_time is not None: + conditions_list.append((table.purchaseTime, + SQLConditionEnum.lower, + end_time)) + return self.get_conditions_rows(table, conditions_list=conditions_list) + + def get_last_lending_purchase_time(self, lending_type: Optional[str] = None): + """ + return the latest time when an lending purchase was made. + If None, return the millistamp corresponding to 2017/01/01 + + :param lending_type: type of lending + :type lending_type: str + :return: millistamp + :rtype: int + """ + conditions_list = [] + table = tables.LENDING_PURCHASE_TABLE + if lending_type is not None: + conditions_list.append((table.lendingType, + SQLConditionEnum.equal, + lending_type)) + selection = f"MAX({table.purchaseTime})" + result = self.get_conditions_rows(table, + selection=selection, + conditions_list=conditions_list) + default = datetime_to_millistamp(datetime.datetime(2017, 1, 1, tzinfo=datetime.timezone.utc)) + try: + result = result[0][0] + except IndexError: + return default + if result is None: + return default + return result + def add_lending_interest(self, time: int, lending_type: str, asset: str, amount: float, auto_commit: bool = True): """ @@ -425,7 +599,7 @@ def add_lending_interest(self, time: int, lending_type: str, asset: str, amount: :type time: int :param lending_type: either 'DAILY', 'ACTIVITY' or 'CUSTOMIZED_FIXED' :type lending_type: str - :param asset: asset that got converted to BNB + :param asset: asset that was received :type asset: str :param amount: amount of asset received :type amount: float diff --git a/BinanceWatch/storage/BinanceManager.py b/BinanceWatch/storage/BinanceManager.py index cff0202..a23ddd3 100644 --- a/BinanceWatch/storage/BinanceManager.py +++ b/BinanceWatch/storage/BinanceManager.py @@ -55,8 +55,8 @@ def update_lending(self): :rtype: None """ self.update_lending_interests() - # TODO add update lending purchase - # TODO add update lending redemption + self.update_lending_purchases() + self.update_lending_redemptions() def update_universal_transfers(self): """ @@ -342,6 +342,83 @@ def update_all_cross_margin_trades(self, limit: int = 1000): pbar.update() pbar.close() + def update_lending_redemptions(self): + """ + update the lending redemptions database. + + sources: + https://python-binance.readthedocs.io/en/latest/binance.html#binance.client.Client.get_lending_redemption_history + https://binance-docs.github.io/apidocs/spot/en/#get-redemption-record-user_data + + :return: None + :rtype: None + """ + lending_types = ['DAILY', 'ACTIVITY', 'CUSTOMIZED_FIXED'] + pbar = tqdm(total=3) + for lending_type in lending_types: + pbar.set_description(f"fetching lending redemptions of type {lending_type}") + latest_time = self.db.get_last_lending_redemption_time(lending_type=lending_type) + 1 + current = 1 + while True: + lending_redemptions = self.client.get_lending_redemption_history(lendingType=lending_type, + startTime=latest_time, + current=current, + size=100) + for li in lending_redemptions: + if li['status'] == 'PAID': + self.db.add_lending_redemption(redemption_time=li['createTime'], + lending_type=lending_type, + asset=li['asset'], + amount=li['amount'] + ) + + if len(lending_redemptions): + current += 1 # next page + self.db.commit() + else: + break + pbar.update() + pbar.close() + + def update_lending_purchases(self): + """ + update the lending purchases database. + + sources: + https://python-binance.readthedocs.io/en/latest/binance.html#binance.client.Client.get_lending_purchase_history + https://binance-docs.github.io/apidocs/spot/en/#get-purchase-record-user_data + + :return: None + :rtype: None + """ + lending_types = ['DAILY', 'ACTIVITY', 'CUSTOMIZED_FIXED'] + pbar = tqdm(total=3) + for lending_type in lending_types: + pbar.set_description(f"fetching lending purchases of type {lending_type}") + latest_time = self.db.get_last_lending_purchase_time(lending_type=lending_type) + 1 + current = 1 + while True: + lending_purchases = self.client.get_lending_purchase_history(lendingType=lending_type, + startTime=latest_time, + current=current, + size=100) + for li in lending_purchases: + if li['status'] == 'SUCCESS': + self.db.add_lending_purchase(purchase_id=li['purchaseId'], + purchase_time=li['createTime'], + lending_type=li['lendingType'], + asset=li['asset'], + amount=li['amount'] + ) + + if len(lending_purchases): + current += 1 # next page + self.db.commit() + else: + break + pbar.update() + pbar.close() + def update_lending_interests(self): """ update the lending interests database. @@ -356,14 +433,14 @@ def update_lending_interests(self): lending_types = ['DAILY', 'ACTIVITY', 'CUSTOMIZED_FIXED'] pbar = tqdm(total=3) for lending_type in lending_types: - pbar.set_description(f"fetching lending type {lending_type}") + pbar.set_description(f"fetching lending interests of type {lending_type}") latest_time = self.db.get_last_lending_interest_time(lending_type=lending_type) + 3600 * 1000 # add 1 hour current = 1 while True: lending_interests = self.client.get_lending_interest_history(lendingType=lending_type, startTime=latest_time, current=current, - limit=100) + size=100) for li in lending_interests: self.db.add_lending_interest(time=li['time'], lending_type=li['lendingType'], diff --git a/BinanceWatch/storage/tables.py b/BinanceWatch/storage/tables.py index da5541f..da25004 100644 --- a/BinanceWatch/storage/tables.py +++ b/BinanceWatch/storage/tables.py @@ -142,6 +142,40 @@ def __init__(self, name: str, columns_names: List[str], columns_sql_types: List[ ] ) +LENDING_PURCHASE_TABLE = Table( + 'lending_purchase_history', + [ + 'purchaseTime', + 'lendingType', + 'asset', + 'amount' + ], + [ + 'INTEGER', + 'TEXT', + 'TEXT', + 'INTEGER' + ], + primary_key='purchaseId', + primary_key_sql_type='INTEGER' +) + +LENDING_REDEMPTION_TABLE = Table( + 'lending_redemption_history', + [ + 'redemptionTime', + 'lendingType', + 'asset', + 'amount' + ], + [ + 'INTEGER', + 'TEXT', + 'TEXT', + 'INTEGER' + ] +) + CROSS_MARGIN_TRADE_TABLE = Table( 'cross_margin_trade', [ diff --git a/README.md b/README.md index 983c7be..fc1b235 100644 --- a/README.md +++ b/README.md @@ -27,11 +27,17 @@ It currently supports: - Spot Crypto Deposits - Spot Crypto Withdraws - Spot Dividends -- Spot Interests - Spot Dusts - Universal Transfers + + + +- Lending Purchases +- Lending Interests +- Lending Redemptions + - Cross Margin Trades - Cross Margin Repayment - Cross Margin Loans diff --git a/setup.py b/setup.py index 21fdc2b..73fbfc2 100644 --- a/setup.py +++ b/setup.py @@ -10,7 +10,7 @@ license='MIT', description='Local tracker of a binance account', install_requires=['numpy', 'tqdm', 'dateparser', 'requests', 'python-binance', 'appdirs'], - keywords='binance exchange wallet save tracking bitcoin ethereum btc eth neo', + keywords='binance exchange wallet save tracking history bitcoin ethereum btc eth', classifiers=[ 'Intended Audience :: Developers', 'License :: OSI Approved :: MIT License', From efa22e7bac8a70be67fda108654d7d47caae1549 Mon Sep 17 00:00:00 2001 From: EtWn <34377743+EtWnn@users.noreply.github.com> Date: Tue, 23 Mar 2021 10:40:51 +0100 Subject: [PATCH 14/17] Delete tables (#14) * add drop_all_tables reformat drop_table to accept table name (str) * remove drop methods from BinanceManager * remove unused import --- BinanceWatch/storage/BinanceManager.py | 93 -------------------------- BinanceWatch/storage/DataBase.py | 37 ++++++++-- 2 files changed, 32 insertions(+), 98 deletions(-) diff --git a/BinanceWatch/storage/BinanceManager.py b/BinanceWatch/storage/BinanceManager.py index a23ddd3..42e506d 100644 --- a/BinanceWatch/storage/BinanceManager.py +++ b/BinanceWatch/storage/BinanceManager.py @@ -8,7 +8,6 @@ from BinanceWatch.utils.time_utils import datetime_to_millistamp from BinanceWatch.storage.BinanceDataBase import BinanceDataBase -from BinanceWatch.storage import tables class BinanceManager: @@ -675,95 +674,3 @@ def update_all_spot_trades(self, limit: int = 1000): pbar.update() pbar.close() - def drop_spot_trade_table(self): - """ - erase the spot trades table - - :return: None - :rtype: None - """ - self.db.drop_table(tables.SPOT_TRADE_TABLE) - - def drop_spot_deposit_table(self): - """ - erase the spot deposits table - - :return: None - :rtype: None - """ - self.db.drop_table(tables.SPOT_DEPOSIT_TABLE) - - def drop_spot_withdraw_table(self): - """ - erase the spot withdraws table - - :return: None - :rtype: None - """ - self.db.drop_table(tables.SPOT_WITHDRAW_TABLE) - - def drop_spot_dividends_table(self): - """ - erase the spot dividends table - - :return: None - :rtype: None - """ - self.db.drop_table(tables.SPOT_DIVIDEND_TABLE) - - def drop_dust_table(self): - """ - erase the spot dust table - - :return: None - :rtype: None - """ - self.db.drop_table(tables.SPOT_DUST_TABLE) - - def drop_lending_interest_table(self): - """ - erase the lending interests - - :return: None - :rtype: None - """ - self.db.drop_table(tables.LENDING_INTEREST_TABLE) - - def drop_cross_margin_trade_table(self): - """ - erase the cross margin trades table - - :return: None - :rtype: None - """ - self.db.drop_table(tables.CROSS_MARGIN_TRADE_TABLE) - - def drop_cross_margin_loan_table(self): - """ - erase the cross margin loan table - - :return: None - :rtype: None - """ - self.db.drop_table(tables.CROSS_MARGIN_LOAN_TABLE) - - def drop_cross_margin_repay_table(self): - """ - erase the cross margin repay table - - :return: None - :rtype: None - """ - self.db.drop_table(tables.CROSS_MARGIN_REPAY_TABLE) - - def drop_all_tables(self): - """ - erase all the tables of the database by calling all the methods having 'drop' and 'table' in their names - - :return: None - :rtype: None - """ - methods = [m for m in dir(self) if 'drop' in m and 'table' in m and callable(getattr(self, m))] - for m in methods: - if m != "drop_all_tables": - getattr(self, m)() diff --git a/BinanceWatch/storage/DataBase.py b/BinanceWatch/storage/DataBase.py index 3c8b3ca..81359c1 100644 --- a/BinanceWatch/storage/DataBase.py +++ b/BinanceWatch/storage/DataBase.py @@ -125,16 +125,43 @@ def create_table(self, table: Table): self.db_cursor.execute(create_cmd) self.db_conn.commit() - def drop_table(self, table: Table): + def drop_table(self, table: Union[Table, str]): """ - delete a table from the database - :param table: Table instance with the config of the table to drop - :return: + delete a table from the database + + :param table: table or table name to drop + :type table: str or Table instance + :return: None + :rtype: None """ - execution_order = f"DROP TABLE IF EXISTS {table.name}" + if isinstance(table, Table): + table = table.name + execution_order = f"DROP TABLE IF EXISTS {table}" self.db_cursor.execute(execution_order) self.db_conn.commit() + def drop_all_tables(self): + """ + drop all the tables existing in the database + + :return: None + :rtype: None + """ + tables_desc = self.get_all_tables() + for table_desc in tables_desc: + self.drop_table(table_desc[1]) + self.commit() + + def get_all_tables(self) -> List[Tuple]: + """ + return all the tables existing in the database + + :return: tables descriptions + :rtype: List[Tuple] + """ + cmd = "SELECT * FROM sqlite_master WHERE type='table';" + return self._fetch_rows(cmd) + def commit(self): """ submit and save the database state From 38056cea0191af47c849987ddeb0268381ea0bda Mon Sep 17 00:00:00 2001 From: EtWn <34377743+EtWnn@users.noreply.github.com> Date: Tue, 23 Mar 2021 11:04:52 +0100 Subject: [PATCH 15/17] add the possibility to filter which universal transfers to update (#15) --- BinanceWatch/storage/BinanceManager.py | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/BinanceWatch/storage/BinanceManager.py b/BinanceWatch/storage/BinanceManager.py index 42e506d..3885fe8 100644 --- a/BinanceWatch/storage/BinanceManager.py +++ b/BinanceWatch/storage/BinanceManager.py @@ -1,6 +1,7 @@ import datetime import math import time +from typing import Optional import dateparser from binance.client import Client @@ -31,7 +32,7 @@ def update_spot(self): self.update_spot_withdraws() self.update_spot_dusts() self.update_spot_dividends() - self.update_universal_transfers() + self.update_universal_transfers(transfer_filter='MAIN') def update_cross_margin(self): """ @@ -44,7 +45,7 @@ def update_cross_margin(self): self.update_cross_margin_loans() self.update_cross_margin_interests() self.update_cross_margin_repays() - self.update_universal_transfers() + self.update_universal_transfers(transfer_filter='MARGIN') def update_lending(self): """ @@ -57,7 +58,7 @@ def update_lending(self): self.update_lending_purchases() self.update_lending_redemptions() - def update_universal_transfers(self): + def update_universal_transfers(self, transfer_filter: Optional[str] = None): """ update the universal transfers database. @@ -65,14 +66,20 @@ def update_universal_transfers(self): https://python-binance.readthedocs.io/en/latest/binance.html#binance.client.Client.query_universal_transfer_history https://binance-docs.github.io/apidocs/spot/en/#query-user-universal-transfer-history + :param transfer_filter: if not None, only the transfers containing this filter will be updated (ex: 'MAIN') + :type transfer_filter: Optional[str] :return: None :rtype: None """ - transfers_types = ['MAIN_C2C', 'MAIN_UMFUTURE', 'MAIN_CMFUTURE', 'MAIN_MARGIN', 'MAIN_MINING', 'C2C_MAIN', - 'C2C_UMFUTURE', 'C2C_MINING', 'C2C_MARGIN', 'UMFUTURE_MAIN', 'UMFUTURE_C2C', - 'UMFUTURE_MARGIN', 'CMFUTURE_MAIN', 'CMFUTURE_MARGIN', 'MARGIN_MAIN', 'MARGIN_UMFUTURE', - 'MARGIN_CMFUTURE', 'MARGIN_MINING', 'MARGIN_C2C', 'MINING_MAIN', 'MINING_UMFUTURE', - 'MINING_C2C', 'MINING_MARGIN'] + all_types = ['MAIN_C2C', 'MAIN_UMFUTURE', 'MAIN_CMFUTURE', 'MAIN_MARGIN', 'MAIN_MINING', 'C2C_MAIN', + 'C2C_UMFUTURE', 'C2C_MINING', 'C2C_MARGIN', 'UMFUTURE_MAIN', 'UMFUTURE_C2C', + 'UMFUTURE_MARGIN', 'CMFUTURE_MAIN', 'CMFUTURE_MARGIN', 'MARGIN_MAIN', 'MARGIN_UMFUTURE', + 'MARGIN_CMFUTURE', 'MARGIN_MINING', 'MARGIN_C2C', 'MINING_MAIN', 'MINING_UMFUTURE', + 'MINING_C2C', 'MINING_MARGIN'] + if transfer_filter is not None: + transfers_types = list(filter(lambda x: transfer_filter in x, all_types)) + else: + transfers_types = all_types pbar = tqdm(total=len(transfers_types)) for transfer_type in transfers_types: pbar.set_description(f"fetching transfer type {transfer_type}") @@ -673,4 +680,3 @@ def update_all_spot_trades(self, limit: int = 1000): limit=limit) pbar.update() pbar.close() - From 374ed51d0afa854b98729f944145cb80db0eec83 Mon Sep 17 00:00:00 2001 From: EtWn <34377743+EtWnn@users.noreply.github.com> Date: Tue, 23 Mar 2021 12:26:28 +0100 Subject: [PATCH 16/17] Bug fix dust (#16) * move BinanceMananger out of storage folder * update the drop table command --- BinanceWatch/{storage => }/BinanceManager.py | 3 ++- README.md | 4 +--- 2 files changed, 3 insertions(+), 4 deletions(-) rename BinanceWatch/{storage => }/BinanceManager.py (99%) diff --git a/BinanceWatch/storage/BinanceManager.py b/BinanceWatch/BinanceManager.py similarity index 99% rename from BinanceWatch/storage/BinanceManager.py rename to BinanceWatch/BinanceManager.py index 3885fe8..1d82500 100644 --- a/BinanceWatch/storage/BinanceManager.py +++ b/BinanceWatch/BinanceManager.py @@ -7,6 +7,7 @@ from binance.client import Client from tqdm import tqdm +from BinanceWatch.storage import tables from BinanceWatch.utils.time_utils import datetime_to_millistamp from BinanceWatch.storage.BinanceDataBase import BinanceDataBase @@ -474,7 +475,7 @@ def update_spot_dusts(self): :return: None :rtype: None """ - self.drop_dust_table() + self.db.drop_table(tables.SPOT_DUST_TABLE) result = self.client.get_dust_log() dusts = result['results'] diff --git a/README.md b/README.md index fc1b235..cf18904 100644 --- a/README.md +++ b/README.md @@ -49,10 +49,8 @@ It currently supports: [Generate an API Key](https://www.binance.com/en/my/settings/api-management) in your binance account. Only read permissions are needed. - - ```python -from BinanceWatch.storage.BinanceManager import BinanceManager +from BinanceWatch.BinanceManager import BinanceManager api_key = "" api_secret = "" From 12e3968b24ced024be870e6f47f051a1da559dca Mon Sep 17 00:00:00 2001 From: EtWn <34377743+EtWnn@users.noreply.github.com> Date: Tue, 23 Mar 2021 15:27:15 +0100 Subject: [PATCH 17/17] Pip (#17) * add version and author * setup version to 0.1.0 add long description for Pypi * add __init__ * add large update example, pip install --- BinanceWatch/__init__.py | 2 ++ BinanceWatch/storage/__init__.py | 0 BinanceWatch/utils/__init__.py | 0 README.md | 19 ++++++++++++++++--- setup.py | 13 +++++++++++-- 5 files changed, 29 insertions(+), 5 deletions(-) create mode 100644 BinanceWatch/storage/__init__.py create mode 100644 BinanceWatch/utils/__init__.py diff --git a/BinanceWatch/__init__.py b/BinanceWatch/__init__.py index e69de29..ff90f68 100644 --- a/BinanceWatch/__init__.py +++ b/BinanceWatch/__init__.py @@ -0,0 +1,2 @@ +__version__ = "0.1.0" +__author__ = 'EtWnn' diff --git a/BinanceWatch/storage/__init__.py b/BinanceWatch/storage/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/BinanceWatch/utils/__init__.py b/BinanceWatch/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/README.md b/README.md index cf18904..1f490b9 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,11 @@ -# Welcome to BinanceWatch v0.1 +# Welcome to BinanceWatch v0.1.0 ## Note -This library is under development by EtWnn, but feel free to drop your suggestions or remarks in -the discussion tab of this repo. You are also welcome to contribute by submitting PRs. +This library is under development by EtWnn, feel free to drop your suggestions or remarks in +the discussion tab of the git repo. You are also welcome to contribute by submitting PRs. This is an unofficial tracker for binance accounts. I am in no way affiliated with Binance, use at your own risk. @@ -49,6 +49,9 @@ It currently supports: [Generate an API Key](https://www.binance.com/en/my/settings/api-management) in your binance account. Only read permissions are needed. +Install this library with pip: +`pip install BinanceWatch` + ```python from BinanceWatch.BinanceManager import BinanceManager @@ -74,6 +77,16 @@ start_time = datetime_to_millistamp(datetime(2018,1,1)) spot_trades = bm.db.get_trades('spot', start_time=start_time) ``` +You can also call update functions at an account-type level, and it will call every update +methods related to this account-type: +```python +bm.update_spot() # (trades, transfers, deposits ...) + +bm.update_cross_margin() # (trades, loans, repays, interests...) + +bm.update_lending() # (purchases, interests, redemptions..) +``` + ## Donation diff --git a/setup.py b/setup.py index 73fbfc2..d1b338a 100644 --- a/setup.py +++ b/setup.py @@ -1,14 +1,23 @@ from setuptools import setup +from os import path +this_directory = path.abspath(path.dirname(__file__)) +with open(path.join(this_directory, 'README.md'), encoding='utf-8') as f: + long_description = f.read() setup( name='BinanceWatch', - version='0.1', - packages=['BinanceWatch', 'tests'], + version='0.1.0', + packages=['BinanceWatch', + 'tests', + 'BinanceWatch.storage', + 'BinanceWatch.utils'], url='https://github.com/EtWnn/BinanceWatch', author='EtWnn', author_email='', license='MIT', description='Local tracker of a binance account', + long_description=long_description, + long_description_content_type='text/markdown', install_requires=['numpy', 'tqdm', 'dateparser', 'requests', 'python-binance', 'appdirs'], keywords='binance exchange wallet save tracking history bitcoin ethereum btc eth', classifiers=[