From 8ef7da8a21ef453a236496af787cc2224dc05eba Mon Sep 17 00:00:00 2001 From: EtWn <34377743+EtWnn@users.noreply.github.com> Date: Mon, 29 Mar 2021 10:14:56 +0200 Subject: [PATCH 1/5] Multiple Accounts (#27) * add name choice for the Binance account * add __init__description for BinanceManager --- BinanceWatch/BinanceManager.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/BinanceWatch/BinanceManager.py b/BinanceWatch/BinanceManager.py index 40d7da7..8dd57bb 100644 --- a/BinanceWatch/BinanceManager.py +++ b/BinanceWatch/BinanceManager.py @@ -17,8 +17,19 @@ class BinanceManager: This class is in charge of filling the database by calling the binance API """ - def __init__(self, api_key: str, api_secret: str): - self.db = BinanceDataBase() + def __init__(self, api_key: str, api_secret: str, account_name: str = 'binance'): + """ + initialise the binance manager. + + :param api_key: key for the Binance api + :type api_key: str + :param api_secret: secret for the Binance api + :type api_secret: str + :param account_name: if you have several accounts to monitor, you need to give them different names or the + database will collide + :type account_name: str + """ + self.db = BinanceDataBase(name=f"{account_name}_db") self.client = Client(api_key=api_key, api_secret=api_secret) def update_spot(self): From 9912fce12b1c7771d18ae0f50c4a3a5c8cb8cea0 Mon Sep 17 00:00:00 2001 From: EtWn <34377743+EtWnn@users.noreply.github.com> Date: Mon, 29 Mar 2021 14:01:27 +0200 Subject: [PATCH 2/5] Rate limits (#28) * Add a logger to the BinanceManager * add a genereric client call for limit rate * correct rate limit error number * integrate rate limit for cross margin api calls * integrate rate limit for lending api calls * integrate rate limit for spot api calls --- BinanceWatch/BinanceManager.py | 205 +++++++++++++++++++++++++-------- 1 file changed, 158 insertions(+), 47 deletions(-) diff --git a/BinanceWatch/BinanceManager.py b/BinanceWatch/BinanceManager.py index 8dd57bb..12ae73d 100644 --- a/BinanceWatch/BinanceManager.py +++ b/BinanceWatch/BinanceManager.py @@ -1,13 +1,15 @@ import datetime import math import time -from typing import Optional +from typing import Optional, Dict import dateparser from binance.client import Client +from binance.exceptions import BinanceAPIException from tqdm import tqdm from BinanceWatch.storage import tables +from BinanceWatch.utils.LoggerGenerator import LoggerGenerator from BinanceWatch.utils.time_utils import datetime_to_millistamp from BinanceWatch.storage.BinanceDataBase import BinanceDataBase @@ -16,8 +18,9 @@ class BinanceManager: """ This class is in charge of filling the database by calling the binance API """ + API_MAX_RETRY = 3 - def __init__(self, api_key: str, api_secret: str, account_name: str = 'binance'): + def __init__(self, api_key: str, api_secret: str, account_name: str = 'default'): """ initialise the binance manager. @@ -29,8 +32,10 @@ def __init__(self, api_key: str, api_secret: str, account_name: str = 'binance') database will collide :type account_name: str """ - self.db = BinanceDataBase(name=f"{account_name}_db") + self.account_name = account_name + self.db = BinanceDataBase(name=f"{self.account_name}_db") self.client = Client(api_key=api_key, api_secret=api_secret) + self.logger = LoggerGenerator.get_logger(f"BinanceManager_{self.account_name}") def update_spot(self): """ @@ -98,10 +103,14 @@ def update_universal_transfers(self, transfer_filter: Optional[str] = None): latest_time = self.db.get_last_universal_transfer_time(transfer_type=transfer_type) + 1 current = 1 while True: - universal_transfers = self.client.query_universal_transfer_history(type=transfer_type, - startTime=latest_time, - current=current, - size=100) + client_params = { + 'type': transfer_type, + 'startTime': latest_time, + 'current': current, + 'size': 100 + } + universal_transfers = self._call_binance_client('query_universal_transfer_history', client_params) + try: universal_transfers = universal_transfers['rows'] except KeyError: @@ -145,8 +154,15 @@ def update_cross_margin_interests(self): 'size': 100, 'archived': archived } + # no built-in method yet in python-binance for margin/interestHistory - interests = self.client._request_margin_api('get', 'margin/interestHistory', signed=True, data=params) + client_params = { + 'method': 'get', + 'path': 'margin/interestHistory', + 'signed': True, + 'data': params + } + interests = self._call_binance_client('_request_margin_api', client_params) for interest in interests['rows']: self.db.add_margin_interest(margin_type=margin_type, @@ -175,7 +191,12 @@ def update_cross_margin_repays(self): :return: None :rtype: None """ - symbols_info = self.client._request_margin_api('get', 'margin/allPairs', data={}) # not built-in yet + client_params = { + 'method': 'get', + 'path': 'margin/allPairs', + 'data': {} + } + symbols_info = self._call_binance_client('_request_margin_api', client_params) # not built-in yet assets = set() for symbol_info in symbols_info: assets.add(symbol_info['base']) @@ -208,12 +229,16 @@ def update_margin_asset_repay(self, asset: str, isolated_symbol=''): archived = 1000 * time.time() - latest_time > 1000 * 3600 * 24 * 30 * 3 current = 1 while True: - repays = self.client.get_margin_repay_details(asset=asset, - current=current, - startTime=latest_time + 1000, - archived=archived, - isolatedSymbol=isolated_symbol, - size=100) + client_params = { + 'asset': asset, + 'current':current, + 'startTime': latest_time + 1000, + 'archived': archived, + 'isolatedSymbol': isolated_symbol, + 'size': 100 + } + repays = self._call_binance_client('get_margin_repay_details', client_params) + for repay in repays['rows']: if repay['status'] == 'CONFIRMED': self.db.add_repay(margin_type=margin_type, @@ -241,7 +266,12 @@ def update_cross_margin_loans(self): :return: None :rtype: None """ - symbols_info = self.client._request_margin_api('get', 'margin/allPairs', data={}) # not built-in yet + client_params = { + 'method': 'get', + 'path': 'margin/allPairs', + 'data': {} + } + symbols_info = self._call_binance_client('_request_margin_api', client_params) # not built-in yet assets = set() for symbol_info in symbols_info: assets.add(symbol_info['base']) @@ -274,12 +304,16 @@ def update_margin_asset_loans(self, asset: str, isolated_symbol=''): archived = 1000 * time.time() - latest_time > 1000 * 3600 * 24 * 30 * 3 current = 1 while True: - loans = self.client.get_margin_loan_details(asset=asset, - current=current, - startTime=latest_time + 1000, - archived=archived, - isolatedSymbol=isolated_symbol, - size=100) + client_params = { + 'asset': asset, + 'current': current, + 'startTime': latest_time + 1000, + 'archived': archived, + 'isolatedSymbol': isolated_symbol, + 'size': 100 + } + loans = self._call_binance_client('get_margin_loan_details', client_params) + for loan in loans['rows']: if loan['status'] == 'CONFIRMED': self.db.add_loan(margin_type=margin_type, @@ -321,7 +355,13 @@ def update_cross_margin_symbol_trades(self, asset: str, ref_asset: str, limit: i symbol = asset + ref_asset last_trade_id = self.db.get_max_trade_id(asset, ref_asset, 'cross_margin') while True: - new_trades = self.client.get_margin_trades(symbol=symbol, fromId=last_trade_id + 1, limit=limit) + client_params = { + 'symbol': symbol, + 'fromId': last_trade_id + 1, + 'limit': limit + } + new_trades = self._call_binance_client('get_margin_trades', client_params) + for trade in new_trades: self.db.add_trade(trade_type='cross_margin', trade_id=int(trade['id']), @@ -350,7 +390,13 @@ def update_all_cross_margin_trades(self, limit: int = 1000): :return: None :rtype: None """ - symbols_info = self.client._request_margin_api('get', 'margin/allPairs', data={}) # not built-in yet + client_params = { + 'method': 'get', + 'path': 'margin/allPairs', + 'data': {} + } + symbols_info = self._call_binance_client('_request_margin_api', client_params) # not built-in yet + pbar = tqdm(total=len(symbols_info)) for symbol_info in symbols_info: pbar.set_description(f"fetching {symbol_info['symbol']} cross margin trades") @@ -378,10 +424,14 @@ def update_lending_redemptions(self): latest_time = self.db.get_last_lending_redemption_time(lending_type=lending_type) + 1 current = 1 while True: - lending_redemptions = self.client.get_lending_redemption_history(lendingType=lending_type, - startTime=latest_time, - current=current, - size=100) + client_params = { + 'lendingType': lending_type, + 'startTime': latest_time, + 'current': current, + 'size': 100 + } + lending_redemptions = self._call_binance_client('get_lending_redemption_history', client_params) + for li in lending_redemptions: if li['status'] == 'PAID': self.db.add_lending_redemption(redemption_time=li['createTime'], @@ -416,10 +466,14 @@ def update_lending_purchases(self): latest_time = self.db.get_last_lending_purchase_time(lending_type=lending_type) + 1 current = 1 while True: - lending_purchases = self.client.get_lending_purchase_history(lendingType=lending_type, - startTime=latest_time, - current=current, - size=100) + client_params = { + 'lendingType': lending_type, + 'startTime': latest_time, + 'current': current, + 'size': 100 + } + lending_purchases = self._call_binance_client('get_lending_purchase_history', client_params) + for li in lending_purchases: if li['status'] == 'SUCCESS': self.db.add_lending_purchase(purchase_id=li['purchaseId'], @@ -455,10 +509,14 @@ def update_lending_interests(self): latest_time = self.db.get_last_lending_interest_time(lending_type=lending_type) + 3600 * 1000 # add 1 hour current = 1 while True: - lending_interests = self.client.get_lending_interest_history(lendingType=lending_type, - startTime=latest_time, - current=current, - size=100) + client_params = { + 'lendingType': lending_type, + 'startTime': latest_time, + 'current': current, + 'size': 100 + } + lending_interests = self._call_binance_client('get_lending_interest_history', client_params) + for li in lending_interests: self.db.add_lending_interest(time=li['time'], lending_type=li['lendingType'], @@ -488,7 +546,7 @@ def update_spot_dusts(self): """ self.db.drop_table(tables.SPOT_DUST_TABLE) - result = self.client.get_dust_log() + result = self._call_binance_client('get_dust_log') dusts = result['results'] pbar = tqdm(total=dusts['total']) pbar.set_description("fetching spot dusts") @@ -528,18 +586,21 @@ def update_spot_dividends(self, day_jump: float = 90, limit: int = 500): pbar = tqdm(total=math.ceil((now_millistamp - start_time) / delta_jump)) pbar.set_description("fetching spot dividends") while start_time < now_millistamp: + # the stable working version of client.get_asset_dividend_history is not released yet, + # for now it has a post error, so this protected member is used in the meantime params = { 'startTime': start_time, 'endTime': start_time + delta_jump, 'limit': limit } - # the stable working version of client.get_asset_dividend_history is not released yet, - # for now it has a post error, so this protected member is used in the meantime - result = self.client._request_margin_api('get', - 'asset/assetDividend', - True, - data=params - ) + client_params = { + 'method': 'get', + 'path': 'asset/assetDividend', + 'signed': True, + 'data': params + } + result = self._call_binance_client('_request_margin_api', client_params) + dividends = result['rows'] for div in dividends: self.db.add_dividend(div_id=int(div['tranId']), @@ -579,7 +640,13 @@ def update_spot_withdraws(self, day_jump: float = 90): pbar = tqdm(total=math.ceil((now_millistamp - start_time) / delta_jump)) pbar.set_description("fetching spot withdraws") while start_time < now_millistamp: - result = self.client.get_withdraw_history(startTime=start_time, endTime=start_time + delta_jump, status=6) + client_params = { + 'startTime': start_time, + 'endTime': start_time + delta_jump, + 'status': 6 + } + result = self._call_binance_client('get_withdraw_history', client_params) + withdraws = result['withdrawList'] for withdraw in withdraws: self.db.add_withdraw(withdraw_id=withdraw['id'], @@ -618,7 +685,13 @@ def update_spot_deposits(self, day_jump: float = 90): pbar = tqdm(total=math.ceil((now_millistamp - start_time) / delta_jump)) pbar.set_description("fetching spot deposits") while start_time < now_millistamp: - result = self.client.get_deposit_history(startTime=start_time, endTime=start_time + delta_jump, status=1) + client_params = { + 'startTime': start_time, + 'endTime': start_time + delta_jump, + 'status': 1 + } + result = self._call_binance_client('get_deposit_history', client_params) + deposits = result['depositList'] for deposit in deposits: self.db.add_deposit(tx_id=deposit['txId'], @@ -654,7 +727,13 @@ def update_spot_symbol_trades(self, asset: str, ref_asset: str, limit: int = 100 symbol = asset + ref_asset last_trade_id = self.db.get_max_trade_id(asset, ref_asset, 'spot') while True: - new_trades = self.client.get_my_trades(symbol=symbol, fromId=last_trade_id + 1, limit=limit) + client_params = { + 'symbol': symbol, + 'fromId': last_trade_id + 1, + 'limit': limit + } + new_trades = self._call_binance_client('get_my_trades', client_params) + for trade in new_trades: self.db.add_trade(trade_type='spot', trade_id=int(trade['id']), @@ -692,3 +771,35 @@ def update_all_spot_trades(self, limit: int = 1000): limit=limit) pbar.update() pbar.close() + + def _call_binance_client(self, method_name: str, params: Optional[Dict] = None, retry_count: int = 0): + """ + This method is used to handle rate limits: if a rate limits is breached, it will wait the necessary time + to call again the API. + + :param method_name: name of the method binance.Client to call + :type method_name: str + :param params: parameters to pass to the above method + :type params: Dict + :param retry_count: internal use only to count the number of retry if rate limits are breached + :type retry_count: int + :return: response of binance.Client method + :rtype: Dict + """ + if params is None: + params = dict() + if retry_count >= BinanceManager.API_MAX_RETRY: + raise RuntimeError(f"The API rate limits has been breached {retry_count} times") + + try: + return getattr(self.client, method_name)(**params) + except BinanceAPIException as err: + if err.code == -1003: # API rate Limits + wait_time = float(err.response.headers['Retry-After']) + if err.response.status_code == 418: # ban + self.logger.error(f"API calls resulted in a ban, retry in {wait_time} seconds") + raise err + self.logger.info(f"API calls resulted in a breach of rate limits, will retry after {wait_time} seconds") + time.sleep(wait_time + 1) + return self._call_binance_client(method_name, params, retry_count + 1) + raise err From e76179d611951db93081e63000f2d4a831e3110d Mon Sep 17 00:00:00 2001 From: EtWn <34377743+EtWnn@users.noreply.github.com> Date: Mon, 29 Mar 2021 16:29:12 +0200 Subject: [PATCH 3/5] Version (#29) * change version to v0.1.2dev * add latest pip install * auto version name for conf.py and setup.py --- BinanceWatch/__init__.py | 2 +- README.rst | 14 +++++++++++--- docs/source/conf.py | 7 +++++-- setup.py | 14 ++++++++++---- 4 files changed, 27 insertions(+), 10 deletions(-) diff --git a/BinanceWatch/__init__.py b/BinanceWatch/__init__.py index 863cfad..ae2aacc 100644 --- a/BinanceWatch/__init__.py +++ b/BinanceWatch/__init__.py @@ -1,2 +1,2 @@ -__version__ = "0.1.1" +__version__ = "0.1.2dev" __author__ = 'EtWnn' diff --git a/README.rst b/README.rst index dbaef29..986fe43 100644 --- a/README.rst +++ b/README.rst @@ -1,6 +1,6 @@ -============================== -Welcome to BinanceWatch v0.1.1 -============================== +=================================== +Welcome to BinanceWatch v0.1.2dev +=================================== Note ---- @@ -62,6 +62,14 @@ permissions are needed. pip install BinanceWatch +If you prefer to install the latest developments use: + +.. code:: bash + + pip install git+https://github.com/EtWnn/BinanceWatch.git@develop + +Use your Binance api keys to initiate the manager: + .. code:: python from BinanceWatch.BinanceManager import BinanceManager diff --git a/docs/source/conf.py b/docs/source/conf.py index f015d09..d697de0 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -21,8 +21,11 @@ author = 'EtWnn' # The full version, including alpha/beta/rc tags -release = '0.1.1' - +this_directory = os.path.abspath(os.path.dirname(__file__)) +about = {} +with open(os.path.join(this_directory, '../../BinanceWatch/__init__.py'), encoding='utf-8') as f: + exec(f.read(), about) +release = about['__version__'] # -- General configuration --------------------------------------------------- diff --git a/setup.py b/setup.py index 25e9216..9fe30bd 100644 --- a/setup.py +++ b/setup.py @@ -1,12 +1,18 @@ +import os from setuptools import setup -from os import path -this_directory = path.abspath(path.dirname(__file__)) -with open(path.join(this_directory, 'README.rst'), encoding='utf-8') as f: + +this_directory = os.path.abspath(os.path.dirname(__file__)) + +with open(os.path.join(this_directory, 'README.rst'), encoding='utf-8') as f: long_description = f.read() +about = {} +with open(os.path.join(this_directory, 'BinanceWatch/__init__.py'), encoding='utf-8') as f: + exec(f.read(), about) + setup( name='BinanceWatch', - version='0.1.1', + version=about['__version__'], packages=['BinanceWatch', 'tests', 'BinanceWatch.storage', From 95a732bd6052fcd27bae3030604f7a345bb8f88d Mon Sep 17 00:00:00 2001 From: EtWn <34377743+EtWnn@users.noreply.github.com> Date: Tue, 30 Mar 2021 10:39:08 +0200 Subject: [PATCH 4/5] add version requirements >=0.7.9 for python-binance (#30) --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 293a748..ec07ed6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ numpy tqdm dateparser requests -python-binance +python-binance>=0.7.9 appdirs sphinx sphinx_rtd_theme diff --git a/setup.py b/setup.py index 9fe30bd..2e45fd6 100644 --- a/setup.py +++ b/setup.py @@ -24,7 +24,7 @@ description='Local tracker of a binance account', long_description=long_description, long_description_content_type='text/x-rst', - install_requires=['numpy', 'tqdm', 'dateparser', 'requests', 'python-binance', 'appdirs'], + install_requires=['numpy', 'tqdm', 'dateparser', 'requests', 'python-binance>=0.7.9', 'appdirs'], keywords='binance exchange wallet save tracking history bitcoin ethereum btc eth', classifiers=[ 'Intended Audience :: Developers', From 8a28a5f722ef962112831647f96e7f8f8805188c Mon Sep 17 00:00:00 2001 From: EtWn <34377743+EtWnn@users.noreply.github.com> Date: Tue, 30 Mar 2021 10:59:50 +0200 Subject: [PATCH 5/5] update version to 0.1.2 (#31) --- BinanceWatch/__init__.py | 2 +- README.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/BinanceWatch/__init__.py b/BinanceWatch/__init__.py index ae2aacc..99277fe 100644 --- a/BinanceWatch/__init__.py +++ b/BinanceWatch/__init__.py @@ -1,2 +1,2 @@ -__version__ = "0.1.2dev" +__version__ = "0.1.2" __author__ = 'EtWnn' diff --git a/README.rst b/README.rst index 986fe43..4889d88 100644 --- a/README.rst +++ b/README.rst @@ -1,5 +1,5 @@ =================================== -Welcome to BinanceWatch v0.1.2dev +Welcome to BinanceWatch v0.1.2 =================================== Note