diff --git a/BinanceWatch/BinanceManager.py b/BinanceWatch/BinanceManager.py index 40d7da7..12ae73d 100644 --- a/BinanceWatch/BinanceManager.py +++ b/BinanceWatch/BinanceManager.py @@ -1,13 +1,15 @@ import datetime import math import time -from typing import Optional +from typing import Optional, Dict import dateparser from binance.client import Client +from binance.exceptions import BinanceAPIException from tqdm import tqdm from BinanceWatch.storage import tables +from BinanceWatch.utils.LoggerGenerator import LoggerGenerator from BinanceWatch.utils.time_utils import datetime_to_millistamp from BinanceWatch.storage.BinanceDataBase import BinanceDataBase @@ -16,10 +18,24 @@ class BinanceManager: """ This class is in charge of filling the database by calling the binance API """ + API_MAX_RETRY = 3 - def __init__(self, api_key: str, api_secret: str): - self.db = BinanceDataBase() + def __init__(self, api_key: str, api_secret: str, account_name: str = 'default'): + """ + initialise the binance manager. + + :param api_key: key for the Binance api + :type api_key: str + :param api_secret: secret for the Binance api + :type api_secret: str + :param account_name: if you have several accounts to monitor, you need to give them different names or the + database will collide + :type account_name: str + """ + self.account_name = account_name + self.db = BinanceDataBase(name=f"{self.account_name}_db") self.client = Client(api_key=api_key, api_secret=api_secret) + self.logger = LoggerGenerator.get_logger(f"BinanceManager_{self.account_name}") def update_spot(self): """ @@ -87,10 +103,14 @@ def update_universal_transfers(self, transfer_filter: Optional[str] = None): latest_time = self.db.get_last_universal_transfer_time(transfer_type=transfer_type) + 1 current = 1 while True: - universal_transfers = self.client.query_universal_transfer_history(type=transfer_type, - startTime=latest_time, - current=current, - size=100) + client_params = { + 'type': transfer_type, + 'startTime': latest_time, + 'current': current, + 'size': 100 + } + universal_transfers = self._call_binance_client('query_universal_transfer_history', client_params) + try: universal_transfers = universal_transfers['rows'] except KeyError: @@ -134,8 +154,15 @@ def update_cross_margin_interests(self): 'size': 100, 'archived': archived } + # no built-in method yet in python-binance for margin/interestHistory - interests = self.client._request_margin_api('get', 'margin/interestHistory', signed=True, data=params) + client_params = { + 'method': 'get', + 'path': 'margin/interestHistory', + 'signed': True, + 'data': params + } + interests = self._call_binance_client('_request_margin_api', client_params) for interest in interests['rows']: self.db.add_margin_interest(margin_type=margin_type, @@ -164,7 +191,12 @@ def update_cross_margin_repays(self): :return: None :rtype: None """ - symbols_info = self.client._request_margin_api('get', 'margin/allPairs', data={}) # not built-in yet + client_params = { + 'method': 'get', + 'path': 'margin/allPairs', + 'data': {} + } + symbols_info = self._call_binance_client('_request_margin_api', client_params) # not built-in yet assets = set() for symbol_info in symbols_info: assets.add(symbol_info['base']) @@ -197,12 +229,16 @@ def update_margin_asset_repay(self, asset: str, isolated_symbol=''): archived = 1000 * time.time() - latest_time > 1000 * 3600 * 24 * 30 * 3 current = 1 while True: - repays = self.client.get_margin_repay_details(asset=asset, - current=current, - startTime=latest_time + 1000, - archived=archived, - isolatedSymbol=isolated_symbol, - size=100) + client_params = { + 'asset': asset, + 'current':current, + 'startTime': latest_time + 1000, + 'archived': archived, + 'isolatedSymbol': isolated_symbol, + 'size': 100 + } + repays = self._call_binance_client('get_margin_repay_details', client_params) + for repay in repays['rows']: if repay['status'] == 'CONFIRMED': self.db.add_repay(margin_type=margin_type, @@ -230,7 +266,12 @@ def update_cross_margin_loans(self): :return: None :rtype: None """ - symbols_info = self.client._request_margin_api('get', 'margin/allPairs', data={}) # not built-in yet + client_params = { + 'method': 'get', + 'path': 'margin/allPairs', + 'data': {} + } + symbols_info = self._call_binance_client('_request_margin_api', client_params) # not built-in yet assets = set() for symbol_info in symbols_info: assets.add(symbol_info['base']) @@ -263,12 +304,16 @@ def update_margin_asset_loans(self, asset: str, isolated_symbol=''): archived = 1000 * time.time() - latest_time > 1000 * 3600 * 24 * 30 * 3 current = 1 while True: - loans = self.client.get_margin_loan_details(asset=asset, - current=current, - startTime=latest_time + 1000, - archived=archived, - isolatedSymbol=isolated_symbol, - size=100) + client_params = { + 'asset': asset, + 'current': current, + 'startTime': latest_time + 1000, + 'archived': archived, + 'isolatedSymbol': isolated_symbol, + 'size': 100 + } + loans = self._call_binance_client('get_margin_loan_details', client_params) + for loan in loans['rows']: if loan['status'] == 'CONFIRMED': self.db.add_loan(margin_type=margin_type, @@ -310,7 +355,13 @@ def update_cross_margin_symbol_trades(self, asset: str, ref_asset: str, limit: i symbol = asset + ref_asset last_trade_id = self.db.get_max_trade_id(asset, ref_asset, 'cross_margin') while True: - new_trades = self.client.get_margin_trades(symbol=symbol, fromId=last_trade_id + 1, limit=limit) + client_params = { + 'symbol': symbol, + 'fromId': last_trade_id + 1, + 'limit': limit + } + new_trades = self._call_binance_client('get_margin_trades', client_params) + for trade in new_trades: self.db.add_trade(trade_type='cross_margin', trade_id=int(trade['id']), @@ -339,7 +390,13 @@ def update_all_cross_margin_trades(self, limit: int = 1000): :return: None :rtype: None """ - symbols_info = self.client._request_margin_api('get', 'margin/allPairs', data={}) # not built-in yet + client_params = { + 'method': 'get', + 'path': 'margin/allPairs', + 'data': {} + } + symbols_info = self._call_binance_client('_request_margin_api', client_params) # not built-in yet + pbar = tqdm(total=len(symbols_info)) for symbol_info in symbols_info: pbar.set_description(f"fetching {symbol_info['symbol']} cross margin trades") @@ -367,10 +424,14 @@ def update_lending_redemptions(self): latest_time = self.db.get_last_lending_redemption_time(lending_type=lending_type) + 1 current = 1 while True: - lending_redemptions = self.client.get_lending_redemption_history(lendingType=lending_type, - startTime=latest_time, - current=current, - size=100) + client_params = { + 'lendingType': lending_type, + 'startTime': latest_time, + 'current': current, + 'size': 100 + } + lending_redemptions = self._call_binance_client('get_lending_redemption_history', client_params) + for li in lending_redemptions: if li['status'] == 'PAID': self.db.add_lending_redemption(redemption_time=li['createTime'], @@ -405,10 +466,14 @@ def update_lending_purchases(self): latest_time = self.db.get_last_lending_purchase_time(lending_type=lending_type) + 1 current = 1 while True: - lending_purchases = self.client.get_lending_purchase_history(lendingType=lending_type, - startTime=latest_time, - current=current, - size=100) + client_params = { + 'lendingType': lending_type, + 'startTime': latest_time, + 'current': current, + 'size': 100 + } + lending_purchases = self._call_binance_client('get_lending_purchase_history', client_params) + for li in lending_purchases: if li['status'] == 'SUCCESS': self.db.add_lending_purchase(purchase_id=li['purchaseId'], @@ -444,10 +509,14 @@ def update_lending_interests(self): latest_time = self.db.get_last_lending_interest_time(lending_type=lending_type) + 3600 * 1000 # add 1 hour current = 1 while True: - lending_interests = self.client.get_lending_interest_history(lendingType=lending_type, - startTime=latest_time, - current=current, - size=100) + client_params = { + 'lendingType': lending_type, + 'startTime': latest_time, + 'current': current, + 'size': 100 + } + lending_interests = self._call_binance_client('get_lending_interest_history', client_params) + for li in lending_interests: self.db.add_lending_interest(time=li['time'], lending_type=li['lendingType'], @@ -477,7 +546,7 @@ def update_spot_dusts(self): """ self.db.drop_table(tables.SPOT_DUST_TABLE) - result = self.client.get_dust_log() + result = self._call_binance_client('get_dust_log') dusts = result['results'] pbar = tqdm(total=dusts['total']) pbar.set_description("fetching spot dusts") @@ -517,18 +586,21 @@ def update_spot_dividends(self, day_jump: float = 90, limit: int = 500): pbar = tqdm(total=math.ceil((now_millistamp - start_time) / delta_jump)) pbar.set_description("fetching spot dividends") while start_time < now_millistamp: + # the stable working version of client.get_asset_dividend_history is not released yet, + # for now it has a post error, so this protected member is used in the meantime params = { 'startTime': start_time, 'endTime': start_time + delta_jump, 'limit': limit } - # the stable working version of client.get_asset_dividend_history is not released yet, - # for now it has a post error, so this protected member is used in the meantime - result = self.client._request_margin_api('get', - 'asset/assetDividend', - True, - data=params - ) + client_params = { + 'method': 'get', + 'path': 'asset/assetDividend', + 'signed': True, + 'data': params + } + result = self._call_binance_client('_request_margin_api', client_params) + dividends = result['rows'] for div in dividends: self.db.add_dividend(div_id=int(div['tranId']), @@ -568,7 +640,13 @@ def update_spot_withdraws(self, day_jump: float = 90): pbar = tqdm(total=math.ceil((now_millistamp - start_time) / delta_jump)) pbar.set_description("fetching spot withdraws") while start_time < now_millistamp: - result = self.client.get_withdraw_history(startTime=start_time, endTime=start_time + delta_jump, status=6) + client_params = { + 'startTime': start_time, + 'endTime': start_time + delta_jump, + 'status': 6 + } + result = self._call_binance_client('get_withdraw_history', client_params) + withdraws = result['withdrawList'] for withdraw in withdraws: self.db.add_withdraw(withdraw_id=withdraw['id'], @@ -607,7 +685,13 @@ def update_spot_deposits(self, day_jump: float = 90): pbar = tqdm(total=math.ceil((now_millistamp - start_time) / delta_jump)) pbar.set_description("fetching spot deposits") while start_time < now_millistamp: - result = self.client.get_deposit_history(startTime=start_time, endTime=start_time + delta_jump, status=1) + client_params = { + 'startTime': start_time, + 'endTime': start_time + delta_jump, + 'status': 1 + } + result = self._call_binance_client('get_deposit_history', client_params) + deposits = result['depositList'] for deposit in deposits: self.db.add_deposit(tx_id=deposit['txId'], @@ -643,7 +727,13 @@ def update_spot_symbol_trades(self, asset: str, ref_asset: str, limit: int = 100 symbol = asset + ref_asset last_trade_id = self.db.get_max_trade_id(asset, ref_asset, 'spot') while True: - new_trades = self.client.get_my_trades(symbol=symbol, fromId=last_trade_id + 1, limit=limit) + client_params = { + 'symbol': symbol, + 'fromId': last_trade_id + 1, + 'limit': limit + } + new_trades = self._call_binance_client('get_my_trades', client_params) + for trade in new_trades: self.db.add_trade(trade_type='spot', trade_id=int(trade['id']), @@ -681,3 +771,35 @@ def update_all_spot_trades(self, limit: int = 1000): limit=limit) pbar.update() pbar.close() + + def _call_binance_client(self, method_name: str, params: Optional[Dict] = None, retry_count: int = 0): + """ + This method is used to handle rate limits: if a rate limits is breached, it will wait the necessary time + to call again the API. + + :param method_name: name of the method binance.Client to call + :type method_name: str + :param params: parameters to pass to the above method + :type params: Dict + :param retry_count: internal use only to count the number of retry if rate limits are breached + :type retry_count: int + :return: response of binance.Client method + :rtype: Dict + """ + if params is None: + params = dict() + if retry_count >= BinanceManager.API_MAX_RETRY: + raise RuntimeError(f"The API rate limits has been breached {retry_count} times") + + try: + return getattr(self.client, method_name)(**params) + except BinanceAPIException as err: + if err.code == -1003: # API rate Limits + wait_time = float(err.response.headers['Retry-After']) + if err.response.status_code == 418: # ban + self.logger.error(f"API calls resulted in a ban, retry in {wait_time} seconds") + raise err + self.logger.info(f"API calls resulted in a breach of rate limits, will retry after {wait_time} seconds") + time.sleep(wait_time + 1) + return self._call_binance_client(method_name, params, retry_count + 1) + raise err diff --git a/BinanceWatch/__init__.py b/BinanceWatch/__init__.py index 863cfad..99277fe 100644 --- a/BinanceWatch/__init__.py +++ b/BinanceWatch/__init__.py @@ -1,2 +1,2 @@ -__version__ = "0.1.1" +__version__ = "0.1.2" __author__ = 'EtWnn' diff --git a/README.rst b/README.rst index dbaef29..4889d88 100644 --- a/README.rst +++ b/README.rst @@ -1,6 +1,6 @@ -============================== -Welcome to BinanceWatch v0.1.1 -============================== +=================================== +Welcome to BinanceWatch v0.1.2 +=================================== Note ---- @@ -62,6 +62,14 @@ permissions are needed. pip install BinanceWatch +If you prefer to install the latest developments use: + +.. code:: bash + + pip install git+https://github.com/EtWnn/BinanceWatch.git@develop + +Use your Binance api keys to initiate the manager: + .. code:: python from BinanceWatch.BinanceManager import BinanceManager diff --git a/docs/source/conf.py b/docs/source/conf.py index f015d09..d697de0 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -21,8 +21,11 @@ author = 'EtWnn' # The full version, including alpha/beta/rc tags -release = '0.1.1' - +this_directory = os.path.abspath(os.path.dirname(__file__)) +about = {} +with open(os.path.join(this_directory, '../../BinanceWatch/__init__.py'), encoding='utf-8') as f: + exec(f.read(), about) +release = about['__version__'] # -- General configuration --------------------------------------------------- diff --git a/requirements.txt b/requirements.txt index 293a748..ec07ed6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ numpy tqdm dateparser requests -python-binance +python-binance>=0.7.9 appdirs sphinx sphinx_rtd_theme diff --git a/setup.py b/setup.py index 25e9216..2e45fd6 100644 --- a/setup.py +++ b/setup.py @@ -1,12 +1,18 @@ +import os from setuptools import setup -from os import path -this_directory = path.abspath(path.dirname(__file__)) -with open(path.join(this_directory, 'README.rst'), encoding='utf-8') as f: + +this_directory = os.path.abspath(os.path.dirname(__file__)) + +with open(os.path.join(this_directory, 'README.rst'), encoding='utf-8') as f: long_description = f.read() +about = {} +with open(os.path.join(this_directory, 'BinanceWatch/__init__.py'), encoding='utf-8') as f: + exec(f.read(), about) + setup( name='BinanceWatch', - version='0.1.1', + version=about['__version__'], packages=['BinanceWatch', 'tests', 'BinanceWatch.storage', @@ -18,7 +24,7 @@ description='Local tracker of a binance account', long_description=long_description, long_description_content_type='text/x-rst', - install_requires=['numpy', 'tqdm', 'dateparser', 'requests', 'python-binance', 'appdirs'], + install_requires=['numpy', 'tqdm', 'dateparser', 'requests', 'python-binance>=0.7.9', 'appdirs'], keywords='binance exchange wallet save tracking history bitcoin ethereum btc eth', classifiers=[ 'Intended Audience :: Developers',