improved api

README.md

@@ -18,16 +18,16 @@ You can use `.get()`, `.get_history()` and `.update()` with all avaliable apis.
 * Binance
 * Bitmex
 
-### Get most recent history with .get()
+### Get history with .get()
 
     import pandas as pd
     from tradingfeatures import bitfinex, bitstamp, binance, bitmex
 
     bitfinex = bitfinex()
 
-    df = bitfinex.get()
+    df = bitfinex.get(2000)
 
-This is useful to get most recent history. But limit is 10000 for Bitfinex and 1000 for others.
+Just pass how much data you want. It will return the amount in most recent 1h data. Currently only 1h data is supported. If history amount is above api limit, `.get()` will run `.get_history()` under the hood, so you don't need to worry about it. But if you want everything and don't want to guess how much data avaliable on each exchange, just run `.get_history()` and get everything.
 
 ### Download all available history with .get_history()
 The tool will download all avaliable history while respecting request per minute limits. Using it easy, and it takes couple of minutes for 1h data.

setup.py

@@ -5,7 +5,7 @@
 
 setup(
     name="tradingfeatures",
-    version="0.5.0",
+    version="0.5.1",
     author="Onur Cetinkol",
     author_email="realiti44@gmail.com",
     description="A useful tool to download market history from popular exchanges.",

tradingfeatures/__init__.py

@@ -6,4 +6,5 @@
 # from tradingfeatures.legacy import *
 
 from tradingfeatures.uber import Uber
+# from tradingfeatures.uber.uber_mp import UberMP
 

tradingfeatures/api_base.py

@@ -15,6 +15,7 @@ def __init__(self, name, per_step, sleep):
 
         self.default_columns = ['high', 'volume', 'low', 'close', 'open']
         self.start = None
+        self.limit = None
 
     def get(self, *args, **kwargs):
         raise NotImplementedError
@@ -40,7 +41,7 @@ def response_handler(self, address, params, timeout=60):
                 if r.status_code == 429:
                     retry_after += int(r.headers['Retry-After'])
 
-                print(f'\nResponse: {r.status_code}, trying after {retry_after}secs')
+                print(f'\nResponse: {r.status_code} for {self.name}, trying after {retry_after}secs')
                 try:
                     print(r.json())
                 except:
@@ -66,7 +67,8 @@ def get_hist(self,
             df_update=None,
             ):      
 
-        # init        
+        # init
+        verbose_after = 10     
         name = f'{self.name}_{interval}'
         columns = columns or self.default_columns
         start = start or self.start
@@ -79,9 +81,9 @@ def get_hist(self,
 
         df = pd.DataFrame(columns=columns)
         df.index.name = 'timestamp'
-        # df = None
 
-        print(f'  Downloading {name}')
+        if steps > verbose_after:
+            print(f'  Downloading {name}')
 
         for i in range(steps):
             start_batch = start + (interval*i*self.per_step)
@@ -95,15 +97,11 @@ def get_hist(self,
                 print('error between timestamps: ', start_batch, end_batch)
                 if steps <= 1: return None
 
-            # if df is None:
-            #     df = df_temp
-            # else:
-
             df_temp = pd.concat([df, df_temp])
             df = df_temp
 
-            print('\r' + f'  {i} of {steps}', end='')
-            # print(f'  {i} of {steps}')
+            if steps > verbose_after:
+                print('\r' + f'  {i} of {steps}', end='')
             time.sleep(self.sleep)
 
         if global_columns:
@@ -124,7 +122,8 @@ def get_hist(self,
             df.replace(0, np.nan, inplace=True)
             df.interpolate(inplace=True)
 
-        print(f'\nCompleted: {self.name}')
+        if steps > verbose_after:
+            print(f'\nCompleted: {self.name}')
         # df['date'] = pd.to_datetime(df.index, unit='s', utc=True)
 
         return df
@@ -149,6 +148,19 @@ def update(self, path):
 
         return df_final
 
+    def calc_start(self, limit, start=None, end=None, interval=3600):
+        current_time = int(time.time())
+        if isinstance(interval, str):
+            interval = self.interval_check(interval)[0]
+
+        limit = self.limit if limit is None else limit
+        limit = int(limit / (interval / 3600))
+        end = current_time if end is None else end
+        if start is None:
+            start = end - (interval * (limit-1))  # limit -1 to be sure to get the latest hour
+        out_of_range = True if limit > self.limit else False
+        return start, end, out_of_range
+
     def to_ts(self, df):   # Convert datetime to timestamp        
         return df.values.astype(np.int64) // 10 ** 9
 

tradingfeatures/apis/binance/base.py

@@ -28,8 +28,13 @@ def get(self,
             start: int = None,
             end: int = None,
             interval: str = '1h',
+            columns: list = None,
             return_r: bool = False,
-            ):      
+            ):
+
+        start, end, out_of_range = self.calc_start(limit, start, end)
+        if out_of_range:
+            return self.get_hist(start=start, end=end)
 
         address = address or self.address
         address = self.base_address + address
@@ -39,16 +44,14 @@ def get(self,
             limit = self.limit if limit is None else limit
             start, end = self.ts_to_mts(start), self.ts_to_mts(end)
 
-            query = {'symbol': symbol, 'interval': interval, 'endTime': end, 'limit': limit}
-            if start is not None:
-                query['startTime'] = start
+            query = {'symbol': symbol, 'interval': interval, 'startTime': start, 'endTime': end, 'limit': limit}
 
         r = requests.get(address, params=query)
 
         result = r.json()
 
-        df = pd.DataFrame(result, columns=['open_time', 'open', 'high', 'low', 'close', 'volume', 'close_time', 'quote_asset_volume',
-                                    'number_of_trades', 'taker_base_asset_volume', 'taker_quote_asset_volume', 'ignore'])
+        df = pd.DataFrame(result, columns=['open_time', 'open', 'high', 'low', 'close', 'volume', 'close_time', 'quote_asset_vol',
+                                    'number_of_trades', 'taker_base_asset_vol', 'taker_quote_asset_vol', 'ignore'])
 
         df = df.astype(float)
         df['timestamp'] = df['open_time'].div(1000).astype(int)
@@ -58,7 +61,11 @@ def get(self,
 
         df = df.set_index('timestamp')
         df.index = df.index.astype(int)
-        return df.astype(float)
+        df = df.astype(float)
+
+        if columns is not None:
+            return df[columns]
+        return df
 
     def get_hist(self, *args, **kwargs):
         # start = 1500000000

tradingfeatures/apis/bitfinex/base.py

@@ -29,10 +29,15 @@ def get(self,
             start: int = None,
             end: int = None,
             interval: str = '1h',
+            columns: list = None,
             return_r: bool = False,
             sort = -1,
             ):      
 
+        start, end, out_of_range = self.calc_start(limit, start, end)
+        if out_of_range:
+            return self.get_hist(start=start, end=end)
+
         address = address or self.address
         address = self.base_address + address
         symbol = symbol or 'tBTCUSD'        
@@ -42,9 +47,7 @@ def get(self,
             start, end = self.ts_to_mts(start), self.ts_to_mts(end)     # Conver for Bitfinex
             address = address + f'/trade:{interval}:{symbol}/hist'   
 
-            query = {'limit': limit, 'end': end, 'sort': sort}
-            if start is not None:
-                query['start'] = start
+            query = {'limit': limit, 'start': start, 'end': end, 'sort': sort}
 
         r = self.response_handler(address, params=query, timeout=60)
 
@@ -55,6 +58,8 @@ def get(self,
         df['timestamp'] = df['timestamp'].div(1000).astype(int)     # Fixing timestamp inside self.get
         df = df.set_index('timestamp')
 
+        if columns is not None:
+            return df[columns]
         return df
 
     def get_hist(self, *args, **kwargs):

tradingfeatures/apis/bitmex/base.py

@@ -28,23 +28,24 @@ def get(self,
             start: int = None,
             end: int = None,
             interval: str = '1h',
+            columns: list = None,
             return_r: bool = False,
-            ):      
+            ):
+
+        start, end, out_of_range = self.calc_start(limit, start, end, interval)
+        if out_of_range:
+            return self.get_hist(start=start, end=end, columns=columns)
 
         address = address or self.address
         address = self.base_address + address
         symbol = symbol or 'XBT'
 
         if query is None:
             limit = self.limit if limit is None else limit
-            start = self.start if start is None else start
-            end = time.time() if end is None else end
             start, end = self.to_date(start), self.to_date(end)
 
-            query = {'symbol': symbol, 'binSize': interval, 'count': limit, 'reverse': 'false', 'startTime': start}
-
-            # if '/trade' in address:
-            #     query['binSize'] = interval
+            query = {'symbol': symbol, 'binSize': interval, 'count': limit, 'startTime': start, 'endTime': end,
+                    'reverse': 'false'}
 
         r = self.response_handler(address, query)
         # Bitmex remaining limit
@@ -62,6 +63,9 @@ def get(self,
         df.index = df.index.astype(int)
         # df = df.astype(float)
 
+        df.index = df.index - 3600  # Compability with other apis, bitmex timestamp indexing is different
+        if columns is not None:
+            return df[columns]
         return df
 
     def get_hist(self, *args, **kwargs):

tradingfeatures/apis/bitmex/funding.py

@@ -14,34 +14,44 @@ def __init__(self):
         self.name = 'bitmex_fundings'
         self.address = '/funding'
         self.start = 1463227200
+        self.limit = 500
+
+    def get(self, limit=None, symbol=None, query=None, start=None, end=None, *args, **kwargs):
+        interval = '8h'
+        start, end, out_of_range = self.calc_start(limit, start, end, interval)
+        if out_of_range:
+            return self.get_hist(start=start, end=end)
+
+        # Get recent funding if getting latest data
+        if (int(end) // 3600) *3600 == (time.time() // 3600) *3600:
+            get_latest = True
+        else:
+            get_latest = False
 
-    def get(self, symbol=None, query=None, start=None, end=None, *args, **kwargs):
-        start = self.start if start is None else start
-        end = time.time() if end is None else end
         start, end = self.to_date(start), self.to_date(end)
         symbol = symbol or 'XBT'
 
         if query is None:
             query = {'symbol': symbol, 'count': self.limit, 'reverse': 'false', 'startTime': start}
 
-        return super(bitmexFunding, self).get(
+        df = super(bitmexFunding, self).get(
             query=query,
             start=start,
             end=end,
+            interval=interval,
             *args, **kwargs
         )
 
-    def get_hist(self, convert_funds=False, *args, **kwargs):  
-        df_fundings = apiBase.get_hist(
-            self,
-            columns=['fundingRate', 'fundingRateDaily'],
-            interval='8h',
-            *args, **kwargs
-        )
+        if get_latest:
+            df = self.get_recent(df)
+
+        # check results for get_history with both, solving convert problem here
+        # return df
+        return self.convert_funding(df)
 
-        # Recent funding
+    def get_recent(self, df):   # Recent Funding
         address = '/instrument'
-        r_current = self.get(address=address, query={'symbol': 'XBT'}, return_r=True)
+        r_current = super(bitmexFunding, self).get(address=address, query={'symbol': 'XBT'}, return_r=True)
         current_data = r_current.json()[0]
 
         funding_ts = current_data['fundingTimestamp']
@@ -50,14 +60,26 @@ def get_hist(self, convert_funds=False, *args, **kwargs):
         df_recent_funding = pd.DataFrame([[funding_ts, funding_rate]], columns=['timestamp', 'fundingRate'])
         df_recent_funding['timestamp'] = self.to_ts(pd.to_datetime(df_recent_funding['timestamp']))
         df_recent_funding.set_index('timestamp', inplace=True)
+        # Compability with other apis, bitmex timestamp indexing is different
+        df_recent_funding.index = df_recent_funding.index - 3600
 
-        df_final = pd.concat([df_fundings, df_recent_funding])
+        df_final = pd.concat([df, df_recent_funding])
         df_final = df_final[['fundingRate']]
+        return df_final
 
-        if convert_funds:       # convert 8h to 1h and backfill
-            aranged_array = np.arange(df_final.index[0], df_final.index[-1] + 1, 3600)
-            df_empty = pd.DataFrame(index=aranged_array)
-            df_final = df_empty.join(df_final)
-            df_final = df_empty.join(df_final).fillna(method='bfill')
+    def get_hist(self, columns=None, convert_funds=False, *args, **kwargs):  
+        columns = ['fundingRate', 'fundingRateDaily'] if columns is None else columns
+        return apiBase.get_hist(
+            self,
+            columns=columns,
+            interval='8h',
+            *args, **kwargs
+        )
 
-        return df_final
+    def convert_funding(self, df):  # convert 8h to 1h and backfill
+        aranged_array = np.arange(df.index[0], df.index[-1] + 1, 3600)
+        df_empty = pd.DataFrame(index=aranged_array)
+        df = df_empty.join(df)
+        df = df_empty.join(df).fillna(method='bfill')
+        return df
+

tradingfeatures/apis/bitmex/quote.py

@@ -16,13 +16,6 @@ def __init__(self):
         self.start = 1423227200        
 
     def get(self, symbol=None, query=None, start=None, end=None, *args, **kwargs):
-        # start = self.start if start is None else start
-        # end = time.time() if end is None else end
-        # start, end = self.to_date(start), self.to_date(end)
-        # symbol = symbol or 'XBT'
-
-        # if query is None:
-        #     query = {'symbol': symbol, 'binSize': interval, 'count': self.limit, 'reverse': 'false', 'startTime': start}
 
         return super(bitmexQuote, self).get(
             query=query,
@@ -31,10 +24,12 @@ def get(self, symbol=None, query=None, start=None, end=None, *args, **kwargs):
             *args, **kwargs
         )
 
-    def get_hist(self, convert_funds=False, *args, **kwargs):
+    def get_hist(self, columns=None, convert_funds=False, *args, **kwargs):
+        columns = ['bidSize', 'bidPrice', 'askPrice', 'askSize'] if columns is None else columns
+
         df = apiBase.get_hist(
             self,
-            columns=['bidSize', 'bidPrice', 'askPrice', 'askSize'],
+            columns=columns,
             # interval='8h',
             *args, **kwargs
         )