From bcaa2084e5f903b5021bd1cf921a2ce6445f5768 Mon Sep 17 00:00:00 2001 From: Niharika Singh Date: Wed, 12 Sep 2018 16:34:02 +0530 Subject: [PATCH 01/54] Added reset check --- ratelimit/decorators.py | 4 ++-- ratelimit/utils.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ratelimit/decorators.py b/ratelimit/decorators.py index 663bce6..3297e81 100644 --- a/ratelimit/decorators.py +++ b/ratelimit/decorators.py @@ -12,7 +12,7 @@ __all__ = ['ratelimit'] -def ratelimit(group=None, key=None, rate=None, method=ALL, block=False): +def ratelimit(group=None, key=None, rate=None, method=ALL, block=False, reset=None): def decorator(fn): @wraps(fn) def _wrapped(*args, **kw): @@ -24,7 +24,7 @@ def _wrapped(*args, **kw): request.limited = getattr(request, 'limited', False) ratelimited = is_ratelimited(request=request, group=group, fn=fn, key=key, rate=rate, method=method, - increment=True) + increment=True, reset=reset) if ratelimited and block: raise Ratelimited() return fn(*args, **kw) diff --git a/ratelimit/utils.py b/ratelimit/utils.py index 285e999..1a2887a 100644 --- a/ratelimit/utils.py +++ b/ratelimit/utils.py @@ -101,7 +101,7 @@ def _make_cache_key(group, rate, value, methods): def is_ratelimited(request, group=None, fn=None, key=None, rate=None, - method=ALL, increment=False): + method=ALL, increment=False, reset=None): if group is None: if hasattr(fn, '__self__'): parts = fn.__module__, fn.__self__.__class__.__name__, fn.__name__ From f5361bdafc75332a869d5dd42886daf6ab351d08 Mon Sep 17 00:00:00 2001 From: Niharika Singh Date: Wed, 12 Sep 2018 17:21:40 +0530 Subject: [PATCH 02/54] Added custom exception message changes --- ratelimit/decorators.py | 2 +- ratelimit/exceptions.py | 16 +++++++++++++--- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/ratelimit/decorators.py b/ratelimit/decorators.py index 3297e81..1c90360 100644 --- a/ratelimit/decorators.py +++ b/ratelimit/decorators.py @@ -26,7 +26,7 @@ def _wrapped(*args, **kw): key=key, rate=rate, method=method, increment=True, reset=reset) if ratelimited and block: - raise Ratelimited() + raise Ratelimited(*args, **kw) return fn(*args, **kw) return _wrapped return decorator diff --git a/ratelimit/exceptions.py b/ratelimit/exceptions.py index f39a0f4..91f56e3 100644 --- a/ratelimit/exceptions.py +++ b/ratelimit/exceptions.py @@ -1,5 +1,15 @@ -from django.core.exceptions import PermissionDenied +# from django.core.exceptions import PermissionDenied -class Ratelimited(PermissionDenied): - pass +class Ratelimited(Exception): + def __init__(self, code): + self.code = code + + def __str__(self): + return repr(self.code) + + +try: + raise Ratelimited(429) +except Ratelimited as e: + print("Too many requests, Please retry after some time:", e.code) \ No newline at end of file From 4f2ca1dde75a9d069786cc062d7c8d0709eb2546 Mon Sep 17 00:00:00 2001 From: Niharika Singh Date: Wed, 12 Sep 2018 18:36:24 +0530 Subject: [PATCH 03/54] changes _init()_ method args --- ratelimit/decorators.py | 2 +- ratelimit/exceptions.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ratelimit/decorators.py b/ratelimit/decorators.py index 1c90360..722ddd1 100644 --- a/ratelimit/decorators.py +++ b/ratelimit/decorators.py @@ -26,7 +26,7 @@ def _wrapped(*args, **kw): key=key, rate=rate, method=method, increment=True, reset=reset) if ratelimited and block: - raise Ratelimited(*args, **kw) + raise Ratelimited(429) return fn(*args, **kw) return _wrapped return decorator diff --git a/ratelimit/exceptions.py b/ratelimit/exceptions.py index 91f56e3..5029fba 100644 --- a/ratelimit/exceptions.py +++ b/ratelimit/exceptions.py @@ -12,4 +12,4 @@ def __str__(self): try: raise Ratelimited(429) except Ratelimited as e: - print("Too many requests, Please retry after some time:", e.code) \ No newline at end of file + print("Too many requests, Please retry after some time:", e.code) \ No newline at end of file From 50dfec83d644f1864c7c7148869fd49742bc4ae7 Mon Sep 17 00:00:00 2001 From: Niharika Singh Date: Wed, 12 Sep 2018 20:41:59 +0530 Subject: [PATCH 04/54] Added exception message --- ratelimit/custom_exceptions.py | 0 ratelimit/decorators.py | 4 ++-- 2 files changed, 2 insertions(+), 2 deletions(-) create mode 100644 ratelimit/custom_exceptions.py diff --git a/ratelimit/custom_exceptions.py b/ratelimit/custom_exceptions.py new file mode 100644 index 0000000..e69de29 diff --git a/ratelimit/decorators.py b/ratelimit/decorators.py index 722ddd1..4cecadd 100644 --- a/ratelimit/decorators.py +++ b/ratelimit/decorators.py @@ -5,7 +5,7 @@ from django.http import HttpRequest from ratelimit import ALL, UNSAFE -from ratelimit.exceptions import Ratelimited +from ratelimit.custom_exceptions import Ratelimited from ratelimit.utils import is_ratelimited @@ -26,7 +26,7 @@ def _wrapped(*args, **kw): key=key, rate=rate, method=method, increment=True, reset=reset) if ratelimited and block: - raise Ratelimited(429) + raise Ratelimited("Too many requests", 429) return fn(*args, **kw) return _wrapped return decorator From 780c62cd5e53ea9fcf0dc1251781a22cdb9a1a77 Mon Sep 17 00:00:00 2001 From: Niharika Singh Date: Wed, 12 Sep 2018 20:58:39 +0530 Subject: [PATCH 05/54] fixes --- ratelimit/decorators.py | 2 +- ratelimit/exceptions.py | 14 +++----------- 2 files changed, 4 insertions(+), 12 deletions(-) diff --git a/ratelimit/decorators.py b/ratelimit/decorators.py index 4cecadd..0904d69 100644 --- a/ratelimit/decorators.py +++ b/ratelimit/decorators.py @@ -5,7 +5,7 @@ from django.http import HttpRequest from ratelimit import ALL, UNSAFE -from ratelimit.custom_exceptions import Ratelimited +from ratelimit.exceptions import Ratelimited from ratelimit.utils import is_ratelimited diff --git a/ratelimit/exceptions.py b/ratelimit/exceptions.py index 5029fba..cc748e1 100644 --- a/ratelimit/exceptions.py +++ b/ratelimit/exceptions.py @@ -1,15 +1,7 @@ -# from django.core.exceptions import PermissionDenied - - class Ratelimited(Exception): - def __init__(self, code): + def __init__(self, message, code): + self.message = message self.code = code def __str__(self): - return repr(self.code) - - -try: - raise Ratelimited(429) -except Ratelimited as e: - print("Too many requests, Please retry after some time:", e.code) \ No newline at end of file + return self.message \ No newline at end of file From 473b81da948a12682148633ffd48ef709653d539 Mon Sep 17 00:00:00 2001 From: Niharika Singh Date: Thu, 13 Sep 2018 23:57:17 +0530 Subject: [PATCH 06/54] revert changes --- ratelimit/custom_exceptions.py | 0 ratelimit/exceptions.py | 10 ++++------ 2 files changed, 4 insertions(+), 6 deletions(-) delete mode 100644 ratelimit/custom_exceptions.py diff --git a/ratelimit/custom_exceptions.py b/ratelimit/custom_exceptions.py deleted file mode 100644 index e69de29..0000000 diff --git a/ratelimit/exceptions.py b/ratelimit/exceptions.py index cc748e1..f39a0f4 100644 --- a/ratelimit/exceptions.py +++ b/ratelimit/exceptions.py @@ -1,7 +1,5 @@ -class Ratelimited(Exception): - def __init__(self, message, code): - self.message = message - self.code = code +from django.core.exceptions import PermissionDenied - def __str__(self): - return self.message \ No newline at end of file + +class Ratelimited(PermissionDenied): + pass From 0ea7204dfa7e893d30178bf4bfe4dc3b25444796 Mon Sep 17 00:00:00 2001 From: Niharika Singh Date: Fri, 14 Sep 2018 03:27:51 +0530 Subject: [PATCH 07/54] Added an optional reset callable that indicates if the current counter should be reset to zero. --- ratelimit/utils.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/ratelimit/utils.py b/ratelimit/utils.py index 1a2887a..016f339 100644 --- a/ratelimit/utils.py +++ b/ratelimit/utils.py @@ -124,7 +124,7 @@ def is_ratelimited(request, group=None, fn=None, key=None, rate=None, if rate is None: request.limited = old_limited return False - usage = get_usage_count(request, group, fn, key, rate, method, increment) + usage = get_usage_count(request, group, fn, key, rate, method, increment,reset) fail_open = getattr(settings, 'RATELIMIT_FAIL_OPEN', False) @@ -141,7 +141,7 @@ def is_ratelimited(request, group=None, fn=None, key=None, rate=None, def get_usage_count(request, group=None, fn=None, key=None, rate=None, - method=ALL, increment=False): + method=ALL, increment=False,reset=None): if not key: raise ImproperlyConfigured('Ratelimit key must be specified') limit, period = _split_rate(rate) @@ -166,6 +166,12 @@ def get_usage_count(request, group=None, fn=None, key=None, rate=None, 'Could not understand ratelimit key: %s' % key) cache_key = _make_cache_key(group, rate, value, method) + + if reset and callable(reset): + should_reset = reset(request) + if should_reset: + cache.delete(cache_key) + time_left = _get_window(value, period) - int(time.time()) initial_value = 1 if increment else 0 added = cache.add(cache_key, initial_value, period + EXPIRATION_FUDGE) From 3595bee2b2f0260f78025297af642ebdd2a70234 Mon Sep 17 00:00:00 2001 From: Niharika Singh Date: Mon, 17 Sep 2018 15:52:12 +0530 Subject: [PATCH 08/54] Added header 'HTTP_X_FORWARDED_FOR' for ip key --- ratelimit/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ratelimit/utils.py b/ratelimit/utils.py index 016f339..0bc069a 100644 --- a/ratelimit/utils.py +++ b/ratelimit/utils.py @@ -27,11 +27,11 @@ def user_or_ip(request): if is_authenticated(request.user): return str(request.user.pk) - return request.META['REMOTE_ADDR'] + return request.META.get('HTTP_X_FORWARDED_FOR') _SIMPLE_KEYS = { - 'ip': lambda r: r.META['REMOTE_ADDR'], + 'ip': lambda r: r.META.get('HTTP_X_FORWARDED_FOR'), 'user': lambda r: str(r.user.pk), 'user_or_ip': user_or_ip, } From b78e616b259dc014633d65a8a617e3b9d981c703 Mon Sep 17 00:00:00 2001 From: Niharika Singh Date: Mon, 17 Sep 2018 16:04:29 +0530 Subject: [PATCH 09/54] fix --- ratelimit/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ratelimit/utils.py b/ratelimit/utils.py index 0bc069a..77e05fe 100644 --- a/ratelimit/utils.py +++ b/ratelimit/utils.py @@ -27,11 +27,11 @@ def user_or_ip(request): if is_authenticated(request.user): return str(request.user.pk) - return request.META.get('HTTP_X_FORWARDED_FOR') + return request.META['HTTP_X_FORWARDED_FOR'] _SIMPLE_KEYS = { - 'ip': lambda r: r.META.get('HTTP_X_FORWARDED_FOR'), + 'ip': lambda r: r.META['HTTP_X_FORWARDED_FOR'], 'user': lambda r: str(r.user.pk), 'user_or_ip': user_or_ip, } From fcacf50ec57391527bb1b60a603f6d4d61b613ad Mon Sep 17 00:00:00 2001 From: Niharika Singh Date: Mon, 17 Sep 2018 16:22:52 +0530 Subject: [PATCH 10/54] print simple key attrs --- ratelimit/utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ratelimit/utils.py b/ratelimit/utils.py index 77e05fe..212b7bd 100644 --- a/ratelimit/utils.py +++ b/ratelimit/utils.py @@ -151,6 +151,7 @@ def get_usage_count(request, group=None, fn=None, key=None, rate=None, if callable(key): value = key(group, request) elif key in _SIMPLE_KEYS: + print(_SIMPLE_KEYS[key](request)) value = _SIMPLE_KEYS[key](request) elif ':' in key: accessor, k = key.split(':', 1) From 62e7a8a8f4c714415c96cabb5c36aae2b949cec8 Mon Sep 17 00:00:00 2001 From: Vikas Chahal Date: Mon, 24 Jun 2019 17:39:13 +0530 Subject: [PATCH 11/54] sliding window algo added --- ratelimit/exceptions.py | 17 ++++++ ratelimit/redis_rate_limit.py | 100 ++++++++++++++++++++++++++++++++++ ratelimit/utils.py | 36 +++++++++++- 3 files changed, 150 insertions(+), 3 deletions(-) create mode 100644 ratelimit/redis_rate_limit.py diff --git a/ratelimit/exceptions.py b/ratelimit/exceptions.py index f39a0f4..e169baa 100644 --- a/ratelimit/exceptions.py +++ b/ratelimit/exceptions.py @@ -3,3 +3,20 @@ class Ratelimited(PermissionDenied): pass + + +class CustomException(Exception): + def __init__(self, message): + super(CustomException, self).__init__(message) + + +class DatastoreConnectionError(CustomException): + def __init__(self): + message = 'Could not establish connection to data store' + super(DatastoreConnectionError, self).__init__(message) + + +class RateLimited(CustomException): + def __init__(self): + message = 'Rate limited' + super(RateLimited, self).__init__(message) diff --git a/ratelimit/redis_rate_limit.py b/ratelimit/redis_rate_limit.py new file mode 100644 index 0000000..d4439b5 --- /dev/null +++ b/ratelimit/redis_rate_limit.py @@ -0,0 +1,100 @@ +import time + +from functools import wraps +from django.conf import settings +import redis +from .exceptions import RateLimited, DatastoreConnectionError + +__author__ = 'vikaschahal' + + +class RateLimiter(object): + """Base class for Rate Limiting""" + + def __init__(self, limit, window, connection, key): + """ + :param limit: number of requests allowed + :type limit: int + :param window: window in secs in which :limit number requests allowed + :type window: int + """ + self._connection = connection + self._key = key + self._limit = limit + self._window = window + + def is_allowed(self, log_current_request=True): + """ + :param log_current_request: Consider the call for rate limiting + :type log_current_request: bool + :return: Whether a requests is allowed or rate limited. + :rtype: bool + """ + raise NotImplementedError + + @property + def remaining_requests(self): + raise NotImplementedError + + def limit(self, func): + """Decorator to check the rate limit.""" + + @wraps(func) + def decorated(*args, **kwargs): + if self.is_allowed(): + return func(*args, **kwargs) + raise RateLimited + + return decorated + + +class RedisRateLimiterConnection(object): + def __init__(self, host=None, port=None, db=0, connection=None): + self.connection = None + if host and port: + connection = redis.StrictRedis(host, port, db) + if not connection.ping(): + raise DatastoreConnectionError + self.connection = connection + elif connection: + if not connection.ping(): + raise DatastoreConnectionError + self.connection = connection + else: + raise DatastoreConnectionError + + +class RedisRateLimiter(RateLimiter): + def __init__(self, limit, window, connection, key): + super(RedisRateLimiter, self).__init__(limit, window, connection, key) + self._pipeline = self._connection.pipeline() + + def _increment_request(self): + key_value = int(time.time()) + self._window + self._pipeline.zadd( + self._key, {key_value: key_value}, + ) + self._pipeline.execute() + + def is_allowed(self, log_current_request=True): + if log_current_request: + self._increment_request() + current_time = time.time() + self._pipeline.zremrangebyscore(self._key, '-inf', current_time) + self._pipeline.zcount(self._key, '-inf', '+inf') + result = self._pipeline.execute() + return result[-1] <= self._limit + + def count(self, log_current_request=True): + if log_current_request: + self._increment_request() + current_time = time.time() + self._pipeline.zremrangebyscore(self._key, '-inf', current_time) + self._pipeline.zcount(self._key, '-inf', '+inf') + result = self._pipeline.execute() + return result[-1] + + +connection = RedisRateLimiterConnection(host=settings.REDIS_HOST_INTERNAL, port=6379, db=0) + +RedisRateLimit = RedisRateLimiter() diff --git a/ratelimit/utils.py b/ratelimit/utils.py index 212b7bd..50eabe8 100644 --- a/ratelimit/utils.py +++ b/ratelimit/utils.py @@ -7,10 +7,10 @@ from django.conf import settings from django.core.cache import caches from django.core.exceptions import ImproperlyConfigured +from redis_rate_limit import connection, RedisRateLimiter from ratelimit import ALL, UNSAFE - __all__ = ['is_ratelimited'] _PERIODS = { @@ -100,6 +100,36 @@ def _make_cache_key(group, rate, value, methods): return prefix + hashlib.md5(u''.join(parts).encode('utf-8')).hexdigest() +def _get_usage_count(request, group=None, fn=None, key=None, rate=None, + method=ALL, increment=False, reset=None): + if not key: + raise ImproperlyConfigured('Ratelimit key must be specified') + limit, period = _split_rate(rate) + + if callable(key): + value = key(group, request) + elif key in _SIMPLE_KEYS: + print(_SIMPLE_KEYS[key](request)) + value = _SIMPLE_KEYS[key](request) + elif ':' in key: + accessor, k = key.split(':', 1) + if accessor not in _ACCESSOR_KEYS: + raise ImproperlyConfigured('Unknown ratelimit key: %s' % key) + value = _ACCESSOR_KEYS[accessor](request, k) + elif '.' in key: + mod, attr = key.rsplit('.', 1) + keyfn = getattr(import_module(mod), attr) + value = keyfn(group, request) + else: + raise ImproperlyConfigured( + 'Could not understand ratelimit key: %s' % key) + + cache_key = _make_cache_key(group, rate, value, method) + redis_limiter = RedisRateLimiter(limit=limit, window=period, connection=connection, key=cache_key) + count = redis_limiter.count() + return {'count': count, 'limit': limit} + + def is_ratelimited(request, group=None, fn=None, key=None, rate=None, method=ALL, increment=False, reset=None): if group is None: @@ -124,7 +154,7 @@ def is_ratelimited(request, group=None, fn=None, key=None, rate=None, if rate is None: request.limited = old_limited return False - usage = get_usage_count(request, group, fn, key, rate, method, increment,reset) + usage = _get_usage_count(request, group, fn, key, rate, method, increment, reset) fail_open = getattr(settings, 'RATELIMIT_FAIL_OPEN', False) @@ -141,7 +171,7 @@ def is_ratelimited(request, group=None, fn=None, key=None, rate=None, def get_usage_count(request, group=None, fn=None, key=None, rate=None, - method=ALL, increment=False,reset=None): + method=ALL, increment=False, reset=None): if not key: raise ImproperlyConfigured('Ratelimit key must be specified') limit, period = _split_rate(rate) From 9ecf5245f40e71b76e2f7071011a2e06d150249b Mon Sep 17 00:00:00 2001 From: Vikas Chahal Date: Mon, 24 Jun 2019 17:46:12 +0530 Subject: [PATCH 12/54] version upgraded --- ratelimit/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ratelimit/__init__.py b/ratelimit/__init__.py index 9babbfb..7eac2f0 100644 --- a/ratelimit/__init__.py +++ b/ratelimit/__init__.py @@ -1,4 +1,4 @@ -VERSION = (1, 1, 0) +VERSION = (1, 2, 0) __version__ = '.'.join(map(str, VERSION)) ALL = (None,) # Sentinel value for all HTTP methods. From ce9dfe723b974ed6242aa6c67ed77aea313c2797 Mon Sep 17 00:00:00 2001 From: Vikas Chahal Date: Mon, 24 Jun 2019 17:49:45 +0530 Subject: [PATCH 13/54] redis object issue fix --- ratelimit/redis_rate_limit.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/ratelimit/redis_rate_limit.py b/ratelimit/redis_rate_limit.py index d4439b5..afd1841 100644 --- a/ratelimit/redis_rate_limit.py +++ b/ratelimit/redis_rate_limit.py @@ -96,5 +96,3 @@ def count(self, log_current_request=True): connection = RedisRateLimiterConnection(host=settings.REDIS_HOST_INTERNAL, port=6379, db=0) - -RedisRateLimit = RedisRateLimiter() From 7e28fe0bae369ecfa29af60b257aa9cd0bf099fd Mon Sep 17 00:00:00 2001 From: Vikas Chahal Date: Tue, 25 Jun 2019 11:29:16 +0530 Subject: [PATCH 14/54] redis connection issue --- ratelimit/redis_rate_limit.py | 6 +++--- ratelimit/utils.py | 11 +++++++---- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/ratelimit/redis_rate_limit.py b/ratelimit/redis_rate_limit.py index afd1841..dd9bc4d 100644 --- a/ratelimit/redis_rate_limit.py +++ b/ratelimit/redis_rate_limit.py @@ -67,12 +67,12 @@ def __init__(self, host=None, port=None, db=0, connection=None): class RedisRateLimiter(RateLimiter): def __init__(self, limit, window, connection, key): super(RedisRateLimiter, self).__init__(limit, window, connection, key) - self._pipeline = self._connection.pipeline() + self._pipeline = self._connection.connection.pipeline() def _increment_request(self): key_value = int(time.time()) + self._window self._pipeline.zadd( - self._key, {key_value: key_value}, + self._key, key_value, key_value ) self._pipeline.execute() @@ -95,4 +95,4 @@ def count(self, log_current_request=True): return result[-1] -connection = RedisRateLimiterConnection(host=settings.REDIS_HOST_INTERNAL, port=6379, db=0) +redis_connection = RedisRateLimiterConnection(host=settings.REDIS_HOST_INTERNAL, port=6379, db=0) diff --git a/ratelimit/utils.py b/ratelimit/utils.py index 50eabe8..5e9843d 100644 --- a/ratelimit/utils.py +++ b/ratelimit/utils.py @@ -7,7 +7,7 @@ from django.conf import settings from django.core.cache import caches from django.core.exceptions import ImproperlyConfigured -from redis_rate_limit import connection, RedisRateLimiter +from redis_rate_limit import redis_connection, RedisRateLimiter from ratelimit import ALL, UNSAFE @@ -125,13 +125,13 @@ def _get_usage_count(request, group=None, fn=None, key=None, rate=None, 'Could not understand ratelimit key: %s' % key) cache_key = _make_cache_key(group, rate, value, method) - redis_limiter = RedisRateLimiter(limit=limit, window=period, connection=connection, key=cache_key) + redis_limiter = RedisRateLimiter(limit=limit, window=period, connection=redis_connection, key=cache_key) count = redis_limiter.count() return {'count': count, 'limit': limit} def is_ratelimited(request, group=None, fn=None, key=None, rate=None, - method=ALL, increment=False, reset=None): + method=ALL, increment=False, reset=None, sliding_window=True): if group is None: if hasattr(fn, '__self__'): parts = fn.__module__, fn.__self__.__class__.__name__, fn.__name__ @@ -154,7 +154,10 @@ def is_ratelimited(request, group=None, fn=None, key=None, rate=None, if rate is None: request.limited = old_limited return False - usage = _get_usage_count(request, group, fn, key, rate, method, increment, reset) + if sliding_window: + usage = _get_usage_count(request, group, fn, key, rate, method, increment, reset) + else: + usage = get_usage_count(request, group, fn, key, rate, method, increment, reset) fail_open = getattr(settings, 'RATELIMIT_FAIL_OPEN', False) From 3c7e1a19079dff9bdb1905d62c26efe70fdd5240 Mon Sep 17 00:00:00 2001 From: Vikas Chahal Date: Tue, 25 Jun 2019 13:03:52 +0530 Subject: [PATCH 15/54] key expiry added --- ratelimit/redis_rate_limit.py | 1 + ratelimit/utils.py | 13 ++++++++----- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/ratelimit/redis_rate_limit.py b/ratelimit/redis_rate_limit.py index dd9bc4d..e4f1ecb 100644 --- a/ratelimit/redis_rate_limit.py +++ b/ratelimit/redis_rate_limit.py @@ -74,6 +74,7 @@ def _increment_request(self): self._pipeline.zadd( self._key, key_value, key_value ) + self._pipeline.expire(self._key, self._window) # set key expiry self._pipeline.execute() def is_allowed(self, log_current_request=True): diff --git a/ratelimit/utils.py b/ratelimit/utils.py index 5e9843d..75be2dd 100644 --- a/ratelimit/utils.py +++ b/ratelimit/utils.py @@ -85,10 +85,13 @@ def _get_window(value, period): return w -def _make_cache_key(group, rate, value, methods): +def _make_cache_key(group, rate, value, methods, sliding_window=False): count, period = _split_rate(rate) safe_rate = '%d/%ds' % (count, period) - window = _get_window(value, period) + if sliding_window: + window = '' + else: + window = _get_window(value, period) parts = [group + safe_rate, value, str(window)] if methods is not None: if methods == ALL: @@ -101,7 +104,7 @@ def _make_cache_key(group, rate, value, methods): def _get_usage_count(request, group=None, fn=None, key=None, rate=None, - method=ALL, increment=False, reset=None): + method=ALL, increment=False, reset=None, sliding_window=True): if not key: raise ImproperlyConfigured('Ratelimit key must be specified') limit, period = _split_rate(rate) @@ -124,7 +127,7 @@ def _get_usage_count(request, group=None, fn=None, key=None, rate=None, raise ImproperlyConfigured( 'Could not understand ratelimit key: %s' % key) - cache_key = _make_cache_key(group, rate, value, method) + cache_key = _make_cache_key(group, rate, value, method, sliding_window) redis_limiter = RedisRateLimiter(limit=limit, window=period, connection=redis_connection, key=cache_key) count = redis_limiter.count() return {'count': count, 'limit': limit} @@ -155,7 +158,7 @@ def is_ratelimited(request, group=None, fn=None, key=None, rate=None, request.limited = old_limited return False if sliding_window: - usage = _get_usage_count(request, group, fn, key, rate, method, increment, reset) + usage = _get_usage_count(request, group, fn, key, rate, method, increment, reset, sliding_window) else: usage = get_usage_count(request, group, fn, key, rate, method, increment, reset) From b6a54d5857dbf7375cd7acaa1cc902043122cc72 Mon Sep 17 00:00:00 2001 From: Pratyush Date: Thu, 18 Jul 2019 12:33:00 +0530 Subject: [PATCH 16/54] Block if exceeded certain limit --- ratelimit/utils.py | 37 ++++++++++++++++++++++++++++++++----- 1 file changed, 32 insertions(+), 5 deletions(-) diff --git a/ratelimit/utils.py b/ratelimit/utils.py index 75be2dd..974be72 100644 --- a/ratelimit/utils.py +++ b/ratelimit/utils.py @@ -103,12 +103,9 @@ def _make_cache_key(group, rate, value, methods, sliding_window=False): return prefix + hashlib.md5(u''.join(parts).encode('utf-8')).hexdigest() -def _get_usage_count(request, group=None, fn=None, key=None, rate=None, - method=ALL, increment=False, reset=None, sliding_window=True): +def _get_value_from_key(request, group=None, key=None): if not key: raise ImproperlyConfigured('Ratelimit key must be specified') - limit, period = _split_rate(rate) - if callable(key): value = key(group, request) elif key in _SIMPLE_KEYS: @@ -126,15 +123,32 @@ def _get_usage_count(request, group=None, fn=None, key=None, rate=None, else: raise ImproperlyConfigured( 'Could not understand ratelimit key: %s' % key) + return value + +def _get_usage_count(request, group=None, fn=None, key=None, rate=None, + method=ALL, increment=False, reset=None, sliding_window=True): + value = _get_value_from_key(request, group=group, key=key) + limit, period = _split_rate(rate) cache_key = _make_cache_key(group, rate, value, method, sliding_window) redis_limiter = RedisRateLimiter(limit=limit, window=period, connection=redis_connection, key=cache_key) count = redis_limiter.count() return {'count': count, 'limit': limit} +def get_offence_count(request, group=None, max_offence_rate=None, + key=None, method=ALL, sliding_window=True, count_current_request=False): + value = _get_value_from_key(request, group=group, key=key) + limit, period = _split_rate(max_offence_rate) + cache_key = _make_cache_key(group, max_offence_rate, value, method, sliding_window) + redis_limiter = RedisRateLimiter(limit=limit, window=period, connection=redis_connection, key=cache_key) + count = redis_limiter.count(log_current_request=count_current_request) + return {'count': count, 'limit': limit} + + def is_ratelimited(request, group=None, fn=None, key=None, rate=None, - method=ALL, increment=False, reset=None, sliding_window=True): + method=ALL, increment=False, reset=None, sliding_window=True, + max_offence_rate=None): if group is None: if hasattr(fn, '__self__'): parts = fn.__module__, fn.__self__.__class__.__name__, fn.__name__ @@ -157,6 +171,15 @@ def is_ratelimited(request, group=None, fn=None, key=None, rate=None, if rate is None: request.limited = old_limited return False + + if max_offence_rate is not None: + offence_report = get_offence_count(group, max_offence_rate, key, method, sliding_window) + offence_count = offence_report.get('count') + if offence_count is not None: + max_offence_count = offence_report.get('limit') + if offence_count >= max_offence_count: + return False + if sliding_window: usage = _get_usage_count(request, group, fn, key, rate, method, increment, reset, sliding_window) else: @@ -173,6 +196,10 @@ def is_ratelimited(request, group=None, fn=None, key=None, rate=None, if increment: request.limited = old_limited or limited + + if limited: + get_offence_count(group, max_offence_rate, key, method, sliding_window, count_current_request=True) + return limited From e7aa89542fc0ac7fe14bcaf9d8aff4a4cfe3b3f5 Mon Sep 17 00:00:00 2001 From: Pratyush Date: Thu, 18 Jul 2019 12:43:51 +0530 Subject: [PATCH 17/54] Add missing param --- ratelimit/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ratelimit/utils.py b/ratelimit/utils.py index 974be72..c97c7c2 100644 --- a/ratelimit/utils.py +++ b/ratelimit/utils.py @@ -173,7 +173,7 @@ def is_ratelimited(request, group=None, fn=None, key=None, rate=None, return False if max_offence_rate is not None: - offence_report = get_offence_count(group, max_offence_rate, key, method, sliding_window) + offence_report = get_offence_count(request, group, max_offence_rate, key, method, sliding_window) offence_count = offence_report.get('count') if offence_count is not None: max_offence_count = offence_report.get('limit') @@ -198,7 +198,7 @@ def is_ratelimited(request, group=None, fn=None, key=None, rate=None, request.limited = old_limited or limited if limited: - get_offence_count(group, max_offence_rate, key, method, sliding_window, count_current_request=True) + get_offence_count(request, group, max_offence_rate, key, method, sliding_window, count_current_request=True) return limited From 900397dad741957d92cbff305da66a2045e7f230 Mon Sep 17 00:00:00 2001 From: Pratyush Date: Thu, 18 Jul 2019 13:05:12 +0530 Subject: [PATCH 18/54] Update count on offence --- ratelimit/utils.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ratelimit/utils.py b/ratelimit/utils.py index c97c7c2..b2ff610 100644 --- a/ratelimit/utils.py +++ b/ratelimit/utils.py @@ -143,7 +143,7 @@ def get_offence_count(request, group=None, max_offence_rate=None, cache_key = _make_cache_key(group, max_offence_rate, value, method, sliding_window) redis_limiter = RedisRateLimiter(limit=limit, window=period, connection=redis_connection, key=cache_key) count = redis_limiter.count(log_current_request=count_current_request) - return {'count': count, 'limit': limit} + return {'count': count, 'limit': limit, 'limiter': redis_limiter} def is_ratelimited(request, group=None, fn=None, key=None, rate=None, @@ -178,7 +178,9 @@ def is_ratelimited(request, group=None, fn=None, key=None, rate=None, if offence_count is not None: max_offence_count = offence_report.get('limit') if offence_count >= max_offence_count: - return False + get_offence_count(request, group, max_offence_rate, key, method, + sliding_window, count_current_request=True) + return True if sliding_window: usage = _get_usage_count(request, group, fn, key, rate, method, increment, reset, sliding_window) From 05130ececfc34f26a0c0547ffd5c942c0878ce9f Mon Sep 17 00:00:00 2001 From: Pratyush Date: Thu, 18 Jul 2019 14:51:13 +0530 Subject: [PATCH 19/54] Increment count only if max offence rate is present --- ratelimit/utils.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/ratelimit/utils.py b/ratelimit/utils.py index b2ff610..1acfb59 100644 --- a/ratelimit/utils.py +++ b/ratelimit/utils.py @@ -143,7 +143,7 @@ def get_offence_count(request, group=None, max_offence_rate=None, cache_key = _make_cache_key(group, max_offence_rate, value, method, sliding_window) redis_limiter = RedisRateLimiter(limit=limit, window=period, connection=redis_connection, key=cache_key) count = redis_limiter.count(log_current_request=count_current_request) - return {'count': count, 'limit': limit, 'limiter': redis_limiter} + return {'count': count, 'limit': limit} def is_ratelimited(request, group=None, fn=None, key=None, rate=None, @@ -199,8 +199,9 @@ def is_ratelimited(request, group=None, fn=None, key=None, rate=None, if increment: request.limited = old_limited or limited - if limited: - get_offence_count(request, group, max_offence_rate, key, method, sliding_window, count_current_request=True) + if max_offence_rate is not None and limited: + get_offence_count(request, group, max_offence_rate, key, method, + sliding_window, count_current_request=True) return limited From ab07c9f5708fa0c94a240dd911f58b7055dd87d1 Mon Sep 17 00:00:00 2001 From: Pratyush Date: Thu, 18 Jul 2019 17:35:32 +0530 Subject: [PATCH 20/54] Update version --- ratelimit/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ratelimit/__init__.py b/ratelimit/__init__.py index 7eac2f0..78c2c9f 100644 --- a/ratelimit/__init__.py +++ b/ratelimit/__init__.py @@ -1,4 +1,4 @@ -VERSION = (1, 2, 0) +VERSION = (1, 2, 1) __version__ = '.'.join(map(str, VERSION)) ALL = (None,) # Sentinel value for all HTTP methods. From 2a6c25f8cdac21cdaddde9ab3b8f0aa798822b61 Mon Sep 17 00:00:00 2001 From: Veerendra Mandava Date: Thu, 12 Sep 2019 22:03:10 +0530 Subject: [PATCH 21/54] added rate limiting on ip by considering different attributes of the request --- ratelimit/__init__.py | 2 +- ratelimit/redis_rate_limit.py | 28 ++++++++++++++++++++++++++++ ratelimit/utils.py | 33 ++++++++++++++++++++++++++++++++- 3 files changed, 61 insertions(+), 2 deletions(-) diff --git a/ratelimit/__init__.py b/ratelimit/__init__.py index 78c2c9f..b3e6efa 100644 --- a/ratelimit/__init__.py +++ b/ratelimit/__init__.py @@ -1,4 +1,4 @@ -VERSION = (1, 2, 1) +VERSION = (1, 3, 2) __version__ = '.'.join(map(str, VERSION)) ALL = (None,) # Sentinel value for all HTTP methods. diff --git a/ratelimit/redis_rate_limit.py b/ratelimit/redis_rate_limit.py index e4f1ecb..440558e 100644 --- a/ratelimit/redis_rate_limit.py +++ b/ratelimit/redis_rate_limit.py @@ -96,4 +96,32 @@ def count(self, log_current_request=True): return result[-1] +class IpRateLimiter(RateLimiter): + def __init__(self, limit, window, connection, key): + super(IpRateLimiter, self).__init__(limit, window, connection, key) + self.connection = self._connection.connection + + def add(self, value): + self.connection.sadd(self._key, value) + self.connection.expire(self._key, self._window) + + def count(self): + return self.connection.scard(self._key) + + def delete(self): + self.connection.delete(self._key) + + def is_allowed(self, log_current_request=True): + if not self.connection.exists(self._key): + return True + if self.connection.ttl(self._key) < 0: + self.delete() + return True + return self.count() < self._limit + + @property + def remaining_requests(self): + return self._limit - self.count() + + redis_connection = RedisRateLimiterConnection(host=settings.REDIS_HOST_INTERNAL, port=6379, db=0) diff --git a/ratelimit/utils.py b/ratelimit/utils.py index 1acfb59..ae53020 100644 --- a/ratelimit/utils.py +++ b/ratelimit/utils.py @@ -1,4 +1,5 @@ import hashlib +import json import re import time import zlib @@ -7,7 +8,8 @@ from django.conf import settings from django.core.cache import caches from django.core.exceptions import ImproperlyConfigured -from redis_rate_limit import redis_connection, RedisRateLimiter + +from redis_rate_limit import redis_connection, RedisRateLimiter, IpRateLimiter from ratelimit import ALL, UNSAFE @@ -265,3 +267,32 @@ def is_authenticated(user): return user.is_authenticated() else: return user.is_authenticated + + +def get_cache_key_for_ip_blocking(request, func): + ip = _SIMPLE_KEYS['ip'](request) + name = func.__name__ + keys = [ip, name] + return 'ip_rl:' + hashlib.md5(u''.join(keys).encode('utf-8')).hexdigest() + + +def is_request_allowed(request, func, rate): + limit, period = _split_rate(rate) + cache_key = get_cache_key_for_ip_blocking(request, func) + redis_set = IpRateLimiter(limit=limit, window=period, connection=redis_connection, key=cache_key) + return redis_set.is_allowed() + + +def block_ip(request, func, function_to_get_attributes, rate): + limit, period = _split_rate(rate) + cache_key = get_cache_key_for_ip_blocking(request, func) + redis_set = IpRateLimiter(limit=limit, window=period, connection=redis_connection, key=cache_key) + hash_value = hashlib.md5(json.dumps(function_to_get_attributes(request))).hexdigest() + redis_set.add(hash_value) + + +def unblock_ip(request, func, rate): + limit, period = _split_rate(rate) + cache_key = get_cache_key_for_ip_blocking(request, func) + redis_set = IpRateLimiter(limit=limit, window=period, connection=redis_connection, key=cache_key) + redis_set.delete() From e05a32549f0f6f5c0c83a328419edc6d68f9b475 Mon Sep 17 00:00:00 2001 From: Veerendra Mandava Date: Thu, 12 Sep 2019 22:10:09 +0530 Subject: [PATCH 22/54] added utils functions to __all__ --- ratelimit/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ratelimit/utils.py b/ratelimit/utils.py index ae53020..df58f2d 100644 --- a/ratelimit/utils.py +++ b/ratelimit/utils.py @@ -13,7 +13,7 @@ from ratelimit import ALL, UNSAFE -__all__ = ['is_ratelimited'] +__all__ = ['is_ratelimited', 'unblock_ip', 'block_ip', 'is_request_allowed'] _PERIODS = { 's': 1, From 7e8b849ce3335341c038b13270a1484d7dab3b75 Mon Sep 17 00:00:00 2001 From: Veerendra Mandava Date: Thu, 12 Sep 2019 22:29:51 +0530 Subject: [PATCH 23/54] added a util function to check if rate is valid or not --- ratelimit/__init__.py | 2 +- ratelimit/utils.py | 19 ++++++++++++++++++- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/ratelimit/__init__.py b/ratelimit/__init__.py index b3e6efa..389e2aa 100644 --- a/ratelimit/__init__.py +++ b/ratelimit/__init__.py @@ -1,4 +1,4 @@ -VERSION = (1, 3, 2) +VERSION = (1, 3, 3) __version__ = '.'.join(map(str, VERSION)) ALL = (None,) # Sentinel value for all HTTP methods. diff --git a/ratelimit/utils.py b/ratelimit/utils.py index df58f2d..17e85fb 100644 --- a/ratelimit/utils.py +++ b/ratelimit/utils.py @@ -13,7 +13,7 @@ from ratelimit import ALL, UNSAFE -__all__ = ['is_ratelimited', 'unblock_ip', 'block_ip', 'is_request_allowed'] +__all__ = ['is_ratelimited', 'unblock_ip', 'block_ip', 'is_request_allowed', 'is_a_valid_rate'] _PERIODS = { 's': 1, @@ -75,6 +75,23 @@ def _split_rate(rate): return count, seconds +def is_a_valid_rate(rate): + if not isinstance(rate, str): + return False + count, multi, period = rate_re.match(rate).groups() + try: + count = int(count) + except ValueError as e: + return False + if multi: + try: + multi = int(multi) + except ValueError as e: + return False + if period not in _PERIODS: + return False + return True + def _get_window(value, period): ts = int(time.time()) if period == 1: From 79d5ff9c15de1d2c5b0489d0fed228f4e2aa0064 Mon Sep 17 00:00:00 2001 From: Veerendra Mandava Date: Thu, 12 Sep 2019 22:38:45 +0530 Subject: [PATCH 24/54] indentation --- ratelimit/utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ratelimit/utils.py b/ratelimit/utils.py index 17e85fb..968c3c8 100644 --- a/ratelimit/utils.py +++ b/ratelimit/utils.py @@ -92,6 +92,7 @@ def is_a_valid_rate(rate): return False return True + def _get_window(value, period): ts = int(time.time()) if period == 1: From da52b2657f20e1111e6f40e5efd6502e3296474d Mon Sep 17 00:00:00 2001 From: Veerendra Mandava Date: Mon, 16 Sep 2019 12:32:59 +0530 Subject: [PATCH 25/54] using pipeline --- ratelimit/__init__.py | 2 +- ratelimit/redis_rate_limit.py | 24 ++++++++++++++++-------- 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/ratelimit/__init__.py b/ratelimit/__init__.py index 389e2aa..950979d 100644 --- a/ratelimit/__init__.py +++ b/ratelimit/__init__.py @@ -1,4 +1,4 @@ -VERSION = (1, 3, 3) +VERSION = (1, 3, 4) __version__ = '.'.join(map(str, VERSION)) ALL = (None,) # Sentinel value for all HTTP methods. diff --git a/ratelimit/redis_rate_limit.py b/ratelimit/redis_rate_limit.py index 440558e..08bd472 100644 --- a/ratelimit/redis_rate_limit.py +++ b/ratelimit/redis_rate_limit.py @@ -99,25 +99,33 @@ def count(self, log_current_request=True): class IpRateLimiter(RateLimiter): def __init__(self, limit, window, connection, key): super(IpRateLimiter, self).__init__(limit, window, connection, key) - self.connection = self._connection.connection + self._pipeline = self._connection.connection.pipeline() def add(self, value): - self.connection.sadd(self._key, value) - self.connection.expire(self._key, self._window) + self._pipeline.sadd(self._key, value) + self._pipeline.expire(self._key, self._window) + self._pipeline.execute() def count(self): - return self.connection.scard(self._key) + self._pipeline.scard(self._key) + result = self._pipeline.execute() + return result[0] def delete(self): - self.connection.delete(self._key) + self._pipeline.delete(self._key) + self._pipeline.execute() def is_allowed(self, log_current_request=True): - if not self.connection.exists(self._key): + self._pipeline.exists(self._key) + self._pipeline.ttl(self._key) + self._pipeline.scard(self._key) + results = self._pipeline.execute() + if not results[0]: return True - if self.connection.ttl(self._key) < 0: + if results[1] < 0: self.delete() return True - return self.count() < self._limit + return results[2], self._limit @property def remaining_requests(self): From 854b369d7b2487c8815effa5d4770657ffd95b82 Mon Sep 17 00:00:00 2001 From: Veerendra Mandava Date: Mon, 16 Sep 2019 12:36:40 +0530 Subject: [PATCH 26/54] removed some useless util functions --- ratelimit/utils.py | 20 +------------------- 1 file changed, 1 insertion(+), 19 deletions(-) diff --git a/ratelimit/utils.py b/ratelimit/utils.py index 968c3c8..df58f2d 100644 --- a/ratelimit/utils.py +++ b/ratelimit/utils.py @@ -13,7 +13,7 @@ from ratelimit import ALL, UNSAFE -__all__ = ['is_ratelimited', 'unblock_ip', 'block_ip', 'is_request_allowed', 'is_a_valid_rate'] +__all__ = ['is_ratelimited', 'unblock_ip', 'block_ip', 'is_request_allowed'] _PERIODS = { 's': 1, @@ -75,24 +75,6 @@ def _split_rate(rate): return count, seconds -def is_a_valid_rate(rate): - if not isinstance(rate, str): - return False - count, multi, period = rate_re.match(rate).groups() - try: - count = int(count) - except ValueError as e: - return False - if multi: - try: - multi = int(multi) - except ValueError as e: - return False - if period not in _PERIODS: - return False - return True - - def _get_window(value, period): ts = int(time.time()) if period == 1: From 4fe8349d8d5b86b96934037a26ec4beb7b015932 Mon Sep 17 00:00:00 2001 From: Veerendra Mandava Date: Mon, 16 Sep 2019 12:59:20 +0530 Subject: [PATCH 27/54] error fix --- ratelimit/__init__.py | 2 +- ratelimit/redis_rate_limit.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ratelimit/__init__.py b/ratelimit/__init__.py index 950979d..fe39662 100644 --- a/ratelimit/__init__.py +++ b/ratelimit/__init__.py @@ -1,4 +1,4 @@ -VERSION = (1, 3, 4) +VERSION = (1, 3, 5) __version__ = '.'.join(map(str, VERSION)) ALL = (None,) # Sentinel value for all HTTP methods. diff --git a/ratelimit/redis_rate_limit.py b/ratelimit/redis_rate_limit.py index 08bd472..66226a4 100644 --- a/ratelimit/redis_rate_limit.py +++ b/ratelimit/redis_rate_limit.py @@ -125,7 +125,7 @@ def is_allowed(self, log_current_request=True): if results[1] < 0: self.delete() return True - return results[2], self._limit + return results[2] < self._limit @property def remaining_requests(self): From 8758025ffbe41b8ef57df4a2868bff1c41df2114 Mon Sep 17 00:00:00 2001 From: Veerendra Mandava Date: Mon, 16 Sep 2019 17:49:33 +0530 Subject: [PATCH 28/54] added a util function for getting the ip from the request --- ratelimit/__init__.py | 2 +- ratelimit/utils.py | 8 ++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/ratelimit/__init__.py b/ratelimit/__init__.py index fe39662..011d0e1 100644 --- a/ratelimit/__init__.py +++ b/ratelimit/__init__.py @@ -1,4 +1,4 @@ -VERSION = (1, 3, 5) +VERSION = (1, 3, 1) __version__ = '.'.join(map(str, VERSION)) ALL = (None,) # Sentinel value for all HTTP methods. diff --git a/ratelimit/utils.py b/ratelimit/utils.py index df58f2d..b60c7a9 100644 --- a/ratelimit/utils.py +++ b/ratelimit/utils.py @@ -13,7 +13,7 @@ from ratelimit import ALL, UNSAFE -__all__ = ['is_ratelimited', 'unblock_ip', 'block_ip', 'is_request_allowed'] +__all__ = ['is_ratelimited', 'unblock_ip', 'block_ip', 'is_request_allowed', 'get_ip_from_request'] _PERIODS = { 's': 1, @@ -270,7 +270,7 @@ def is_authenticated(user): def get_cache_key_for_ip_blocking(request, func): - ip = _SIMPLE_KEYS['ip'](request) + ip = get_ip_from_request(request) name = func.__name__ keys = [ip, name] return 'ip_rl:' + hashlib.md5(u''.join(keys).encode('utf-8')).hexdigest() @@ -296,3 +296,7 @@ def unblock_ip(request, func, rate): cache_key = get_cache_key_for_ip_blocking(request, func) redis_set = IpRateLimiter(limit=limit, window=period, connection=redis_connection, key=cache_key) redis_set.delete() + + +def get_ip_from_request(request): + return _SIMPLE_KEYS['ip'](request) From eacad1d82be8d583d2b29aabba21a9e65ff0dc1b Mon Sep 17 00:00:00 2001 From: Veerendra Mandava Date: Tue, 15 Oct 2019 14:41:20 +0530 Subject: [PATCH 29/54] considering the custom header --- ratelimit/utils.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/ratelimit/utils.py b/ratelimit/utils.py index b60c7a9..658f061 100644 --- a/ratelimit/utils.py +++ b/ratelimit/utils.py @@ -13,7 +13,7 @@ from ratelimit import ALL, UNSAFE -__all__ = ['is_ratelimited', 'unblock_ip', 'block_ip', 'is_request_allowed', 'get_ip_from_request'] +__all__ = ['is_ratelimited', 'unblock_ip', 'block_ip', 'is_request_allowed', 'get_custom_ip_from_request'] _PERIODS = { 's': 1, @@ -33,6 +33,7 @@ def user_or_ip(request): _SIMPLE_KEYS = { + 'custom_ip': lambda r: r.META.get('HTTP_X_' + settings.PROXY_PASS_CUSTOM_HEADER_NAME + "_CLIENT_IP", None), 'ip': lambda r: r.META['HTTP_X_FORWARDED_FOR'], 'user': lambda r: str(r.user.pk), 'user_or_ip': user_or_ip, @@ -270,7 +271,7 @@ def is_authenticated(user): def get_cache_key_for_ip_blocking(request, func): - ip = get_ip_from_request(request) + ip = get_custom_ip_from_request(request) name = func.__name__ keys = [ip, name] return 'ip_rl:' + hashlib.md5(u''.join(keys).encode('utf-8')).hexdigest() @@ -298,5 +299,5 @@ def unblock_ip(request, func, rate): redis_set.delete() -def get_ip_from_request(request): - return _SIMPLE_KEYS['ip'](request) +def get_custom_ip_from_request(request): + return _SIMPLE_KEYS['custom_ip'](request) From c1d5d7ab20028bb887f6b413844d5dab4bbf65e6 Mon Sep 17 00:00:00 2001 From: Veerendra Mandava Date: Tue, 15 Oct 2019 14:43:02 +0530 Subject: [PATCH 30/54] changed the version --- ratelimit/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ratelimit/__init__.py b/ratelimit/__init__.py index 011d0e1..b3e6efa 100644 --- a/ratelimit/__init__.py +++ b/ratelimit/__init__.py @@ -1,4 +1,4 @@ -VERSION = (1, 3, 1) +VERSION = (1, 3, 2) __version__ = '.'.join(map(str, VERSION)) ALL = (None,) # Sentinel value for all HTTP methods. From 9152987a783cf8695d09f5ce96e2a372b06f1141 Mon Sep 17 00:00:00 2001 From: Veerendra Mandava Date: Wed, 16 Oct 2019 10:42:46 +0530 Subject: [PATCH 31/54] changed the util function to get the custom ip --- ratelimit/__init__.py | 2 +- ratelimit/utils.py | 16 ++++++---------- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/ratelimit/__init__.py b/ratelimit/__init__.py index b3e6efa..389e2aa 100644 --- a/ratelimit/__init__.py +++ b/ratelimit/__init__.py @@ -1,4 +1,4 @@ -VERSION = (1, 3, 2) +VERSION = (1, 3, 3) __version__ = '.'.join(map(str, VERSION)) ALL = (None,) # Sentinel value for all HTTP methods. diff --git a/ratelimit/utils.py b/ratelimit/utils.py index 658f061..34141bf 100644 --- a/ratelimit/utils.py +++ b/ratelimit/utils.py @@ -13,7 +13,7 @@ from ratelimit import ALL, UNSAFE -__all__ = ['is_ratelimited', 'unblock_ip', 'block_ip', 'is_request_allowed', 'get_custom_ip_from_request'] +__all__ = ['is_ratelimited', 'block_ip', 'is_request_allowed', 'get_custom_ip_from_request'] _PERIODS = { 's': 1, @@ -33,7 +33,6 @@ def user_or_ip(request): _SIMPLE_KEYS = { - 'custom_ip': lambda r: r.META.get('HTTP_X_' + settings.PROXY_PASS_CUSTOM_HEADER_NAME + "_CLIENT_IP", None), 'ip': lambda r: r.META['HTTP_X_FORWARDED_FOR'], 'user': lambda r: str(r.user.pk), 'user_or_ip': user_or_ip, @@ -292,12 +291,9 @@ def block_ip(request, func, function_to_get_attributes, rate): redis_set.add(hash_value) -def unblock_ip(request, func, rate): - limit, period = _split_rate(rate) - cache_key = get_cache_key_for_ip_blocking(request, func) - redis_set = IpRateLimiter(limit=limit, window=period, connection=redis_connection, key=cache_key) - redis_set.delete() - - def get_custom_ip_from_request(request): - return _SIMPLE_KEYS['custom_ip'](request) + ips = request.META.get('HTTP_X_' + settings.PROXY_PASS_CUSTOM_HEADER_NAME + "_CLIENT_IP", None) + ips = ips.split(",") + if len(ips) == 0: + return None + return ips[-1] From 3d958b39f15f3a65e52a914445b74844c8d02f22 Mon Sep 17 00:00:00 2001 From: Veerendra Mandava Date: Wed, 16 Oct 2019 10:45:29 +0530 Subject: [PATCH 32/54] considering the request url for making cache key for ip blocking --- ratelimit/utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ratelimit/utils.py b/ratelimit/utils.py index 34141bf..e4119ad 100644 --- a/ratelimit/utils.py +++ b/ratelimit/utils.py @@ -272,7 +272,8 @@ def is_authenticated(user): def get_cache_key_for_ip_blocking(request, func): ip = get_custom_ip_from_request(request) name = func.__name__ - keys = [ip, name] + url = request.path + keys = [ip, name, url] return 'ip_rl:' + hashlib.md5(u''.join(keys).encode('utf-8')).hexdigest() From 0fb22f29f29c3b661a1a81d6b0f78caef089aae3 Mon Sep 17 00:00:00 2001 From: Veerendra Mandava Date: Wed, 16 Oct 2019 10:51:40 +0530 Subject: [PATCH 33/54] not updating the ttl for add to set --- ratelimit/redis_rate_limit.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ratelimit/redis_rate_limit.py b/ratelimit/redis_rate_limit.py index 66226a4..4511464 100644 --- a/ratelimit/redis_rate_limit.py +++ b/ratelimit/redis_rate_limit.py @@ -102,8 +102,11 @@ def __init__(self, limit, window, connection, key): self._pipeline = self._connection.connection.pipeline() def add(self, value): + self._pipeline.exists(self._key) + results = self._pipeline.execute() self._pipeline.sadd(self._key, value) - self._pipeline.expire(self._key, self._window) + if not results[0]: + self._pipeline.expire(self._key, self._window) self._pipeline.execute() def count(self): From b631aac1a3e900a9645bd7c428e7c407d41ad234 Mon Sep 17 00:00:00 2001 From: Veerendra Mandava Date: Wed, 16 Oct 2019 10:55:05 +0530 Subject: [PATCH 34/54] removed the not used arguments --- ratelimit/redis_rate_limit.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ratelimit/redis_rate_limit.py b/ratelimit/redis_rate_limit.py index 4511464..c03d314 100644 --- a/ratelimit/redis_rate_limit.py +++ b/ratelimit/redis_rate_limit.py @@ -118,7 +118,7 @@ def delete(self): self._pipeline.delete(self._key) self._pipeline.execute() - def is_allowed(self, log_current_request=True): + def is_allowed(self): self._pipeline.exists(self._key) self._pipeline.ttl(self._key) self._pipeline.scard(self._key) From 470124944ea3635aac37b2ed53d77b86a7065cea Mon Sep 17 00:00:00 2001 From: Veerendra Mandava Date: Wed, 16 Oct 2019 11:39:54 +0530 Subject: [PATCH 35/54] key error fix --- ratelimit/__init__.py | 2 +- ratelimit/utils.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/ratelimit/__init__.py b/ratelimit/__init__.py index 389e2aa..193df4c 100644 --- a/ratelimit/__init__.py +++ b/ratelimit/__init__.py @@ -1,4 +1,4 @@ -VERSION = (1, 3, 3) +VERSION = (1, 3, 10) __version__ = '.'.join(map(str, VERSION)) ALL = (None,) # Sentinel value for all HTTP methods. diff --git a/ratelimit/utils.py b/ratelimit/utils.py index e4119ad..b0e83bd 100644 --- a/ratelimit/utils.py +++ b/ratelimit/utils.py @@ -294,6 +294,8 @@ def block_ip(request, func, function_to_get_attributes, rate): def get_custom_ip_from_request(request): ips = request.META.get('HTTP_X_' + settings.PROXY_PASS_CUSTOM_HEADER_NAME + "_CLIENT_IP", None) + if ips in None: + return None ips = ips.split(",") if len(ips) == 0: return None From 7f3c542547368c2f0c6114f54fe6657e09ce44ec Mon Sep 17 00:00:00 2001 From: Veerendra Mandava Date: Wed, 16 Oct 2019 11:52:49 +0530 Subject: [PATCH 36/54] error fix --- ratelimit/__init__.py | 2 +- ratelimit/utils.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ratelimit/__init__.py b/ratelimit/__init__.py index 193df4c..bac6889 100644 --- a/ratelimit/__init__.py +++ b/ratelimit/__init__.py @@ -1,4 +1,4 @@ -VERSION = (1, 3, 10) +VERSION = (1, 3, 11) __version__ = '.'.join(map(str, VERSION)) ALL = (None,) # Sentinel value for all HTTP methods. diff --git a/ratelimit/utils.py b/ratelimit/utils.py index b0e83bd..c8554c3 100644 --- a/ratelimit/utils.py +++ b/ratelimit/utils.py @@ -294,7 +294,7 @@ def block_ip(request, func, function_to_get_attributes, rate): def get_custom_ip_from_request(request): ips = request.META.get('HTTP_X_' + settings.PROXY_PASS_CUSTOM_HEADER_NAME + "_CLIENT_IP", None) - if ips in None: + if ips is None: return None ips = ips.split(",") if len(ips) == 0: From 9702c4988a5c81e053d2a800009a5cc59873ec52 Mon Sep 17 00:00:00 2001 From: Veerendra Mandava Date: Wed, 16 Oct 2019 15:15:03 +0530 Subject: [PATCH 37/54] added an util function to get the right most public ip --- ratelimit/__init__.py | 2 +- ratelimit/utils.py | 13 ++++++++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/ratelimit/__init__.py b/ratelimit/__init__.py index bac6889..2c173d1 100644 --- a/ratelimit/__init__.py +++ b/ratelimit/__init__.py @@ -1,4 +1,4 @@ -VERSION = (1, 3, 11) +VERSION = (1, 3, 12) __version__ = '.'.join(map(str, VERSION)) ALL = (None,) # Sentinel value for all HTTP methods. diff --git a/ratelimit/utils.py b/ratelimit/utils.py index c8554c3..a06d672 100644 --- a/ratelimit/utils.py +++ b/ratelimit/utils.py @@ -60,6 +60,7 @@ def _method_match(request, method=ALL): rate_re = re.compile('([\d]+)/([\d]*)([smhd])?') +private_ip = re.compile("^172\.(1[6-9]|2[0-9]|3[0-1])\.[0-9]{1,3}\.[0-9]{1,3}$") def _split_rate(rate): @@ -292,6 +293,16 @@ def block_ip(request, func, function_to_get_attributes, rate): redis_set.add(hash_value) +def get_right_most_public_ip(ips): + index = len(ips) + while index > 0: + ip = ips[index - 1] + if not private_ip.match(ip): + return ip + index -= 1 + return None + + def get_custom_ip_from_request(request): ips = request.META.get('HTTP_X_' + settings.PROXY_PASS_CUSTOM_HEADER_NAME + "_CLIENT_IP", None) if ips is None: @@ -299,4 +310,4 @@ def get_custom_ip_from_request(request): ips = ips.split(",") if len(ips) == 0: return None - return ips[-1] + return get_right_most_public_ip(ips) From 64c01aecf66190391026ece00d45bcb8a1c75f34 Mon Sep 17 00:00:00 2001 From: Veerendra Mandava Date: Wed, 16 Oct 2019 15:29:32 +0530 Subject: [PATCH 38/54] changed the cache header to upper case --- ratelimit/__init__.py | 2 +- ratelimit/utils.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ratelimit/__init__.py b/ratelimit/__init__.py index 2c173d1..ab4b037 100644 --- a/ratelimit/__init__.py +++ b/ratelimit/__init__.py @@ -1,4 +1,4 @@ -VERSION = (1, 3, 12) +VERSION = (1, 3, 13) __version__ = '.'.join(map(str, VERSION)) ALL = (None,) # Sentinel value for all HTTP methods. diff --git a/ratelimit/utils.py b/ratelimit/utils.py index a06d672..67d9cc5 100644 --- a/ratelimit/utils.py +++ b/ratelimit/utils.py @@ -304,7 +304,7 @@ def get_right_most_public_ip(ips): def get_custom_ip_from_request(request): - ips = request.META.get('HTTP_X_' + settings.PROXY_PASS_CUSTOM_HEADER_NAME + "_CLIENT_IP", None) + ips = request.META.get('HTTP_X_' + settings.PROXY_PASS_CUSTOM_HEADER_NAME.upper() + "_CLIENT_IP", None) if ips is None: return None ips = ips.split(",") From 721c536e03866945614af8deda145a16997509e4 Mon Sep 17 00:00:00 2001 From: Veerendra Mandava Date: Wed, 16 Oct 2019 16:04:01 +0530 Subject: [PATCH 39/54] removed the ttl check --- ratelimit/__init__.py | 2 +- ratelimit/redis_rate_limit.py | 4 ---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/ratelimit/__init__.py b/ratelimit/__init__.py index ab4b037..011d0e1 100644 --- a/ratelimit/__init__.py +++ b/ratelimit/__init__.py @@ -1,4 +1,4 @@ -VERSION = (1, 3, 13) +VERSION = (1, 3, 1) __version__ = '.'.join(map(str, VERSION)) ALL = (None,) # Sentinel value for all HTTP methods. diff --git a/ratelimit/redis_rate_limit.py b/ratelimit/redis_rate_limit.py index c03d314..28a4606 100644 --- a/ratelimit/redis_rate_limit.py +++ b/ratelimit/redis_rate_limit.py @@ -120,14 +120,10 @@ def delete(self): def is_allowed(self): self._pipeline.exists(self._key) - self._pipeline.ttl(self._key) self._pipeline.scard(self._key) results = self._pipeline.execute() if not results[0]: return True - if results[1] < 0: - self.delete() - return True return results[2] < self._limit @property From f9e3a944f753b1b595f30db638eff5b6dc6fb98e Mon Sep 17 00:00:00 2001 From: Veerendra Mandava Date: Wed, 16 Oct 2019 16:04:43 +0530 Subject: [PATCH 40/54] error fix --- ratelimit/redis_rate_limit.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ratelimit/redis_rate_limit.py b/ratelimit/redis_rate_limit.py index 28a4606..611ecf9 100644 --- a/ratelimit/redis_rate_limit.py +++ b/ratelimit/redis_rate_limit.py @@ -124,7 +124,7 @@ def is_allowed(self): results = self._pipeline.execute() if not results[0]: return True - return results[2] < self._limit + return results[1] < self._limit @property def remaining_requests(self): From f425e5d293aba0f742051cab36e5e8791b63c61a Mon Sep 17 00:00:00 2001 From: Veerendra Mandava Date: Thu, 17 Oct 2019 13:00:33 +0530 Subject: [PATCH 41/54] using custom header everywhere for ratelimiting --- ratelimit/__init__.py | 2 +- ratelimit/utils.py | 11 +++++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/ratelimit/__init__.py b/ratelimit/__init__.py index 011d0e1..b3e6efa 100644 --- a/ratelimit/__init__.py +++ b/ratelimit/__init__.py @@ -1,4 +1,4 @@ -VERSION = (1, 3, 1) +VERSION = (1, 3, 2) __version__ = '.'.join(map(str, VERSION)) ALL = (None,) # Sentinel value for all HTTP methods. diff --git a/ratelimit/utils.py b/ratelimit/utils.py index 67d9cc5..aa238f4 100644 --- a/ratelimit/utils.py +++ b/ratelimit/utils.py @@ -26,14 +26,21 @@ EXPIRATION_FUDGE = 5 +def get_ip(request): + custom_ip = get_custom_ip_from_request(request) + if custom_ip: + return custom_ip + return request.META['HTTP_X_FORWARDED_FOR'] + + def user_or_ip(request): if is_authenticated(request.user): return str(request.user.pk) - return request.META['HTTP_X_FORWARDED_FOR'] + return get_ip(request) _SIMPLE_KEYS = { - 'ip': lambda r: r.META['HTTP_X_FORWARDED_FOR'], + 'ip': get_ip, 'user': lambda r: str(r.user.pk), 'user_or_ip': user_or_ip, } From 9fbcbf50274f2c532afac0146efdb57d64dd503d Mon Sep 17 00:00:00 2001 From: Veerendra Mandava Date: Thu, 17 Oct 2019 13:02:29 +0530 Subject: [PATCH 42/54] revertred the version --- ratelimit/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ratelimit/__init__.py b/ratelimit/__init__.py index b3e6efa..011d0e1 100644 --- a/ratelimit/__init__.py +++ b/ratelimit/__init__.py @@ -1,4 +1,4 @@ -VERSION = (1, 3, 2) +VERSION = (1, 3, 1) __version__ = '.'.join(map(str, VERSION)) ALL = (None,) # Sentinel value for all HTTP methods. From 3696f87c391bbc55da59c81b0619878ad6419baf Mon Sep 17 00:00:00 2001 From: Veerendra Mandava Date: Thu, 17 Oct 2019 13:44:53 +0530 Subject: [PATCH 43/54] changed the version --- ratelimit/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ratelimit/__init__.py b/ratelimit/__init__.py index 011d0e1..b3e6efa 100644 --- a/ratelimit/__init__.py +++ b/ratelimit/__init__.py @@ -1,4 +1,4 @@ -VERSION = (1, 3, 1) +VERSION = (1, 3, 2) __version__ = '.'.join(map(str, VERSION)) ALL = (None,) # Sentinel value for all HTTP methods. From c3e54af021eb3b1815de4019c4ddd0ea3a2188a9 Mon Sep 17 00:00:00 2001 From: Veerendra Mandava Date: Thu, 17 Oct 2019 15:05:06 +0530 Subject: [PATCH 44/54] changed the cache key --- ratelimit/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ratelimit/utils.py b/ratelimit/utils.py index aa238f4..64da8f7 100644 --- a/ratelimit/utils.py +++ b/ratelimit/utils.py @@ -282,7 +282,7 @@ def get_cache_key_for_ip_blocking(request, func): name = func.__name__ url = request.path keys = [ip, name, url] - return 'ip_rl:' + hashlib.md5(u''.join(keys).encode('utf-8')).hexdigest() + return 'ip_rl_v2:' + hashlib.md5(u''.join(keys).encode('utf-8')).hexdigest() def is_request_allowed(request, func, rate): From 8894cade160c3b1449c37245eaf3ddf6f99fbed6 Mon Sep 17 00:00:00 2001 From: Veerendra Mandava Date: Thu, 2 Apr 2020 14:19:10 +0530 Subject: [PATCH 45/54] added utils for region based ratelimiting --- ratelimit/utils.py | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/ratelimit/utils.py b/ratelimit/utils.py index 64da8f7..9d1ed90 100644 --- a/ratelimit/utils.py +++ b/ratelimit/utils.py @@ -13,7 +13,8 @@ from ratelimit import ALL, UNSAFE -__all__ = ['is_ratelimited', 'block_ip', 'is_request_allowed', 'get_custom_ip_from_request'] +__all__ = ['is_ratelimited', 'block_ip', 'is_request_allowed', 'get_custom_ip_from_request', + 'get_region_code_from_request', 'is_request_from_region_allowed', 'block_region'] _PERIODS = { 's': 1, @@ -318,3 +319,30 @@ def get_custom_ip_from_request(request): if len(ips) == 0: return None return get_right_most_public_ip(ips) + + +def get_cache_key_for_region_blocking(request, func): + country_code = get_region_code_from_request(request) + name = func.__name__ + url = request.path + keys = [country_code, name, url] + return 'rl_region:' + hashlib.md5(u''.join(keys).encode('utf-8')).hexdigest() + + +def get_region_code_from_request(request): + return request.META.get('HTTP_X_' + settings.PROXY_PASS_CUSTOM_HEADER_NAME.upper() + "_COUNTRY_CODE", None) + + +def is_request_from_region_allowed(request, func, rate): + limit, period = _split_rate(rate) + cache_key = get_cache_key_for_region_blocking(request, func) + redis_set = IpRateLimiter(limit=limit, window=period, connection=redis_connection, key=cache_key) + return redis_set.is_allowed() + + +def block_region(request, func, function_to_get_attributes, rate): + limit, period = _split_rate(rate) + cache_key = get_cache_key_for_region_blocking(request, func) + redis_set = IpRateLimiter(limit=limit, window=period, connection=redis_connection, key=cache_key) + hash_value = hashlib.md5(json.dumps(function_to_get_attributes(request))).hexdigest() + redis_set.add(hash_value) From 374b5cfd1d3fed1134d3f35091c66b2efe8e3ef6 Mon Sep 17 00:00:00 2001 From: Veerendra Mandava Date: Thu, 2 Apr 2020 15:03:23 +0530 Subject: [PATCH 46/54] changed the version --- ratelimit/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ratelimit/__init__.py b/ratelimit/__init__.py index b3e6efa..389e2aa 100644 --- a/ratelimit/__init__.py +++ b/ratelimit/__init__.py @@ -1,4 +1,4 @@ -VERSION = (1, 3, 2) +VERSION = (1, 3, 3) __version__ = '.'.join(map(str, VERSION)) ALL = (None,) # Sentinel value for all HTTP methods. From c243e75a192bff2de56c94e189c92c533e0677fa Mon Sep 17 00:00:00 2001 From: Alok Maurya Date: Fri, 3 Sep 2021 00:50:05 +0530 Subject: [PATCH 47/54] use redis cluster --- ratelimit/redis_rate_limit.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ratelimit/redis_rate_limit.py b/ratelimit/redis_rate_limit.py index 611ecf9..bd701b8 100644 --- a/ratelimit/redis_rate_limit.py +++ b/ratelimit/redis_rate_limit.py @@ -2,7 +2,7 @@ from functools import wraps from django.conf import settings -import redis +from rediscluster import StrictRedisCluster from .exceptions import RateLimited, DatastoreConnectionError __author__ = 'vikaschahal' @@ -49,10 +49,10 @@ def decorated(*args, **kwargs): class RedisRateLimiterConnection(object): - def __init__(self, host=None, port=None, db=0, connection=None): + def __init__(self, host=None, connection=None): self.connection = None if host and port: - connection = redis.StrictRedis(host, port, db) + connection = StrictRedisCluster(startup_nodes=[{'host': host, 'port': 6379}]) if not connection.ping(): raise DatastoreConnectionError self.connection = connection @@ -131,4 +131,4 @@ def remaining_requests(self): return self._limit - self.count() -redis_connection = RedisRateLimiterConnection(host=settings.REDIS_HOST_INTERNAL, port=6379, db=0) +redis_connection = RedisRateLimiterConnection(host=settings.REDIS_HOST_INTERNAL_NEW) From 1b5b61519b1a9c38a60ff809a2dd9e278f88e411 Mon Sep 17 00:00:00 2001 From: Alok Maurya Date: Fri, 3 Sep 2021 01:01:50 +0530 Subject: [PATCH 48/54] Update __init__.py version update --- ratelimit/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ratelimit/__init__.py b/ratelimit/__init__.py index 389e2aa..950979d 100644 --- a/ratelimit/__init__.py +++ b/ratelimit/__init__.py @@ -1,4 +1,4 @@ -VERSION = (1, 3, 3) +VERSION = (1, 3, 4) __version__ = '.'.join(map(str, VERSION)) ALL = (None,) # Sentinel value for all HTTP methods. From 1ab39755878f1650a41bc2eff82521cf98225f24 Mon Sep 17 00:00:00 2001 From: Alok Maurya Date: Fri, 3 Sep 2021 01:05:45 +0530 Subject: [PATCH 49/54] cluster issue fix --- ratelimit/redis_rate_limit.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ratelimit/redis_rate_limit.py b/ratelimit/redis_rate_limit.py index bd701b8..dd85d9c 100644 --- a/ratelimit/redis_rate_limit.py +++ b/ratelimit/redis_rate_limit.py @@ -52,7 +52,7 @@ class RedisRateLimiterConnection(object): def __init__(self, host=None, connection=None): self.connection = None if host and port: - connection = StrictRedisCluster(startup_nodes=[{'host': host, 'port': 6379}]) + connection = StrictRedisCluster(startup_nodes=[{'host': host, 'port': 6379}], skip_full_coverage_check=True) if not connection.ping(): raise DatastoreConnectionError self.connection = connection From 6d7bfa5beaebc0e33a7527517d71b6fb62691f80 Mon Sep 17 00:00:00 2001 From: Alok Maurya Date: Fri, 3 Sep 2021 01:06:02 +0530 Subject: [PATCH 50/54] version update --- ratelimit/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ratelimit/__init__.py b/ratelimit/__init__.py index 950979d..fe39662 100644 --- a/ratelimit/__init__.py +++ b/ratelimit/__init__.py @@ -1,4 +1,4 @@ -VERSION = (1, 3, 4) +VERSION = (1, 3, 5) __version__ = '.'.join(map(str, VERSION)) ALL = (None,) # Sentinel value for all HTTP methods. From d7774484bc9ee997857b5236ad3b421adaaaf629 Mon Sep 17 00:00:00 2001 From: Alok Maurya Date: Fri, 3 Sep 2021 01:08:11 +0530 Subject: [PATCH 51/54] port variable removed --- ratelimit/redis_rate_limit.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ratelimit/redis_rate_limit.py b/ratelimit/redis_rate_limit.py index dd85d9c..9b2017a 100644 --- a/ratelimit/redis_rate_limit.py +++ b/ratelimit/redis_rate_limit.py @@ -51,7 +51,7 @@ def decorated(*args, **kwargs): class RedisRateLimiterConnection(object): def __init__(self, host=None, connection=None): self.connection = None - if host and port: + if host: connection = StrictRedisCluster(startup_nodes=[{'host': host, 'port': 6379}], skip_full_coverage_check=True) if not connection.ping(): raise DatastoreConnectionError From bad1f5bcc62baf83e6b807c76d0a61fafc0316b5 Mon Sep 17 00:00:00 2001 From: Alok Maurya Date: Fri, 3 Sep 2021 01:08:31 +0530 Subject: [PATCH 52/54] version update --- ratelimit/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ratelimit/__init__.py b/ratelimit/__init__.py index fe39662..8aa5b9b 100644 --- a/ratelimit/__init__.py +++ b/ratelimit/__init__.py @@ -1,4 +1,4 @@ -VERSION = (1, 3, 5) +VERSION = (1, 3, 6) __version__ = '.'.join(map(str, VERSION)) ALL = (None,) # Sentinel value for all HTTP methods. From d5ed17460edda1322f09c1ba2e74c9e6d1c4d1ce Mon Sep 17 00:00:00 2001 From: sagarunacademy <56915697+sagarunacademy@users.noreply.github.com> Date: Fri, 3 Sep 2021 21:28:42 +0530 Subject: [PATCH 53/54] redis connection and redis cluster connection --- .gitignore | 1 + ratelimit/__init__.py | 2 +- ratelimit/redis_rate_limit.py | 13 ++++++++++--- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index c47828c..0bfc0eb 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ dist *.egg-info docs/_build/* .tox/ +.idea/ diff --git a/ratelimit/__init__.py b/ratelimit/__init__.py index 8aa5b9b..5362284 100644 --- a/ratelimit/__init__.py +++ b/ratelimit/__init__.py @@ -1,4 +1,4 @@ -VERSION = (1, 3, 6) +VERSION = (1, 3, 7) __version__ = '.'.join(map(str, VERSION)) ALL = (None,) # Sentinel value for all HTTP methods. diff --git a/ratelimit/redis_rate_limit.py b/ratelimit/redis_rate_limit.py index 9b2017a..643f1ca 100644 --- a/ratelimit/redis_rate_limit.py +++ b/ratelimit/redis_rate_limit.py @@ -1,4 +1,5 @@ import time +import redis from functools import wraps from django.conf import settings @@ -49,10 +50,13 @@ def decorated(*args, **kwargs): class RedisRateLimiterConnection(object): - def __init__(self, host=None, connection=None): + def __init__(self, host=None, port=None, db=0, connection=None): self.connection = None if host: - connection = StrictRedisCluster(startup_nodes=[{'host': host, 'port': 6379}], skip_full_coverage_check=True) + if settings.REDIS_HOST_INTERNAL_NEW_IS_CLUSTER: + connection = StrictRedisCluster(startup_nodes=[{'host': host, 'port': 6379}], skip_full_coverage_check=True) + else: + connection = redis.StrictRedis(host, port, db) if not connection.ping(): raise DatastoreConnectionError self.connection = connection @@ -131,4 +135,7 @@ def remaining_requests(self): return self._limit - self.count() -redis_connection = RedisRateLimiterConnection(host=settings.REDIS_HOST_INTERNAL_NEW) +if settings.REDIS_HOST_INTERNAL_NEW_IS_CLUSTER: + redis_connection = RedisRateLimiterConnection(host=settings.REDIS_HOST_INTERNAL_NEW) +else: + redis_connection = RedisRateLimiterConnection(host=settings.REDIS_HOST_INTERNAL, port=6379, db=0) \ No newline at end of file From 10f61241539605eaf5bb0b0bac788a160199c6b8 Mon Sep 17 00:00:00 2001 From: adityakumarsarvaiya <92515255+adityakumarsarvaiya@users.noreply.github.com> Date: Tue, 20 Jun 2023 18:23:45 +0530 Subject: [PATCH 54/54] Added codeowners file. --- CODEOWNERS | 1 + 1 file changed, 1 insertion(+) create mode 100644 CODEOWNERS diff --git a/CODEOWNERS b/CODEOWNERS new file mode 100644 index 0000000..860d1e1 --- /dev/null +++ b/CODEOWNERS @@ -0,0 +1 @@ +.fabric/ @unacademy/sre \ No newline at end of file